Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nick Sweeting
|
5750f9be76 |
No files matched your search
@@ -10,7 +10,6 @@ on:
|
||||
- bin/collect_ui_screenshots.*
|
||||
- bin/generate_ui_screenshot_gallery.py
|
||||
- bin/take_screenshot.js
|
||||
- docs/Screenshots.md
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
- .github/workflows/deploy-publicsite.yml
|
||||
|
||||
@@ -64,4 +64,4 @@ repos:
|
||||
args: ["--maxkb=600"]
|
||||
- id: name-tests-test
|
||||
args: ["--pytest-test-first"]
|
||||
exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
|
||||
exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$|import_helpers\.py$)
|
||||
@@ -1,5 +1,5 @@
|
||||
<div align="center" style="text-align: center; width: 100%">
|
||||
<img src="https://archivebox.io/icon.png" height="90px"/>
|
||||
<img style="height: 90px" src="https://archivebox.io/icon.png"/>
|
||||
<h1>ArchiveBox<br/><sub>Open-source self-hosted web archiving.</sub></h1>
|
||||
|
||||
<br/>
|
||||
@@ -108,7 +108,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
|
||||
|
||||
<div align="center" style="text-align: center">
|
||||
<br/><br/>
|
||||
<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
|
||||
<img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic"> <img style="height: 75px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" alt="logo" align="top"/> <img style="height: 70px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" alt="bookshelf graphic">
|
||||
<br/><br/>
|
||||
<small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
|
||||
<br/>
|
||||
@@ -152,7 +152,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
|
||||
#### ✳️ Easy Setup
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b> (macOS/Linux) <b>👈 recommended</b> <i>(click to expand)</i></summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker-compose</code></b> (macOS/Linux) <b>👈 recommended</b> <i>(click to expand)</i></summary>
|
||||
<br/>
|
||||
<i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all <a href="#dependencies">extras</a> out-of-the-box.</i>
|
||||
<br/><br/>
|
||||
@@ -181,7 +181,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b> (macOS/Linux)</summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker run</code></b> (macOS/Linux)</summary>
|
||||
<br/>
|
||||
<ol>
|
||||
<li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
|
||||
@@ -205,7 +205,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" height="28px" align="top"/> <code>bash</code> uv install shortcut</b> (macOS/Linux/BSD)</summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh uv install shortcut" align="top"/> <code>bash</code> uv install shortcut</b> (macOS/Linux/BSD)</summary>
|
||||
<br/>
|
||||
<ol>
|
||||
<li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
|
||||
@@ -228,7 +228,7 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
|
||||
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
|
||||
<br/>
|
||||
<ol>
|
||||
|
||||
@@ -262,7 +262,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
|
||||
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
|
||||
<br/>
|
||||
<ol>
|
||||
<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
|
||||
@@ -295,7 +295,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
|
||||
<br/>
|
||||
Run Homebrew as your normal non-root user on both macOS and Linux; do not use <code>sudo brew</code>.
|
||||
<ol>
|
||||
@@ -329,7 +329,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
|
||||
<summary><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" align="top"/> <code>pacman</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" align="top"/> <code>pkg</code> / <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
|
||||
<br/>
|
||||
|
||||
> *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
|
||||
@@ -350,7 +350,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
|
||||
#### 🎗 Other Options
|
||||
|
||||
<details>
|
||||
<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker</code> + <img src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" height="28px" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker</code> + <img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447263-4316c980-af0b-11eb-928d-eaf1292ac646.png" alt="Electron" align="top"/> <code>electron</code> Desktop App</b> (macOS/Linux/Windows)</summary>
|
||||
<br/>
|
||||
<ol>
|
||||
<li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
|
||||
@@ -369,7 +369,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" height="28px" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
|
||||
<summary><img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
|
||||
<br/>
|
||||
|
||||
> *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
|
||||
@@ -393,26 +393,26 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
|
||||
<summary><img style="height: 27px" src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" align="top"/> Third-party paid hosting options (cloud VPS)</summary>
|
||||
<br/>
|
||||
<ul>
|
||||
<li><a href="https://elest.io/open-source/archivebox"><img src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat" height="22px"/></a></li>
|
||||
<li><a href="https://www.stellarhosted.com/archivebox/"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat" height="22px"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
|
||||
<li><a href="https://www.pikapods.com/pods?run=archivebox"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat" height="22px"/></a> (from USD $2.6/mo)</li>
|
||||
<li><a href="https://elest.io/open-source/archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat"/></a></li>
|
||||
<li><a href="https://www.stellarhosted.com/archivebox/"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
|
||||
<li><a href="https://www.pikapods.com/pods?run=archivebox"><img style="height: 22px" src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat"/></a> (from USD $2.6/mo)</li>
|
||||
<li><a href="https://m.do.co/c/cbc4c0c17840">
|
||||
<img src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat" height="22px"/>
|
||||
<img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-DigitalOcean.com-%232f7cf7.svg?style=flat"/>
|
||||
</a> (USD $5-50+/mo, <a href="https://m.do.co/c/cbc4c0c17840">🎗 referral link</a>, <a href="https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
|
||||
<li><a href="https://www.vultr.com/?ref=7130289">
|
||||
<img src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat" height="22px"/>
|
||||
<img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Vultr.com-%232337a8.svg?style=flat"/>
|
||||
</a> (USD $2.5-50+/mo, <a href="https://www.vultr.com/?ref=7130289">🎗 referral link</a>, <a href="https://www.vultr.com/docs/install-docker-compose-on-ubuntu-20-04">instructions</a>)</li>
|
||||
<li><a href="https://fly.io/">
|
||||
<img src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat" height="22px"/>
|
||||
<img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat"/>
|
||||
</a> (USD $10-50+/mo, <a href="https://fly.io/docs/hands-on/start/">instructions</a>)</li>
|
||||
<li><a href="https://railway.app/template/2Vvhmy">
|
||||
<img src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat" height="22px"/>
|
||||
<img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat"/>
|
||||
</a> (USD $0-5+/mo)</li>
|
||||
<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
|
||||
<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
|
||||
<li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat"/></a> (USD $60-200+/mo)</li>
|
||||
<li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img style="height: 22px" src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat"/></a> (USD $60-200+/mo)</li>
|
||||
<br/>
|
||||
<sub><i>Referral links marked 🎗 provide $5-10 of free credit for new users and help pay for our <a href="https://demo.archivebox.io">demo server</a> hosting costs.</i></sub>
|
||||
</ul>
|
||||
@@ -465,7 +465,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev help
|
||||
|
||||
<br/>
|
||||
<details>
|
||||
<summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
|
||||
<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
|
||||
<br/>
|
||||
<pre lang="bash"><code style="white-space: pre-line">
|
||||
# make sure you have uv-installed ArchiveBox and it's available in your $PATH first
|
||||
@@ -484,7 +484,7 @@ archivebox help # get list of archivebox subcommands that can be ru
|
||||
<br/>
|
||||
|
||||
<details>
|
||||
<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
|
||||
<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker Compose</b></summary>
|
||||
<br/>
|
||||
<pre lang="bash"><code style="white-space: pre-line">
|
||||
# make sure you have `docker-compose.yml` from the Quickstart instructions first
|
||||
@@ -501,7 +501,7 @@ docker compose exec archivebox archivebox add 'https://example.com'
|
||||
<br/>
|
||||
|
||||
<details>
|
||||
<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
|
||||
<summary><img style="height: 22px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
|
||||
<br/>
|
||||
<pre lang="bash"><code style="white-space: pre-line">
|
||||
# make sure the `archivebox` server container from the Quickstart is running first
|
||||
@@ -613,19 +613,19 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
|
||||
## Input Formats: How to pass URLs into ArchiveBox for saving
|
||||
|
||||
|
||||
- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="28px"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>
|
||||
- <img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b><br>
|
||||
<i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox browsers.</i>
|
||||
|
||||
- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files
|
||||
- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files<br>
|
||||
<i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
|
||||
|
||||
- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)
|
||||
- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)<br>
|
||||
<i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
|
||||
|
||||
- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12" height="22px"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)
|
||||
- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4f7bd318-265c-4235-ad25-38be89946b12"/> From URLs visited through a [MITM Proxy](https://mitmproxy.org/) with [`archivebox-proxy`](https://github.com/ArchiveBox/archivebox-proxy)<br>
|
||||
<i>Provides [realtime archiving](https://github.com/ArchiveBox/ArchiveBox/issues/577) of all traffic from any device going through the proxy.</i>
|
||||
|
||||
- <img src="https://getpocket.com/favicon.ico" height="22px"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)
|
||||
- <img style="height: 22px" src="https://getpocket.com/favicon.ico"/> From bookmarking services or social media (e.g. Twitter bookmarks, Reddit saved posts, etc.)<br>
|
||||
<i>Instructions: <a href="https://getpocket.com/export">Pocket</a>, <a href="https://pinboard.in/export/">Pinboard</a>, <a href="https://www.instapaper.com/user">Instapaper</a>, <a href="https://shaarli.readthedocs.io/en/master/Usage/#importexport">Shaarli</a>, <a href="https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/">Delicious</a>, <a href="https://github.com/csu/export-saved-reddit">Reddit Saved</a>, <a href="https://doc.wallabag.org/en/user/import/wallabagv2.html">Wallabag</a>, <a href="http://help.unmark.it/import-export">Unmark.it</a>, <a href="https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/">OneTab</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/issues/648">Firefox Sync</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
|
||||
|
||||
|
||||
@@ -1009,7 +1009,7 @@ Each re-archive creates a distinct Snapshot row for the same URL:
|
||||
archivebox add --no-only-new 'https://example.com'
|
||||
</code></pre>
|
||||
|
||||
The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
|
||||
The <img style="height: 24px" src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button"/> button in the Admin UI performs the same explicit re-archive.
|
||||
|
||||
<h4>Learn More</h4>
|
||||
|
||||
@@ -1328,7 +1328,7 @@ archivebox server 0.0.0.0:8000
|
||||
|
||||
```bash
|
||||
# Optional: develop via docker by mounting the code dir into the container
|
||||
# if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
|
||||
# if you edit e.g. ./archivebox/core/models/snapshots.py on the docker host, runserver
|
||||
# inside the container will reload and pick up your changes
|
||||
./bin/build_docker.sh dev
|
||||
|
||||
|
||||
+9
-57
@@ -8,7 +8,7 @@ from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
|
||||
from signal_webhooks.settings import webhook_settings
|
||||
from signal_webhooks.utils import get_webhook_model, model_from_reference
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
|
||||
|
||||
from archivebox.api.models import APIToken
|
||||
|
||||
@@ -25,27 +25,9 @@ class APITokenAdmin(BaseModelAdmin):
|
||||
search_fields = ("id", "created_by__username", "token")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Token",
|
||||
{
|
||||
"fields": ("token", "expires"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Owner",
|
||||
{
|
||||
"fields": ("created_by",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Token", ("token", "expires")),
|
||||
card_fieldset("Owner", ("created_by",)),
|
||||
card_fieldset("Timestamps", ("created_at", "modified_at")),
|
||||
)
|
||||
|
||||
list_filter = ("created_by",)
|
||||
@@ -73,41 +55,11 @@ class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
|
||||
readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Webhook",
|
||||
{
|
||||
"fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Authentication",
|
||||
{
|
||||
"fields": _webhook_fields("auth_token"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Status",
|
||||
{
|
||||
"fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Owner",
|
||||
{
|
||||
"fields": _webhook_fields("created_by"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": _webhook_fields("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Webhook", _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"), wide=True),
|
||||
card_fieldset("Authentication", _webhook_fields("auth_token")),
|
||||
card_fieldset("Status", _webhook_fields("enabled", "last_success", "last_failure", "last_response")),
|
||||
card_fieldset("Owner", _webhook_fields("created_by")),
|
||||
card_fieldset("Timestamps", _webhook_fields("created_at", "modified_at")),
|
||||
)
|
||||
|
||||
def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Fields shared by responses for user-owned model objects."""
|
||||
|
||||
from ninja import Schema
|
||||
|
||||
|
||||
class OwnedObjectSchema(Schema):
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by.pk)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj) -> str:
|
||||
user = obj.created_by
|
||||
return user.username if isinstance(user.username, str) else str(user)
|
||||
+34
-90
@@ -117,16 +117,27 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
|
||||
return kwargs
|
||||
|
||||
|
||||
def cli_response(request, result, *, result_format="json", success=True, errors=()):
|
||||
"""Use one response envelope for CLI-backed endpoints, including captured output."""
|
||||
payload = {"success": success, "errors": list(errors), "result": result, "result_format": result_format}
|
||||
for stream_name in ("stdout", "stderr"):
|
||||
stream = request.__dict__.get(stream_name)
|
||||
payload[stream_name] = ansi_to_html(stream.getvalue().strip()) if isinstance(stream, StringIO) else ""
|
||||
return payload
|
||||
|
||||
|
||||
def command_config(args) -> dict[str, object] | None:
|
||||
"""Translate shared legacy add/schedule flags into runtime config overrides."""
|
||||
if args.update or args.overwrite:
|
||||
return {"ONLY_NEW": False}
|
||||
return {"ONLY_NEW": bool(args.only_new)} if args.only_new is not None else None
|
||||
|
||||
|
||||
@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
|
||||
def cli_add(request: HttpRequest, args: AddCommandSchema):
|
||||
from archivebox.cli.archivebox_add import add
|
||||
from archivebox.misc.util import validate_url
|
||||
|
||||
config_overrides: dict[str, object] = {}
|
||||
if args.only_new is not None:
|
||||
config_overrides["ONLY_NEW"] = bool(args.only_new)
|
||||
if args.update or args.overwrite:
|
||||
config_overrides["ONLY_NEW"] = False
|
||||
submitted_urls: str | list[str] = args.urls
|
||||
if len(args.urls) == 1:
|
||||
try:
|
||||
@@ -144,20 +155,10 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
|
||||
raise HttpError(400, "No valid URLs were submitted")
|
||||
crawl, snapshots = add(
|
||||
urls=submitted_urls,
|
||||
snapshot_ids=args.snapshot_ids,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
max_urls=args.max_urls,
|
||||
crawl_max_size=args.crawl_max_size,
|
||||
crawl_timeout=args.crawl_timeout,
|
||||
snapshot_max_size=args.snapshot_max_size,
|
||||
index_only=args.index_only,
|
||||
plugins=args.plugins,
|
||||
persona=args.persona,
|
||||
parser=args.parser,
|
||||
**args.model_dump(exclude={"urls", "only_new", "update", "overwrite"}),
|
||||
bg=True, # Always run in background for API calls
|
||||
created_by_id=request.user.pk,
|
||||
config=config_overrides or None,
|
||||
config=command_config(args),
|
||||
)
|
||||
|
||||
snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
|
||||
@@ -167,17 +168,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
|
||||
"snapshot_ids": snapshot_ids,
|
||||
"queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
|
||||
}
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result_payload,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
return cli_response(request, result_payload, result_format="json")
|
||||
|
||||
|
||||
@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
|
||||
@@ -204,55 +195,20 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
|
||||
]
|
||||
|
||||
update(**update_kwargs)
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": {
|
||||
"matched_count": len(matched_snapshot_ids),
|
||||
"snapshot_ids": matched_snapshot_ids,
|
||||
}
|
||||
if is_filtered_update
|
||||
else None,
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
return cli_response(
|
||||
request,
|
||||
{"matched_count": len(matched_snapshot_ids), "snapshot_ids": matched_snapshot_ids} if is_filtered_update else None,
|
||||
result_format="str",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
|
||||
def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
|
||||
from archivebox.cli.archivebox_schedule import schedule
|
||||
|
||||
config_overrides: dict[str, object] = {}
|
||||
if args.only_new is not None:
|
||||
config_overrides["ONLY_NEW"] = bool(args.only_new)
|
||||
if args.update or args.overwrite:
|
||||
config_overrides["ONLY_NEW"] = False
|
||||
result = schedule(
|
||||
import_path=args.import_path,
|
||||
add=args.add,
|
||||
show=args.show,
|
||||
foreground=args.foreground,
|
||||
run_all=args.run_all,
|
||||
quiet=args.quiet,
|
||||
clear=args.clear,
|
||||
every=args.every,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
config=config_overrides or None,
|
||||
)
|
||||
result = schedule(**args.model_dump(exclude={"only_new", "update", "overwrite"}), config=command_config(args))
|
||||
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
return cli_response(request, result, result_format="json")
|
||||
|
||||
|
||||
@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
|
||||
@@ -287,16 +243,7 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
|
||||
else:
|
||||
result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
|
||||
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": result_format,
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
return cli_response(request, result, result_format=result_format)
|
||||
|
||||
|
||||
@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
|
||||
@@ -317,13 +264,10 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
|
||||
snapshots=snapshots_to_remove,
|
||||
timeout=timeout,
|
||||
)
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": bool(result["success"]),
|
||||
"errors": [str(result["error"])] if result["error"] else [],
|
||||
"result": result,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
return cli_response(
|
||||
request,
|
||||
result,
|
||||
result_format="json",
|
||||
success=bool(result["success"]),
|
||||
errors=[str(result["error"])] if result["error"] else [],
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,47 @@
|
||||
"""Core REST routes, composed from entity-specific routers."""
|
||||
|
||||
from typing import Union
|
||||
|
||||
from django.db.models import Model
|
||||
from django.http import HttpRequest
|
||||
from django.shortcuts import redirect
|
||||
from ninja import Router
|
||||
from ninja.errors import HttpError
|
||||
|
||||
from archivebox.api.v1_crawls import CrawlSchema, get_crawl
|
||||
|
||||
from . import archiveresults, snapshots, tags
|
||||
from .archiveresults import get_archiveresult
|
||||
from .pagination import CustomPagination as CustomPagination
|
||||
from .schemas import ArchiveResultSchema, SnapshotSchema, TagSchema
|
||||
from .snapshots import get_snapshot
|
||||
from .tags import get_tag
|
||||
|
||||
router = Router(tags=["Core Models"])
|
||||
router.add_router("", archiveresults.router)
|
||||
router.add_router("", snapshots.router)
|
||||
router.add_router("", tags.router)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/any/{id}",
|
||||
response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema],
|
||||
url_name="get_any",
|
||||
summary="Get any object by its ID",
|
||||
)
|
||||
def get_any(request: HttpRequest, id: str):
|
||||
"""Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
|
||||
setattr(request, "with_snapshots", False)
|
||||
setattr(request, "with_archiveresults", False)
|
||||
|
||||
for getter in (get_snapshot, get_archiveresult, get_tag, get_crawl):
|
||||
try:
|
||||
response = getter(request, id)
|
||||
if isinstance(response, Model):
|
||||
return redirect(
|
||||
f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
raise HttpError(404, "Object with given ID not found")
|
||||
@@ -0,0 +1,310 @@
|
||||
import json
|
||||
import mimetypes
|
||||
import re
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from django.core.files.storage import FileSystemStorage
|
||||
from django.http import HttpRequest
|
||||
from django.http.multipartparser import MultiPartParser, MultiPartParserError
|
||||
from ninja import Form, Query, Router, UploadedFile
|
||||
from ninja.errors import HttpError
|
||||
from ninja.pagination import paginate
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
|
||||
|
||||
from .lookup import _get_snapshot_by_ref
|
||||
from .pagination import CustomPagination
|
||||
from .schemas import ArchiveResultFilterSchema, ArchiveResultSchema
|
||||
|
||||
router = Router(tags=["Core Models"])
|
||||
|
||||
ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
|
||||
ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
|
||||
|
||||
|
||||
@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
|
||||
@paginate(CustomPagination)
|
||||
def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
|
||||
"""List all ArchiveResult entries matching these filters."""
|
||||
queryset = filters.filter(ArchiveResult.objects.all())
|
||||
if filters.search or filters.snapshot_tag:
|
||||
return queryset.distinct()
|
||||
return queryset
|
||||
|
||||
|
||||
@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
|
||||
def get_archiveresult(request: HttpRequest, archiveresult_id: str):
|
||||
"""Get a specific ArchiveResult by id."""
|
||||
return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
|
||||
|
||||
|
||||
def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
|
||||
normalized = str(plugin or "").strip().strip("/")
|
||||
if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
|
||||
raise HttpError(400, "Invalid ArchiveResult plugin name")
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_uploaded_archiveresult_output_path(output_path: str, *, filename: str) -> str:
|
||||
raw_path = str(output_path or filename or "").strip().replace("\\", "/")
|
||||
if not raw_path:
|
||||
raise HttpError(400, "ArchiveResult output path is required")
|
||||
|
||||
path = PurePosixPath(raw_path)
|
||||
if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
|
||||
raise HttpError(400, "Invalid ArchiveResult output path")
|
||||
|
||||
return str(path)
|
||||
|
||||
|
||||
def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any] | None:
|
||||
if not output_json:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(output_json)
|
||||
except json.JSONDecodeError as err:
|
||||
raise HttpError(400, "ArchiveResult output_json must be valid JSON") from err
|
||||
if parsed is None:
|
||||
return None
|
||||
if not isinstance(parsed, dict):
|
||||
raise HttpError(400, "ArchiveResult output_json must be a JSON object")
|
||||
return parsed
|
||||
|
||||
|
||||
def _get_archiveresult_upload_data(request: HttpRequest):
|
||||
cached = request.__dict__.get("_archiveresult_upload_data")
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
if request.method.upper() == "PATCH" and request.content_type.startswith("multipart/"):
|
||||
try:
|
||||
data = MultiPartParser(request.META, request, request.upload_handlers, request.encoding).parse()
|
||||
except MultiPartParserError as err:
|
||||
raise HttpError(400, f"Invalid ArchiveResult multipart upload: {err}") from err
|
||||
else:
|
||||
data = (request.POST, request.FILES)
|
||||
|
||||
setattr(request, "_archiveresult_upload_data", data)
|
||||
return data
|
||||
|
||||
|
||||
def _get_archiveresult_upload_files(request: HttpRequest, *, allow_empty: bool = False) -> list[UploadedFile]:
|
||||
_post, request_files = _get_archiveresult_upload_data(request)
|
||||
files = [*request_files.getlist("files"), *request_files.getlist("file")]
|
||||
if not files and not allow_empty:
|
||||
raise HttpError(400, "At least one ArchiveResult file is required")
|
||||
return files
|
||||
|
||||
|
||||
def _get_archiveresult_upload_form_values(request: HttpRequest, *field_names: str) -> list[str]:
|
||||
request_post, _files = _get_archiveresult_upload_data(request)
|
||||
values: list[str] = []
|
||||
for field_name in field_names:
|
||||
values.extend(str(value) for value in request_post.getlist(field_name))
|
||||
if len(values) == 1:
|
||||
value = values[0].strip()
|
||||
if value.startswith("["):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
if isinstance(parsed, list):
|
||||
return [str(item) for item in parsed]
|
||||
return values
|
||||
|
||||
|
||||
def _get_archiveresult_upload_form_value(request: HttpRequest, *field_names: str) -> str:
|
||||
request_post, _files = _get_archiveresult_upload_data(request)
|
||||
for field_name in field_names:
|
||||
value = request_post.get(field_name)
|
||||
if value is not None:
|
||||
return str(value)
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int | None = None) -> int:
|
||||
if value == "" and default is not None:
|
||||
return default
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError) as err:
|
||||
raise HttpError(400, f"ArchiveResult {field_name} must be an integer") from err
|
||||
if parsed < 0:
|
||||
raise HttpError(400, f"ArchiveResult {field_name} must be non-negative")
|
||||
return parsed
|
||||
|
||||
|
||||
def _write_archiveresult_files(
|
||||
request: HttpRequest,
|
||||
snapshot: Snapshot,
|
||||
plugin_name: str,
|
||||
*,
|
||||
existing_output_files: dict[str, dict[str, Any]] | None = None,
|
||||
allow_empty: bool = False,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
|
||||
output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
|
||||
mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
|
||||
chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
|
||||
|
||||
snapshot_dir = snapshot.output_dir
|
||||
plugin_dir = snapshot_dir / plugin_name
|
||||
storage = FileSystemStorage(location=str(plugin_dir))
|
||||
output_files = dict(existing_output_files or {})
|
||||
|
||||
if not files:
|
||||
return output_files
|
||||
|
||||
if chunk_output_path:
|
||||
if len(files) != 1:
|
||||
raise HttpError(400, "Exactly one ArchiveResult file chunk is required")
|
||||
|
||||
uploaded_file = files[0]
|
||||
relative_output_path = _normalize_uploaded_archiveresult_output_path(
|
||||
chunk_output_path,
|
||||
filename=uploaded_file.name,
|
||||
)
|
||||
chunk_index = _parse_archiveresult_upload_int(
|
||||
_get_archiveresult_upload_form_value(request, "chunk_index"),
|
||||
"chunk_index",
|
||||
)
|
||||
chunk_count = _parse_archiveresult_upload_int(
|
||||
_get_archiveresult_upload_form_value(request, "chunk_count"),
|
||||
"chunk_count",
|
||||
)
|
||||
chunk_offset = _parse_archiveresult_upload_int(
|
||||
_get_archiveresult_upload_form_value(request, "chunk_offset"),
|
||||
"chunk_offset",
|
||||
)
|
||||
chunk_total_size = _parse_archiveresult_upload_int(
|
||||
_get_archiveresult_upload_form_value(request, "chunk_total_size"),
|
||||
"chunk_total_size",
|
||||
)
|
||||
|
||||
if chunk_count < 1:
|
||||
raise HttpError(400, "ArchiveResult chunk_count must be at least 1")
|
||||
if chunk_index >= chunk_count:
|
||||
raise HttpError(400, "ArchiveResult chunk_index must be less than chunk_count")
|
||||
if chunk_total_size and chunk_offset > chunk_total_size:
|
||||
raise HttpError(400, "ArchiveResult chunk_offset cannot exceed chunk_total_size")
|
||||
|
||||
if chunk_index == 0 and chunk_offset == 0 and storage.exists(relative_output_path):
|
||||
storage.delete(relative_output_path)
|
||||
|
||||
current_size = storage.size(relative_output_path) if storage.exists(relative_output_path) else 0
|
||||
if current_size != chunk_offset:
|
||||
raise HttpError(
|
||||
409,
|
||||
f"ArchiveResult chunk offset mismatch for {relative_output_path}: expected {current_size}, got {chunk_offset}",
|
||||
)
|
||||
|
||||
Path(storage.path(relative_output_path)).parent.mkdir(parents=True, exist_ok=True)
|
||||
with storage.open(relative_output_path, "ab") as destination:
|
||||
for chunk in uploaded_file.chunks():
|
||||
destination.write(chunk)
|
||||
|
||||
size = storage.size(relative_output_path)
|
||||
upload_complete = chunk_index + 1 == chunk_count
|
||||
if upload_complete and size != chunk_total_size:
|
||||
raise HttpError(
|
||||
409,
|
||||
f"ArchiveResult chunk size mismatch for {relative_output_path}: expected {chunk_total_size}, got {size}",
|
||||
)
|
||||
|
||||
guessed_mime = mimetypes.guess_type(relative_output_path)[0]
|
||||
output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
|
||||
output_files[relative_output_path] = {
|
||||
"extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
|
||||
"mimetype": output_mime_type,
|
||||
"size": size,
|
||||
"upload": {
|
||||
"chunked": True,
|
||||
"chunk_index": chunk_index,
|
||||
"chunk_count": chunk_count,
|
||||
"chunks_received": chunk_index + 1,
|
||||
"complete": upload_complete,
|
||||
},
|
||||
}
|
||||
|
||||
return output_files
|
||||
|
||||
for index, uploaded_file in enumerate(files):
|
||||
relative_output_path = _normalize_uploaded_archiveresult_output_path(
|
||||
output_paths[index] if index < len(output_paths) else "",
|
||||
filename=uploaded_file.name,
|
||||
)
|
||||
if storage.exists(relative_output_path):
|
||||
storage.delete(relative_output_path)
|
||||
saved_output_path = storage.save(relative_output_path, uploaded_file)
|
||||
size = storage.size(saved_output_path)
|
||||
guessed_mime = mimetypes.guess_type(saved_output_path)[0]
|
||||
output_mime_type = (
|
||||
(mime_types[index] if index < len(mime_types) else "")
|
||||
or uploaded_file.content_type
|
||||
or guessed_mime
|
||||
or "application/octet-stream"
|
||||
)
|
||||
output_files[saved_output_path] = {
|
||||
"extension": PurePosixPath(saved_output_path).suffix.lower().lstrip("."),
|
||||
"mimetype": output_mime_type,
|
||||
"size": size,
|
||||
}
|
||||
|
||||
return output_files
|
||||
|
||||
|
||||
@router.post(
|
||||
"/archiveresults",
|
||||
response=ArchiveResultSchema,
|
||||
url_name="create_archiveresult",
|
||||
)
|
||||
def create_archiveresult(
|
||||
request: HttpRequest,
|
||||
snapshot_id: str = Form(...),
|
||||
plugin: str = Form(...),
|
||||
output_str: str = Form(""),
|
||||
hook_name: str = Form(ARCHIVERESULT_UPLOAD_HOOK_NAME),
|
||||
status: str = Form(str(ArchiveResult.StatusChoices.SUCCEEDED)),
|
||||
output_json: str = Form(""),
|
||||
):
|
||||
"""Create or update an ArchiveResult with one or more output files."""
|
||||
snapshot = _get_snapshot_by_ref(snapshot_id)
|
||||
plugin_name = _normalize_uploaded_archiveresult_plugin(plugin)
|
||||
result = ArchiveResult.objects.filter(
|
||||
snapshot=snapshot,
|
||||
plugin=plugin_name,
|
||||
hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME,
|
||||
).first()
|
||||
result = result or ArchiveResult(snapshot=snapshot, plugin=plugin_name, hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME)
|
||||
metadata = {"output_str": output_str, "status": status, "output_json": _parse_archiveresult_output_json(output_json)}
|
||||
files = _write_archiveresult_files(request, snapshot, plugin_name, allow_empty=True)
|
||||
try:
|
||||
return result.apply_upload(files, metadata=metadata, replace_metadata=True)
|
||||
except ArchiveResult.UploadConflict as err:
|
||||
raise HttpError(409, str(err)) from err
|
||||
|
||||
|
||||
@router.patch("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="patch_archiveresult")
|
||||
def patch_archiveresult(
|
||||
request: HttpRequest,
|
||||
archiveresult_id: str,
|
||||
):
|
||||
"""Append or replace files on an existing ArchiveResult."""
|
||||
result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
|
||||
uploaded_output_files = _write_archiveresult_files(
|
||||
request,
|
||||
result.snapshot,
|
||||
result.plugin,
|
||||
)
|
||||
output_str = _get_archiveresult_upload_form_value(request, "output_str")
|
||||
status = _get_archiveresult_upload_form_value(request, "status")
|
||||
output_json = _get_archiveresult_upload_form_value(request, "output_json")
|
||||
metadata = {key: value for key, value in {"output_str": output_str, "status": status}.items() if value}
|
||||
if output_json:
|
||||
metadata["output_json"] = _parse_archiveresult_output_json(output_json)
|
||||
try:
|
||||
return result.apply_upload(uploaded_output_files, metadata=metadata)
|
||||
except ArchiveResult.UploadConflict as err:
|
||||
raise HttpError(409, str(err)) from err
|
||||
@@ -0,0 +1,12 @@
|
||||
from django.db.models import Q
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
|
||||
|
||||
|
||||
def _get_snapshot_by_ref(snapshot_id: str):
|
||||
queryset = Snapshot.objects.select_related("crawl__created_by")
|
||||
try:
|
||||
return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
|
||||
except Snapshot.DoesNotExist:
|
||||
return queryset.get(_uuid_ref_query("id", snapshot_id))
|
||||
@@ -0,0 +1,40 @@
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from django.http import HttpRequest
|
||||
from ninja.pagination import PaginationBase
|
||||
|
||||
|
||||
class CustomPagination(PaginationBase):
|
||||
class Input(PaginationBase.Input):
|
||||
limit: int = 200
|
||||
offset: int = 0
|
||||
page: int = 0
|
||||
|
||||
class Output(PaginationBase.Output):
|
||||
count: int
|
||||
total_items: int
|
||||
total_pages: int
|
||||
page: int
|
||||
limit: int
|
||||
offset: int
|
||||
num_items: int
|
||||
items: list[Any]
|
||||
|
||||
def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
|
||||
limit = min(pagination.limit, 500)
|
||||
offset = pagination.offset or (pagination.page * limit)
|
||||
total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
|
||||
total_pages = math.ceil(total / limit)
|
||||
current_page = math.ceil(offset / (limit + 1))
|
||||
items = queryset[offset : offset + limit]
|
||||
return {
|
||||
"count": total,
|
||||
"total_items": total,
|
||||
"total_pages": total_pages,
|
||||
"page": current_page,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"num_items": len(items),
|
||||
"items": items,
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Any
|
||||
from uuid import UUID
|
||||
|
||||
from django.db.models import Q
|
||||
from archivebox.api.schemas import OwnedObjectSchema
|
||||
|
||||
from ninja import FilterLookup, FilterSchema, Schema
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
|
||||
|
||||
class MinimalArchiveResultSchema(OwnedObjectSchema):
|
||||
TYPE: str = "core.models.ArchiveResult"
|
||||
id: UUID
|
||||
created_at: datetime | None
|
||||
modified_at: datetime | None
|
||||
status: str
|
||||
retry_at: datetime | None = None
|
||||
plugin: str
|
||||
hook_name: str
|
||||
process_id: UUID | None
|
||||
cmd_version: str | None
|
||||
cmd: list[str] | None
|
||||
pwd: str | None
|
||||
output_str: str
|
||||
output_json: dict[str, Any] | None
|
||||
output_files: dict[str, dict[str, Any]] | None
|
||||
output_size: int
|
||||
output_mimetypes: str
|
||||
start_ts: datetime | None
|
||||
end_ts: datetime | None
|
||||
|
||||
@staticmethod
|
||||
def resolve_output_files(obj):
|
||||
return obj.output_file_map()
|
||||
|
||||
@staticmethod
|
||||
def resolve_output_mimetypes(obj) -> str:
|
||||
mime_sizes: dict[str, int] = defaultdict(int)
|
||||
for metadata in obj.output_file_map().values():
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
mimetype = str(metadata.get("mimetype") or "").strip()
|
||||
try:
|
||||
size = max(int(metadata.get("size") or 0), 0)
|
||||
except (TypeError, ValueError):
|
||||
size = 0
|
||||
if mimetype and size:
|
||||
mime_sizes[mimetype] += size
|
||||
if mime_sizes:
|
||||
return ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
|
||||
return obj.output_mimetypes or ""
|
||||
|
||||
|
||||
class ArchiveResultSchema(MinimalArchiveResultSchema):
|
||||
TYPE: str = "core.models.ArchiveResult"
|
||||
snapshot_id: UUID
|
||||
snapshot_timestamp: str
|
||||
snapshot_url: str
|
||||
snapshot_tags: list[str]
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshot_timestamp(obj):
|
||||
return obj.snapshot.timestamp
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshot_url(obj):
|
||||
return obj.snapshot.url
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshot_id(obj):
|
||||
return obj.snapshot_id
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshot_tags(obj):
|
||||
return sorted(tag.name for tag in obj.snapshot.tags.all())
|
||||
|
||||
|
||||
class ArchiveResultFilterSchema(FilterSchema):
|
||||
id: Annotated[str | None, FilterLookup(["id__startswith", "snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
|
||||
search: Annotated[
|
||||
str | None,
|
||||
FilterLookup(
|
||||
[
|
||||
"snapshot__url__icontains",
|
||||
"snapshot__title__icontains",
|
||||
"snapshot__tags__name__icontains",
|
||||
"plugin",
|
||||
"output_str__icontains",
|
||||
"id__startswith",
|
||||
"snapshot__id__startswith",
|
||||
"snapshot__timestamp__startswith",
|
||||
],
|
||||
),
|
||||
] = None
|
||||
snapshot_id: Annotated[str | None, FilterLookup(["snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
|
||||
snapshot_url: Annotated[str | None, FilterLookup("snapshot__url__icontains")] = None
|
||||
snapshot_tag: Annotated[str | None, FilterLookup("snapshot__tags__name__icontains")] = None
|
||||
status: Annotated[str | None, FilterLookup("status")] = None
|
||||
output_str: Annotated[str | None, FilterLookup("output_str__icontains")] = None
|
||||
plugin: Annotated[str | None, FilterLookup("plugin__icontains")] = None
|
||||
hook_name: Annotated[str | None, FilterLookup("hook_name__icontains")] = None
|
||||
process_id: Annotated[str | None, FilterLookup("process__id__startswith")] = None
|
||||
cmd: Annotated[str | None, FilterLookup("cmd__0__icontains")] = None
|
||||
pwd: Annotated[str | None, FilterLookup("pwd__icontains")] = None
|
||||
cmd_version: Annotated[str | None, FilterLookup("cmd_version")] = None
|
||||
created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
|
||||
created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
|
||||
created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
|
||||
|
||||
|
||||
class SnapshotSchema(OwnedObjectSchema):
|
||||
TYPE: str = "core.models.Snapshot"
|
||||
id: UUID
|
||||
created_at: datetime
|
||||
modified_at: datetime
|
||||
status: str
|
||||
retry_at: datetime | None
|
||||
bookmarked_at: datetime
|
||||
downloaded_at: datetime | None
|
||||
url: str
|
||||
tags: list[str]
|
||||
title: str | None
|
||||
timestamp: str
|
||||
archive_path: str
|
||||
archive_size: int
|
||||
output_size: int
|
||||
num_archiveresults: int
|
||||
archiveresults: list[MinimalArchiveResultSchema]
|
||||
|
||||
@staticmethod
|
||||
def resolve_tags(obj):
|
||||
return sorted(tag.name for tag in obj.tags.all())
|
||||
|
||||
@staticmethod
|
||||
def resolve_archive_size(obj):
|
||||
return int(obj.archive_size or 0)
|
||||
|
||||
@staticmethod
|
||||
def resolve_output_size(obj):
|
||||
return SnapshotSchema.resolve_archive_size(obj)
|
||||
|
||||
@staticmethod
|
||||
def resolve_num_archiveresults(obj, context):
|
||||
return obj.archiveresult_set.all().distinct().count()
|
||||
|
||||
@staticmethod
|
||||
def resolve_archiveresults(obj, context):
|
||||
if bool(context["request"].__dict__.get("with_archiveresults", False)):
|
||||
return obj.archiveresult_set.all().distinct()
|
||||
return ArchiveResult.objects.none()
|
||||
|
||||
|
||||
class SnapshotUpdateSchema(Schema):
|
||||
action: str | None = None
|
||||
status: str | None = None
|
||||
retry_at: datetime | None = None
|
||||
tags: list[str] | None = None
|
||||
|
||||
|
||||
class SnapshotCreateSchema(Schema):
|
||||
url: str
|
||||
crawl_id: str | None = None
|
||||
depth: int = 0
|
||||
title: str | None = None
|
||||
tags: list[str] | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class SnapshotDeleteResponseSchema(Schema):
|
||||
success: bool
|
||||
snapshot_id: str
|
||||
crawl_id: str
|
||||
deleted_count: int
|
||||
|
||||
|
||||
class SnapshotFilterSchema(FilterSchema):
|
||||
id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
|
||||
created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
|
||||
created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
|
||||
created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
|
||||
created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
|
||||
created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
|
||||
modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
|
||||
modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
|
||||
modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
|
||||
search: str | None = None
|
||||
search_mode: str | None = None
|
||||
status: str | None = None
|
||||
url: Annotated[str | None, FilterLookup("url")] = None
|
||||
tag: Annotated[str | None, FilterLookup("tags__name")] = None
|
||||
title: Annotated[str | None, FilterLookup("title__icontains")] = None
|
||||
timestamp: Annotated[str | None, FilterLookup("timestamp__startswith")] = None
|
||||
bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
|
||||
bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
|
||||
|
||||
def filter_search(self, value: str | None) -> Q:
|
||||
return Q()
|
||||
|
||||
def filter_search_mode(self, value: str | None) -> Q:
|
||||
return Q()
|
||||
|
||||
def filter_status(self, value: str | None) -> Q:
|
||||
return Q()
|
||||
|
||||
|
||||
class TagSchema(OwnedObjectSchema):
|
||||
TYPE: str = "core.models.Tag"
|
||||
id: int
|
||||
modified_at: datetime
|
||||
created_at: datetime
|
||||
name: str
|
||||
num_snapshots: int
|
||||
snapshots: list[SnapshotSchema]
|
||||
|
||||
@staticmethod
|
||||
def resolve_num_snapshots(obj, context):
|
||||
return obj.snapshot_set.all().distinct().count()
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshots(obj, context):
|
||||
if bool(context["request"].__dict__.get("with_snapshots", False)):
|
||||
return obj.snapshot_set.all().distinct()
|
||||
return Snapshot.objects.none()
|
||||
|
||||
|
||||
class TagAutocompleteSchema(Schema):
|
||||
tags: list[dict]
|
||||
|
||||
|
||||
class TagCreateSchema(Schema):
|
||||
name: str
|
||||
|
||||
|
||||
class TagCreateResponseSchema(Schema):
|
||||
success: bool
|
||||
tag_id: int
|
||||
tag_name: str
|
||||
created: bool
|
||||
|
||||
|
||||
class TagSearchSnapshotSchema(Schema):
|
||||
id: str
|
||||
title: str
|
||||
url: str
|
||||
favicon_url: str
|
||||
admin_url: str
|
||||
archive_url: str
|
||||
downloaded_at: str | None = None
|
||||
|
||||
|
||||
class TagSearchCardSchema(Schema):
|
||||
id: int
|
||||
name: str
|
||||
slug: str
|
||||
num_snapshots: int
|
||||
filter_url: str
|
||||
edit_url: str
|
||||
export_urls_url: str
|
||||
export_jsonl_url: str
|
||||
rename_url: str
|
||||
delete_url: str
|
||||
snapshots: list[TagSearchSnapshotSchema]
|
||||
|
||||
|
||||
class TagSearchResponseSchema(Schema):
|
||||
tags: list[TagSearchCardSchema]
|
||||
sort: str
|
||||
created_by: str
|
||||
year: str
|
||||
has_snapshots: str
|
||||
|
||||
|
||||
class TagUpdateSchema(Schema):
|
||||
name: str
|
||||
|
||||
|
||||
class TagUpdateResponseSchema(Schema):
|
||||
success: bool
|
||||
tag_id: int
|
||||
tag_name: str
|
||||
|
||||
|
||||
class TagDeleteResponseSchema(Schema):
|
||||
success: bool
|
||||
tag_id: int
|
||||
deleted_count: int
|
||||
|
||||
|
||||
class TagSnapshotRequestSchema(Schema):
|
||||
snapshot_id: str
|
||||
tag_name: str | None = None
|
||||
tag_id: int | None = None
|
||||
|
||||
|
||||
class TagSnapshotResponseSchema(Schema):
|
||||
success: bool
|
||||
tag_id: int
|
||||
tag_name: str
|
||||
@@ -0,0 +1,356 @@
|
||||
from datetime import datetime, time
|
||||
from html import unescape
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.contrib.auth.models import User
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import IntegrityError
|
||||
from django.db.models import Q
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
from django.utils import timezone
|
||||
from django.utils.dateparse import parse_date, parse_datetime
|
||||
from django.utils.feedgenerator import Rss201rev2Feed
|
||||
from ninja import Query, Router
|
||||
from ninja.errors import HttpError
|
||||
from ninja.pagination import paginate
|
||||
|
||||
from archivebox.api.v1_crawls import get_crawl_by_ref
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.routes_util import build_web_url
|
||||
from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
|
||||
from archivebox.crawls.locks import crawl_lifecycle_lock
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
|
||||
from archivebox.search.config import get_search_mode, get_search_mode_backend
|
||||
from archivebox.search.query import apply_snapshot_search
|
||||
|
||||
from .lookup import _get_snapshot_by_ref
|
||||
from .pagination import CustomPagination
|
||||
from .schemas import SnapshotCreateSchema, SnapshotDeleteResponseSchema, SnapshotFilterSchema, SnapshotSchema, SnapshotUpdateSchema
|
||||
|
||||
router = Router(tags=["Core Models"])
|
||||
|
||||
|
||||
def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
|
||||
return [tag.strip() for tag in (tags or []) if tag and tag.strip()]
|
||||
|
||||
|
||||
def _parse_rss_before(before: str | None) -> datetime:
|
||||
if not before:
|
||||
return timezone.now()
|
||||
|
||||
value = before.strip()
|
||||
parsed_dt = None
|
||||
|
||||
if len(value) == 8 and value.isdigit():
|
||||
parsed_date = datetime.strptime(value, "%Y%m%d").date()
|
||||
else:
|
||||
parsed_dt = parse_datetime(value)
|
||||
parsed_date = None if parsed_dt else parse_date(value)
|
||||
|
||||
if parsed_dt is None:
|
||||
if parsed_date is None:
|
||||
raise HttpError(400, "before must be an ISO datetime, YYYY-MM-DD, or YYYYMMDD")
|
||||
parsed_dt = datetime.combine(parsed_date, time.max)
|
||||
|
||||
if timezone.is_naive(parsed_dt):
|
||||
parsed_dt = timezone.make_aware(parsed_dt, timezone.get_current_timezone())
|
||||
return parsed_dt
|
||||
|
||||
|
||||
def _filter_snapshots_for_rss(
|
||||
*,
|
||||
crawl_id: str = "",
|
||||
created_by: str = "",
|
||||
before: str | None = None,
|
||||
limit: int = 50,
|
||||
):
|
||||
limit = max(1, min(int(limit or 50), 500))
|
||||
before_dt = _parse_rss_before(before)
|
||||
queryset = (
|
||||
Snapshot.objects.select_related("crawl__created_by")
|
||||
.prefetch_related("tags")
|
||||
.only(
|
||||
"id",
|
||||
"url",
|
||||
"title",
|
||||
"timestamp",
|
||||
"bookmarked_at",
|
||||
"created_at",
|
||||
"modified_at",
|
||||
"fs_version",
|
||||
"crawl_id",
|
||||
"crawl__id",
|
||||
"crawl__created_by_id",
|
||||
"crawl__created_by__id",
|
||||
"crawl__created_by__username",
|
||||
)
|
||||
.filter(bookmarked_at__lte=before_dt)
|
||||
)
|
||||
crawl_id = crawl_id.strip()
|
||||
if crawl_id:
|
||||
matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
|
||||
queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
|
||||
|
||||
created_by = created_by.strip()
|
||||
if created_by:
|
||||
created_by_query = Q(crawl__created_by__username__iexact=created_by)
|
||||
user_model = get_user_model()
|
||||
try:
|
||||
prepared_pk = user_model._meta.pk.get_prep_value(created_by)
|
||||
except (TypeError, ValueError, ValidationError):
|
||||
prepared_pk = None
|
||||
if prepared_pk not in (None, ""):
|
||||
created_by_query |= Q(crawl__created_by_id=prepared_pk)
|
||||
queryset = queryset.filter(created_by_query)
|
||||
|
||||
return queryset.order_by("-bookmarked_at", "-created_at", "-id")[:limit]
|
||||
|
||||
|
||||
def _snapshots_rss_response(
|
||||
request: HttpRequest,
|
||||
*,
|
||||
snapshots,
|
||||
title: str = "ArchiveBox Snapshots",
|
||||
) -> HttpResponse:
|
||||
web_base_url = build_web_url("/", request=request).rstrip("/")
|
||||
feed_query = request.GET.copy()
|
||||
for sensitive_param in ("api_key", "token", "password"):
|
||||
feed_query.pop(sensitive_param, None)
|
||||
feed_path = request.path
|
||||
feed_url = request.build_absolute_uri(f"{feed_path}?{feed_query.urlencode()}" if feed_query else feed_path)
|
||||
|
||||
feed = Rss201rev2Feed(
|
||||
title=title,
|
||||
link=build_web_url("/public/", request=request),
|
||||
description="Recently added ArchiveBox snapshots.",
|
||||
language="en",
|
||||
feed_url=feed_url,
|
||||
)
|
||||
|
||||
for snapshot in snapshots:
|
||||
archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
|
||||
tags = [tag.name for tag in snapshot.tags.all()]
|
||||
crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
|
||||
description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
|
||||
feed.add_item(
|
||||
title=unescape(snapshot.title or snapshot.url),
|
||||
link=archived_url or web_base_url,
|
||||
description=description,
|
||||
unique_id=str(snapshot.id),
|
||||
unique_id_is_permalink=False,
|
||||
pubdate=snapshot.bookmarked_at or snapshot.created_at,
|
||||
updateddate=snapshot.modified_at,
|
||||
author_name=crawl_user.username if crawl_user else None,
|
||||
categories=tags,
|
||||
)
|
||||
|
||||
return HttpResponse(feed.writeString("utf-8"), content_type="application/rss+xml; charset=utf-8")
|
||||
|
||||
|
||||
@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
|
||||
@paginate(CustomPagination)
|
||||
def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
|
||||
"""List all Snapshot entries matching these filters."""
|
||||
setattr(request, "with_archiveresults", with_archiveresults)
|
||||
try:
|
||||
queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
queryset = filters.filter(queryset).distinct()
|
||||
query = (filters.search or "").strip()
|
||||
if not query:
|
||||
return queryset
|
||||
|
||||
runtime_config = request.archivebox_config
|
||||
search_mode = get_search_mode(filters.search_mode, config=runtime_config)
|
||||
try:
|
||||
return apply_snapshot_search(
|
||||
queryset,
|
||||
query,
|
||||
search_mode=search_mode,
|
||||
config=runtime_config,
|
||||
include_id_matches=True,
|
||||
)
|
||||
except Exception:
|
||||
if get_search_mode_backend(search_mode, config=runtime_config):
|
||||
return queryset.none()
|
||||
return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
|
||||
|
||||
|
||||
@router.get("/snapshots.rss", url_name="get_snapshots_rss")
|
||||
def get_snapshots_rss(
|
||||
request: HttpRequest,
|
||||
crawl_id: str = "",
|
||||
created_by: str = "",
|
||||
limit: int = 50,
|
||||
before: str | None = None,
|
||||
):
|
||||
"""Return matching snapshots as an RSS feed, newest first."""
|
||||
snapshots = _filter_snapshots_for_rss(
|
||||
crawl_id=crawl_id,
|
||||
created_by=created_by,
|
||||
limit=limit,
|
||||
before=before,
|
||||
)
|
||||
return _snapshots_rss_response(request, snapshots=snapshots)
|
||||
|
||||
|
||||
@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
|
||||
def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
|
||||
"""Get a specific Snapshot by id."""
|
||||
setattr(request, "with_archiveresults", with_archiveresults)
|
||||
return _get_snapshot_by_ref(snapshot_id)
|
||||
|
||||
|
||||
@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
|
||||
def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
|
||||
tags = normalize_tag_list(data.tags)
|
||||
try:
|
||||
status = normalize_snapshot_status(data.status)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
if not data.url.strip():
|
||||
raise HttpError(400, "URL is required")
|
||||
try:
|
||||
validate_url_length(data.url.strip())
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
if data.depth not in (0, 1, 2, 3, 4):
|
||||
raise HttpError(400, "depth must be between 0 and 4")
|
||||
|
||||
if data.crawl_id:
|
||||
crawl = get_crawl_by_ref(data.crawl_id)
|
||||
crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
|
||||
tags = tags or crawl_tags
|
||||
else:
|
||||
crawl = Crawl.objects.create(
|
||||
urls=data.url,
|
||||
max_depth=max(data.depth, 0),
|
||||
tags_str=",".join(tags),
|
||||
status=Crawl.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
created_by=request.user if isinstance(request.user, User) else None,
|
||||
)
|
||||
|
||||
# Browser uploads must not wait behind the runner's crawl-wide lifecycle
|
||||
# lock. The unique insert recovery and CAS update below are the request-side
|
||||
# coordination boundary for this idempotent metadata sync.
|
||||
snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
|
||||
if snapshot is None:
|
||||
try:
|
||||
snapshot = Snapshot.objects.create(
|
||||
url=data.url,
|
||||
crawl=crawl,
|
||||
depth=data.depth,
|
||||
title=data.title,
|
||||
timestamp=str(timezone.now().timestamp()),
|
||||
status=status or Snapshot.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
except IntegrityError:
|
||||
snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
|
||||
if snapshot is None:
|
||||
raise
|
||||
|
||||
for _attempt in range(3):
|
||||
updates: dict[str, Any] = {}
|
||||
if data.title is not None and snapshot.title != data.title:
|
||||
updates["title"] = data.title
|
||||
if status is not None and snapshot.status != status:
|
||||
updates["status"] = status
|
||||
if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
|
||||
break
|
||||
else:
|
||||
raise HttpError(409, "Snapshot changed while metadata was being updated")
|
||||
|
||||
if tags:
|
||||
snapshot.save_tags(
|
||||
tags,
|
||||
created_by=request.user if isinstance(request.user, User) else None,
|
||||
)
|
||||
|
||||
try:
|
||||
snapshot.ensure_crawl_symlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
setattr(request, "with_archiveresults", False)
|
||||
return snapshot
|
||||
|
||||
|
||||
@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
|
||||
def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
|
||||
"""Update a snapshot (e.g., set status=sealed to cancel queued work)."""
|
||||
snapshot = _get_snapshot_by_ref(snapshot_id)
|
||||
crawl_id = str(snapshot.crawl_id)
|
||||
|
||||
payload = data.dict(exclude_unset=True)
|
||||
update_fields = ["modified_at"]
|
||||
action = payload.pop("action", None)
|
||||
tags = payload.pop("tags", None)
|
||||
|
||||
if action:
|
||||
if action == "pause":
|
||||
snapshot.pause()
|
||||
setattr(request, "with_archiveresults", False)
|
||||
return snapshot
|
||||
if action in ("resume", "unpause"):
|
||||
snapshot.resume()
|
||||
setattr(request, "with_archiveresults", False)
|
||||
return snapshot
|
||||
if action == "cancel":
|
||||
snapshot.cancel()
|
||||
setattr(request, "with_archiveresults", False)
|
||||
return snapshot
|
||||
raise HttpError(400, f"Invalid action: {action}")
|
||||
|
||||
with crawl_lifecycle_lock(crawl_id):
|
||||
snapshot = _get_snapshot_by_ref(snapshot_id)
|
||||
if "status" in payload:
|
||||
try:
|
||||
snapshot.status = normalize_snapshot_status(payload["status"])
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
|
||||
snapshot.retry_at = None
|
||||
update_fields.append("status")
|
||||
|
||||
if "retry_at" in payload:
|
||||
snapshot.retry_at = payload["retry_at"]
|
||||
update_fields.append("retry_at")
|
||||
|
||||
if tags is not None:
|
||||
snapshot.save_tags(
|
||||
normalize_tag_list(tags),
|
||||
created_by=request.user if isinstance(request.user, User) else None,
|
||||
)
|
||||
|
||||
if payload.get("status") == Snapshot.StatusChoices.SEALED:
|
||||
snapshot.cancel()
|
||||
else:
|
||||
snapshot.save(update_fields=update_fields)
|
||||
setattr(request, "with_archiveresults", False)
|
||||
return snapshot
|
||||
|
||||
|
||||
@router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
|
||||
def delete_snapshot(request: HttpRequest, snapshot_id: str):
|
||||
snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
|
||||
snapshot_id_str = str(snapshot.id)
|
||||
crawl_id_str = str(snapshot.crawl.pk)
|
||||
snapshot.cancel()
|
||||
|
||||
from archivebox.services.runner import run_pending_crawls
|
||||
|
||||
with crawl_lifecycle_lock(crawl_id_str):
|
||||
run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
|
||||
snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
|
||||
deleted_count, _ = snapshot.delete()
|
||||
return {
|
||||
"success": True,
|
||||
"snapshot_id": snapshot_id_str,
|
||||
"crawl_id": crawl_id_str,
|
||||
"deleted_count": deleted_count,
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db.models import Q
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
from ninja import Router
|
||||
from ninja.errors import HttpError
|
||||
from ninja.pagination import paginate
|
||||
|
||||
from archivebox.api.auth import authenticated_user_from_request
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.models import Snapshot, Tag
|
||||
from archivebox.core.permissions import public_snapshots_queryset
|
||||
from archivebox.core.tag_util import (
|
||||
add_snapshot_counts,
|
||||
build_tag_cards,
|
||||
export_tag_snapshots_jsonl,
|
||||
export_tag_urls,
|
||||
get_matching_tags,
|
||||
get_tag_by_ref,
|
||||
normalize_created_by_filter,
|
||||
normalize_created_year_filter,
|
||||
normalize_has_snapshots_filter,
|
||||
normalize_tag_sort,
|
||||
)
|
||||
|
||||
from .pagination import CustomPagination
|
||||
from .schemas import (
|
||||
TagAutocompleteSchema,
|
||||
TagCreateResponseSchema,
|
||||
TagCreateSchema,
|
||||
TagDeleteResponseSchema,
|
||||
TagSchema,
|
||||
TagSearchResponseSchema,
|
||||
TagSnapshotRequestSchema,
|
||||
TagSnapshotResponseSchema,
|
||||
TagUpdateResponseSchema,
|
||||
TagUpdateSchema,
|
||||
)
|
||||
|
||||
router = Router(tags=["Core Models"])
|
||||
|
||||
|
||||
@router.get("/tags", response=list[TagSchema], url_name="get_tags")
|
||||
@paginate(CustomPagination)
|
||||
def get_tags(request: HttpRequest):
|
||||
setattr(request, "with_snapshots", False)
|
||||
setattr(request, "with_archiveresults", False)
|
||||
return get_matching_tags()
|
||||
|
||||
|
||||
@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
|
||||
def get_tag(request: HttpRequest, tag_id: str, with_snapshots: bool = True):
|
||||
setattr(request, "with_snapshots", with_snapshots)
|
||||
setattr(request, "with_archiveresults", False)
|
||||
try:
|
||||
return get_tag_by_ref(tag_id)
|
||||
except (Tag.DoesNotExist, ValidationError):
|
||||
raise HttpError(404, "Tag not found")
|
||||
|
||||
|
||||
def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
|
||||
snapshot_ref = str(snapshot_ref or "").strip().lower()
|
||||
if not snapshot_ref:
|
||||
raise HttpError(400, "Snapshot id is required")
|
||||
|
||||
snapshot_qs = Snapshot.objects.only("id")
|
||||
is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
|
||||
if is_full_uuid:
|
||||
try:
|
||||
return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
|
||||
except (Snapshot.DoesNotExist, ValueError):
|
||||
pass
|
||||
|
||||
if len(snapshot_ref) >= 14:
|
||||
try:
|
||||
return snapshot_qs.get(timestamp=snapshot_ref)
|
||||
except Snapshot.DoesNotExist:
|
||||
pass
|
||||
except Snapshot.MultipleObjectsReturned:
|
||||
snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
|
||||
if snapshot is not None:
|
||||
return snapshot
|
||||
|
||||
try:
|
||||
return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
|
||||
except Snapshot.DoesNotExist:
|
||||
raise HttpError(404, "Snapshot not found") from None
|
||||
except Snapshot.MultipleObjectsReturned:
|
||||
snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
|
||||
if snapshot is None:
|
||||
raise HttpError(404, "Snapshot not found")
|
||||
return snapshot
|
||||
|
||||
|
||||
@router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
|
||||
def search_tags(
|
||||
request: HttpRequest,
|
||||
q: str = "",
|
||||
sort: str = "created_desc",
|
||||
created_by: str = "",
|
||||
year: str = "",
|
||||
has_snapshots: str = "all",
|
||||
):
|
||||
"""Return detailed tag cards for admin/live-search UIs."""
|
||||
normalized_sort = normalize_tag_sort(sort)
|
||||
normalized_created_by = normalize_created_by_filter(created_by)
|
||||
normalized_year = normalize_created_year_filter(year)
|
||||
normalized_has_snapshots = normalize_has_snapshots_filter(has_snapshots)
|
||||
return {
|
||||
"tags": build_tag_cards(
|
||||
query=q,
|
||||
request=request,
|
||||
preview_limit=0,
|
||||
sort=normalized_sort,
|
||||
created_by=normalized_created_by,
|
||||
year=normalized_year,
|
||||
has_snapshots=normalized_has_snapshots,
|
||||
),
|
||||
"sort": normalized_sort,
|
||||
"created_by": normalized_created_by,
|
||||
"year": normalized_year,
|
||||
"has_snapshots": normalized_has_snapshots,
|
||||
}
|
||||
|
||||
|
||||
def _public_tag_listing_enabled() -> bool:
|
||||
return get_config().PUBLIC_INDEX
|
||||
|
||||
|
||||
def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
|
||||
if authenticated_user_from_request(request):
|
||||
return True
|
||||
|
||||
return _public_tag_listing_enabled()
|
||||
|
||||
|
||||
@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete", auth=None)
|
||||
def tags_autocomplete(request: HttpRequest, q: str = ""):
|
||||
"""Return tags matching the query for autocomplete."""
|
||||
if not _request_has_tag_autocomplete_access(request):
|
||||
raise HttpError(401, "Authentication required")
|
||||
|
||||
public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
|
||||
queryset = get_matching_tags(q)
|
||||
public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
|
||||
if public_only:
|
||||
queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
|
||||
tags = list(queryset[: 50 if not q else 20])
|
||||
add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
|
||||
|
||||
return {
|
||||
"tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
|
||||
def tags_create(request: HttpRequest, data: TagCreateSchema):
|
||||
"""Create a new tag or return existing one."""
|
||||
try:
|
||||
tag, created = Tag.get_or_create_by_name(
|
||||
data.name,
|
||||
created_by=request.user if request.user.is_authenticated else None,
|
||||
)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"tag_id": tag.pk,
|
||||
"tag_name": tag.name,
|
||||
"created": created,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tag/{tag_id}/rename", response=TagUpdateResponseSchema, url_name="rename_tag")
|
||||
def rename_tag(request: HttpRequest, tag_id: int, data: TagUpdateSchema):
|
||||
try:
|
||||
tag = get_tag_by_ref(tag_id).rename(data.name)
|
||||
except Tag.DoesNotExist as err:
|
||||
raise HttpError(404, "Tag not found") from err
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"tag_id": tag.pk,
|
||||
"tag_name": tag.name,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/tag/{tag_id}", response=TagDeleteResponseSchema, url_name="delete_tag")
|
||||
def delete_tag(request: HttpRequest, tag_id: int):
|
||||
try:
|
||||
tag = get_tag_by_ref(tag_id)
|
||||
except Tag.DoesNotExist as err:
|
||||
raise HttpError(404, "Tag not found") from err
|
||||
|
||||
deleted_count, _ = tag.delete()
|
||||
return {
|
||||
"success": True,
|
||||
"tag_id": int(tag_id),
|
||||
"deleted_count": deleted_count,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/tag/{tag_id}/urls.txt", url_name="tag_urls_export")
|
||||
def tag_urls_export(request: HttpRequest, tag_id: int):
|
||||
try:
|
||||
tag = get_tag_by_ref(tag_id)
|
||||
except Tag.DoesNotExist as err:
|
||||
raise HttpError(404, "Tag not found") from err
|
||||
|
||||
response = HttpResponse(export_tag_urls(tag), content_type="text/plain; charset=utf-8")
|
||||
response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-urls.txt"'
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/tag/{tag_id}/snapshots.jsonl", url_name="tag_snapshots_export")
|
||||
def tag_snapshots_export(request: HttpRequest, tag_id: int):
|
||||
try:
|
||||
tag = get_tag_by_ref(tag_id)
|
||||
except Tag.DoesNotExist as err:
|
||||
raise HttpError(404, "Tag not found") from err
|
||||
|
||||
response = HttpResponse(export_tag_snapshots_jsonl(tag), content_type="application/x-ndjson; charset=utf-8")
|
||||
response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-snapshots.jsonl"'
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
|
||||
def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
|
||||
"""Add a tag to a snapshot. Creates the tag if it doesn't exist."""
|
||||
snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
|
||||
|
||||
# Get or create the tag
|
||||
if data.tag_name:
|
||||
try:
|
||||
tag, _ = Tag.get_or_create_by_name(
|
||||
data.tag_name,
|
||||
created_by=request.user if request.user.is_authenticated else None,
|
||||
)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
elif data.tag_id:
|
||||
try:
|
||||
tag = get_tag_by_ref(data.tag_id)
|
||||
except Tag.DoesNotExist:
|
||||
raise HttpError(404, "Tag not found")
|
||||
else:
|
||||
raise HttpError(400, "Either tag_name or tag_id is required")
|
||||
|
||||
snapshot.add_tag_ids([tag.pk])
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"tag_id": tag.pk,
|
||||
"tag_name": tag.name,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
|
||||
def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
|
||||
"""Remove a tag from a snapshot."""
|
||||
snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
|
||||
|
||||
# Get the tag
|
||||
if data.tag_id:
|
||||
try:
|
||||
tag = Tag.objects.get(pk=data.tag_id)
|
||||
except Tag.DoesNotExist:
|
||||
raise HttpError(404, "Tag not found")
|
||||
elif data.tag_name:
|
||||
try:
|
||||
tag = Tag.objects.get(name__iexact=data.tag_name.strip())
|
||||
except Tag.DoesNotExist:
|
||||
raise HttpError(404, "Tag not found")
|
||||
else:
|
||||
raise HttpError(400, "Either tag_name or tag_id is required")
|
||||
|
||||
snapshot.remove_tag_ids([tag.pk])
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"tag_id": tag.pk,
|
||||
"tag_name": tag.name,
|
||||
}
|
||||
@@ -7,9 +7,10 @@ from django.http import FileResponse, HttpRequest
|
||||
from django.shortcuts import redirect
|
||||
from django.utils import timezone
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from archivebox.api.schemas import OwnedObjectSchema
|
||||
|
||||
from ninja import Router, Schema
|
||||
from ninja.errors import HttpError
|
||||
|
||||
@@ -29,15 +30,13 @@ from .auth import API_AUTH_METHODS, authenticated_user_from_request
|
||||
router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
|
||||
|
||||
|
||||
class CrawlSchema(Schema):
|
||||
class CrawlSchema(OwnedObjectSchema):
|
||||
TYPE: str = "crawls.models.Crawl"
|
||||
|
||||
id: UUID
|
||||
|
||||
modified_at: datetime
|
||||
created_at: datetime
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
|
||||
status: str
|
||||
retry_at: datetime | None
|
||||
@@ -48,17 +47,6 @@ class CrawlSchema(Schema):
|
||||
tags_str: str
|
||||
config: dict
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by_id)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj):
|
||||
user_model = get_user_model()
|
||||
user = user_model.objects.get(id=obj.created_by_id)
|
||||
username = user.username
|
||||
return username if isinstance(username, str) else str(user)
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses can never leak the raw
|
||||
|
||||
@@ -7,6 +7,8 @@ from uuid import UUID
|
||||
|
||||
from django.db.models import Q
|
||||
from django.http import HttpRequest
|
||||
from archivebox.api.schemas import OwnedObjectSchema
|
||||
|
||||
from ninja import Router, Schema
|
||||
from ninja.pagination import paginate
|
||||
from pydantic import Field
|
||||
@@ -36,23 +38,13 @@ class PersonaSyncSchema(Schema):
|
||||
auth_json: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class PersonaSchema(Schema):
|
||||
class PersonaSchema(OwnedObjectSchema):
|
||||
TYPE: str = "personas.models.Persona"
|
||||
id: UUID
|
||||
name: str
|
||||
created_at: datetime
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
config: dict[str, Any] | None
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by.pk)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj) -> str:
|
||||
return obj.created_by.username
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses don't leak the raw
|
||||
|
||||
+59
-640
@@ -13,10 +13,16 @@ from django.db import DatabaseError, models
|
||||
from django.forms.renderers import BaseRenderer
|
||||
from django.http import HttpRequest, QueryDict
|
||||
from django.urls import path, register_converter
|
||||
from django.utils.html import format_html
|
||||
from django.utils.safestring import SafeString, mark_safe
|
||||
from django_object_actions import DjangoObjectActions
|
||||
|
||||
|
||||
def card_fieldset(title: str | None, fields: tuple, *, wide: bool = False, **options):
|
||||
"""Build a Django fieldset using the shared card layout."""
|
||||
return title, {"fields": fields, "classes": ("card", "wide") if wide else ("card",), **options}
|
||||
|
||||
|
||||
class HexUUIDConverter:
|
||||
"""URL path converter that canonicalizes UUIDs to their 32-char hex form.
|
||||
|
||||
@@ -81,6 +87,11 @@ class KeyValueWidget(forms.Widget):
|
||||
options: dict[str, ConfigOption] = {}
|
||||
for key, metadata in config_field_metadata().items():
|
||||
option_type = metadata.get("type", "string")
|
||||
# Core fields expose Python type names; plugins use JSON Schema.
|
||||
# The editor should apply the same typed validation to both.
|
||||
type_names = {"bool": "boolean", "int": "integer", "float": "number", "str": "string", "dict": "object", "list": "array"}
|
||||
if isinstance(option_type, str):
|
||||
option_type = type_names.get(option_type, option_type)
|
||||
option: ConfigOption = {
|
||||
"plugin": str(metadata.get("plugin", "archivebox")),
|
||||
"type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
|
||||
@@ -118,650 +129,36 @@ class KeyValueWidget(forms.Widget):
|
||||
attrs: Mapping[str, str] | None = None,
|
||||
renderer: BaseRenderer | None = None,
|
||||
) -> SafeString:
|
||||
data = self._parse_value(value)
|
||||
from django.template.loader import render_to_string
|
||||
|
||||
widget_id = attrs.get("id", name) if attrs else name
|
||||
config_options = self._get_config_options()
|
||||
|
||||
# Build datalist options
|
||||
datalist_options = "\n".join(
|
||||
f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
|
||||
for key, opt in sorted(config_options.items())
|
||||
options = self._get_config_options()
|
||||
rows = [
|
||||
mark_safe(self._render_row(widget_id, key, val if isinstance(val, str) else json.dumps(val)))
|
||||
for key, val in self._parse_value(value).items()
|
||||
]
|
||||
empty_row = mark_safe(self._render_row(widget_id, "", ""))
|
||||
# JSON is embedded in a script, so escape HTML delimiters without changing data.
|
||||
metadata = json.dumps(options).translate({ord("<"): r"\u003C", ord(">"): r"\u003E", ord("&"): r"\u0026"})
|
||||
return mark_safe(
|
||||
render_to_string(
|
||||
"admin/widgets/config.html",
|
||||
{
|
||||
"name": name,
|
||||
"widget_id": widget_id,
|
||||
"datalist_options": mark_safe(
|
||||
"\n".join(
|
||||
f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
|
||||
for key, opt in sorted(options.items())
|
||||
),
|
||||
),
|
||||
"config_meta_json": metadata,
|
||||
"rows": [*rows, empty_row],
|
||||
"empty_row": empty_row,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
# Build config metadata as JSON for JS
|
||||
config_meta_json = json.dumps(config_options)
|
||||
|
||||
html = f'''
|
||||
<div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
|
||||
<datalist id="{widget_id}_keys">
|
||||
{datalist_options}
|
||||
</datalist>
|
||||
<div id="{widget_id}_rows" class="key-value-rows">
|
||||
'''
|
||||
|
||||
# Render existing key-value pairs
|
||||
for key, val in data.items():
|
||||
val_str = json.dumps(val) if not isinstance(val, str) else val
|
||||
html += self._render_row(widget_id, key, val_str)
|
||||
|
||||
# Always add one empty row for new entries
|
||||
html += self._render_row(widget_id, "", "")
|
||||
|
||||
html += f'''
|
||||
</div>
|
||||
<div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
|
||||
<button type="button" onclick="addKeyValueRow_{widget_id}()"
|
||||
style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
|
||||
+ Add Row
|
||||
</button>
|
||||
</div>
|
||||
<input type="hidden" name="{name}" id="{widget_id}" value="">
|
||||
<script>
|
||||
(function() {{
|
||||
var configMeta_{widget_id} = {config_meta_json};
|
||||
var rowCounter_{widget_id} = 0;
|
||||
|
||||
function stringifyValue_{widget_id}(value) {{
|
||||
return typeof value === 'string' ? value : JSON.stringify(value);
|
||||
}}
|
||||
|
||||
function getTypes_{widget_id}(meta) {{
|
||||
if (!meta || meta.type === undefined || meta.type === null) {{
|
||||
return [];
|
||||
}}
|
||||
return Array.isArray(meta.type) ? meta.type : [meta.type];
|
||||
}}
|
||||
|
||||
function getMetaForKey_{widget_id}(key) {{
|
||||
if (!key) {{
|
||||
return null;
|
||||
}}
|
||||
|
||||
var explicitMeta = configMeta_{widget_id}[key];
|
||||
if (explicitMeta) {{
|
||||
return Object.assign({{ key: key }}, explicitMeta);
|
||||
}}
|
||||
|
||||
if (key.endsWith('_BINARY')) {{
|
||||
return {{
|
||||
key: key,
|
||||
plugin: 'custom',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Path to binary executable',
|
||||
}};
|
||||
}}
|
||||
|
||||
if (isRegexConfigKey_{widget_id}(key)) {{
|
||||
return {{
|
||||
key: key,
|
||||
plugin: 'custom',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Regex pattern list',
|
||||
}};
|
||||
}}
|
||||
|
||||
return null;
|
||||
}}
|
||||
|
||||
function describeMeta_{widget_id}(meta) {{
|
||||
if (!meta) {{
|
||||
return '';
|
||||
}}
|
||||
|
||||
var details = '';
|
||||
if (Array.isArray(meta.enum) && meta.enum.length) {{
|
||||
details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
|
||||
}} else {{
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (types.length) {{
|
||||
details = 'Expected: ' + types.join(' or ');
|
||||
}}
|
||||
}}
|
||||
|
||||
if (meta.minimum !== undefined || meta.maximum !== undefined) {{
|
||||
var bounds = [];
|
||||
if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
|
||||
if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
|
||||
details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
|
||||
}}
|
||||
|
||||
return [meta.description || '', details].filter(Boolean).join(' ');
|
||||
}}
|
||||
|
||||
function getExampleInput_{widget_id}(key, meta) {{
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (key.endsWith('_BINARY')) {{
|
||||
return 'Example: wget or /usr/bin/wget';
|
||||
}}
|
||||
if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
|
||||
return 'Example: ["--extra-arg"]';
|
||||
}}
|
||||
if (types.includes('array')) {{
|
||||
return 'Example: ["value"]';
|
||||
}}
|
||||
if (types.includes('object')) {{
|
||||
return 'Example: {{"key": "value"}}';
|
||||
}}
|
||||
return '';
|
||||
}}
|
||||
|
||||
function isRegexConfigKey_{widget_id}(key) {{
|
||||
return key === 'URL_ALLOWLIST' ||
|
||||
key === 'URL_DENYLIST' ||
|
||||
key.endsWith('_PATTERN') ||
|
||||
key.includes('REGEX');
|
||||
}}
|
||||
|
||||
function isSimpleFilterPattern_{widget_id}(pattern) {{
|
||||
return /^[\\w.*:-]+$/.test(pattern);
|
||||
}}
|
||||
|
||||
function validateRegexPattern_{widget_id}(pattern) {{
|
||||
if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
|
||||
return '';
|
||||
}}
|
||||
|
||||
try {{
|
||||
new RegExp(pattern);
|
||||
}} catch (error) {{
|
||||
return error && error.message ? error.message : 'Invalid regex';
|
||||
}}
|
||||
return '';
|
||||
}}
|
||||
|
||||
function validateRegexConfig_{widget_id}(key, raw, typeName) {{
|
||||
if (typeName === 'object') {{
|
||||
var parsed;
|
||||
try {{
|
||||
parsed = JSON.parse(raw);
|
||||
}} catch (error) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
|
||||
}}
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
|
||||
return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
|
||||
}}
|
||||
for (var regexKey in parsed) {{
|
||||
var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
|
||||
if (objectRegexError) {{
|
||||
return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
|
||||
}}
|
||||
}}
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
|
||||
var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
|
||||
return pattern.trim();
|
||||
}}).filter(Boolean);
|
||||
for (var i = 0; i < patterns.length; i++) {{
|
||||
var regexError = validateRegexPattern_{widget_id}(patterns[i]);
|
||||
if (regexError) {{
|
||||
return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
|
||||
}}
|
||||
}}
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
function validateBinaryValue_{widget_id}(raw) {{
|
||||
if (!raw) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
if (/['"`]/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
|
||||
}}
|
||||
|
||||
if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
|
||||
}}
|
||||
|
||||
if (raw.startsWith('/')) {{
|
||||
if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
|
||||
}}
|
||||
|
||||
if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
|
||||
}}
|
||||
|
||||
function parseValue_{widget_id}(raw) {{
|
||||
try {{
|
||||
if (raw === 'true') return true;
|
||||
if (raw === 'false') return false;
|
||||
if (raw === 'null') return null;
|
||||
if (raw !== '' && !isNaN(raw)) return Number(raw);
|
||||
if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
|
||||
(raw.startsWith('[') && raw.endsWith(']')) ||
|
||||
(raw.startsWith('"') && raw.endsWith('"'))) {{
|
||||
return JSON.parse(raw);
|
||||
}}
|
||||
}} catch (error) {{
|
||||
return raw;
|
||||
}}
|
||||
return raw;
|
||||
}}
|
||||
|
||||
function sameValue_{widget_id}(left, right) {{
|
||||
return left === right || JSON.stringify(left) === JSON.stringify(right);
|
||||
}}
|
||||
|
||||
function parseTypedValue_{widget_id}(raw, typeName, meta) {{
|
||||
var numberValue;
|
||||
var parsed;
|
||||
|
||||
if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
|
||||
return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
|
||||
}}
|
||||
|
||||
if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
|
||||
return validateBinaryValue_{widget_id}(raw);
|
||||
}}
|
||||
|
||||
if (typeName === 'string') {{
|
||||
if (meta.pattern) {{
|
||||
try {{
|
||||
if (!(new RegExp(meta.pattern)).test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
|
||||
}}
|
||||
}} catch (error) {{}}
|
||||
}}
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'integer') {{
|
||||
if (!/^-?\\d+$/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be an integer' }};
|
||||
}}
|
||||
numberValue = Number(raw);
|
||||
if (meta.minimum !== undefined && numberValue < meta.minimum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
|
||||
}}
|
||||
if (meta.maximum !== undefined && numberValue > meta.maximum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
|
||||
}}
|
||||
return {{ ok: true, value: numberValue, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'number') {{
|
||||
if (raw === '' || isNaN(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be a number' }};
|
||||
}}
|
||||
numberValue = Number(raw);
|
||||
if (meta.minimum !== undefined && numberValue < meta.minimum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
|
||||
}}
|
||||
if (meta.maximum !== undefined && numberValue > meta.maximum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
|
||||
}}
|
||||
return {{ ok: true, value: numberValue, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'boolean') {{
|
||||
var lowered = raw.toLowerCase();
|
||||
if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
|
||||
if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
|
||||
return {{ ok: false, value: raw, message: 'Must be true or false' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'null') {{
|
||||
return raw === 'null'
|
||||
? {{ ok: true, value: null, message: '' }}
|
||||
: {{ ok: false, value: raw, message: 'Must be null' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'array' || typeName === 'object') {{
|
||||
try {{
|
||||
parsed = JSON.parse(raw);
|
||||
}} catch (error) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'array' && Array.isArray(parsed)) {{
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
|
||||
return {{
|
||||
ok: false,
|
||||
value: parsed,
|
||||
message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
|
||||
}};
|
||||
}}
|
||||
|
||||
return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
|
||||
}}
|
||||
|
||||
function validateValueAgainstMeta_{widget_id}(raw, meta) {{
|
||||
if (!meta || raw === '') {{
|
||||
return {{ state: 'neutral', value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (!types.length) {{
|
||||
types = ['string'];
|
||||
}}
|
||||
|
||||
var error = 'Invalid value';
|
||||
for (var i = 0; i < types.length; i++) {{
|
||||
var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
|
||||
if (!candidate.ok) {{
|
||||
error = candidate.message || error;
|
||||
continue;
|
||||
}}
|
||||
if (enumValues.length && !enumValues.some(function(enumValue) {{
|
||||
return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
|
||||
}})) {{
|
||||
error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
|
||||
continue;
|
||||
}}
|
||||
return {{ state: 'valid', value: candidate.value, message: '' }};
|
||||
}}
|
||||
|
||||
return {{ state: 'invalid', value: raw, message: error }};
|
||||
}}
|
||||
|
||||
function ensureRowId_{widget_id}(row) {{
|
||||
if (!row.dataset.rowId) {{
|
||||
row.dataset.rowId = String(rowCounter_{widget_id}++);
|
||||
}}
|
||||
return row.dataset.rowId;
|
||||
}}
|
||||
|
||||
function setRowHelp_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var help = row.querySelector('.kv-help');
|
||||
if (!keyInput || !help) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var key = keyInput.value.trim();
|
||||
if (!key) {{
|
||||
help.textContent = '';
|
||||
return;
|
||||
}}
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (meta) {{
|
||||
var extra = isRegexConfigKey_{widget_id}(key)
|
||||
? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
|
||||
? ' Expected: JSON object with regex keys.'
|
||||
: ' Expected: valid regex.')
|
||||
: '';
|
||||
var example = getExampleInput_{widget_id}(key, meta);
|
||||
help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
|
||||
}} else {{
|
||||
help.textContent = 'Custom key';
|
||||
}}
|
||||
}}
|
||||
|
||||
function configureValueInput_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valueInput = row.querySelector('.kv-value');
|
||||
var datalist = row.querySelector('.kv-value-options');
|
||||
if (!keyInput || !valueInput || !datalist) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var rowId = ensureRowId_{widget_id}(row);
|
||||
datalist.id = '{widget_id}_value_options_' + rowId;
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
|
||||
var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (!enumValues.length && types.includes('boolean')) {{
|
||||
enumValues = ['True', 'False'];
|
||||
}}
|
||||
if (enumValues.length) {{
|
||||
datalist.innerHTML = enumValues.map(function(enumValue) {{
|
||||
return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '"') + '"></option>';
|
||||
}}).join('');
|
||||
valueInput.setAttribute('list', datalist.id);
|
||||
}} else {{
|
||||
datalist.innerHTML = '';
|
||||
valueInput.removeAttribute('list');
|
||||
}}
|
||||
}}
|
||||
|
||||
function setValueValidationState_{widget_id}(input, state, message) {{
|
||||
if (!input) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
if (state === 'valid') {{
|
||||
input.style.borderColor = '#2da44e';
|
||||
input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
|
||||
input.style.backgroundColor = '#f6ffed';
|
||||
}} else if (state === 'invalid') {{
|
||||
input.style.borderColor = '#cf222e';
|
||||
input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
|
||||
input.style.backgroundColor = '#fff8f8';
|
||||
}} else {{
|
||||
input.style.borderColor = '#ccc';
|
||||
input.style.boxShadow = 'none';
|
||||
input.style.backgroundColor = '';
|
||||
}}
|
||||
input.title = message || '';
|
||||
}}
|
||||
|
||||
function applyValueValidation_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valueInput = row.querySelector('.kv-value');
|
||||
if (!keyInput || !valueInput) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var key = keyInput.value.trim();
|
||||
if (!key) {{
|
||||
setValueValidationState_{widget_id}(valueInput, 'neutral', '');
|
||||
return;
|
||||
}}
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (!meta) {{
|
||||
setValueValidationState_{widget_id}(valueInput, 'neutral', '');
|
||||
return;
|
||||
}}
|
||||
|
||||
var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
|
||||
setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
|
||||
}}
|
||||
|
||||
function coerceValueForStorage_{widget_id}(key, raw) {{
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (!meta) {{
|
||||
return parseValue_{widget_id}(raw);
|
||||
}}
|
||||
|
||||
var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
|
||||
return validation.state === 'valid' ? validation.value : raw;
|
||||
}}
|
||||
|
||||
function initializeRows_{widget_id}() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
container.querySelectorAll('.key-value-row').forEach(function(row) {{
|
||||
ensureRowId_{widget_id}(row);
|
||||
configureValueInput_{widget_id}(row);
|
||||
setRowHelp_{widget_id}(row);
|
||||
applyValueValidation_{widget_id}(row);
|
||||
}});
|
||||
}}
|
||||
|
||||
function updateHiddenField_{widget_id}() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
var rows = container.querySelectorAll('.key-value-row');
|
||||
var result = {{}};
|
||||
rows.forEach(function(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valInput = row.querySelector('.kv-value');
|
||||
if (keyInput && valInput && keyInput.value.trim()) {{
|
||||
var key = keyInput.value.trim();
|
||||
var val = valInput.value.trim();
|
||||
result[key] = coerceValueForStorage_{widget_id}(key, val);
|
||||
}}
|
||||
}});
|
||||
document.getElementById('{widget_id}').value = JSON.stringify(result);
|
||||
}}
|
||||
|
||||
window.addKeyValueRow_{widget_id} = function() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
var newRow = document.createElement('div');
|
||||
newRow.className = 'key-value-row';
|
||||
newRow.style.cssText = 'margin-bottom: 6px;';
|
||||
newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
|
||||
'<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
|
||||
'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
|
||||
'<input type="text" class="kv-value" placeholder="value" ' +
|
||||
'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
|
||||
'<datalist class="kv-value-options"></datalist>' +
|
||||
'<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
|
||||
'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
|
||||
'</div>' +
|
||||
'<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
|
||||
container.appendChild(newRow);
|
||||
ensureRowId_{widget_id}(newRow);
|
||||
configureValueInput_{widget_id}(newRow);
|
||||
setRowHelp_{widget_id}(newRow);
|
||||
applyValueValidation_{widget_id}(newRow);
|
||||
updateHiddenField_{widget_id}();
|
||||
newRow.querySelector('.kv-key').focus();
|
||||
}};
|
||||
|
||||
window.removeKeyValueRow_{widget_id} = function(btn) {{
|
||||
var row = btn.closest('.key-value-row');
|
||||
row.remove();
|
||||
updateHiddenField_{widget_id}();
|
||||
}};
|
||||
|
||||
window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
|
||||
|
||||
function configRowForKey_{widget_id}(key) {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
if (!container) {{
|
||||
return null;
|
||||
}}
|
||||
var match = null;
|
||||
container.querySelectorAll('.key-value-row').forEach(function(row) {{
|
||||
if (match) {{ return; }}
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
if (keyInput && keyInput.value.trim() === key) {{
|
||||
match = row;
|
||||
}}
|
||||
}});
|
||||
if (!match) {{
|
||||
window.addKeyValueRow_{widget_id}();
|
||||
var rows = container.querySelectorAll('.key-value-row');
|
||||
match = rows[rows.length - 1];
|
||||
var keyInput = match.querySelector('.kv-key');
|
||||
if (keyInput) {{
|
||||
keyInput.value = key;
|
||||
keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
}}
|
||||
}}
|
||||
return match;
|
||||
}}
|
||||
|
||||
function prefillConfigFromQuery_{widget_id}() {{
|
||||
var params = new URLSearchParams(window.location.search);
|
||||
var consumedConfigKeys = [];
|
||||
params.forEach(function(value, key) {{
|
||||
if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
|
||||
return;
|
||||
}}
|
||||
consumedConfigKeys.push(key);
|
||||
var match = configRowForKey_{widget_id}(key);
|
||||
var valueInput = match && match.querySelector('.kv-value');
|
||||
if (valueInput) {{
|
||||
valueInput.value = value;
|
||||
valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
}}
|
||||
}});
|
||||
updateHiddenField_{widget_id}();
|
||||
if (consumedConfigKeys.length) {{
|
||||
var cleanUrl = new URL(window.location.href);
|
||||
consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
|
||||
window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
|
||||
}}
|
||||
}}
|
||||
|
||||
function focusConfigKeyFromHash_{widget_id}() {{
|
||||
// Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
|
||||
// to (or creates) the matching row in this editor. Used by
|
||||
// the setup wizard and live-config detail pages.
|
||||
var hash = (window.location.hash || '').replace(/^#/, '').trim();
|
||||
if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
|
||||
return;
|
||||
}}
|
||||
var match = configRowForKey_{widget_id}(hash);
|
||||
if (!match) {{
|
||||
return;
|
||||
}}
|
||||
match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
|
||||
var prevOutline = match.style.outline;
|
||||
match.style.outline = '2px solid #f59e0b';
|
||||
match.style.outlineOffset = '2px';
|
||||
match.style.transition = 'outline 1.2s ease-out';
|
||||
setTimeout(function() {{
|
||||
match.style.outline = prevOutline || 'none';
|
||||
}}, 1400);
|
||||
var valueInput = match.querySelector('.kv-value');
|
||||
if (valueInput) {{
|
||||
valueInput.focus();
|
||||
try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
|
||||
}}
|
||||
}}
|
||||
|
||||
// Initialize on load
|
||||
document.addEventListener('DOMContentLoaded', function() {{
|
||||
initializeRows_{widget_id}();
|
||||
prefillConfigFromQuery_{widget_id}();
|
||||
updateHiddenField_{widget_id}();
|
||||
focusConfigKeyFromHash_{widget_id}();
|
||||
}});
|
||||
// Also run immediately in case DOM is already ready
|
||||
if (document.readyState !== 'loading') {{
|
||||
initializeRows_{widget_id}();
|
||||
prefillConfigFromQuery_{widget_id}();
|
||||
updateHiddenField_{widget_id}();
|
||||
focusConfigKeyFromHash_{widget_id}();
|
||||
}}
|
||||
|
||||
window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
|
||||
|
||||
// Update on any input change
|
||||
var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
|
||||
|
||||
rowsEl_{widget_id}.addEventListener('input', function(event) {{
|
||||
var row = event.target.closest('.key-value-row');
|
||||
if (!row) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
if (event.target.classList.contains('kv-key')) {{
|
||||
configureValueInput_{widget_id}(row);
|
||||
setRowHelp_{widget_id}(row);
|
||||
}}
|
||||
|
||||
if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
|
||||
applyValueValidation_{widget_id}(row);
|
||||
updateHiddenField_{widget_id}();
|
||||
}}
|
||||
}});
|
||||
}})();
|
||||
</script>
|
||||
</div>
|
||||
'''
|
||||
return mark_safe(html)
|
||||
|
||||
def _render_row(self, widget_id: str, key: str, value: str) -> str:
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
@@ -872,6 +269,28 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
|
||||
show_search_mode_selector = False
|
||||
change_form_template = "admin/archivebox_change_form.html"
|
||||
|
||||
@admin.display(description="Health", ordering="health")
|
||||
def health_display(self, obj):
|
||||
h = obj.health
|
||||
color = "green" if h >= 80 else "orange" if h >= 50 else "red"
|
||||
return format_html('<span style="color: {};">{}</span>', color, h)
|
||||
|
||||
def get_ordering_fields(self, request):
|
||||
ordering = request.GET.get("o")
|
||||
if not ordering:
|
||||
return set()
|
||||
fields = set()
|
||||
for part in ordering.split("."):
|
||||
if not part:
|
||||
continue
|
||||
try:
|
||||
idx = abs(int(part)) - 1
|
||||
except ValueError:
|
||||
continue
|
||||
if 0 <= idx < len(self.list_display):
|
||||
fields.add(self.list_display[idx])
|
||||
return fields
|
||||
|
||||
def get_admin_toolbar_actions(self, request, obj):
|
||||
"""Return extra action button dicts for the shared change-form toolbar.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ __package__ = "archivebox.base_models"
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
from pathlib import Path
|
||||
@@ -23,6 +23,10 @@ from django_stubs_ext.db.models import TypedModelMeta
|
||||
from archivebox.config import CONSTANTS
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from archivebox.config.common import ArchiveBoxBaseConfig
|
||||
|
||||
|
||||
def normalize_config_json_values(config: Any) -> Any:
|
||||
if not isinstance(config, dict):
|
||||
return config
|
||||
@@ -55,17 +59,6 @@ def get_or_create_system_user_pk(username="system"):
|
||||
return user.pk
|
||||
|
||||
|
||||
class AutoDateTimeField(models.DateTimeField):
|
||||
"""DateTimeField that automatically updates on save (legacy compatibility)."""
|
||||
|
||||
def pre_save(self, model_instance, add):
|
||||
if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
|
||||
value = timezone.now()
|
||||
setattr(model_instance, self.attname, value)
|
||||
return value
|
||||
return super().pre_save(model_instance, add)
|
||||
|
||||
|
||||
class ModelWithUUID(models.Model):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
@@ -138,6 +131,7 @@ class ModelWithHealthStats(models.Model):
|
||||
class ModelWithConfig(models.Model):
|
||||
"""Mixin for models with a JSON config field."""
|
||||
|
||||
_runtime_config: "ArchiveBoxBaseConfig"
|
||||
config = models.JSONField(default=dict, null=True, blank=True, editable=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
|
||||
@@ -72,7 +72,7 @@ def create_archiveresults(
|
||||
"""
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.plugins.hooks import discover_hooks
|
||||
from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
|
||||
from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
@@ -99,26 +99,9 @@ def create_archiveresults(
|
||||
for record in records:
|
||||
record_type = record.get("type", "")
|
||||
|
||||
if record_type == TYPE_SNAPSHOT:
|
||||
# Pass through the Snapshot record itself
|
||||
if record_type:
|
||||
pass_through_records.append(record)
|
||||
if record.get("id"):
|
||||
snapshot_ids.append(record["id"])
|
||||
|
||||
elif record_type == TYPE_ARCHIVERESULT:
|
||||
# ArchiveResult records: pass through if they have an id
|
||||
if record.get("id"):
|
||||
pass_through_records.append(record)
|
||||
# If no id, we could create it, but for now just pass through
|
||||
else:
|
||||
pass_through_records.append(record)
|
||||
|
||||
elif record_type:
|
||||
# Other typed records (Crawl, Tag, etc): pass through
|
||||
pass_through_records.append(record)
|
||||
|
||||
elif record.get("id"):
|
||||
# Untyped record with id - assume it's a snapshot ID
|
||||
if record.get("id") and record_type in ("", TYPE_SNAPSHOT):
|
||||
snapshot_ids.append(record["id"])
|
||||
|
||||
# Output pass-through records first
|
||||
@@ -169,48 +152,20 @@ def list_archiveresults(
|
||||
snapshot_id: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List ArchiveResults as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
"""List archive results as JSONL, or formatted rows in a terminal."""
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.cli.cli_util import list_records, format_status
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = ArchiveResult.objects.all().order_by("-start_ts")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"status": status,
|
||||
"plugin": plugin,
|
||||
"snapshot_id": snapshot_id,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for result in queryset:
|
||||
if is_tty:
|
||||
status_color = {
|
||||
"queued": "yellow",
|
||||
"started": "blue",
|
||||
"succeeded": "green",
|
||||
"failed": "red",
|
||||
"skipped": "dim",
|
||||
"noresults": "dim",
|
||||
"backoff": "magenta",
|
||||
}.get(result.status, "dim")
|
||||
rprint(
|
||||
f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
|
||||
)
|
||||
else:
|
||||
write_record(result.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
queryset = apply_filters(
|
||||
ArchiveResult.objects.order_by("-start_ts"),
|
||||
{"status": status, "plugin": plugin, "snapshot_id": snapshot_id},
|
||||
limit=limit,
|
||||
)
|
||||
return list_records(
|
||||
queryset,
|
||||
plural="archive results",
|
||||
render=lambda result: f"{format_status(result.status, 10)} {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -218,54 +173,17 @@ def list_archiveresults(
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_archiveresults(
|
||||
status: str | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update ArchiveResults from stdin JSONL.
|
||||
|
||||
Reads ArchiveResult records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
def update_archiveresults(status: str | None = None) -> int:
|
||||
"""Apply supplied fields to each JSONL-selected ArchiveResult."""
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.cli.cli_util import update_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
def update(archiveresult):
|
||||
if status:
|
||||
archiveresult.status = status
|
||||
archiveresult.save()
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
result_id = record.get("id")
|
||||
if not result_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
result = ArchiveResult.objects.get(id=result_id)
|
||||
|
||||
# Apply updates from CLI flags
|
||||
if status:
|
||||
result.status = status
|
||||
|
||||
result.save()
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
write_record(result.to_json())
|
||||
|
||||
except ArchiveResult.DoesNotExist:
|
||||
rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return update_records(ArchiveResult, update, plural="archive results")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -274,52 +192,19 @@ def update_archiveresults(
|
||||
|
||||
|
||||
def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete ArchiveResults from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
"""Delete archive results selected by stdin JSONL; --yes confirms, --dry-run previews."""
|
||||
from archivebox.cli.cli_util import delete_records
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
result_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not result_ids:
|
||||
rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results = ArchiveResult.objects.filter(id__in=result_ids)
|
||||
count = results.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
|
||||
for result in results[:10]:
|
||||
rprint(f" [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
|
||||
if count > 10:
|
||||
rprint(f" ... and {count - 10} more", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = results.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return delete_records(
|
||||
ArchiveResult,
|
||||
label="archive result",
|
||||
plural="archive results",
|
||||
preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.plugin} {obj.snapshot.url[:40]}",
|
||||
yes=yes,
|
||||
dry_run=dry_run,
|
||||
preview_limit=10,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -337,9 +222,9 @@ def main():
|
||||
@click.option("--snapshot-id", help="Snapshot ID to create results for")
|
||||
@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
|
||||
@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
|
||||
def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
|
||||
def create_cmd(**kwargs):
|
||||
"""Emit Snapshot plugin extraction requests as JSONL."""
|
||||
sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
|
||||
sys.exit(create_archiveresults(**kwargs))
|
||||
|
||||
|
||||
@main.command("list")
|
||||
@@ -347,36 +232,24 @@ def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
|
||||
@click.option("--plugin", "-p", help="Filter by plugin name")
|
||||
@click.option("--snapshot-id", help="Filter by snapshot ID")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(
|
||||
status: str | None,
|
||||
plugin: str | None,
|
||||
snapshot_id: str | None,
|
||||
limit: int | None,
|
||||
):
|
||||
def list_cmd(**kwargs):
|
||||
"""List ArchiveResults as JSONL."""
|
||||
sys.exit(
|
||||
list_archiveresults(
|
||||
status=status,
|
||||
plugin=plugin,
|
||||
snapshot_id=snapshot_id,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
sys.exit(list_archiveresults(**kwargs))
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--status", "-s", help="Set status")
|
||||
def update_cmd(status: str | None):
|
||||
def update_cmd(**kwargs):
|
||||
"""Update ArchiveResults from stdin JSONL."""
|
||||
sys.exit(update_archiveresults(status=status))
|
||||
sys.exit(update_archiveresults(**kwargs))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
def delete_cmd(**kwargs):
|
||||
"""Delete ArchiveResults from stdin JSONL."""
|
||||
sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
|
||||
sys.exit(delete_archiveresults(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -114,37 +114,20 @@ def list_binaries(
|
||||
version__icontains: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Binaries as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
"""List binaries as JSONL, or formatted rows in a terminal."""
|
||||
from archivebox.machine.models import Binary
|
||||
from archivebox.cli.cli_util import list_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"name": name,
|
||||
"abspath__icontains": abspath__icontains,
|
||||
"version__icontains": version__icontains,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for binary in queryset:
|
||||
if is_tty:
|
||||
rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
|
||||
else:
|
||||
write_record(binary.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
queryset = apply_filters(
|
||||
Binary.objects.order_by("name", "-modified_at", "-created_at"),
|
||||
{"name": name, "abspath__icontains": abspath__icontains, "version__icontains": version__icontains},
|
||||
limit=limit,
|
||||
)
|
||||
return list_records(
|
||||
queryset,
|
||||
plural="binaries",
|
||||
render=lambda binary: f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}",
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -152,57 +135,19 @@ def list_binaries(
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_binaries(
|
||||
version: str | None = None,
|
||||
abspath: str | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update Binaries from stdin JSONL.
|
||||
|
||||
Reads Binary records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
def update_binaries(version: str | None = None, abspath: str | None = None) -> int:
|
||||
"""Apply supplied fields to each JSONL-selected Binary."""
|
||||
from archivebox.machine.models import Binary
|
||||
from archivebox.cli.cli_util import update_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
def update(binary):
|
||||
if version:
|
||||
binary.version = version
|
||||
if abspath:
|
||||
binary.abspath = abspath
|
||||
binary.save()
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
binary_id = record.get("id")
|
||||
if not binary_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
binary = Binary.objects.get(id=binary_id)
|
||||
|
||||
# Apply updates from CLI flags
|
||||
if version:
|
||||
binary.version = version
|
||||
if abspath:
|
||||
binary.abspath = abspath
|
||||
|
||||
binary.save()
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
write_record(binary.to_json())
|
||||
|
||||
except Binary.DoesNotExist:
|
||||
rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return update_records(Binary, update, plural="binaries")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -211,50 +156,18 @@ def update_binaries(
|
||||
|
||||
|
||||
def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete Binaries from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
"""Delete binaries selected by stdin JSONL; --yes confirms, --dry-run previews."""
|
||||
from archivebox.cli.cli_util import delete_records
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
binary_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not binary_ids:
|
||||
rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
binaries = Binary.objects.filter(id__in=binary_ids)
|
||||
count = binaries.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
|
||||
for binary in binaries:
|
||||
rprint(f" {binary.name} {binary.abspath}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = binaries.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return delete_records(
|
||||
Binary,
|
||||
label="binary",
|
||||
plural="binaries",
|
||||
preview=lambda obj: f"{obj.name} {obj.abspath}",
|
||||
yes=yes,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -272,9 +185,9 @@ def main():
|
||||
@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
|
||||
@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
|
||||
@click.option("--version", "-v", default="", help="Binary version")
|
||||
def create_cmd(name: str, abspath: str, version: str):
|
||||
def create_cmd(**kwargs):
|
||||
"""Create/register a Binary."""
|
||||
sys.exit(create_binary(name=name, abspath=abspath, version=version))
|
||||
sys.exit(create_binary(**kwargs))
|
||||
|
||||
|
||||
@main.command("list")
|
||||
@@ -282,37 +195,25 @@ def create_cmd(name: str, abspath: str, version: str):
|
||||
@click.option("--abspath__icontains", help="Filter by path contains")
|
||||
@click.option("--version__icontains", help="Filter by version contains")
|
||||
@click.option("--limit", type=int, help="Limit number of results")
|
||||
def list_cmd(
|
||||
name: str | None,
|
||||
abspath__icontains: str | None,
|
||||
version__icontains: str | None,
|
||||
limit: int | None,
|
||||
):
|
||||
def list_cmd(**kwargs):
|
||||
"""List Binaries as JSONL."""
|
||||
sys.exit(
|
||||
list_binaries(
|
||||
name=name,
|
||||
abspath__icontains=abspath__icontains,
|
||||
version__icontains=version__icontains,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
sys.exit(list_binaries(**kwargs))
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--version", "-v", help="Set version")
|
||||
@click.option("--abspath", "-p", help="Set path")
|
||||
def update_cmd(version: str | None, abspath: str | None):
|
||||
def update_cmd(**kwargs):
|
||||
"""Update Binaries from stdin JSONL."""
|
||||
sys.exit(update_binaries(version=version, abspath=abspath))
|
||||
sys.exit(update_binaries(**kwargs))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
def delete_cmd(**kwargs):
|
||||
"""Delete Binaries from stdin JSONL."""
|
||||
sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
|
||||
sys.exit(delete_binaries(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -177,43 +177,20 @@ def list_crawls(
|
||||
max_depth: int | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Crawls as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
"""List crawls as JSONL, or formatted rows in a terminal."""
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.cli.cli_util import list_records, format_status
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Crawl.objects.all().order_by("-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"status": status,
|
||||
"urls__icontains": urls__icontains,
|
||||
"max_depth": max_depth,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for crawl in queryset:
|
||||
if is_tty:
|
||||
status_color = {
|
||||
"queued": "yellow",
|
||||
"started": "blue",
|
||||
"sealed": "green",
|
||||
}.get(crawl.status, "dim")
|
||||
url_preview = crawl.urls[:50].replace("\n", " ")
|
||||
rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
|
||||
else:
|
||||
write_record(crawl.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
queryset = apply_filters(
|
||||
Crawl.objects.order_by("-created_at"),
|
||||
{"status": status, "urls__icontains": urls__icontains, "max_depth": max_depth},
|
||||
limit=limit,
|
||||
)
|
||||
return list_records(
|
||||
queryset,
|
||||
plural="crawls",
|
||||
render=lambda crawl: f"{format_status(crawl.status, 8)} [dim]{crawl.id}[/dim] {crawl.urls[:50].replace(chr(10), chr(32))}...",
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -221,73 +198,26 @@ def list_crawls(
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_crawls(
|
||||
status: str | None = None,
|
||||
max_depth: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update Crawls from stdin JSONL.
|
||||
|
||||
Reads Crawl records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
def update_crawls(status: str | None = None, max_depth: int | None = None) -> int:
|
||||
"""Update JSONL-selected crawls through their lifecycle operations."""
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.cli.cli_util import update_records, update_record_status
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
def update(crawl):
|
||||
if status:
|
||||
try:
|
||||
update_record_status(crawl, status)
|
||||
except ValueError as err:
|
||||
rprint(f"[red]{err}[/red]", file=sys.stderr)
|
||||
return False
|
||||
if max_depth is not None:
|
||||
crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
|
||||
crawl.max_depth = max_depth
|
||||
elif not status:
|
||||
crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
crawl_id = record.get("id")
|
||||
if not crawl_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
crawl = Crawl.objects.get(id=crawl_id)
|
||||
|
||||
if status:
|
||||
if status not in Crawl.StatusChoices.values:
|
||||
rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
|
||||
continue
|
||||
if status == Crawl.StatusChoices.SEALED:
|
||||
crawl.cancel()
|
||||
elif status == Crawl.StatusChoices.PAUSED:
|
||||
crawl.pause()
|
||||
elif status == Crawl.StatusChoices.QUEUED:
|
||||
if crawl.status == Crawl.StatusChoices.PAUSED:
|
||||
crawl.resume()
|
||||
else:
|
||||
crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
|
||||
elif status == Crawl.StatusChoices.STARTED:
|
||||
crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
|
||||
if max_depth is not None:
|
||||
crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
|
||||
crawl.max_depth = max_depth
|
||||
elif not status:
|
||||
crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
crawl.refresh_from_db()
|
||||
write_record(crawl.to_json())
|
||||
|
||||
except Crawl.DoesNotExist:
|
||||
rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return update_records(Crawl, update, plural="crawls", refresh=True)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -296,51 +226,18 @@ def update_crawls(
|
||||
|
||||
|
||||
def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete Crawls from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
"""Delete crawls selected by stdin JSONL; --yes confirms, --dry-run previews."""
|
||||
from archivebox.cli.cli_util import delete_records
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
crawl_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not crawl_ids:
|
||||
rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
crawls = Crawl.objects.filter(id__in=crawl_ids)
|
||||
count = crawls.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
|
||||
for crawl in crawls:
|
||||
url_preview = crawl.urls[:50].replace("\n", " ")
|
||||
rprint(f" [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = crawls.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return delete_records(
|
||||
Crawl,
|
||||
label="crawl",
|
||||
plural="crawls",
|
||||
preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.urls[:50].replace(chr(10), chr(32))}...",
|
||||
yes=yes,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -369,37 +266,25 @@ def create_cmd(urls: tuple, depth: int, tag: str, status: str):
|
||||
@click.option("--urls__icontains", help="Filter by URLs contains")
|
||||
@click.option("--max-depth", type=int, help="Filter by max depth")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(
|
||||
status: str | None,
|
||||
urls__icontains: str | None,
|
||||
max_depth: int | None,
|
||||
limit: int | None,
|
||||
):
|
||||
def list_cmd(**kwargs):
|
||||
"""List Crawls as JSONL."""
|
||||
sys.exit(
|
||||
list_crawls(
|
||||
status=status,
|
||||
urls__icontains=urls__icontains,
|
||||
max_depth=max_depth,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
sys.exit(list_crawls(**kwargs))
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--status", "-s", help="Set status")
|
||||
@click.option("--max-depth", type=int, help="Set max depth")
|
||||
def update_cmd(status: str | None, max_depth: int | None):
|
||||
def update_cmd(**kwargs):
|
||||
"""Update Crawls from stdin JSONL."""
|
||||
sys.exit(update_crawls(status=status, max_depth=max_depth))
|
||||
sys.exit(update_crawls(**kwargs))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
def delete_cmd(**kwargs):
|
||||
"""Delete Crawls from stdin JSONL."""
|
||||
sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
|
||||
sys.exit(delete_crawls(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -18,7 +18,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
|
||||
ArchiveResult input is accepted as a convenient reference to its parent
|
||||
Snapshot and plugin. It is never reset or converted into queued work.
|
||||
"""
|
||||
from archivebox.api.v1_core import _uuid_ref_query
|
||||
from archivebox.misc.db import uuid_ref_query
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
|
||||
|
||||
@@ -36,7 +36,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
|
||||
continue
|
||||
|
||||
if record_type == TYPE_ARCHIVERESULT:
|
||||
result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
|
||||
result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
|
||||
if result is not None:
|
||||
requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
|
||||
continue
|
||||
@@ -50,7 +50,7 @@ def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[
|
||||
# convenience of passing an ArchiveResult ID by trying that reference
|
||||
# only after the Snapshot lookup misses.
|
||||
if snapshot is None and record_id and not record.get("url"):
|
||||
result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
|
||||
result = ArchiveResult.objects.filter(uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
|
||||
if result is not None:
|
||||
requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
|
||||
continue
|
||||
@@ -177,11 +177,6 @@ def run_plugins(
|
||||
return 0
|
||||
|
||||
|
||||
def process_archiveresult_by_id(archiveresult_id: str) -> int:
|
||||
"""Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
|
||||
return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
|
||||
@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
|
||||
|
||||
@@ -25,7 +25,6 @@ __command__ = "archivebox machine"
|
||||
import sys
|
||||
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
@@ -35,41 +34,21 @@ from archivebox.cli.cli_util import apply_filters
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def list_machines(
|
||||
hostname__icontains: str | None = None,
|
||||
os_platform: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Machines as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
def list_machines(hostname__icontains: str | None = None, os_platform: str | None = None, limit: int | None = None) -> int:
|
||||
"""List machines as JSONL, or formatted rows in a terminal."""
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.cli.cli_util import list_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Machine.objects.all().order_by("-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"hostname__icontains": hostname__icontains,
|
||||
"os_platform": os_platform,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for machine in queryset:
|
||||
if is_tty:
|
||||
rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
|
||||
else:
|
||||
write_record(machine.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
queryset = apply_filters(
|
||||
Machine.objects.order_by("-created_at"),
|
||||
{"hostname__icontains": hostname__icontains, "os_platform": os_platform},
|
||||
limit=limit,
|
||||
)
|
||||
return list_records(
|
||||
queryset,
|
||||
plural="machines",
|
||||
render=lambda machine: f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}",
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -87,15 +66,9 @@ def main():
|
||||
@click.option("--hostname__icontains", help="Filter by hostname contains")
|
||||
@click.option("--os-platform", help="Filter by OS platform")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
|
||||
def list_cmd(**kwargs):
|
||||
"""List Machines as JSONL."""
|
||||
sys.exit(
|
||||
list_machines(
|
||||
hostname__icontains=hostname__icontains,
|
||||
os_platform=os_platform,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
sys.exit(list_machines(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -36,44 +36,10 @@ from collections.abc import Iterable
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters, list_records
|
||||
from archivebox.personas import importers as persona_importers
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Validation Helpers
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def validate_persona_name(name: str) -> tuple[bool, str]:
|
||||
"""
|
||||
Validate persona name to prevent path traversal attacks.
|
||||
|
||||
Returns:
|
||||
(is_valid, error_message): tuple indicating if name is valid
|
||||
"""
|
||||
if not name or not name.strip():
|
||||
return False, "Persona name cannot be empty"
|
||||
|
||||
# Check for path separators
|
||||
if "/" in name or "\\" in name:
|
||||
return False, "Persona name cannot contain path separators (/ or \\)"
|
||||
|
||||
# Check for parent directory references
|
||||
if ".." in name:
|
||||
return False, "Persona name cannot contain parent directory references (..)"
|
||||
|
||||
# Check for hidden files/directories
|
||||
if name.startswith("."):
|
||||
return False, "Persona name cannot start with a dot (.)"
|
||||
|
||||
# Ensure name doesn't contain null bytes or other dangerous chars
|
||||
if "\x00" in name or "\n" in name or "\r" in name:
|
||||
return False, "Persona name contains invalid characters"
|
||||
|
||||
return True, ""
|
||||
|
||||
|
||||
def ensure_path_within_personas_dir(persona_path: Path) -> bool:
|
||||
"""
|
||||
Verify that a persona path is within PERSONAS_DIR.
|
||||
@@ -204,15 +170,7 @@ def create_personas(
|
||||
if created:
|
||||
rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
|
||||
if not is_tty:
|
||||
write_record(
|
||||
{
|
||||
"id": str(persona.id),
|
||||
"name": persona.name,
|
||||
"path": str(persona.path),
|
||||
"CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
|
||||
"COOKIES_FILE": persona.COOKIES_FILE,
|
||||
},
|
||||
)
|
||||
write_record(persona.to_json())
|
||||
|
||||
rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
|
||||
return 0
|
||||
@@ -234,41 +192,20 @@ def list_personas(
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
queryset = apply_filters(
|
||||
Persona.objects.order_by("name"),
|
||||
{"name": name, "name__icontains": name__icontains},
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
queryset = Persona.objects.all().order_by("name")
|
||||
def render(persona):
|
||||
cookies = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
|
||||
chrome = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
|
||||
return f"[cyan]{persona.name:20}[/cyan] cookies:{cookies} chrome:{chrome} [dim]{persona.path}[/dim]"
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"name": name,
|
||||
"name__icontains": name__icontains,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for persona in queryset:
|
||||
cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
|
||||
chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
|
||||
|
||||
if is_tty:
|
||||
rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
|
||||
else:
|
||||
write_record(
|
||||
{
|
||||
"id": str(persona.id),
|
||||
"name": persona.name,
|
||||
"path": str(persona.path),
|
||||
"CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
|
||||
"COOKIES_FILE": persona.COOKIES_FILE,
|
||||
},
|
||||
)
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
return list_records(queryset, plural="persona(s)", render=render)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -332,13 +269,7 @@ def update_personas(name: str | None = None) -> int:
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
write_record(
|
||||
{
|
||||
"id": str(persona.id),
|
||||
"name": persona.name,
|
||||
"path": str(persona.path),
|
||||
},
|
||||
)
|
||||
write_record(persona.to_json())
|
||||
|
||||
except Persona.DoesNotExist:
|
||||
rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
|
||||
@@ -458,9 +389,9 @@ def create_cmd(names: tuple, import_from: str | None, profile: str | None, sourc
|
||||
@click.option("--name", help="Filter by exact name")
|
||||
@click.option("--name__icontains", help="Filter by name contains")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
|
||||
def list_cmd(**kwargs):
|
||||
"""List Personas as JSONL."""
|
||||
sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
|
||||
sys.exit(list_personas(**kwargs))
|
||||
|
||||
|
||||
@main.command("open")
|
||||
@@ -477,7 +408,7 @@ def open_cmd(name: str):
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
valid, reason = validate_persona_name(name)
|
||||
valid, reason = persona_importers.validate_persona_name(name)
|
||||
if not valid:
|
||||
raise click.ClickException(reason)
|
||||
persona = Persona.get_or_create_named(name)
|
||||
@@ -543,17 +474,17 @@ def open_cmd(name: str):
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--name", "-n", help="Set new name")
|
||||
def update_cmd(name: str | None):
|
||||
def update_cmd(**kwargs):
|
||||
"""Update Personas from stdin JSONL."""
|
||||
sys.exit(update_personas(name=name))
|
||||
sys.exit(update_personas(**kwargs))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
def delete_cmd(**kwargs):
|
||||
"""Delete Personas from stdin JSONL."""
|
||||
sys.exit(delete_personas(yes=yes, dry_run=dry_run))
|
||||
sys.exit(delete_personas(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -28,7 +28,6 @@ __command__ = "archivebox process"
|
||||
import sys
|
||||
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
@@ -38,46 +37,20 @@ from archivebox.cli.cli_util import apply_filters
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def list_processes(
|
||||
binary_name: str | None = None,
|
||||
machine_id: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Processes as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
def list_processes(binary_name: str | None = None, machine_id: str | None = None, limit: int | None = None) -> int:
|
||||
"""List processes as JSONL, or exit codes and binary names in a terminal."""
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.cli.cli_util import list_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
def render(process):
|
||||
name = process.binary.name if process.binary else "unknown"
|
||||
exit_code = process.exit_code if process.exit_code is not None else "?"
|
||||
color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
|
||||
return f"[{color}]exit={exit_code:3}[/{color}] [cyan]{name:15}[/cyan] [dim]{process.id}[/dim]"
|
||||
|
||||
queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {}
|
||||
if binary_name:
|
||||
filter_kwargs["binary__name"] = binary_name
|
||||
if machine_id:
|
||||
filter_kwargs["machine_id"] = machine_id
|
||||
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for process in queryset:
|
||||
if is_tty:
|
||||
binary_name_str = process.binary.name if process.binary else "unknown"
|
||||
exit_code = process.exit_code if process.exit_code is not None else "?"
|
||||
status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
|
||||
rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
|
||||
else:
|
||||
write_record(process.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
queryset = Process.objects.select_related("binary", "machine").order_by("-started_at", "-created_at")
|
||||
filters = {"binary__name": binary_name or None, "machine_id": machine_id or None}
|
||||
return list_records(apply_filters(queryset, filters, limit=limit), plural="processes", render=render)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -95,15 +68,9 @@ def main():
|
||||
@click.option("--binary-name", "-b", help="Filter by binary name")
|
||||
@click.option("--machine-id", "-m", help="Filter by machine ID")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
|
||||
def list_cmd(**kwargs):
|
||||
"""List Processes as JSONL."""
|
||||
sys.exit(
|
||||
list_processes(
|
||||
binary_name=binary_name,
|
||||
machine_id=machine_id,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
sys.exit(list_processes(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -4,14 +4,17 @@ __package__ = "archivebox.cli"
|
||||
__command__ = "archivebox remove"
|
||||
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from collections.abc import Iterable
|
||||
|
||||
import rich_click as click
|
||||
from rich.console import Console
|
||||
|
||||
from django.db.models import QuerySet
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.config.django import setup_django
|
||||
from archivebox.misc.util import enforce_types, docstring
|
||||
from archivebox.misc.checks import check_data_folder
|
||||
@@ -20,7 +23,6 @@ from archivebox.misc.logging_util import (
|
||||
log_list_finished,
|
||||
log_removal_started,
|
||||
log_removal_finished,
|
||||
TimedProgress,
|
||||
)
|
||||
from archivebox.cli.archivebox_snapshot import snapshot_filter_options
|
||||
|
||||
@@ -56,16 +58,14 @@ def remove(
|
||||
pattern_list = list(filter_patterns)
|
||||
|
||||
log_list_started(pattern_list or None, filter_type)
|
||||
timer = TimedProgress(360, prefix=" ")
|
||||
try:
|
||||
progress = Console().status("Finding matching snapshots...") if get_config().SHOW_PROGRESS else nullcontext()
|
||||
with progress:
|
||||
if snapshots is None:
|
||||
snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
|
||||
# Freeze the target set up-front so a concurrent daemon writing new
|
||||
# snapshots can't extend the deletion under us, and so the cursor isn't
|
||||
# held open across the per-row deletes below.
|
||||
snapshot_pks = list(snapshots.values_list("pk", flat=True))
|
||||
finally:
|
||||
timer.end()
|
||||
|
||||
if not snapshot_pks:
|
||||
log_removal_finished(0, 0)
|
||||
|
||||
@@ -90,7 +90,7 @@ def process_stdin_records() -> int:
|
||||
)
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.core.models import Snapshot, ArchiveResult
|
||||
from archivebox.api.v1_core import _uuid_ref_query
|
||||
from archivebox.misc.db import uuid_ref_query
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
|
||||
from archivebox.machine.models import Binary
|
||||
@@ -117,15 +117,7 @@ def process_stdin_records() -> int:
|
||||
|
||||
try:
|
||||
if record_type == TYPE_CRAWL:
|
||||
if record_id:
|
||||
# Existing crawl - re-queue
|
||||
try:
|
||||
crawl = Crawl.objects.get(id=record_id)
|
||||
except Crawl.DoesNotExist:
|
||||
crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
else:
|
||||
# New crawl - create it
|
||||
crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
|
||||
if crawl:
|
||||
crawl.update_and_requeue(
|
||||
@@ -138,14 +130,8 @@ def process_stdin_records() -> int:
|
||||
queued_count += 1
|
||||
|
||||
elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
|
||||
if record_id:
|
||||
# Existing snapshot - re-queue
|
||||
try:
|
||||
snapshot = Snapshot.objects.get(id=record_id)
|
||||
except Snapshot.DoesNotExist:
|
||||
snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
else:
|
||||
# New snapshot - create it
|
||||
snapshot = Snapshot.objects.filter(id=record_id).first() if record_id else None
|
||||
if snapshot is None:
|
||||
snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
|
||||
if snapshot:
|
||||
@@ -161,7 +147,7 @@ def process_stdin_records() -> int:
|
||||
plugin_name = str(record.get("plugin") or "")
|
||||
archiveresult = None
|
||||
if not snapshot_id and record_id:
|
||||
archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
|
||||
archiveresult = ArchiveResult.objects.filter(uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
|
||||
if archiveresult:
|
||||
snapshot_id = str(archiveresult.snapshot_id)
|
||||
plugin_name = plugin_name or archiveresult.plugin
|
||||
@@ -177,12 +163,8 @@ def process_stdin_records() -> int:
|
||||
queued_count += 1
|
||||
|
||||
elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
|
||||
if record_id:
|
||||
try:
|
||||
binary = Binary.objects.get(id=record_id)
|
||||
except Binary.DoesNotExist:
|
||||
binary = Binary.from_json(record)
|
||||
else:
|
||||
binary = Binary.objects.filter(id=record_id).first() if record_id else None
|
||||
if binary is None:
|
||||
binary = Binary.from_json(record)
|
||||
|
||||
if binary:
|
||||
|
||||
@@ -144,15 +144,7 @@ def create_snapshots(
|
||||
write_record(record)
|
||||
|
||||
# Input is a Crawl - get or create it, then create Snapshots for its URLs
|
||||
crawl = None
|
||||
crawl_id = record.get("id")
|
||||
if crawl_id:
|
||||
try:
|
||||
crawl = Crawl.objects.get(id=crawl_id)
|
||||
except Crawl.DoesNotExist:
|
||||
crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
else:
|
||||
crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
|
||||
|
||||
if not crawl:
|
||||
continue
|
||||
@@ -174,7 +166,7 @@ def create_snapshots(
|
||||
"depth": depth,
|
||||
"status": status,
|
||||
}
|
||||
snapshot = Snapshot.from_json(snapshot_record, overrides={"created_by_id": created_by_id})
|
||||
snapshot = Snapshot.from_json(snapshot_record, overrides={"crawl": crawl, "created_by_id": created_by_id})
|
||||
if snapshot:
|
||||
created_snapshots.append(snapshot)
|
||||
if not is_tty:
|
||||
@@ -368,77 +360,29 @@ def list_snapshots(
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_snapshots(
|
||||
status: str | None = None,
|
||||
tag: str | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update Snapshots from stdin JSONL.
|
||||
|
||||
Reads Snapshot records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
def update_snapshots(status: str | None = None, tag: str | None = None) -> int:
|
||||
"""Update JSONL-selected snapshots through their lifecycle operations."""
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.cli.cli_util import update_records, update_record_status
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
def update(snapshot):
|
||||
if status:
|
||||
try:
|
||||
update_record_status(snapshot, status)
|
||||
except ValueError as err:
|
||||
rprint(f"[red]{err}[/red]", file=sys.stderr)
|
||||
return False
|
||||
if tag:
|
||||
from archivebox.core.models import Tag
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
tag_obj, _ = Tag.objects.get_or_create(name=tag)
|
||||
snapshot.tags.add(tag_obj)
|
||||
snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
if not status and not tag:
|
||||
snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
snapshot_id = record.get("id")
|
||||
if not snapshot_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
snapshot = Snapshot.objects.get(id=snapshot_id)
|
||||
|
||||
if status:
|
||||
if status not in Snapshot.StatusChoices.values:
|
||||
rprint(f"[red]Invalid snapshot status: {status}[/red]", file=sys.stderr)
|
||||
continue
|
||||
if status == Snapshot.StatusChoices.SEALED:
|
||||
snapshot.cancel()
|
||||
elif status == Snapshot.StatusChoices.PAUSED:
|
||||
snapshot.pause()
|
||||
elif status == Snapshot.StatusChoices.QUEUED:
|
||||
if snapshot.status == Snapshot.StatusChoices.PAUSED:
|
||||
snapshot.resume()
|
||||
else:
|
||||
snapshot.update_and_requeue(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
|
||||
elif status == Snapshot.StatusChoices.STARTED:
|
||||
snapshot.update_and_requeue(status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now())
|
||||
if tag:
|
||||
from archivebox.core.models import Tag
|
||||
|
||||
tag_obj, _ = Tag.objects.get_or_create(name=tag)
|
||||
snapshot.tags.add(tag_obj)
|
||||
snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
|
||||
if not status and not tag:
|
||||
snapshot.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
snapshot.refresh_from_db()
|
||||
write_record(snapshot.to_json())
|
||||
|
||||
except Snapshot.DoesNotExist:
|
||||
rprint(f"[yellow]Snapshot not found: {snapshot_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} snapshots[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return update_records(Snapshot, update, plural="snapshots", refresh=True)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -447,50 +391,18 @@ def update_snapshots(
|
||||
|
||||
|
||||
def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete Snapshots from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
"""Delete snapshots selected by stdin JSONL; --yes confirms, --dry-run previews."""
|
||||
from archivebox.cli.cli_util import delete_records
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
snapshot_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not snapshot_ids:
|
||||
rprint("[yellow]No valid snapshot IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
|
||||
count = snapshots.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} snapshots (dry run)[/yellow]", file=sys.stderr)
|
||||
for snapshot in snapshots:
|
||||
rprint(f" [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = snapshots.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} snapshots[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return delete_records(
|
||||
Snapshot,
|
||||
label="snapshot",
|
||||
plural="snapshots",
|
||||
preview=lambda obj: f"[dim]{obj.id}[/dim] {obj.url[:60]}",
|
||||
yes=yes,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -525,17 +437,17 @@ def list_cmd(**kwargs):
|
||||
@main.command("update")
|
||||
@click.option("--status", "-s", help="Set status")
|
||||
@click.option("--tag", "-t", help="Add tag")
|
||||
def update_cmd(status: str | None, tag: str | None):
|
||||
def update_cmd(**kwargs):
|
||||
"""Update Snapshots from stdin JSONL."""
|
||||
sys.exit(update_snapshots(status=status, tag=tag))
|
||||
sys.exit(update_snapshots(**kwargs))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
def delete_cmd(**kwargs):
|
||||
"""Delete Snapshots from stdin JSONL."""
|
||||
sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
|
||||
sys.exit(delete_snapshots(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -70,7 +70,7 @@ def create_tags(names: Iterable[str]) -> int:
|
||||
if not name:
|
||||
continue
|
||||
|
||||
tag, created = Tag.objects.get_or_create(name=name)
|
||||
tag, created = Tag.get_or_create_by_name(name)
|
||||
|
||||
if not is_tty:
|
||||
write_record(tag.to_json())
|
||||
@@ -90,42 +90,17 @@ def create_tags(names: Iterable[str]) -> int:
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def list_tags(
|
||||
name: str | None = None,
|
||||
name__icontains: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Tags as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
def list_tags(name: str | None = None, name__icontains: str | None = None, limit: int | None = None) -> int:
|
||||
"""List tags as JSONL, or formatted rows in a terminal."""
|
||||
from archivebox.core.models import Tag
|
||||
from archivebox.cli.cli_util import list_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Tag.objects.all().order_by("name")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"name": name,
|
||||
"name__icontains": name__icontains,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for tag in queryset:
|
||||
snapshot_count = tag.snapshot_set.count()
|
||||
if is_tty:
|
||||
rprint(f"[cyan]{tag.name:30}[/cyan] [dim]({snapshot_count} snapshots)[/dim]")
|
||||
else:
|
||||
write_record(tag.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} tags[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
queryset = apply_filters(Tag.objects.order_by("name"), {"name": name, "name__icontains": name__icontains}, limit=limit)
|
||||
return list_records(
|
||||
queryset,
|
||||
plural="tags",
|
||||
render=lambda tag: f"[cyan]{tag.name:30}[/cyan] [dim]({tag.snapshot_set.count()} snapshots)[/dim]",
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -134,56 +109,15 @@ def list_tags(
|
||||
|
||||
|
||||
def update_tags(name: str | None = None) -> int:
|
||||
"""
|
||||
Update Tags from stdin JSONL.
|
||||
|
||||
Reads Tag records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
"""Apply supplied fields to each JSONL-selected Tag."""
|
||||
from archivebox.core.models import Tag
|
||||
from archivebox.cli.cli_util import update_records
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
def update(tag):
|
||||
if name:
|
||||
tag.rename(name)
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
tag_id = record.get("id")
|
||||
old_name = record.get("name")
|
||||
|
||||
if not tag_id and not old_name:
|
||||
continue
|
||||
|
||||
try:
|
||||
if tag_id:
|
||||
tag = Tag.objects.get(id=tag_id)
|
||||
else:
|
||||
tag = Tag.objects.get(name=old_name)
|
||||
|
||||
# Apply updates from CLI flags
|
||||
if name:
|
||||
tag.name = name
|
||||
tag.save()
|
||||
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
write_record(tag.to_json())
|
||||
|
||||
except Tag.DoesNotExist:
|
||||
rprint(f"[yellow]Tag not found: {tag_id or old_name}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} tags[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return update_records(Tag, update, plural="tags", by_name=True)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -192,65 +126,19 @@ def update_tags(name: str | None = None) -> int:
|
||||
|
||||
|
||||
def delete_tags(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete Tags from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
"""Delete tags selected by stdin JSONL; --yes confirms, --dry-run previews."""
|
||||
from archivebox.cli.cli_util import delete_records
|
||||
from archivebox.core.models import Tag
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Collect tag IDs or names
|
||||
tag_ids = []
|
||||
tag_names = []
|
||||
for r in records:
|
||||
if r.get("id"):
|
||||
tag_ids.append(r["id"])
|
||||
elif r.get("name"):
|
||||
tag_names.append(r["name"])
|
||||
|
||||
if not tag_ids and not tag_names:
|
||||
rprint("[yellow]No valid tag IDs or names in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
from django.db.models import Q
|
||||
|
||||
query = Q()
|
||||
if tag_ids:
|
||||
query |= Q(id__in=tag_ids)
|
||||
if tag_names:
|
||||
query |= Q(name__in=tag_names)
|
||||
|
||||
tags = Tag.objects.filter(query)
|
||||
count = tags.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching tags found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} tags (dry run)[/yellow]", file=sys.stderr)
|
||||
for tag in tags:
|
||||
rprint(f" {tag.name}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = tags.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} tags[/green]", file=sys.stderr)
|
||||
return 0
|
||||
return delete_records(
|
||||
Tag,
|
||||
label="tag",
|
||||
plural="tags",
|
||||
preview=lambda obj: obj.name,
|
||||
yes=yes,
|
||||
dry_run=dry_run,
|
||||
by_name=True,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -275,24 +163,24 @@ def create_cmd(names: tuple):
|
||||
@click.option("--name", help="Filter by exact name")
|
||||
@click.option("--name__icontains", help="Filter by name contains")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
|
||||
def list_cmd(**kwargs):
|
||||
"""List Tags as JSONL."""
|
||||
sys.exit(list_tags(name=name, name__icontains=name__icontains, limit=limit))
|
||||
sys.exit(list_tags(**kwargs))
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--name", "-n", help="Set new name")
|
||||
def update_cmd(name: str | None):
|
||||
def update_cmd(**kwargs):
|
||||
"""Update Tags from stdin JSONL."""
|
||||
sys.exit(update_tags(name=name))
|
||||
sys.exit(update_tags(**kwargs))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
def delete_cmd(**kwargs):
|
||||
"""Delete Tags from stdin JSONL."""
|
||||
sys.exit(delete_tags(yes=yes, dry_run=dry_run))
|
||||
sys.exit(delete_tags(**kwargs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -229,21 +229,20 @@ def update(
|
||||
continue
|
||||
raise SystemExit(exit_code)
|
||||
|
||||
is_filtered_update = any(
|
||||
(
|
||||
filter_patterns,
|
||||
status,
|
||||
url__icontains,
|
||||
url__istartswith,
|
||||
tag,
|
||||
crawl_id,
|
||||
limit,
|
||||
sort,
|
||||
search,
|
||||
before,
|
||||
after,
|
||||
),
|
||||
)
|
||||
filters = {
|
||||
"filter_patterns": filter_patterns,
|
||||
"status": status,
|
||||
"url__icontains": url__icontains,
|
||||
"url__istartswith": url__istartswith,
|
||||
"tag": tag,
|
||||
"crawl_id": crawl_id,
|
||||
"limit": limit,
|
||||
"sort": sort,
|
||||
"search": search,
|
||||
"before": before,
|
||||
"after": after,
|
||||
}
|
||||
is_filtered_update = any(filters.values())
|
||||
touched_snapshot_ids: set[str] = set()
|
||||
exit_code = 0
|
||||
|
||||
@@ -256,33 +255,11 @@ def update(
|
||||
do_index = index_only or not migrate_only
|
||||
|
||||
if do_migrate:
|
||||
if (
|
||||
filter_patterns
|
||||
or status
|
||||
or url__icontains
|
||||
or url__istartswith
|
||||
or tag
|
||||
or crawl_id
|
||||
or limit
|
||||
or sort
|
||||
or search
|
||||
or before
|
||||
or after
|
||||
):
|
||||
if is_filtered_update:
|
||||
print("[*] Processing filtered snapshots from database...")
|
||||
stats = process_filtered_snapshots(
|
||||
filter_patterns=filter_patterns,
|
||||
**filters,
|
||||
filter_type=filter_type,
|
||||
status=status,
|
||||
url__icontains=url__icontains,
|
||||
url__istartswith=url__istartswith,
|
||||
tag=tag,
|
||||
crawl_id=crawl_id,
|
||||
limit=limit,
|
||||
sort=sort,
|
||||
search=search,
|
||||
before=before,
|
||||
after=after,
|
||||
resume=resume,
|
||||
batch_size=batch_size,
|
||||
queue_for_archiving=True,
|
||||
@@ -324,18 +301,8 @@ def update(
|
||||
print("[*] No search indexing plugins are available, nothing to backfill.")
|
||||
else:
|
||||
snapshots = _build_filtered_snapshots_queryset(
|
||||
filter_patterns=filter_patterns,
|
||||
**filters,
|
||||
filter_type=filter_type,
|
||||
status=status,
|
||||
url__icontains=url__icontains,
|
||||
url__istartswith=url__istartswith,
|
||||
tag=tag,
|
||||
crawl_id=crawl_id,
|
||||
limit=limit,
|
||||
sort=sort,
|
||||
search=search,
|
||||
before=before,
|
||||
after=after,
|
||||
resume=resume,
|
||||
)
|
||||
stats = reindex_snapshots(
|
||||
@@ -362,32 +329,17 @@ def update(
|
||||
resume_cmd.append("--migrate-only")
|
||||
if index_only:
|
||||
resume_cmd.append("--index-only")
|
||||
if batch_size != 500:
|
||||
resume_cmd.extend(["--batch-size", str(batch_size)])
|
||||
if exact_resume or resume:
|
||||
resume_cmd.extend(["--resume", str(exact_resume or resume)])
|
||||
if before is not None:
|
||||
resume_cmd.extend(["--before", str(before)])
|
||||
if after is not None:
|
||||
resume_cmd.extend(["--after", str(after)])
|
||||
if filter_type != "exact":
|
||||
resume_cmd.extend(["--filter-type", filter_type])
|
||||
if status:
|
||||
resume_cmd.extend(["--status", status])
|
||||
if url__icontains:
|
||||
resume_cmd.extend(["--url__icontains", url__icontains])
|
||||
if url__istartswith:
|
||||
resume_cmd.extend(["--url__istartswith", url__istartswith])
|
||||
if tag:
|
||||
resume_cmd.extend(["--tag", tag])
|
||||
if crawl_id:
|
||||
resume_cmd.extend(["--crawl-id", crawl_id])
|
||||
if limit:
|
||||
resume_cmd.extend(["--limit", str(limit)])
|
||||
if sort:
|
||||
resume_cmd.extend(["--sort", sort])
|
||||
if search:
|
||||
resume_cmd.extend(["--search", search])
|
||||
resume_options = {
|
||||
"batch-size": batch_size if batch_size != 500 else None,
|
||||
"resume": exact_resume or resume,
|
||||
"filter-type": filter_type if filter_type != "exact" else None,
|
||||
**{key: value for key, value in filters.items() if key != "filter_patterns"},
|
||||
}
|
||||
for name, value in resume_options.items():
|
||||
# Zero is a meaningful date boundary, but remains an unset limit.
|
||||
if value or (name in {"before", "after"} and value is not None):
|
||||
option = "crawl-id" if name == "crawl_id" else name
|
||||
resume_cmd.extend([f"--{option}", str(value)])
|
||||
resume_cmd.extend(str(pattern) for pattern in filter_patterns)
|
||||
print("\n[red][X] archivebox update interrupted.[/red]")
|
||||
print("[yellow]Hint: resume this idempotent update with:[/yellow]")
|
||||
|
||||
@@ -47,13 +47,7 @@ def _format_binary_abspath(
|
||||
continue
|
||||
|
||||
relative_str = relative.as_posix()
|
||||
if prefix == "./":
|
||||
return "." if not relative_str else f"./{relative_str}"
|
||||
if prefix == "ABXPKG_LIB_DIR/":
|
||||
return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
|
||||
if prefix == "PERSONAS_DIR/":
|
||||
return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
|
||||
return "~" if not relative_str else f"~/{relative_str}"
|
||||
return prefix + relative_str if relative_str else prefix.rstrip("/")
|
||||
|
||||
return normalized.as_posix()
|
||||
|
||||
@@ -61,22 +55,9 @@ def _format_binary_abspath(
|
||||
def _render_binary_abspath(abspath: str):
|
||||
from rich.text import Text
|
||||
|
||||
if abspath.startswith("ABXPKG_LIB_DIR/"):
|
||||
return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
|
||||
if abspath == "ABXPKG_LIB_DIR":
|
||||
return Text("ABXPKG_LIB_DIR", style="bright_blue")
|
||||
if abspath.startswith("PERSONAS_DIR/"):
|
||||
return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
|
||||
if abspath == "PERSONAS_DIR":
|
||||
return Text("PERSONAS_DIR", style="medium_purple")
|
||||
if abspath.startswith("~/"):
|
||||
return Text.assemble(("~", "cyan"), (abspath.removeprefix("~"), "green"))
|
||||
if abspath == "~":
|
||||
return Text("~", style="cyan")
|
||||
if abspath.startswith("./"):
|
||||
return Text.assemble((".", "cyan"), (abspath.removeprefix("."), "green"))
|
||||
if abspath == ".":
|
||||
return Text(".", style="cyan")
|
||||
for prefix, color in (("ABXPKG_LIB_DIR", "bright_blue"), ("PERSONAS_DIR", "medium_purple"), ("~", "cyan"), (".", "cyan")):
|
||||
if abspath == prefix or abspath.startswith(prefix + "/"):
|
||||
return Text.assemble((prefix, color), (abspath.removeprefix(prefix), "green"))
|
||||
return Text(abspath, style="green")
|
||||
|
||||
|
||||
@@ -201,9 +182,6 @@ def version(
|
||||
else:
|
||||
requested_names = {name for name in (binaries or ()) if name}
|
||||
|
||||
def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
|
||||
return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
|
||||
|
||||
def plugin_may_have_requested_binary(plugin) -> bool:
|
||||
if not requested_names:
|
||||
return True
|
||||
@@ -262,7 +240,6 @@ def version(
|
||||
|
||||
prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
|
||||
failures = []
|
||||
seen_failures: set[str] = set()
|
||||
seen_rows: set[tuple[str, str, str, str]] = set()
|
||||
|
||||
from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
|
||||
@@ -298,10 +275,10 @@ def version(
|
||||
prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
|
||||
|
||||
declared_binary_specs: dict[str, dict[str, object]] = {}
|
||||
plugin_binaries = []
|
||||
for plugin_name, plugin in plugins.items():
|
||||
if not plugin_may_have_requested_binary(plugin):
|
||||
continue
|
||||
plugin_requested = bool(requested_names)
|
||||
plugin_enabled = plugin_name in enabled_plugin_names
|
||||
binary_records = get_required_binary_requests(
|
||||
plugin,
|
||||
@@ -318,8 +295,7 @@ def version(
|
||||
else actual_name
|
||||
)
|
||||
display_name = logical_name
|
||||
if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
|
||||
continue
|
||||
plugin_binaries.append((plugin_name, plugin_enabled, display_name, binary_record))
|
||||
if not plugin_enabled and not requested_names:
|
||||
continue
|
||||
if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
|
||||
@@ -342,156 +318,81 @@ def version(
|
||||
loaded_binaries = asyncio.run(resolve_declared_binaries())
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
any_rows = False
|
||||
any_available = False
|
||||
compact_paths = console.is_terminal
|
||||
live_enabled = console.is_terminal
|
||||
live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
|
||||
live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if console.is_terminal else None
|
||||
|
||||
def emit_row(row: dict[str, object]) -> None:
|
||||
rows.append(row)
|
||||
def emit_binary(plugin_name, plugin_enabled, display_name, binary, *, dim=False):
|
||||
nonlocal any_available
|
||||
valid = binary is not None
|
||||
abspath = binary.abspath if valid else ""
|
||||
version_str = str(binary.version or "unknown")[:15] if valid else "-"
|
||||
provider = str(binary.binprovider or "env")[:8] if valid else "-"
|
||||
any_available |= valid
|
||||
if not valid and plugin_enabled and display_name not in failures:
|
||||
failures.append(display_name)
|
||||
row_key = _binary_row_dedupe_key(
|
||||
display_name=display_name,
|
||||
valid=valid,
|
||||
version=version_str,
|
||||
provider=provider,
|
||||
abspath=abspath,
|
||||
)
|
||||
if row_key in seen_rows:
|
||||
return
|
||||
seen_rows.add(row_key)
|
||||
rendered_path = "[grey53]not installed[/grey53]"
|
||||
if valid:
|
||||
display_path = (
|
||||
_format_binary_abspath(
|
||||
abspath,
|
||||
pwd=Path.cwd(),
|
||||
lib_dir=config.ABXPKG_LIB_DIR,
|
||||
personas_dir=CONSTANTS.PERSONAS_DIR,
|
||||
home=Path.home(),
|
||||
)
|
||||
if compact_paths
|
||||
else abspath
|
||||
)
|
||||
rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
|
||||
rows.append(
|
||||
{
|
||||
"plugin": plugin_name,
|
||||
"status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
|
||||
"binary": display_name,
|
||||
"version": version_str,
|
||||
"provider": provider,
|
||||
"path": rendered_path,
|
||||
"style": "dim" if dim or not plugin_enabled else "",
|
||||
},
|
||||
)
|
||||
if live_cm is not None:
|
||||
live_cm.update(binary_dependency_table(rows), refresh=True)
|
||||
|
||||
if live_cm is not None:
|
||||
live_cm.start()
|
||||
try:
|
||||
for plugin_name, plugin in plugins.items():
|
||||
if not plugin_may_have_requested_binary(plugin):
|
||||
continue
|
||||
plugin_requested = bool(requested_names)
|
||||
plugin_enabled = plugin_name in enabled_plugin_names
|
||||
binary_records = get_required_binary_requests(
|
||||
plugin,
|
||||
plugin.config.required_binaries,
|
||||
overrides=runtime_config,
|
||||
derived_overrides=derived_config,
|
||||
run_output_dir=CONSTANTS.DATA_DIR,
|
||||
)
|
||||
for binary_record in binary_records:
|
||||
actual_name = str(binary_record["name"])
|
||||
logical_name = (
|
||||
Path(actual_name).expanduser().name
|
||||
if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
|
||||
else actual_name
|
||||
)
|
||||
display_name = logical_name
|
||||
if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
|
||||
for plugin_name, plugin_enabled, display_name, binary_record in plugin_binaries:
|
||||
installed = db_binaries.get(display_name) if db_available else None
|
||||
if not _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
|
||||
# Resolve only enabled or explicitly requested dependencies. Cold-loading
|
||||
# every optional provider makes status and migration checks needlessly slow.
|
||||
if not plugin_enabled and not requested_names:
|
||||
continue
|
||||
|
||||
installed = db_binaries.get(logical_name) if db_available else None
|
||||
if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
|
||||
abspath = installed.abspath
|
||||
version_str = (installed.version or "unknown")[:15]
|
||||
provider = (installed.binprovider or "env")[:8]
|
||||
valid = True
|
||||
elif not plugin_enabled and not requested_names:
|
||||
# `archivebox version` is expected to verify the active runtime, not
|
||||
# cold-load every optional plugin provider. Migration and status
|
||||
# checks often run with PLUGINS narrowed to a tiny set; resolving
|
||||
# disabled plugin binaries there can spend most of the command on
|
||||
# providers the current collection will never execute.
|
||||
continue
|
||||
else:
|
||||
loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
|
||||
abspath = loaded.abspath if loaded is not None else ""
|
||||
version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
|
||||
provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
|
||||
valid = loaded is not None
|
||||
|
||||
any_rows = True
|
||||
if valid:
|
||||
display_path = (
|
||||
_format_binary_abspath(
|
||||
abspath,
|
||||
pwd=Path.cwd(),
|
||||
lib_dir=config.ABXPKG_LIB_DIR,
|
||||
personas_dir=CONSTANTS.PERSONAS_DIR,
|
||||
home=Path.home(),
|
||||
)
|
||||
if compact_paths
|
||||
else abspath
|
||||
)
|
||||
rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
|
||||
any_available = True
|
||||
else:
|
||||
rendered_path = "[grey53]not installed[/grey53]"
|
||||
if plugin_enabled and display_name not in seen_failures:
|
||||
failures.append(display_name)
|
||||
seen_failures.add(display_name)
|
||||
|
||||
row_key = _binary_row_dedupe_key(
|
||||
display_name=display_name,
|
||||
valid=valid,
|
||||
version=version_str if valid else "-",
|
||||
provider=provider if valid else "-",
|
||||
abspath=abspath,
|
||||
)
|
||||
if row_key in seen_rows:
|
||||
continue
|
||||
seen_rows.add(row_key)
|
||||
|
||||
emit_row(
|
||||
{
|
||||
"plugin": plugin_name,
|
||||
"status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
|
||||
"binary": display_name,
|
||||
"version": version_str if valid else "-",
|
||||
"provider": provider if valid else "-",
|
||||
"path": rendered_path,
|
||||
"style": "" if plugin_enabled else "dim",
|
||||
},
|
||||
)
|
||||
installed = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
|
||||
emit_binary(plugin_name, plugin_enabled, display_name, installed)
|
||||
|
||||
if db_available:
|
||||
for binary_name, installed in db_binaries.items():
|
||||
if requested_names and binary_name not in requested_names:
|
||||
continue
|
||||
abspath = installed.abspath
|
||||
version_str = (installed.version or "unknown")[:15]
|
||||
provider = (installed.binprovider or "env")[:8]
|
||||
row_key = _binary_row_dedupe_key(
|
||||
display_name=binary_name,
|
||||
valid=True,
|
||||
version=version_str,
|
||||
provider=provider,
|
||||
abspath=abspath,
|
||||
)
|
||||
if row_key in seen_rows:
|
||||
continue
|
||||
seen_rows.add(row_key)
|
||||
|
||||
display_path = (
|
||||
_format_binary_abspath(
|
||||
abspath,
|
||||
pwd=Path.cwd(),
|
||||
lib_dir=config.ABXPKG_LIB_DIR,
|
||||
personas_dir=CONSTANTS.PERSONAS_DIR,
|
||||
home=Path.home(),
|
||||
)
|
||||
if compact_paths
|
||||
else abspath
|
||||
)
|
||||
emit_row(
|
||||
{
|
||||
"plugin": "(database)",
|
||||
"status": binary_dependency_status(enabled=True, valid=True),
|
||||
"binary": binary_name,
|
||||
"version": version_str,
|
||||
"provider": provider,
|
||||
"path": _render_binary_abspath(display_path) if compact_paths else display_path,
|
||||
"style": "dim",
|
||||
},
|
||||
)
|
||||
any_rows = True
|
||||
any_available = True
|
||||
if not requested_names or binary_name in requested_names:
|
||||
emit_binary("(database)", True, binary_name, installed, dim=True)
|
||||
finally:
|
||||
if live_cm is not None:
|
||||
live_cm.stop()
|
||||
else:
|
||||
prnt(binary_dependency_table(rows))
|
||||
|
||||
if not any_rows:
|
||||
if not rows:
|
||||
prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
|
||||
elif not any_available:
|
||||
prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
|
||||
|
||||
@@ -42,3 +42,139 @@ def apply_filters(queryset, filter_kwargs: dict, limit: int | None = None):
|
||||
queryset = queryset[:limit]
|
||||
|
||||
return queryset
|
||||
|
||||
|
||||
def delete_records(
|
||||
model,
|
||||
*,
|
||||
label: str,
|
||||
plural: str,
|
||||
preview,
|
||||
yes: bool,
|
||||
dry_run: bool,
|
||||
preview_limit: int | None = None,
|
||||
by_name: bool = False,
|
||||
) -> int:
|
||||
"""Delete a JSONL selection with one confirmation and dry-run contract.
|
||||
|
||||
Keep queryset deletion so Django's cascades and model cleanup signals run.
|
||||
Persona deletion has additional filesystem policy and stays with that command.
|
||||
"""
|
||||
import sys
|
||||
|
||||
from django.db.models import Q
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
ids = [record["id"] for record in records if record.get("id")]
|
||||
names = [record["name"] for record in records if by_name and not record.get("id") and record.get("name")]
|
||||
if not ids and not names:
|
||||
rprint(f"[yellow]No valid {label} IDs{' or names' if by_name else ''} in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
query = Q(id__in=ids)
|
||||
if names:
|
||||
query |= Q(name__in=names)
|
||||
queryset = model.objects.filter(query)
|
||||
count = queryset.count()
|
||||
if not count:
|
||||
rprint(f"[yellow]No matching {plural} found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} {plural} (dry run)[/yellow]", file=sys.stderr)
|
||||
for obj in queryset[:preview_limit] if preview_limit is not None else queryset:
|
||||
rprint(f" {preview(obj)}", file=sys.stderr)
|
||||
if preview_limit is not None and count > preview_limit:
|
||||
rprint(f" ... and {count - preview_limit} more", file=sys.stderr)
|
||||
return 0
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
deleted_count, _ = queryset.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} {plural}[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def update_record_status(record, status: str) -> None:
|
||||
"""Route CLI status changes through the model's lifecycle operations."""
|
||||
from django.utils import timezone
|
||||
|
||||
if status not in record.StatusChoices.values:
|
||||
raise ValueError(f"Invalid {record._meta.model_name} status: {status}")
|
||||
transitions = {"sealed": record.cancel, "paused": record.pause}
|
||||
if status == "queued" and record.is_paused:
|
||||
record.resume()
|
||||
elif transition := transitions.get(status):
|
||||
transition()
|
||||
else:
|
||||
record.update_and_requeue(status=status, retry_at=timezone.now())
|
||||
|
||||
|
||||
def list_records(queryset, *, plural: str, render) -> int:
|
||||
"""Stream JSONL when piped; render rows and a count in a terminal."""
|
||||
import sys
|
||||
from rich import print as rprint
|
||||
from archivebox.misc.jsonl import write_record
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
count = 0
|
||||
for record in queryset:
|
||||
if is_tty:
|
||||
rprint(render(record))
|
||||
else:
|
||||
write_record(record.to_json())
|
||||
count += 1
|
||||
rprint(f"[dim]Listed {count} {plural}[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def format_status(status: str, width: int) -> str:
|
||||
colors = {
|
||||
"queued": "yellow",
|
||||
"started": "blue",
|
||||
"sealed": "green",
|
||||
"succeeded": "green",
|
||||
"failed": "red",
|
||||
"backoff": "magenta",
|
||||
}
|
||||
color = colors.get(status, "dim")
|
||||
return f"[{color}]{status:{width}}[/{color}]"
|
||||
|
||||
|
||||
def update_records(model, update, *, plural: str, by_name: bool = False, refresh: bool = False) -> int:
|
||||
"""Apply a command's PATCH operation to each JSONL-selected model.
|
||||
|
||||
Returning False from the operation skips that row. Model operations stay in
|
||||
their commands so lifecycle changes and ordinary field saves remain explicit.
|
||||
"""
|
||||
import sys
|
||||
from rich import print as rprint
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
is_tty = sys.stdout.isatty()
|
||||
count = 0
|
||||
for record in records:
|
||||
key = "id" if record.get("id") else "name" if by_name and record.get("name") else None
|
||||
if key is None:
|
||||
continue
|
||||
try:
|
||||
obj = model.objects.get(**{key: record[key]})
|
||||
if update(obj) is False:
|
||||
continue
|
||||
count += 1
|
||||
if not is_tty:
|
||||
if refresh:
|
||||
obj.refresh_from_db()
|
||||
write_record(obj.to_json())
|
||||
except model.DoesNotExist:
|
||||
rprint(f"[yellow]{model.__name__} not found: {record[key]}[/yellow]", file=sys.stderr)
|
||||
rprint(f"[green]Updated {count} {plural}[/green]", file=sys.stderr)
|
||||
return 0
|
||||
@@ -1,7 +1,5 @@
|
||||
__package__ = "archivebox.core"
|
||||
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
from functools import reduce
|
||||
@@ -16,16 +14,16 @@ from django.core.exceptions import PermissionDenied, SuspiciousOperation, Valida
|
||||
from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
|
||||
from django.db.models.functions import Cast
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import resolve, reverse
|
||||
from django.utils import timezone
|
||||
from django.template.loader import render_to_string
|
||||
from django.urls import reverse
|
||||
from django.utils.html import format_html
|
||||
from django.utils.safestring import mark_safe
|
||||
from django.utils.text import smart_split
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.core.routes_util import build_snapshot_url
|
||||
from archivebox.core.widgets import InlineTagEditorWidget
|
||||
from archivebox.core.widgets import InlineTagEditorWidget, render_copy_block
|
||||
from archivebox.machine.env_util import env_to_shell_exports
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.misc.paginators import AcceleratedPaginator
|
||||
@@ -116,302 +114,58 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
|
||||
}
|
||||
|
||||
rows = []
|
||||
delete_url = html.escape(reverse("admin:core_archiveresult_changelist"), quote=True)
|
||||
for idx, result in enumerate(results):
|
||||
status = result.status or "queued"
|
||||
color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
|
||||
output_files = result.output_files or {}
|
||||
if isinstance(output_files, (dict, list, tuple, set)):
|
||||
output_file_count = len(output_files)
|
||||
elif isinstance(output_files, str):
|
||||
try:
|
||||
parsed = json.loads(output_files)
|
||||
output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0
|
||||
except (TypeError, ValueError):
|
||||
output_file_count = 0
|
||||
else:
|
||||
output_file_count = 0
|
||||
output_size = int(result.output_size or 0)
|
||||
output_size_display = html.escape(printable_filesize(output_size))
|
||||
|
||||
# Get plugin icon
|
||||
icon = get_plugin_icon(result.plugin)
|
||||
|
||||
# Keep each timestamp component intact while allowing one wrap between them.
|
||||
if result.end_ts:
|
||||
end_time = (
|
||||
f'<span style="white-space: nowrap;">{result.end_ts:%Y-%m-%d}</span>'
|
||||
f'<wbr> <span style="white-space: nowrap;">{result.end_ts:%H:%M:%S}</span>'
|
||||
)
|
||||
else:
|
||||
end_time = "-"
|
||||
|
||||
process = result.process_record
|
||||
process_display = "-"
|
||||
if process:
|
||||
process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
|
||||
process_label = html.escape(get_process_link_label(process), quote=True)
|
||||
process_display = f'''
|
||||
<a href="{process_url}"
|
||||
style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
|
||||
title="View process">{process_label}</a>
|
||||
'''
|
||||
|
||||
machine_display = "-"
|
||||
if process and process.machine_id:
|
||||
machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
|
||||
machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
|
||||
machine_display = f'''
|
||||
<a href="{machine_url}"
|
||||
style="color: #2563eb; text-decoration: none; font-size: 12px;"
|
||||
title="View machine">{machine_label}</a>
|
||||
'''
|
||||
|
||||
# Truncate output for display
|
||||
full_output_raw = result.output_str_for_display() or "-"
|
||||
output_display_raw = full_output_raw[:60]
|
||||
if len(full_output_raw) > 60:
|
||||
output_display_raw += "..."
|
||||
full_output = html.escape(full_output_raw)
|
||||
output_display = html.escape(output_display_raw)
|
||||
|
||||
display_cmd = build_abx_dl_display_command(result)
|
||||
replay_cmd = build_abx_dl_replay_command(result, config=config)
|
||||
cmd_str_escaped = html.escape(display_cmd)
|
||||
cmd_attr = html.escape(replay_cmd, quote=True)
|
||||
|
||||
# Build output link - use embed_path() which checks output_files first
|
||||
output = result.output_str_for_display() or "-"
|
||||
embed_path = result.embed_path()
|
||||
snapshot_id = str(result.snapshot_id)
|
||||
if embed_path and result.status == "succeeded":
|
||||
output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
|
||||
else:
|
||||
output_link = build_snapshot_url(snapshot_id, "", config=config)
|
||||
output_link_attr = html.escape(output_link, quote=True)
|
||||
|
||||
# Get version - try cmd_version field
|
||||
version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
|
||||
plugin_text = html.escape(str(result.plugin or ""), quote=True)
|
||||
status_text = html.escape(str(status), quote=True)
|
||||
pwd_text = html.escape(str(result.pwd or "-"), quote=True)
|
||||
|
||||
# Unique ID for this row's expandable output
|
||||
row_id = f"output_{idx}_{str(result.id)[:8]}"
|
||||
delete_button = ""
|
||||
if can_delete:
|
||||
delete_button = f'''
|
||||
<button type="button" data-archive-result-ids="{result.id}" data-delete-url="{delete_url}"
|
||||
title="Delete this output">×</button>
|
||||
'''
|
||||
|
||||
rows.append(f'''
|
||||
<tr data-output-size="{output_size}" style="border-bottom: 1px solid #f1f5f9; transition: background 0.15s;" onmouseover="this.style.background='#f8fafc'" onmouseout="this.style.background='transparent'">
|
||||
<td style="padding: 10px 12px; white-space: nowrap;">
|
||||
<a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
|
||||
style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 11px;"
|
||||
title="View/edit archive result">
|
||||
<code>{str(result.id)[-8:]}</code>
|
||||
</a>
|
||||
</td>
|
||||
<td style="padding: 10px 12px; white-space: nowrap;">
|
||||
<span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
|
||||
font-size: 11px; font-weight: 600; text-transform: uppercase;
|
||||
color: {color}; background: {bg};">{status_text}</span>
|
||||
</td>
|
||||
<td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
|
||||
{icon}
|
||||
</td>
|
||||
<td class="archive-results-plugin" style="padding: 10px 12px; font-weight: 500; color: #334155;">
|
||||
<a href="{output_link_attr}" target="_blank"
|
||||
style="color: #334155; text-decoration: none;"
|
||||
title="View output fullscreen"
|
||||
onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
|
||||
onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
|
||||
{plugin_text}
|
||||
</a>
|
||||
</td>
|
||||
<td class="archive-results-output" style="padding: 10px 12px; max-width: 280px;">
|
||||
<span onclick="document.getElementById('{row_id}').open = !document.getElementById('{row_id}').open"
|
||||
style="display: block; max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
|
||||
title="Click to expand full output">
|
||||
{output_display}
|
||||
</span>
|
||||
</td>
|
||||
<td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
|
||||
{output_file_count}
|
||||
</td>
|
||||
<td class="archive-results-size" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right; white-space: nowrap;">
|
||||
{output_size_display}
|
||||
</td>
|
||||
<td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
|
||||
{end_time}
|
||||
</td>
|
||||
<td style="padding: 10px 12px; white-space: nowrap;">
|
||||
{process_display}
|
||||
</td>
|
||||
<td style="padding: 10px 12px; white-space: nowrap;">
|
||||
{machine_display}
|
||||
</td>
|
||||
<td style="padding: 10px 12px; white-space: nowrap; font-family: ui-monospace, monospace; font-size: 11px; color: #64748b;">
|
||||
{version}
|
||||
</td>
|
||||
<td class="archive-results-actions-cell">
|
||||
<div class="archive-results-actions">
|
||||
<a href="{output_link_attr}" target="_blank"
|
||||
title="View output">📄</a>
|
||||
<a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
|
||||
title="Edit">✏️</a>
|
||||
{delete_button}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr style="border-bottom: 1px solid #e2e8f0;">
|
||||
<td colspan="12" style="padding: 0 12px 10px 12px;">
|
||||
<details id="{row_id}" style="margin: 0;">
|
||||
<summary style="cursor: pointer; font-size: 11px; color: #94a3b8; user-select: none;">
|
||||
Details & Output
|
||||
</summary>
|
||||
<div style="margin-top: 8px; padding: 10px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; max-height: 200px; overflow: auto;">
|
||||
<div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
|
||||
<span style="margin-right: 16px;"><b>ID:</b> <code>{result.id!s}</code></span>
|
||||
<span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
|
||||
<span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
|
||||
</div>
|
||||
<div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
|
||||
<b>Output:</b>
|
||||
</div>
|
||||
<pre style="margin: 0; padding: 8px; background: #1e293b; border-radius: 4px; color: #e2e8f0; font-size: 12px; white-space: pre-wrap; word-break: break-all; max-height: 120px; overflow: auto;">{full_output}</pre>
|
||||
<div style="font-size: 11px; color: #64748b; margin-top: 8px;">
|
||||
<b>Command:</b>
|
||||
</div>
|
||||
<div style="position: relative; margin: 0; padding: 8px 56px 8px 8px; background: #1e293b; border-radius: 4px;">
|
||||
<button type="button"
|
||||
data-command="{cmd_attr}"
|
||||
onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
|
||||
style="position: absolute; top: 6px; right: 6px; padding: 2px 8px; border: 0; border-radius: 4px; background: #334155; color: #e2e8f0; font-size: 11px; cursor: pointer;">
|
||||
Copy
|
||||
</button>
|
||||
<code title="{cmd_attr}" style="display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: #e2e8f0; font-size: 11px;">{cmd_str_escaped}</code>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</td>
|
||||
</tr>
|
||||
''')
|
||||
|
||||
total_count = results[0]._inline_total_count
|
||||
footer = ""
|
||||
if total_count > limit:
|
||||
footer = f"""
|
||||
<tr data-output-footer>
|
||||
<td colspan="12" style="padding: 12px; text-align: center; color: #64748b; font-size: 13px; background: #f8fafc;">
|
||||
Showing {limit} of {total_count} results
|
||||
<a href="/admin/core/archiveresult/?snapshot__id__exact={results[0].snapshot_id if results else ""}"
|
||||
style="color: #2563eb;">View all →</a>
|
||||
</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
return mark_safe(f"""
|
||||
<div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow-x: auto; overflow-y: hidden; background: #fff; width: 100%;">
|
||||
<table class="archive-results-table" style="width: 100%; min-width: 1100px; border-collapse: collapse; font-size: 14px;">
|
||||
<thead>
|
||||
<tr style="background: #f8fafc; border-bottom: 2px solid #e2e8f0;">
|
||||
<th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Details</th>
|
||||
<th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Status</th>
|
||||
<th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; width: 32px;"></th>
|
||||
<th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
|
||||
<th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
|
||||
<th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
|
||||
<th class="archive-results-size" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">
|
||||
<button type="button" data-output-size-sort style="all: unset; cursor: pointer;">Size ↕</button>
|
||||
</th>
|
||||
<th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
|
||||
<th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
|
||||
<th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
|
||||
<th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
|
||||
<th class="archive-results-actions-cell" style="text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{"".join(rows)}
|
||||
{footer}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
""")
|
||||
|
||||
|
||||
class ArchiveResultInline(admin.TabularInline):
|
||||
name = "Archive Results"
|
||||
model = ArchiveResult
|
||||
parent_model = Snapshot
|
||||
extra = 0
|
||||
sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
|
||||
readonly_fields = ("id", "result_id", "completed", "command", "version")
|
||||
fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
|
||||
ordering = ("end_ts",)
|
||||
show_change_link = True
|
||||
|
||||
def get_parent_object_from_request(self, request):
|
||||
resolved = resolve(request.path_info)
|
||||
try:
|
||||
return self.parent_model.objects.get(pk=resolved.kwargs["object_id"])
|
||||
except (self.parent_model.DoesNotExist, ValidationError):
|
||||
return None
|
||||
|
||||
@admin.display(
|
||||
description="Completed",
|
||||
ordering="end_ts",
|
||||
)
|
||||
def completed(self, obj):
|
||||
return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
|
||||
def result_id(self, obj):
|
||||
return format_html(
|
||||
'<a href="{}"><code style="font-size: 10px">[{}]</code></a>',
|
||||
reverse("admin:core_archiveresult_change", args=(obj.id,)),
|
||||
str(obj.id)[:8],
|
||||
rows.append(
|
||||
{
|
||||
"result": result,
|
||||
"status": status,
|
||||
"color": color,
|
||||
"bg": bg,
|
||||
"icon": get_plugin_icon(result.plugin),
|
||||
"file_count": result.output_file_stats()[0],
|
||||
"output_size": int(result.output_size or 0),
|
||||
"output_size_display": printable_filesize(int(result.output_size or 0)),
|
||||
"end_date": result.end_ts.strftime("%Y-%m-%d") if result.end_ts else "",
|
||||
"end_time": result.end_ts.strftime("%H:%M:%S") if result.end_ts else "",
|
||||
"process": process,
|
||||
"process_url": reverse("admin:machine_process_change", args=[process.id]) if process else "",
|
||||
"process_label": get_process_link_label(process) if process else "",
|
||||
"machine_url": reverse("admin:machine_machine_change", args=[process.machine_id]) if process and process.machine_id else "",
|
||||
"output": output,
|
||||
"output_preview": output[:60] + ("..." if len(output) > 60 else ""),
|
||||
"display_command": build_abx_dl_display_command(result),
|
||||
"replay_command": build_abx_dl_replay_command(result, config=config),
|
||||
"output_url": build_snapshot_url(
|
||||
str(result.snapshot_id),
|
||||
embed_path if embed_path and status == "succeeded" else "",
|
||||
config=config,
|
||||
),
|
||||
"change_url": reverse("admin:core_archiveresult_change", args=[result.id]),
|
||||
"version": result.cmd_version or "-",
|
||||
"pwd": str(result.pwd or "-"),
|
||||
"dom_id": f"output_{idx}_{str(result.id)[:8]}",
|
||||
"short_id": str(result.id)[-8:],
|
||||
},
|
||||
)
|
||||
|
||||
def command(self, obj):
|
||||
return format_html("<small><code>{}</code></small>", " ".join(obj.cmd or []))
|
||||
|
||||
def version(self, obj):
|
||||
return format_html("<small><code>{}</code></small>", obj.cmd_version or "-")
|
||||
|
||||
def get_formset(self, request, obj=None, **kwargs):
|
||||
formset = super().get_formset(request, obj, **kwargs)
|
||||
snapshot = self.get_parent_object_from_request(request)
|
||||
base_fields = formset.form.base_fields
|
||||
snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
|
||||
|
||||
# formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
|
||||
|
||||
# default values for new entries
|
||||
base_fields["status"].initial = "succeeded"
|
||||
base_fields["start_ts"].initial = timezone.now()
|
||||
base_fields["end_ts"].initial = timezone.now()
|
||||
base_fields["cmd_version"].initial = "-"
|
||||
base_fields["pwd"].initial = snapshot_output_dir
|
||||
base_fields["cmd"].initial = '["-"]'
|
||||
base_fields["output_str"].initial = "Manually recorded cmd output..."
|
||||
|
||||
if obj is not None:
|
||||
# hidden values for existing entries and new entries
|
||||
base_fields["start_ts"].widget = base_fields["start_ts"].hidden_widget()
|
||||
base_fields["end_ts"].widget = base_fields["end_ts"].hidden_widget()
|
||||
base_fields["cmd"].widget = base_fields["cmd"].hidden_widget()
|
||||
base_fields["pwd"].widget = base_fields["pwd"].hidden_widget()
|
||||
base_fields["cmd_version"].widget = base_fields["cmd_version"].hidden_widget()
|
||||
return formset
|
||||
|
||||
def get_readonly_fields(self, request, obj=None):
|
||||
if obj is not None:
|
||||
return self.readonly_fields
|
||||
else:
|
||||
return []
|
||||
return mark_safe(
|
||||
render_to_string(
|
||||
"admin/core/archiveresult/inline.html",
|
||||
{
|
||||
"rows": rows,
|
||||
"limit": limit,
|
||||
"total_count": results[0]._inline_total_count,
|
||||
"snapshot_id": results[0].snapshot_id,
|
||||
"can_delete": can_delete,
|
||||
"delete_url": reverse("admin:core_archiveresult_changelist"),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ArchiveResultAdmin(BaseModelAdmin):
|
||||
@@ -459,40 +213,14 @@ class ArchiveResultAdmin(BaseModelAdmin):
|
||||
autocomplete_fields = ("snapshot",)
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Snapshot",
|
||||
{
|
||||
"fields": ("snapshot", "snapshot_info", "tags_str", "admin_actions"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Plugin",
|
||||
{
|
||||
"fields": ("plugin_with_icon", "process_link", "status"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timing",
|
||||
{
|
||||
"fields": ("start_ts", "end_ts", "created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Command",
|
||||
{
|
||||
"fields": ("cmd", "cmd_str", "cmd_version", "pwd"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
card_fieldset("Snapshot", ("snapshot", "snapshot_info", "tags_str", "admin_actions"), wide=True),
|
||||
card_fieldset("Plugin", ("plugin_with_icon", "process_link", "status")),
|
||||
card_fieldset("Timing", ("start_ts", "end_ts", "created_at", "modified_at")),
|
||||
card_fieldset("Command", ("cmd", "cmd_str", "cmd_version", "pwd")),
|
||||
card_fieldset(
|
||||
"Output",
|
||||
{
|
||||
"fields": ("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
("output_str", "output_json", "output_files", "output_size", "output_mimetypes", "output_summary"),
|
||||
wide=True,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -588,18 +316,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
|
||||
self.list_per_page = saved_list_per_page
|
||||
|
||||
def get_queryset(self, request):
|
||||
ordering = request.GET.get("o")
|
||||
ordering_fields = set()
|
||||
if ordering:
|
||||
for part in ordering.split("."):
|
||||
if not part:
|
||||
continue
|
||||
try:
|
||||
idx = abs(int(part)) - 1
|
||||
except ValueError:
|
||||
continue
|
||||
if 0 <= idx < len(self.list_display):
|
||||
ordering_fields.add(self.list_display[idx])
|
||||
ordering_fields = self.get_ordering_fields(request)
|
||||
|
||||
qs = (
|
||||
super()
|
||||
@@ -800,26 +517,9 @@ class ArchiveResultAdmin(BaseModelAdmin):
|
||||
|
||||
@admin.display(description="Command")
|
||||
def cmd_str(self, result):
|
||||
request = self.request
|
||||
display_cmd = build_abx_dl_display_command(result)
|
||||
replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
|
||||
return format_html(
|
||||
"""
|
||||
<div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
|
||||
<button type="button"
|
||||
data-command="{}"
|
||||
onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
|
||||
style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
|
||||
Copy
|
||||
</button>
|
||||
<code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
|
||||
{}
|
||||
</code>
|
||||
</div>
|
||||
""",
|
||||
replay_cmd,
|
||||
replay_cmd,
|
||||
display_cmd,
|
||||
return render_copy_block(
|
||||
build_abx_dl_display_command(result),
|
||||
copy_text=build_abx_dl_replay_command(result, config=self.request.archivebox_config),
|
||||
)
|
||||
|
||||
def output_display(self, result):
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
__package__ = "archivebox.core"
|
||||
|
||||
import json
|
||||
from copy import copy
|
||||
from urllib.parse import urlencode
|
||||
from functools import lru_cache
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -9,11 +11,12 @@ from django.urls import path, reverse
|
||||
from django.shortcuts import get_object_or_404, redirect
|
||||
from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
|
||||
from django.utils import timezone
|
||||
from django.utils.html import format_html, format_html_join
|
||||
from django.utils.html import format_html
|
||||
from django.utils.safestring import mark_safe
|
||||
from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
|
||||
from django import forms
|
||||
from django.template import Template, RequestContext
|
||||
from django.template.loader import render_to_string
|
||||
from django.contrib.admin.helpers import ActionForm
|
||||
|
||||
from archivebox.config.common import get_config
|
||||
@@ -23,11 +26,11 @@ from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
|
||||
from archivebox.search.views import admin_snapshot_search_stream_view
|
||||
from archivebox.core.routes_util import build_snapshot_url, build_web_url
|
||||
from archivebox.core.tag_util import get_or_create_tag
|
||||
from archivebox.plugins.hooks import discover_hooks
|
||||
from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.core.widgets import render_permissions_badge, render_snapshot_progress
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
|
||||
|
||||
from archivebox.core.models import Tag, Snapshot, ArchiveResult
|
||||
from archivebox.crawls.models import Crawl
|
||||
@@ -38,7 +41,6 @@ from archivebox.core.permissions import (
|
||||
PERMISSIONS_CHOICES,
|
||||
PERMISSIONS_META,
|
||||
get_snapshot_permissions,
|
||||
normalize_permissions,
|
||||
)
|
||||
from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
|
||||
|
||||
@@ -230,10 +232,12 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
|
||||
|
||||
|
||||
class SnapshotChangeList(SearchResultsChangeList):
|
||||
snapshot_status_choices = Snapshot.StatusChoices.choices
|
||||
|
||||
def __init__(self, request, *args, **kwargs):
|
||||
super().__init__(request, *args, **kwargs)
|
||||
resolver_name = request.resolver_match.url_name
|
||||
self.embedded_changelist = request.GET.get("_embedded") == "crawl"
|
||||
self.embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
|
||||
self.snapshot_is_grid_view = not self.embedded_changelist and (
|
||||
resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
|
||||
)
|
||||
@@ -297,7 +301,7 @@ class SnapshotChangeList(SearchResultsChangeList):
|
||||
|
||||
def get_results(self, request):
|
||||
super().get_results(request)
|
||||
if request.GET.get("_embedded") == "crawl":
|
||||
if request.GET.get("_embedded") in {"crawl", "snapshot"}:
|
||||
self.full_result_count = self.result_count
|
||||
self.show_full_result_count = True
|
||||
self._attach_archiveresult_summaries()
|
||||
@@ -411,70 +415,19 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
"classes": ("card", "actions-card"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Snapshot",
|
||||
{
|
||||
"fields": ("snapshot_summary",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"URL",
|
||||
{
|
||||
"fields": (("url_favicon", "url"), ("title", "tags_badges")),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Tags",
|
||||
{
|
||||
"fields": ("tags_editor", "permissions_config"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Status",
|
||||
{
|
||||
"fields": ("status", "retry_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("bookmarked_at", "created_at", "modified_at", "downloaded_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Relations",
|
||||
{
|
||||
"fields": ("crawl",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
card_fieldset("Snapshot", ("snapshot_summary",)),
|
||||
card_fieldset("URL", (("url_favicon", "url"), ("title", "tags_badges")), wide=True),
|
||||
card_fieldset("Tags", ("tags_editor", "permissions_config")),
|
||||
card_fieldset("Status", ("status", "retry_at")),
|
||||
card_fieldset("Timestamps", ("bookmarked_at", "created_at", "modified_at", "downloaded_at")),
|
||||
card_fieldset("Relations", ("crawl",)),
|
||||
card_fieldset(
|
||||
"Config",
|
||||
{
|
||||
"fields": ("config",),
|
||||
"description": '<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Files",
|
||||
{
|
||||
"fields": ("output_dir",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Archive Results",
|
||||
{
|
||||
"fields": ("archiveresults_list",),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
("config",),
|
||||
description='<span style="display:block; margin:-4px 0 6px; font-size:11px; line-height:1.35; color:#94a3b8;">Uses <code>Crawl.config</code> by default. Only set per-snapshot overrides here when needed.</span>',
|
||||
),
|
||||
card_fieldset("Files", ("output_dir",)),
|
||||
card_fieldset("Archive Results", ("archiveresults_list",), wide=True),
|
||||
)
|
||||
|
||||
ordering = ["-created_at"]
|
||||
@@ -520,7 +473,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
def changelist_view(self, request, extra_context=None):
|
||||
self.request = request
|
||||
saved_list_per_page = self.list_per_page
|
||||
embedded_changelist = request.GET.get("_embedded") == "crawl"
|
||||
embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
|
||||
if embedded_changelist:
|
||||
try:
|
||||
requested_per_page = int(request.GET.get("per_page", "200"))
|
||||
@@ -541,6 +494,27 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
finally:
|
||||
self.list_per_page = saved_list_per_page
|
||||
|
||||
def render_embedded_changelist(self, request, *, filters, title, default_search_mode=None):
|
||||
"""Reuse snapshot search, rows, pagination, and actions in parent admin pages."""
|
||||
changelist_path = reverse(f"{self.admin_site.name}:core_snapshot_changelist")
|
||||
full_url = f"{changelist_path}?{urlencode(filters)}"
|
||||
changelist_request = copy(request)
|
||||
changelist_request.method = "GET"
|
||||
changelist_request.path = changelist_path
|
||||
changelist_request.GET = request.GET.copy()
|
||||
if default_search_mode:
|
||||
changelist_request.GET.setdefault("search_mode", default_search_mode)
|
||||
changelist_request.GET.update({**filters, "_embedded": "snapshot", "per_page": "200"})
|
||||
changelist_request.POST = request.POST.copy()
|
||||
changelist_request.POST.clear()
|
||||
response = self.changelist_view(changelist_request)
|
||||
context = {
|
||||
**response.context_data,
|
||||
"snapshot_changelist_url": full_url,
|
||||
"snapshot_changelist_title": title,
|
||||
}
|
||||
return mark_safe(render_to_string("admin/core/snapshot/embedded_changelist.html", context, request=request))
|
||||
|
||||
def get_actions(self, request):
|
||||
actions = super().get_actions(request)
|
||||
if not actions:
|
||||
@@ -549,7 +523,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
return actions
|
||||
|
||||
def lookup_allowed(self, lookup, value, request=None):
|
||||
if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}:
|
||||
if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id", "crawl__schedule__id__exact"}:
|
||||
return True
|
||||
return super().lookup_allowed(lookup, value, request=request)
|
||||
|
||||
@@ -650,7 +624,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
|
||||
def get_queryset(self, request):
|
||||
self.request = request
|
||||
ordering_fields = self._get_ordering_fields(request)
|
||||
ordering_fields = self.get_ordering_fields(request)
|
||||
needs_files_sort = "files" in ordering_fields
|
||||
needs_tags_sort = "tags_inline" in ordering_fields
|
||||
is_change_view = request.resolver_match.url_name == "core_snapshot_change"
|
||||
@@ -714,45 +688,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
permissions = obj.__dict__.get("snapshot_permissions")
|
||||
if permissions is None:
|
||||
permissions = obj.permissions
|
||||
permissions = normalize_permissions(permissions)
|
||||
icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
|
||||
menu_items = format_html_join(
|
||||
"",
|
||||
(
|
||||
'<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
|
||||
'<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
|
||||
"<span>{}</span>"
|
||||
"</button>"
|
||||
),
|
||||
(
|
||||
(
|
||||
" is-active" if choice_value == permissions else "",
|
||||
choice_value,
|
||||
choice_fg,
|
||||
choice_bg,
|
||||
choice_icon,
|
||||
choice_label,
|
||||
)
|
||||
for choice_value, choice_label in PERMISSIONS_CHOICES
|
||||
for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]]
|
||||
),
|
||||
)
|
||||
return format_html(
|
||||
'<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
|
||||
'<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change snapshot permissions: {}" aria-expanded="false">'
|
||||
'<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
|
||||
"</button>"
|
||||
'<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
|
||||
"</span>",
|
||||
return render_permissions_badge(
|
||||
permissions,
|
||||
reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
|
||||
permissions,
|
||||
label,
|
||||
label,
|
||||
fg,
|
||||
bg,
|
||||
icon,
|
||||
menu_items,
|
||||
url=reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
|
||||
object_name="snapshot",
|
||||
)
|
||||
|
||||
@admin.display(description="Imported Timestamp")
|
||||
@@ -852,23 +791,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
obj.pk,
|
||||
)
|
||||
|
||||
def status_info(self, obj):
|
||||
request = self.request
|
||||
config = request.archivebox_config
|
||||
favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
|
||||
return format_html(
|
||||
"""
|
||||
Archived: {} ({} files {})
|
||||
Favicon: <img src="{}" style="height: 20px"/>
|
||||
Extension: {}
|
||||
""",
|
||||
"✅" if obj.is_archived else "❌",
|
||||
obj.num_outputs,
|
||||
self.size(obj) or "0kb",
|
||||
favicon_url,
|
||||
obj.extension or "-",
|
||||
)
|
||||
|
||||
@admin.display(description="Archive Results")
|
||||
def archiveresults_list(self, obj):
|
||||
request = self.request
|
||||
@@ -1189,36 +1111,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
|
||||
# For started snapshots, show progress bar
|
||||
if obj.status == "started" and stats["total"] > 0:
|
||||
percent = stats["percent"]
|
||||
running = stats["running"]
|
||||
succeeded = stats["succeeded"]
|
||||
failed = stats["failed"]
|
||||
|
||||
return format_html(
|
||||
"""<div style="min-width: 90px;">
|
||||
<div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">
|
||||
<span class="snapshot-progress-spinner"></span>
|
||||
<span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>
|
||||
</div>
|
||||
<div style="background: #e2e8f0; border-radius: 4px; height: 6px; overflow: hidden;">
|
||||
<div style="background: linear-gradient(90deg, #10b981 0%, #10b981 {}%, #ef4444 {}%, #ef4444 {}%, #3b82f6 {}%, #3b82f6 100%);
|
||||
width: {}%; height: 100%; transition: width 0.3s;"></div>
|
||||
</div>
|
||||
<div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">
|
||||
✓{} ✗{} ⏳{}
|
||||
</div>
|
||||
</div>""",
|
||||
succeeded + failed + stats["skipped"],
|
||||
stats["total"],
|
||||
int(succeeded / stats["total"] * 100) if stats["total"] else 0,
|
||||
int(succeeded / stats["total"] * 100) if stats["total"] else 0,
|
||||
int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
|
||||
int((succeeded + failed) / stats["total"] * 100) if stats["total"] else 0,
|
||||
percent,
|
||||
succeeded,
|
||||
failed,
|
||||
running,
|
||||
)
|
||||
return render_snapshot_progress(stats)
|
||||
|
||||
# For other statuses, show simple badge
|
||||
return format_html(
|
||||
@@ -1266,48 +1159,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
)
|
||||
|
||||
def _get_progress_stats(self, obj):
|
||||
cached_stats = obj.__dict__.get("_admin_progress_stats")
|
||||
if cached_stats is not None:
|
||||
return cached_stats
|
||||
|
||||
results = self._get_prefetched_results(obj)
|
||||
if results is None:
|
||||
stats = obj.get_progress_stats()
|
||||
expected_total = self._get_expected_hook_total(obj)
|
||||
total = max(stats["total"], expected_total)
|
||||
completed = stats["succeeded"] + stats["failed"] + stats.get("skipped", 0) + stats.get("noresults", 0)
|
||||
stats["total"] = total
|
||||
stats["pending"] = max(total - completed - stats["running"], 0)
|
||||
stats["percent"] = int((completed / total * 100) if total > 0 else 0)
|
||||
obj._admin_progress_stats = stats
|
||||
return stats
|
||||
|
||||
expected_total = self._get_expected_hook_total(obj)
|
||||
observed_total = len(results)
|
||||
total = max(observed_total, expected_total)
|
||||
succeeded = sum(1 for r in results if r.status == "succeeded")
|
||||
failed = sum(1 for r in results if r.status == "failed")
|
||||
running = sum(1 for r in results if r.status == "started")
|
||||
skipped = sum(1 for r in results if r.status == "skipped")
|
||||
noresults = sum(1 for r in results if r.status == "noresults")
|
||||
pending = max(total - succeeded - failed - running - skipped - noresults, 0)
|
||||
completed = succeeded + failed + skipped + noresults
|
||||
percent = int((completed / total * 100) if total > 0 else 0)
|
||||
is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED)
|
||||
stats = {
|
||||
"total": total,
|
||||
"succeeded": succeeded,
|
||||
"failed": failed,
|
||||
"running": running,
|
||||
"pending": pending,
|
||||
"skipped": skipped,
|
||||
"noresults": noresults,
|
||||
"percent": percent,
|
||||
"output_size": obj.output_size or 0,
|
||||
"is_sealed": is_sealed,
|
||||
}
|
||||
obj._admin_progress_stats = stats
|
||||
return stats
|
||||
if "_admin_progress_stats" not in obj.__dict__:
|
||||
obj._admin_progress_stats = obj.get_progress_stats(
|
||||
results=self._get_prefetched_results(obj),
|
||||
expected_total=self._get_expected_hook_total(obj),
|
||||
)
|
||||
return obj._admin_progress_stats
|
||||
|
||||
def _get_prefetched_results(self, obj):
|
||||
if "_admin_output_results" in obj.__dict__:
|
||||
@@ -1363,22 +1220,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
return list(prefetched_cache["tags"])
|
||||
return None
|
||||
|
||||
def _get_ordering_fields(self, request):
|
||||
ordering = request.GET.get("o")
|
||||
if not ordering:
|
||||
return set()
|
||||
fields = set()
|
||||
for part in ordering.split("."):
|
||||
if not part:
|
||||
continue
|
||||
try:
|
||||
idx = abs(int(part)) - 1
|
||||
except ValueError:
|
||||
continue
|
||||
if 0 <= idx < len(self.list_display):
|
||||
fields.add(self.list_display[idx])
|
||||
return fields
|
||||
|
||||
@admin.display(
|
||||
description="Original URL",
|
||||
ordering="url",
|
||||
@@ -1390,12 +1231,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
obj.url[:128],
|
||||
)
|
||||
|
||||
@admin.display(description="Health", ordering="health")
|
||||
def health_display(self, obj):
|
||||
h = obj.health
|
||||
color = "green" if h >= 80 else "orange" if h >= 50 else "red"
|
||||
return format_html('<span style="color: {};">{}</span>', color, h)
|
||||
|
||||
def grid_view(self, request, extra_context=None):
|
||||
extra_context = extra_context or {}
|
||||
extra_context["snapshot_is_grid_view"] = True
|
||||
@@ -1499,7 +1334,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
|
||||
tags = []
|
||||
for name in tag_names:
|
||||
tag, _ = get_or_create_tag(
|
||||
tag, _ = Tag.get_or_create_by_name(
|
||||
name,
|
||||
created_by=request.user if request.user.is_authenticated else None,
|
||||
)
|
||||
|
||||
@@ -11,7 +11,7 @@ from django.urls import reverse
|
||||
from django.utils.html import format_html
|
||||
from django.utils.safestring import mark_safe
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
|
||||
from archivebox.core.models import SnapshotTag, Tag
|
||||
from archivebox.core.tag_util import (
|
||||
TAG_HAS_SNAPSHOTS_CHOICES,
|
||||
@@ -69,44 +69,14 @@ class TagAdmin(BaseModelAdmin):
|
||||
ordering = ["name", "id"]
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Tag",
|
||||
{
|
||||
"fields": ("name",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Metadata",
|
||||
{
|
||||
"fields": ("id", "created_by", "created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Recent Snapshots",
|
||||
{
|
||||
"fields": ("snapshots",),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
card_fieldset("Tag", ("name",)),
|
||||
card_fieldset("Metadata", ("id", "created_by", "created_at", "modified_at")),
|
||||
card_fieldset("Recent Snapshots", ("snapshots",), wide=True),
|
||||
)
|
||||
|
||||
add_fieldsets = (
|
||||
(
|
||||
"Tag",
|
||||
{
|
||||
"fields": ("name",),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Metadata",
|
||||
{
|
||||
"fields": ("created_by",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Tag", ("name",), wide=True),
|
||||
card_fieldset("Metadata", ("created_by",)),
|
||||
)
|
||||
|
||||
def get_fieldsets(self, request: HttpRequest, obj: Tag | None = None):
|
||||
|
||||
+17
-56
@@ -9,6 +9,7 @@ from archivebox.base_models.admin import KeyValueWidget
|
||||
from archivebox.config.common import get_config, parse_delete_after
|
||||
from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
|
||||
from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
|
||||
from archivebox.crawls.forms import URLFiltersField
|
||||
from archivebox.crawls.schedule_util import validate_schedule
|
||||
from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
|
||||
from archivebox.personas.models import Persona
|
||||
@@ -90,6 +91,18 @@ def edit_string_for_tag_names(tags) -> str:
|
||||
return ", ".join(sorted(names))
|
||||
|
||||
|
||||
class FileSizeField(forms.CharField):
|
||||
def to_python(self, value):
|
||||
raw_value = super().to_python(value)
|
||||
try:
|
||||
size = parse_filesize_to_bytes(raw_value) if raw_value else 0
|
||||
except ValueError as err:
|
||||
raise forms.ValidationError(str(err)) from err
|
||||
if size < 0:
|
||||
raise forms.ValidationError(f"{self.label} must be 0 or a positive number of bytes.")
|
||||
return size
|
||||
|
||||
|
||||
class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
allow_crawl_execution_config_fields = False
|
||||
|
||||
@@ -136,7 +149,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
},
|
||||
),
|
||||
)
|
||||
crawl_max_size = forms.CharField(
|
||||
crawl_max_size = FileSizeField(
|
||||
label="Max crawl size",
|
||||
required=False,
|
||||
initial="0",
|
||||
@@ -169,7 +182,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
},
|
||||
),
|
||||
)
|
||||
snapshot_max_size = forms.CharField(
|
||||
snapshot_max_size = FileSizeField(
|
||||
label="Max snapshot size",
|
||||
required=False,
|
||||
initial="0",
|
||||
@@ -210,7 +223,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
},
|
||||
),
|
||||
)
|
||||
url_filters = forms.Field(
|
||||
url_filters = URLFiltersField(
|
||||
label="URL allowlist / denylist",
|
||||
required=False,
|
||||
widget=URLFiltersWidget(source_selector='textarea[name="url"]'),
|
||||
@@ -347,23 +360,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
cleaned_data["config"] = {}
|
||||
return cleaned_data
|
||||
|
||||
# Combine all plugin groups into single list
|
||||
all_selected_plugins = []
|
||||
for field in [
|
||||
"main_plugins",
|
||||
"page_setup_plugins",
|
||||
"media_plugins",
|
||||
"text_plugins",
|
||||
"metadata_plugins",
|
||||
"postprocessing_plugins",
|
||||
"other_plugins",
|
||||
]:
|
||||
selected = cleaned_data.get(field)
|
||||
if isinstance(selected, list):
|
||||
all_selected_plugins.extend(selected)
|
||||
|
||||
# Store combined list for easy access
|
||||
cleaned_data["plugins"] = all_selected_plugins
|
||||
cleaned_data["plugins"] = [plugin for _category, field, _title in PLUGIN_GROUPS for plugin in cleaned_data.get(field, [])]
|
||||
|
||||
plugin_config_overrides = self.clean_plugin_config_overrides(get_config(persona=cleaned_data.get("persona")))
|
||||
custom_config = cleaned_data.get("config") or {}
|
||||
@@ -380,34 +377,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
raise forms.ValidationError("Enter at least one valid URL.")
|
||||
return value
|
||||
|
||||
def clean_url_filters(self):
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
value = self.cleaned_data.get("url_filters") or {}
|
||||
return {
|
||||
"allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
|
||||
"denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
|
||||
"same_domain_only": bool(value.get("same_domain_only")),
|
||||
"subpaths_only": bool(value.get("subpaths_only")),
|
||||
"only_new": bool(value.get("only_new")),
|
||||
}
|
||||
|
||||
def clean_max_urls(self):
|
||||
value = self.cleaned_data.get("max_urls")
|
||||
return int(value or 0)
|
||||
|
||||
def clean_crawl_max_size(self):
|
||||
raw_value = str(self.cleaned_data.get("crawl_max_size") or "").strip()
|
||||
if not raw_value:
|
||||
return 0
|
||||
try:
|
||||
value = parse_filesize_to_bytes(raw_value)
|
||||
except ValueError as err:
|
||||
raise forms.ValidationError(str(err))
|
||||
if value < 0:
|
||||
raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
|
||||
return value
|
||||
|
||||
def clean_crawl_timeout(self):
|
||||
return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
|
||||
|
||||
@@ -439,18 +412,6 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
|
||||
raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
|
||||
return value
|
||||
|
||||
def clean_snapshot_max_size(self):
|
||||
raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
|
||||
if not raw_value:
|
||||
return 0
|
||||
try:
|
||||
value = parse_filesize_to_bytes(raw_value)
|
||||
except ValueError as err:
|
||||
raise forms.ValidationError(str(err))
|
||||
if value < 0:
|
||||
raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
|
||||
return value
|
||||
|
||||
def clean_delete_after(self):
|
||||
raw_value = str(self.cleaned_data.get("delete_after") or "0").strip() or "0"
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Core persistence models, grouped by entity and queryset responsibility."""
|
||||
|
||||
from .archiveresults import ArchiveResult
|
||||
from .querysets import SnapshotManager, SnapshotQuerySet, UngroupedSubquery
|
||||
from .snapshots import Snapshot, SnapshotMigrationError
|
||||
from .tags import SnapshotTag, Tag
|
||||
|
||||
__all__ = [
|
||||
"ArchiveResult",
|
||||
"Snapshot",
|
||||
"SnapshotManager",
|
||||
"SnapshotMigrationError",
|
||||
"SnapshotQuerySet",
|
||||
"SnapshotTag",
|
||||
"Tag",
|
||||
"UngroupedSubquery",
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,361 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from django.core.exceptions import FieldDoesNotExist
|
||||
from django.db import models
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.misc.system import atomic_write
|
||||
from archivebox.misc.util import (
|
||||
to_json,
|
||||
)
|
||||
|
||||
|
||||
class UngroupedSubquery(models.Subquery):
|
||||
"""Scalar subquery that should not be copied into the outer GROUP BY."""
|
||||
|
||||
def get_group_by_cols(self):
|
||||
return []
|
||||
|
||||
|
||||
class SnapshotQuerySet(models.QuerySet):
|
||||
"""Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
|
||||
|
||||
def bulk_create(self, objs, *args, **kwargs):
|
||||
objs = list(objs)
|
||||
missing_crawl_ids = set()
|
||||
from archivebox.core.permissions import PERMISSIONS_VALUES
|
||||
|
||||
for obj in objs:
|
||||
if isinstance(obj, self.model):
|
||||
config = dict(obj.config or {})
|
||||
permission = str(config.get("PERMISSIONS") or "").strip().lower()
|
||||
if permission not in PERMISSIONS_VALUES and obj.crawl_id:
|
||||
crawl = getattr(obj, "crawl", None)
|
||||
if not getattr(crawl, "permissions", None):
|
||||
missing_crawl_ids.add(str(obj.crawl_id))
|
||||
|
||||
crawl_permissions_by_id = {}
|
||||
if missing_crawl_ids:
|
||||
crawl_permissions_by_id = {
|
||||
str(crawl_id): permissions
|
||||
for crawl_id, permissions in Crawl.objects.filter(pk__in=missing_crawl_ids).values_list("pk", "permissions")
|
||||
}
|
||||
|
||||
from archivebox.misc.db import truncate_overlong_charfields
|
||||
|
||||
for obj in objs:
|
||||
if isinstance(obj, self.model):
|
||||
obj.ensure_permissions_config(crawl_permissions=crawl_permissions_by_id.get(str(obj.crawl_id)))
|
||||
# bulk_create bypasses pre_save, so clamp CharFields here too
|
||||
# (e.g. page titles) to stay within postgres VARCHAR limits.
|
||||
truncate_overlong_charfields(obj)
|
||||
return super().bulk_create(objs, *args, **kwargs)
|
||||
|
||||
def paged_iterator(self, chunk_size: int = 500):
|
||||
"""
|
||||
Iterate snapshots using bounded keyset pages instead of one streaming cursor.
|
||||
|
||||
Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
|
||||
cursor open until the full queryset is exhausted. That is fine for
|
||||
read-only exports, but update/migration code does filesystem work and
|
||||
writes while iterating; a long-lived read cursor there can stretch lock
|
||||
waits across thousands of rows. This respects the queryset's existing
|
||||
filters, order_by(), select_related(), and prefetch_related() state; if
|
||||
no ordering is defined, it falls back to primary-key order.
|
||||
"""
|
||||
pk_field = self.model._meta.pk.name
|
||||
raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
|
||||
|
||||
ordering = []
|
||||
for term in raw_ordering:
|
||||
if not isinstance(term, str) or term == "?":
|
||||
ordering = []
|
||||
break
|
||||
field_name = term.removeprefix("-")
|
||||
field_name = pk_field if field_name == "pk" else field_name
|
||||
ordering.append(f"-{field_name}" if term.startswith("-") else field_name)
|
||||
|
||||
ordered_field_names = [term.removeprefix("-") for term in ordering]
|
||||
unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
|
||||
try:
|
||||
use_keyset = any(name in unique_field_names for name in ordered_field_names) and not any(
|
||||
self.model._meta.get_field(name).null for name in ordered_field_names
|
||||
)
|
||||
except (AttributeError, FieldDoesNotExist):
|
||||
use_keyset = False
|
||||
# Expressions, nullable/related fields and non-unique orderings retain
|
||||
# Django's ordering through bounded offset pages.
|
||||
if not use_keyset:
|
||||
offset = 0
|
||||
while batch := list(self[offset : offset + chunk_size]):
|
||||
yield from batch
|
||||
offset += chunk_size
|
||||
return
|
||||
|
||||
last_values = None
|
||||
value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
|
||||
while True:
|
||||
batch_qs = self.order_by(*ordering)
|
||||
if last_values is not None:
|
||||
page_filter = models.Q()
|
||||
for idx, term in enumerate(ordering):
|
||||
descending = term.startswith("-")
|
||||
field_name = term[1:] if descending else term
|
||||
prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
|
||||
comparison = "lt" if descending else "gt"
|
||||
page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
|
||||
batch_qs = batch_qs.filter(page_filter)
|
||||
|
||||
batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
|
||||
if not batch_rows:
|
||||
break
|
||||
|
||||
pk_idx = value_field_names.index(pk_field)
|
||||
snapshot_ids = [row[pk_idx] for row in batch_rows]
|
||||
snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
|
||||
|
||||
for row in batch_rows:
|
||||
snapshot_id = row[pk_idx]
|
||||
snapshot = snapshots_by_id.get(snapshot_id)
|
||||
if snapshot is not None:
|
||||
yield snapshot
|
||||
|
||||
last_values = batch_rows[-1][: len(ordered_field_names)]
|
||||
|
||||
# =========================================================================
|
||||
# Filtering Methods
|
||||
# =========================================================================
|
||||
|
||||
FILTER_TYPES: ClassVar[dict[str, Any]] = {
|
||||
"exact": lambda pattern: models.Q(url=pattern),
|
||||
"substring": lambda pattern: models.Q(url__icontains=pattern),
|
||||
"regex": lambda pattern: models.Q(url__iregex=pattern),
|
||||
"domain": lambda pattern: (
|
||||
models.Q(url__istartswith=f"http://{pattern}")
|
||||
| models.Q(url__istartswith=f"https://{pattern}")
|
||||
| models.Q(url__istartswith=f"ftp://{pattern}")
|
||||
),
|
||||
"tag": lambda pattern: models.Q(tags__name=pattern),
|
||||
"timestamp": lambda pattern: models.Q(timestamp=pattern),
|
||||
}
|
||||
FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
|
||||
FILTER_ARG_KEYS = (
|
||||
"after",
|
||||
"before",
|
||||
"filter_type",
|
||||
"filter_patterns",
|
||||
"status",
|
||||
"url__icontains",
|
||||
"url__istartswith",
|
||||
"tag",
|
||||
"crawl_id",
|
||||
"limit",
|
||||
"sort",
|
||||
"search",
|
||||
)
|
||||
SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})
|
||||
|
||||
def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> SnapshotQuerySet:
|
||||
"""Filter snapshots by URL patterns using specified filter type"""
|
||||
from archivebox.misc.logging import stderr
|
||||
|
||||
q_filter = models.Q()
|
||||
for pattern in patterns:
|
||||
try:
|
||||
q_filter = q_filter | self.FILTER_TYPES[filter_type](pattern)
|
||||
except KeyError:
|
||||
stderr()
|
||||
stderr(f"[X] Got invalid pattern for --filter-type={filter_type}:", color="red")
|
||||
stderr(f" {pattern}")
|
||||
raise SystemExit(2)
|
||||
return self.filter(q_filter)
|
||||
|
||||
def search(self, **kwargs) -> SnapshotQuerySet:
|
||||
|
||||
from archivebox.core.snapshot_status import filter_snapshots_by_status
|
||||
from archivebox.search.query import apply_snapshot_search
|
||||
|
||||
queryset = self
|
||||
filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
|
||||
filter_type = kwargs.get("filter_type") or "substring"
|
||||
query = kwargs.get("query")
|
||||
if isinstance(query, (list, tuple)):
|
||||
query = " ".join(str(part) for part in query)
|
||||
query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
|
||||
|
||||
field_names = {field.name for field in self.model._meta.get_fields()}
|
||||
field_names.update(field.attname for field in self.model._meta.fields)
|
||||
field_filters = {
|
||||
key: value
|
||||
for key, value in kwargs.items()
|
||||
if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
|
||||
}
|
||||
status = field_filters.pop("status", None)
|
||||
queryset = filter_snapshots_by_status(queryset, status)
|
||||
if field_filters:
|
||||
queryset = queryset.filter(**field_filters)
|
||||
if kwargs.get("tag"):
|
||||
queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
|
||||
if kwargs.get("before") is not None:
|
||||
queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=UTC))
|
||||
if kwargs.get("after") is not None:
|
||||
queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=UTC))
|
||||
|
||||
if query:
|
||||
queryset = apply_snapshot_search(
|
||||
queryset,
|
||||
query,
|
||||
search_mode=kwargs.get("search"),
|
||||
ordering=("-created_at",) if not kwargs.get("sort") else None,
|
||||
max_results=kwargs.get("limit"),
|
||||
skip_backend_when_metadata_satisfies_limit=True,
|
||||
include_metadata_for_forced_backend=True,
|
||||
)
|
||||
elif filter_patterns:
|
||||
queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
|
||||
|
||||
if kwargs.get("sort"):
|
||||
queryset = queryset.order_by(kwargs["sort"])
|
||||
elif not queryset.query.order_by:
|
||||
queryset = queryset.order_by("-created_at")
|
||||
|
||||
limit = kwargs.get("limit")
|
||||
if limit is not None and limit > 0:
|
||||
queryset = queryset[:limit]
|
||||
|
||||
return queryset
|
||||
|
||||
# =========================================================================
|
||||
# Export Methods
|
||||
# =========================================================================
|
||||
|
||||
def to_json(self, with_headers: bool = False) -> str:
|
||||
"""Generate JSON index from snapshots"""
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
from archivebox.config import VERSION
|
||||
|
||||
config = get_config()
|
||||
|
||||
MAIN_INDEX_HEADER = (
|
||||
{
|
||||
"info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
|
||||
"schema": "archivebox.index.json",
|
||||
"copyright_info": config.FOOTER_INFO,
|
||||
"meta": {
|
||||
"project": "ArchiveBox",
|
||||
"version": VERSION,
|
||||
"git_sha": VERSION,
|
||||
"website": "https://ArchiveBox.io",
|
||||
"docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
|
||||
"source": "https://github.com/ArchiveBox/ArchiveBox",
|
||||
"issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
|
||||
"dependencies": {},
|
||||
},
|
||||
}
|
||||
if with_headers
|
||||
else {}
|
||||
)
|
||||
|
||||
snapshot_dicts = [s.to_dict(extended=True, static_export=True) for s in self.iterator(chunk_size=500)]
|
||||
|
||||
if with_headers:
|
||||
output = {
|
||||
**MAIN_INDEX_HEADER,
|
||||
"num_links": len(snapshot_dicts),
|
||||
"updated": datetime.now(UTC),
|
||||
"last_run_cmd": sys.argv,
|
||||
"links": snapshot_dicts,
|
||||
}
|
||||
else:
|
||||
output = snapshot_dicts
|
||||
return to_json(output, indent=4, sort_keys=True)
|
||||
|
||||
def to_csv(self, cols: list[str] | None = None, header: bool = True, separator: str = ",", ljust: int = 0) -> str:
|
||||
"""Generate CSV output from snapshots"""
|
||||
cols = cols or ["timestamp", "is_archived", "url"]
|
||||
header_str = separator.join(col.ljust(ljust) for col in cols) if header else ""
|
||||
row_strs = (s.to_csv(cols=cols, ljust=ljust, separator=separator) for s in self.iterator(chunk_size=500))
|
||||
return "\n".join((header_str, *row_strs))
|
||||
|
||||
def to_html(self, with_headers: bool = True) -> str:
|
||||
"""Generate main index HTML from snapshots"""
|
||||
from datetime import datetime
|
||||
|
||||
from django.template.loader import render_to_string
|
||||
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.version import get_COMMIT_HASH
|
||||
|
||||
config = get_config()
|
||||
|
||||
template = "static_index.html" if with_headers else "minimal_index.html"
|
||||
snapshot_list = list(self.iterator(chunk_size=500))
|
||||
manifest_records = []
|
||||
for snapshot in snapshot_list:
|
||||
outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
|
||||
output_paths = [str(output.get("path") or "") for output in outputs]
|
||||
snapshot._public_preview_paths = [
|
||||
path for preferred in ("screenshot/screenshot.png", "screenshot.png") for path in output_paths if path == preferred
|
||||
]
|
||||
snapshot._public_favicon_paths = [path for path in output_paths if path in ("favicon/favicon.ico", "favicon.ico")]
|
||||
snapshot.write_html_details()
|
||||
if with_headers:
|
||||
# Use the same portable schema as the JSON export. Rendering
|
||||
# above has already populated result-count caches, archive_size
|
||||
# reuses the sealed output_size field, and tags are prefetched.
|
||||
manifest_records.append(snapshot.to_dict(extended=True, static_export=True))
|
||||
|
||||
if with_headers:
|
||||
manifest = "".join(f"{to_json(record, indent=None, sort_keys=True)}\n" for record in manifest_records)
|
||||
atomic_write(str(CONSTANTS.DATA_DIR / CONSTANTS.JSONL_INDEX_FILENAME), manifest)
|
||||
|
||||
return render_to_string(
|
||||
template,
|
||||
{
|
||||
"version": VERSION,
|
||||
"git_sha": get_COMMIT_HASH() or VERSION,
|
||||
"num_links": str(len(snapshot_list)),
|
||||
"date_updated": datetime.now(UTC).strftime("%Y-%m-%d"),
|
||||
"time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"),
|
||||
"links": snapshot_list,
|
||||
"FOOTER_INFO": config.FOOTER_INFO,
|
||||
"STATIC_EXPORT": True,
|
||||
"STATIC_EXPORT_DIR": CONSTANTS.DATA_DIR,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)): # ty: ignore[unsupported-base]
|
||||
"""Manager for Snapshot model - uses SnapshotQuerySet for chainable methods"""
|
||||
|
||||
def filter(self, *args, **kwargs):
|
||||
domain = kwargs.pop("domain", None)
|
||||
qs = super().filter(*args, **kwargs)
|
||||
if domain:
|
||||
qs = qs.filter(url__icontains=f"://{domain}")
|
||||
return qs
|
||||
|
||||
def get_queryset(self):
|
||||
# Don't prefetch by default - it causes "too many open files" during bulk operations
|
||||
# Views/templates can add .prefetch_related('tags', 'archiveresult_set') where needed
|
||||
return super().get_queryset()
|
||||
|
||||
# =========================================================================
|
||||
# Import Methods
|
||||
# =========================================================================
|
||||
|
||||
def remove(self, atomic: bool = False) -> tuple:
|
||||
"""Remove snapshots from the database"""
|
||||
from django.db import transaction
|
||||
|
||||
if atomic:
|
||||
with transaction.atomic():
|
||||
return self.get_queryset().delete()
|
||||
return self.get_queryset().delete()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import IntegrityError, models
|
||||
from django.urls import reverse_lazy
|
||||
from django.utils import timezone
|
||||
from django.utils.text import slugify
|
||||
|
||||
from archivebox.base_models.models import (
|
||||
ModelWithUUID,
|
||||
get_or_create_system_user_pk,
|
||||
)
|
||||
from archivebox.misc.util import (
|
||||
sanitize_html_text,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .snapshots import Snapshot
|
||||
|
||||
|
||||
class Tag(ModelWithUUID):
|
||||
# Keep AutoField for compatibility with main branch migrations
|
||||
# Don't use UUIDField here - requires complex FK transformation
|
||||
id = models.AutoField(primary_key=True, serialize=False, verbose_name="ID")
|
||||
created_by = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
default=get_or_create_system_user_pk,
|
||||
null=True,
|
||||
related_name="tag_set",
|
||||
)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True, null=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
name = models.CharField(unique=True, blank=False, max_length=100)
|
||||
|
||||
@classmethod
|
||||
def get_or_create_by_name(
|
||||
cls,
|
||||
name: str,
|
||||
*,
|
||||
defaults: Mapping[str, Any] | None = None,
|
||||
created_by=None,
|
||||
) -> tuple[Tag, bool]:
|
||||
name = cls.normalize_name(name)
|
||||
defaults = dict(defaults or {})
|
||||
if created_by is not None:
|
||||
defaults["created_by"] = created_by
|
||||
tag = cls.objects.filter(name__iexact=name).first()
|
||||
if tag:
|
||||
return tag, False
|
||||
try:
|
||||
return cls.objects.create(name=name, **(defaults or {})), True
|
||||
except IntegrityError:
|
||||
tag = cls.objects.filter(name__iexact=name).first()
|
||||
if tag is None:
|
||||
raise
|
||||
return tag, False
|
||||
|
||||
@staticmethod
|
||||
def normalize_name(name: str) -> str:
|
||||
name = sanitize_html_text(name).strip()
|
||||
if not name:
|
||||
raise ValueError("Tag name is required")
|
||||
return name
|
||||
|
||||
def rename(self, name: str) -> Tag:
|
||||
name = self.normalize_name(name)
|
||||
existing = type(self).objects.filter(name__iexact=name).exclude(pk=self.pk).first()
|
||||
if existing:
|
||||
raise ValueError(f'Tag "{existing.name}" already exists')
|
||||
if self.name != name:
|
||||
self.name = name
|
||||
self.save()
|
||||
return self
|
||||
|
||||
snapshot_set: models.Manager[Snapshot]
|
||||
|
||||
class Meta(ModelWithUUID.Meta):
|
||||
app_label = "core"
|
||||
verbose_name = "Tag"
|
||||
verbose_name_plural = "Tags"
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if update_fields is None or "name" in update_fields:
|
||||
self.name = sanitize_html_text(self.name).strip()
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def slug(self) -> str:
|
||||
"""ASCII-safe slugified form of the tag name (derived, not stored)."""
|
||||
return slugify(self.name or "") or "tag"
|
||||
|
||||
@property
|
||||
def api_url(self) -> str:
|
||||
return str(reverse_lazy("api-1:get_tag", args=[self.id]))
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""
|
||||
Convert Tag model instance to a JSON-serializable dict.
|
||||
"""
|
||||
from archivebox.config import VERSION
|
||||
|
||||
return {
|
||||
"type": "Tag",
|
||||
"schema_version": VERSION,
|
||||
"id": str(self.id),
|
||||
"name": self.name,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
|
||||
"""
|
||||
Create/update Tag from JSON dict.
|
||||
|
||||
Args:
|
||||
record: JSON dict with 'name' field
|
||||
overrides: Optional dict with 'snapshot' to auto-attach tag
|
||||
|
||||
Returns:
|
||||
Tag instance or None
|
||||
"""
|
||||
name = record.get("name")
|
||||
if not name:
|
||||
return None
|
||||
|
||||
tag, _ = Tag.get_or_create_by_name(name)
|
||||
|
||||
# Auto-attach to snapshot if in overrides
|
||||
if overrides and "snapshot" in overrides and tag:
|
||||
overrides["snapshot"].add_tag_ids([tag.pk])
|
||||
|
||||
return tag
|
||||
|
||||
|
||||
class SnapshotTag(models.Model):
|
||||
id = models.AutoField(primary_key=True)
|
||||
snapshot = models.ForeignKey("Snapshot", db_column="snapshot_id", on_delete=models.CASCADE, to_field="id")
|
||||
tag = models.ForeignKey(Tag, db_column="tag_id", on_delete=models.CASCADE, to_field="id")
|
||||
|
||||
class Meta:
|
||||
app_label = "core"
|
||||
db_table = "core_snapshot_tags"
|
||||
unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]
|
||||
@@ -60,13 +60,6 @@ def _csrf_trusted_origins(config) -> list[str]:
|
||||
return seen
|
||||
|
||||
|
||||
def _allowed_hosts(config) -> set[str]:
|
||||
raw = (config.ALLOWED_HOSTS or "").strip()
|
||||
if not raw:
|
||||
return set()
|
||||
return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
|
||||
|
||||
|
||||
def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
|
||||
"""Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from collections import defaultdict
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from django.db.models import Count, Exists, F, OuterRef, QuerySet
|
||||
from django.db.models.functions import Lower
|
||||
from django.http import HttpRequest
|
||||
@@ -147,34 +146,6 @@ def get_tag_by_ref(tag_ref: str | int) -> Tag:
|
||||
return Tag.objects.get(name__iexact=decoded)
|
||||
|
||||
|
||||
def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, bool]:
|
||||
normalized_name = normalize_tag_name(name)
|
||||
if not normalized_name:
|
||||
raise ValueError("Tag name is required")
|
||||
|
||||
defaults = {"created_by": created_by} if created_by is not None else None
|
||||
return Tag.get_or_create_by_name(normalized_name, defaults=defaults)
|
||||
|
||||
|
||||
def rename_tag(tag: Tag, name: str) -> Tag:
|
||||
normalized_name = normalize_tag_name(name)
|
||||
if not normalized_name:
|
||||
raise ValueError("Tag name is required")
|
||||
|
||||
existing = Tag.objects.filter(name__iexact=normalized_name).exclude(pk=tag.pk).first()
|
||||
if existing:
|
||||
raise ValueError(f'Tag "{existing.name}" already exists')
|
||||
|
||||
if tag.name != normalized_name:
|
||||
tag.name = normalized_name
|
||||
tag.save()
|
||||
return tag
|
||||
|
||||
|
||||
def delete_tag(tag: Tag) -> tuple[int, dict[str, int]]:
|
||||
return tag.delete()
|
||||
|
||||
|
||||
def export_tag_urls(tag: Tag) -> str:
|
||||
urls = tag.snapshot_set.order_by("-downloaded_at", "-created_at", "-pk").values_list("url", flat=True)
|
||||
return "\n".join(urls)
|
||||
@@ -250,10 +221,10 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No
|
||||
"num_snapshots": count,
|
||||
"filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}",
|
||||
"edit_url": f"/admin/core/tag/{tag.pk}/change/",
|
||||
"export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt",
|
||||
"export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl",
|
||||
"rename_url": f"/api/v1/core/tag/{tag.pk}/rename",
|
||||
"delete_url": f"/api/v1/core/tag/{tag.pk}/",
|
||||
"export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]),
|
||||
"export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]),
|
||||
"rename_url": reverse("api-1:rename_tag", args=[tag.pk]),
|
||||
"delete_url": reverse("api-1:delete_tag", args=[tag.pk]),
|
||||
"snapshots": snapshot_previews or [],
|
||||
}
|
||||
|
||||
|
||||
@@ -131,20 +131,10 @@ def ensure_daemon_stack(*, reason: str = ""):
|
||||
rprint(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
|
||||
worker = start_worker(supervisor, sonic_worker)
|
||||
|
||||
# Supervisord marks this worker RUNNING with startsecs=0, before Sonic has
|
||||
# opened its search socket. A query launched at that point fails even though
|
||||
# the daemon is about to become ready.
|
||||
deadline = time.monotonic() + 30.0
|
||||
while time.monotonic() < deadline:
|
||||
if is_port_listening(sonic_event.host, sonic_event.port):
|
||||
if time.monotonic() < deadline:
|
||||
return worker
|
||||
break
|
||||
worker = get_worker(supervisor, sonic_worker["name"])
|
||||
if not isinstance(worker, dict) or worker.get("statename") not in ("STARTING", "RUNNING"):
|
||||
raise RuntimeError(f"Sonic search backend worker failed to start: {worker}")
|
||||
time.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
|
||||
raise RuntimeError(f"Sonic search backend is not listening at {sonic_event.url}")
|
||||
from archivebox.search.sonic_daemon import wait_for_sonic_daemon
|
||||
|
||||
wait_for_sonic_daemon(sonic_event)
|
||||
return worker
|
||||
|
||||
|
||||
def live_runner_processes(*, data_dir: str | Path):
|
||||
|
||||
@@ -47,47 +47,6 @@ _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml",
|
||||
_IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
|
||||
_STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
|
||||
|
||||
_VIDEO_FILE_EXTS = {
|
||||
".mp4",
|
||||
".webm",
|
||||
".mkv",
|
||||
".avi",
|
||||
".mov",
|
||||
".flv",
|
||||
".wmv",
|
||||
".m4v",
|
||||
".mpg",
|
||||
".mpeg",
|
||||
".ts",
|
||||
".m2ts",
|
||||
".mts",
|
||||
".3gp",
|
||||
".3g2",
|
||||
".ogv",
|
||||
}
|
||||
|
||||
_AUDIO_FILE_EXTS = {
|
||||
".mp3",
|
||||
".m4a",
|
||||
".aac",
|
||||
".ogg",
|
||||
".oga",
|
||||
".opus",
|
||||
".wav",
|
||||
".flac",
|
||||
".alac",
|
||||
".aiff",
|
||||
".wma",
|
||||
".mka",
|
||||
".ac3",
|
||||
".eac3",
|
||||
".dts",
|
||||
}
|
||||
|
||||
_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
|
||||
_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
|
||||
_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
|
||||
|
||||
|
||||
def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
|
||||
"""Hide the informational mismatch banner on the conventional public web alias.
|
||||
@@ -104,117 +63,12 @@ def _is_web_subdomain_alias(browser_url: str, configured_base_url: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
|
||||
from abx_dl.output_files import OutputManifest
|
||||
|
||||
return OutputManifest.from_value(output_files).as_mapping()
|
||||
|
||||
|
||||
def _snapshot_id(value: Any) -> Any:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
return value.id if isinstance(value, Snapshot) else value
|
||||
|
||||
|
||||
def _coerce_output_file_size(value: Any) -> int | None:
|
||||
try:
|
||||
return max(int(value or 0), 0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _count_media_files(result) -> int:
|
||||
try:
|
||||
output_files = _normalize_output_files(result.output_files or {})
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
output_files = {}
|
||||
|
||||
if output_files:
|
||||
return sum(1 for path in output_files if Path(path).suffix.lower() in _MEDIA_FILE_EXTS)
|
||||
|
||||
try:
|
||||
plugin_dir = Path(result.snapshot_dir) / result.plugin
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
if not plugin_dir.exists():
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
scanned = 0
|
||||
max_scan = 500
|
||||
for file_path in plugin_dir.rglob("*"):
|
||||
if scanned >= max_scan:
|
||||
break
|
||||
scanned += 1
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _list_media_files(result) -> list[dict]:
|
||||
media_files: list[dict] = []
|
||||
try:
|
||||
plugin_dir = Path(result.snapshot_dir) / result.plugin
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return media_files
|
||||
|
||||
output_files = _normalize_output_files(result.output_files or {})
|
||||
candidates: list[tuple[Path, int | None]] = []
|
||||
if output_files:
|
||||
for path, metadata in output_files.items():
|
||||
rel_path = Path(path)
|
||||
if rel_path.suffix.lower() in _MEDIA_FILE_EXTS:
|
||||
candidates.append((rel_path, _coerce_output_file_size(metadata.get("size"))))
|
||||
|
||||
if not candidates and plugin_dir.exists():
|
||||
scanned = 0
|
||||
max_scan = 2000
|
||||
for file_path in plugin_dir.rglob("*"):
|
||||
if scanned >= max_scan:
|
||||
break
|
||||
scanned += 1
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
if file_path.suffix.lower() in _MEDIA_FILE_EXTS:
|
||||
try:
|
||||
rel_path = file_path.relative_to(plugin_dir)
|
||||
except ValueError:
|
||||
continue
|
||||
try:
|
||||
size = file_path.stat().st_size
|
||||
except OSError:
|
||||
size = None
|
||||
candidates.append((rel_path, size))
|
||||
|
||||
for rel_path, size in candidates:
|
||||
href = str(Path(result.plugin) / rel_path)
|
||||
suffix = rel_path.suffix.lower()
|
||||
media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
|
||||
media_files.append(
|
||||
{
|
||||
"name": rel_path.name,
|
||||
"path": href,
|
||||
"size": size,
|
||||
"media_type": media_type,
|
||||
"is_video": media_type == "video",
|
||||
"is_audio": media_type == "audio",
|
||||
"is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
|
||||
},
|
||||
)
|
||||
|
||||
media_files.sort(
|
||||
key=lambda item: (
|
||||
0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
|
||||
-int(item.get("size") or 0),
|
||||
item["name"].lower(),
|
||||
),
|
||||
)
|
||||
return media_files
|
||||
|
||||
|
||||
def _resolve_snapshot_output_file(snapshot_dir: str | Path | None, raw_output_path: str | None) -> Path | None:
|
||||
if not snapshot_dir or not raw_output_path or str(raw_output_path).strip() in (".", "/", "./"):
|
||||
return None
|
||||
@@ -820,8 +674,7 @@ def plugin_card(context, result) -> str:
|
||||
|
||||
icon_html = get_plugin_icon(plugin)
|
||||
plugin_lower = (plugin or "").lower()
|
||||
media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
|
||||
media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
|
||||
media_files = result.media_files() if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
|
||||
if context.get("STATIC_EXPORT") and media_files:
|
||||
media_files = [
|
||||
item
|
||||
@@ -831,7 +684,6 @@ def plugin_card(context, result) -> str:
|
||||
and ".." not in media_path.parts
|
||||
and (Path(result.snapshot_dir) / media_path).is_file()
|
||||
]
|
||||
media_file_count = len(media_files)
|
||||
if media_files:
|
||||
for item in media_files:
|
||||
path = item.get("path") or ""
|
||||
@@ -852,7 +704,7 @@ def plugin_card(context, result) -> str:
|
||||
"output_path_raw": raw_output_path,
|
||||
"plugin": plugin,
|
||||
"plugin_icon": icon_html,
|
||||
"media_file_count": media_file_count,
|
||||
"media_file_count": len(media_files),
|
||||
"media_files": media_files,
|
||||
},
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,39 @@
|
||||
"""HTTP views grouped by public browsing, ingestion, replay, and configuration."""
|
||||
|
||||
from archivebox.config.common import find_config_source as find_config_source
|
||||
from archivebox.plugins.views import get_config_definition_link as get_config_definition_link
|
||||
|
||||
from .add import AddView as AddView
|
||||
from .add import WebAddView as WebAddView
|
||||
from .config import live_config_list_view as live_config_list_view
|
||||
from .config import live_config_value_view as live_config_value_view
|
||||
from .lookup import _files_index_target as _files_index_target
|
||||
from .lookup import _find_snapshot_by_ref as _find_snapshot_by_ref
|
||||
from .public import HealthCheckView as HealthCheckView
|
||||
from .public import HomepageView as HomepageView
|
||||
from .public import PublicIndexView as PublicIndexView
|
||||
from .replay import OriginalDomainHostView as OriginalDomainHostView
|
||||
from .replay import OriginalDomainReplayView as OriginalDomainReplayView
|
||||
from .replay import SnapshotHostView as SnapshotHostView
|
||||
from .replay import SnapshotPathView as SnapshotPathView
|
||||
from .replay import SnapshotReplayView as SnapshotReplayView
|
||||
from .replay import SnapshotView as SnapshotView
|
||||
from .replay import _latest_response_match as _latest_response_match
|
||||
from .replay import _latest_responses_root as _latest_responses_root
|
||||
from .replay import _original_request_url as _original_request_url
|
||||
from .replay import _plugin_full_preview_response as _plugin_full_preview_response
|
||||
from .replay import _resolve_archiveresult_relpath as _resolve_archiveresult_relpath
|
||||
from .replay import _safe_archive_relpath as _safe_archive_relpath
|
||||
from .replay import _serve_original_domain_replay as _serve_original_domain_replay
|
||||
from .replay import _serve_responses_path as _serve_responses_path
|
||||
from .replay import _serve_snapshot_replay as _serve_snapshot_replay
|
||||
from .replay import _visible_response_snapshots_for_domain as _visible_response_snapshots_for_domain
|
||||
from .replay_auth import REPLAY_AUTH_SALT as REPLAY_AUTH_SALT
|
||||
from .replay_auth import SnapshotReplayAuthView as SnapshotReplayAuthView
|
||||
from .replay_auth import _admin_login_redirect_or_forbidden as _admin_login_redirect_or_forbidden
|
||||
from .replay_auth import _clean_replay_next as _clean_replay_next
|
||||
from .replay_auth import _has_replay_cookie as _has_replay_cookie
|
||||
from .replay_auth import _private_snapshot_auth_redirect as _private_snapshot_auth_redirect
|
||||
from .replay_auth import _replay_auth_response as _replay_auth_response
|
||||
from .replay_auth import _replay_cookie_name as _replay_cookie_name
|
||||
from .replay_auth import _replay_payload_is_valid as _replay_payload_is_valid
|
||||
@@ -0,0 +1,363 @@
|
||||
import json
|
||||
from urllib.parse import quote
|
||||
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.mixins import UserPassesTestMixin
|
||||
from django.http import Http404, HttpRequest, HttpResponse, QueryDict
|
||||
from django.shortcuts import redirect
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.utils.html import format_html
|
||||
from django.views.decorators.csrf import csrf_exempt, csrf_protect
|
||||
from django.views.generic import FormView
|
||||
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.common import (
|
||||
get_config,
|
||||
get_request_config,
|
||||
redact_sensitive_config,
|
||||
)
|
||||
from archivebox.core.forms import AddLinkForm
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_PUBLIC,
|
||||
direct_snapshots_queryset,
|
||||
filter_personas_by_permissions,
|
||||
)
|
||||
from archivebox.core.routes_util import (
|
||||
build_admin_url,
|
||||
build_web_url,
|
||||
get_admin_host,
|
||||
get_web_host,
|
||||
host_matches,
|
||||
)
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.misc.util import (
|
||||
sanitize_html_text,
|
||||
urldecode,
|
||||
validate_url,
|
||||
)
|
||||
from archivebox.plugins.forms import get_plugin_config_binary_urls
|
||||
|
||||
from .replay import SnapshotView
|
||||
|
||||
|
||||
@method_decorator(csrf_exempt, name="dispatch")
|
||||
class AddView(UserPassesTestMixin, FormView):
|
||||
template_name = "add.html"
|
||||
form_class = AddLinkForm
|
||||
|
||||
def get_initial(self):
|
||||
"""Prefill the AddLinkForm with the 'url' GET parameter"""
|
||||
if self.request.method == "GET":
|
||||
url = self.request.GET.get("url", None)
|
||||
if url:
|
||||
return {"url": url if "://" in url else f"https://{url}"}
|
||||
|
||||
return super().get_initial()
|
||||
|
||||
def get_form_kwargs(self):
|
||||
kwargs = super().get_form_kwargs()
|
||||
kwargs["request"] = self.request
|
||||
return kwargs
|
||||
|
||||
def test_func(self):
|
||||
return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
|
||||
|
||||
def post(self, request: HttpRequest, *args: object, **kwargs: object):
|
||||
if request.user.is_authenticated:
|
||||
return csrf_protect(super().post)(request, *args, **kwargs)
|
||||
return super().post(request, *args, **kwargs)
|
||||
|
||||
def _can_override_crawl_config(self) -> bool:
|
||||
user = self.request.user
|
||||
return bool(user.is_authenticated and user.is_active and user.is_superuser)
|
||||
|
||||
def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
|
||||
custom_config = form.cleaned_data.get("config") or {}
|
||||
|
||||
if not isinstance(custom_config, dict):
|
||||
return {}
|
||||
|
||||
if not self._can_override_crawl_config():
|
||||
return {}
|
||||
|
||||
return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
request_config = get_request_config(self.request, resolve_plugins=True)
|
||||
required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
|
||||
can_override_crawl_config = self._can_override_crawl_config()
|
||||
public_persona_config_keys = {
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS",
|
||||
"DELETE_AFTER",
|
||||
"ONLY_NEW",
|
||||
"PERMISSIONS",
|
||||
"TIMEOUT",
|
||||
}
|
||||
persona_queryset = context["form"].fields["persona"].queryset
|
||||
if not can_override_crawl_config:
|
||||
persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
|
||||
persona_config_map = {}
|
||||
for persona in persona_queryset.order_by("name"):
|
||||
effective_config = get_config(persona=persona)
|
||||
effective_config_redacted = redact_sensitive_config(effective_config.model_dump(mode="json"))
|
||||
if can_override_crawl_config:
|
||||
raw_config = redact_sensitive_config(persona.config or {})
|
||||
effective_config_json = effective_config_redacted
|
||||
binary_urls = get_plugin_config_binary_urls(effective_config)
|
||||
else:
|
||||
raw_config = {}
|
||||
effective_config_json = {key: effective_config_redacted.get(key) for key in public_persona_config_keys}
|
||||
binary_urls = {}
|
||||
persona_config_map[persona.name] = {
|
||||
"config": raw_config,
|
||||
"effective_config": effective_config_json,
|
||||
"binary_urls": binary_urls,
|
||||
}
|
||||
recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
|
||||
return {
|
||||
**context,
|
||||
"title": "Create Crawl",
|
||||
# We can't just call request.build_absolute_uri in the template, because it would include query parameters
|
||||
"absolute_add_path": self.request.build_absolute_uri(self.request.path),
|
||||
"web_base_url": build_web_url("", request=self.request),
|
||||
"VERSION": VERSION,
|
||||
"FOOTER_INFO": request_config.FOOTER_INFO,
|
||||
"required_search_plugin": required_search_plugin,
|
||||
"persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
|
||||
"recent_personas": recent_personas,
|
||||
"can_override_crawl_config": can_override_crawl_config,
|
||||
"stdout": "",
|
||||
}
|
||||
|
||||
def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
|
||||
from archivebox.cli.archivebox_add import add
|
||||
|
||||
urls_input = form.cleaned_data["url"]
|
||||
urls = urls_input
|
||||
submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
|
||||
if len(submitted_lines) == 1:
|
||||
try:
|
||||
# A lone URL pasted into /add/ is the same user-facing input as
|
||||
# `archivebox add https://...`: queue that URL directly so a
|
||||
# narrow plugin selection like `wget` can archive it without
|
||||
# also needing parser plugins. Multi-line or formatted text
|
||||
# remains verbatim import content for the internal parser root.
|
||||
urls = [validate_url(submitted_lines[0])]
|
||||
except ValueError:
|
||||
pass
|
||||
print(f"[+] Adding URL: {urls_input}")
|
||||
|
||||
# Extract all form fields
|
||||
tag = form.cleaned_data["tag"]
|
||||
depth = int(form.cleaned_data["depth"])
|
||||
max_urls = int(form.cleaned_data.get("max_urls") or 0)
|
||||
crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
|
||||
crawl_timeout = int(form.cleaned_data.get("crawl_timeout") or 0)
|
||||
timeout = form.cleaned_data.get("timeout")
|
||||
snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
|
||||
delete_after = str(form.cleaned_data.get("delete_after") or "0").strip() or "0"
|
||||
crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
|
||||
permissions = str(form.cleaned_data.get("permissions") or "public").strip().lower()
|
||||
can_override_crawl_config = self._can_override_crawl_config()
|
||||
plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else ""
|
||||
schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else ""
|
||||
persona = form.cleaned_data.get("persona")
|
||||
start_paused = form.cleaned_data.get("start_paused", False) if can_override_crawl_config else False
|
||||
notes = form.cleaned_data.get("notes", "")
|
||||
url_filters = form.cleaned_data.get("url_filters") or {}
|
||||
plugin_config = form.cleaned_data.get("plugin_config") or {}
|
||||
if not isinstance(plugin_config, dict):
|
||||
plugin_config = {}
|
||||
if not can_override_crawl_config:
|
||||
plugin_config = {}
|
||||
custom_config = self._get_custom_config_overrides(form)
|
||||
custom_config.pop("DEFAULT_PERSONA", None)
|
||||
custom_config.pop("PERMISSIONS", None)
|
||||
if persona:
|
||||
persona.ensure_dirs()
|
||||
|
||||
if created_by_id is None:
|
||||
if self.request.user.is_authenticated:
|
||||
created_by_id = self.request.user.pk
|
||||
else:
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
|
||||
created_by_id = get_or_create_system_user_pk()
|
||||
|
||||
config = {}
|
||||
effective_config = get_config(persona=persona) if persona else get_config()
|
||||
if delete_after != str(effective_config.DELETE_AFTER):
|
||||
config["DELETE_AFTER"] = delete_after
|
||||
if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
|
||||
config["TIMEOUT"] = int(timeout)
|
||||
if permissions:
|
||||
config["PERMISSIONS"] = permissions
|
||||
|
||||
config.update(plugin_config)
|
||||
config.update(custom_config)
|
||||
if bool(url_filters.get("only_new")) != bool(effective_config.ONLY_NEW):
|
||||
config["ONLY_NEW"] = bool(url_filters.get("only_new"))
|
||||
crawl, _snapshots = add(
|
||||
urls=urls,
|
||||
depth=depth,
|
||||
max_urls=max_urls,
|
||||
crawl_max_size=crawl_max_size,
|
||||
crawl_timeout=crawl_timeout,
|
||||
snapshot_max_size=snapshot_max_size,
|
||||
crawl_max_concurrent_snapshots=crawl_max_concurrent_snapshots,
|
||||
tag=tag,
|
||||
url_allowlist=url_filters.get("allowlist") or "",
|
||||
url_denylist=url_filters.get("denylist") or "",
|
||||
plugins=plugins,
|
||||
persona=persona.name if persona else "Default",
|
||||
bg=True,
|
||||
created_by_id=created_by_id,
|
||||
config=config,
|
||||
)
|
||||
if notes:
|
||||
crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
|
||||
if permissions and crawl.config.get("PERMISSIONS") != permissions:
|
||||
next_config = {**crawl.config, "PERMISSIONS": permissions}
|
||||
crawl.safe_update({"config": next_config}, refresh=True)
|
||||
if start_paused:
|
||||
crawl.pause()
|
||||
|
||||
# 3. create a CrawlSchedule if schedule is provided
|
||||
if schedule:
|
||||
from archivebox.crawls.models import CrawlSchedule
|
||||
|
||||
crawl_schedule = CrawlSchedule.objects.create(
|
||||
template=crawl,
|
||||
schedule=schedule,
|
||||
is_enabled=True,
|
||||
config=config,
|
||||
label=crawl.label,
|
||||
notes=f"Auto-created from add page. {notes}".strip(),
|
||||
created_by_id=created_by_id,
|
||||
)
|
||||
crawl.schedule = crawl_schedule
|
||||
crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
|
||||
|
||||
return crawl
|
||||
|
||||
def form_valid(self, form):
|
||||
crawl = self._create_crawl_from_form(form)
|
||||
|
||||
urls = form.cleaned_data["url"]
|
||||
schedule = form.cleaned_data.get("schedule", "").strip()
|
||||
rough_url_count = len([url for url in urls.splitlines() if url.strip()])
|
||||
|
||||
schedule_msg = ""
|
||||
if schedule and crawl.schedule_id:
|
||||
schedule_msg = format_html(" and <a href='{}'>scheduled to repeat {}</a>", crawl.schedule.admin_change_url, schedule)
|
||||
|
||||
messages.success(
|
||||
self.request,
|
||||
format_html(
|
||||
"Created crawl with {} starting URL(s){}. Snapshots will be created and archived in the background. <a href='{}'>View Crawl →</a>",
|
||||
rough_url_count,
|
||||
schedule_msg,
|
||||
crawl.admin_change_url,
|
||||
),
|
||||
)
|
||||
|
||||
# Orchestrator (managed by supervisord) will pick up the queued crawl
|
||||
return redirect(crawl.admin_change_url)
|
||||
|
||||
|
||||
class WebAddView(AddView):
|
||||
def _latest_snapshot_for_url(self, requested_url: str):
|
||||
return (
|
||||
direct_snapshots_queryset(
|
||||
self.request,
|
||||
SnapshotView.find_snapshots_for_url(requested_url),
|
||||
)
|
||||
.order_by("-bookmarked_at", "-created_at", "-timestamp")
|
||||
.first()
|
||||
)
|
||||
|
||||
def _normalize_add_url(self, requested_url: str) -> str:
|
||||
if requested_url.startswith(("http://", "https://")):
|
||||
return requested_url
|
||||
return f"https://{requested_url}"
|
||||
|
||||
def dispatch(self, request, *args, **kwargs):
|
||||
requested_url = urldecode(kwargs.get("url", "") or "")
|
||||
if requested_url:
|
||||
snapshot = self._latest_snapshot_for_url(requested_url)
|
||||
if snapshot:
|
||||
return redirect(f"/{snapshot.url_path}")
|
||||
|
||||
request_host = (request.get_host() or "").lower()
|
||||
request_config = get_request_config(request)
|
||||
web_host = get_web_host(config=request_config, request=request)
|
||||
admin_host = get_admin_host(config=request_config, request=request)
|
||||
is_web_host = host_matches(request_host, web_host)
|
||||
is_admin_host = host_matches(request_host, admin_host)
|
||||
if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and is_web_host and not is_admin_host:
|
||||
return redirect(build_admin_url(request.get_full_path(), request=request))
|
||||
|
||||
if not self.test_func():
|
||||
if is_web_host and not is_admin_host:
|
||||
return redirect(build_admin_url(request.get_full_path(), request=request))
|
||||
if is_admin_host:
|
||||
next_url = quote(request.get_full_path(), safe="/:?=&")
|
||||
return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={next_url}")
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
(
|
||||
"<center><br/><br/><br/>"
|
||||
"No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
|
||||
'Return to the <a href="/" target="_top">Main Index</a>'
|
||||
"</center>"
|
||||
),
|
||||
requested_url or "",
|
||||
),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
|
||||
return super().dispatch(request, *args, **kwargs)
|
||||
|
||||
def get(self, request: HttpRequest, *args: object, **kwargs: object):
|
||||
requested_url = urldecode(str(kwargs.get("url") or (args[0] if args else "")))
|
||||
if not requested_url:
|
||||
raise Http404
|
||||
|
||||
snapshot = self._latest_snapshot_for_url(requested_url)
|
||||
if snapshot:
|
||||
return redirect(f"/{snapshot.url_path}")
|
||||
|
||||
add_url = self._normalize_add_url(requested_url)
|
||||
assert self.form_class is not None
|
||||
defaults_form = self.form_class()
|
||||
form_data = QueryDict(mutable=True)
|
||||
form_data.update(
|
||||
{
|
||||
"url": add_url,
|
||||
"depth": defaults_form.fields["depth"].initial or "0",
|
||||
"max_urls": defaults_form.fields["max_urls"].initial or 0,
|
||||
"crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
|
||||
"crawl_timeout": defaults_form.fields["crawl_timeout"].initial or 0,
|
||||
"timeout": defaults_form.fields["timeout"].initial or 0,
|
||||
"snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
|
||||
"delete_after": defaults_form.fields["delete_after"].initial or "0",
|
||||
"crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
|
||||
"persona": defaults_form.fields["persona"].initial or "Default",
|
||||
"permissions": defaults_form.fields["permissions"].initial or "public",
|
||||
"config": "{}",
|
||||
},
|
||||
)
|
||||
if defaults_form.fields["start_paused"].initial:
|
||||
form_data["start_paused"] = "on"
|
||||
|
||||
form = self.form_class(data=form_data)
|
||||
if not form.is_valid():
|
||||
return self.form_invalid(form)
|
||||
|
||||
crawl = self._create_crawl_from_form(form)
|
||||
snapshot = Snapshot.from_json({"url": add_url, "tags": form.cleaned_data.get("tag", "")}, overrides={"crawl": crawl})
|
||||
assert snapshot is not None
|
||||
return redirect(f"/{snapshot.url_path}")
|
||||
@@ -0,0 +1,261 @@
|
||||
import os
|
||||
from typing import cast
|
||||
|
||||
from admin_data_views.typing import ItemContext, SectionData, TableContext
|
||||
from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
|
||||
from django.http import HttpRequest
|
||||
from django.utils.html import format_html, format_html_join
|
||||
from django.utils.safestring import mark_safe
|
||||
|
||||
from archivebox.config import CONSTANTS, CONSTANTS_CONFIG
|
||||
from archivebox.config.common import (
|
||||
PLUGIN_CONFIG_SCHEMAS,
|
||||
SENSITIVE_CONFIG_VALUE_REDACTED,
|
||||
_plugin_config_properties,
|
||||
find_config_default,
|
||||
find_config_section,
|
||||
find_config_source,
|
||||
find_config_type,
|
||||
get_all_configs,
|
||||
get_config,
|
||||
redact_sensitive_config,
|
||||
)
|
||||
from archivebox.config.configset import BaseConfigSet
|
||||
from archivebox.plugins.views import get_config_definition_link
|
||||
|
||||
|
||||
@render_with_table_view
|
||||
def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
CONFIGS = get_all_configs()
|
||||
|
||||
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
|
||||
|
||||
merged_config = get_config(redact_sensitive=True)
|
||||
|
||||
rows = {
|
||||
"Section": [],
|
||||
"Key": [],
|
||||
"Type": [],
|
||||
"Value": [],
|
||||
"Source": [],
|
||||
"Default": [],
|
||||
# "Documentation": [],
|
||||
# "Aliases": [],
|
||||
}
|
||||
|
||||
for section_id, section in reversed(list(CONFIGS.items())):
|
||||
for key in dict(section):
|
||||
rows["Section"].append(section_id) # section.replace('_', ' ').title().replace(' Config', '')
|
||||
rows["Key"].append(ItemLink(key, key=key))
|
||||
rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))
|
||||
|
||||
# Use merged config value (includes machine overrides)
|
||||
actual_value = merged_config.get(key, dict(section)[key])
|
||||
rows["Value"].append(format_html("<code>{}</code>", actual_value))
|
||||
|
||||
# Show where the value comes from
|
||||
source = find_config_source(key, merged_config)
|
||||
source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
|
||||
rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
|
||||
|
||||
rows["Default"].append(
|
||||
format_html(
|
||||
'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
|
||||
key,
|
||||
find_config_default(key) or "See here...",
|
||||
),
|
||||
)
|
||||
|
||||
section = "CONSTANT"
|
||||
for key in CONSTANTS_CONFIG:
|
||||
rows["Section"].append(section) # section.replace('_', ' ').title().replace(' Config', '')
|
||||
rows["Key"].append(ItemLink(key, key=key))
|
||||
rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
|
||||
rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
|
||||
rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
|
||||
rows["Default"].append(
|
||||
format_html(
|
||||
'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{}&type=code"><code style="text-decoration: underline">{}</code></a>',
|
||||
key,
|
||||
find_config_default(key) or "See here...",
|
||||
),
|
||||
)
|
||||
|
||||
return TableContext(
|
||||
title="Computed Configuration Values",
|
||||
table=rows,
|
||||
)
|
||||
|
||||
|
||||
@render_with_item_view
|
||||
def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
CONFIGS = get_all_configs()
|
||||
|
||||
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
|
||||
|
||||
merged_config = get_config(redact_sensitive=True)
|
||||
|
||||
# Determine all sources for this config value
|
||||
sources_info = []
|
||||
|
||||
# Machine config
|
||||
machine = Machine.current()
|
||||
machine_admin_url = machine.admin_change_url
|
||||
if machine.config and key in machine.config:
|
||||
sources_info.append(("Machine", redact_sensitive_config(machine.config).get(key), "purple"))
|
||||
|
||||
# Environment variable
|
||||
if key in os.environ:
|
||||
sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
|
||||
|
||||
# Config file value
|
||||
if CONSTANTS.CONFIG_FILE.exists():
|
||||
file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
|
||||
if key in file_config:
|
||||
sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
|
||||
|
||||
# Default value
|
||||
default_val = find_config_default(key)
|
||||
if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
|
||||
sources_info.append(("Plugin Default", default_val, "gray"))
|
||||
elif default_val:
|
||||
sources_info.append(("Default", default_val, "gray"))
|
||||
|
||||
# Final computed value
|
||||
config_source = find_config_source(key, merged_config)
|
||||
final_value = merged_config.get(key, CONFIGS.get(key, None))
|
||||
is_redacted = final_value == SENSITIVE_CONFIG_VALUE_REDACTED
|
||||
|
||||
# Build sources display
|
||||
sources_html = format_html_join(
|
||||
mark_safe("<br/>"),
|
||||
'<b style="color: {}">{}:</b> <code>{}</code>',
|
||||
((color, source, value) for source, value, color in sources_info),
|
||||
)
|
||||
|
||||
aliases = []
|
||||
|
||||
if key in CONSTANTS_CONFIG:
|
||||
section_header = format_html(
|
||||
'[CONSTANTS] <b><code style="color: lightgray">{}</code></b> <small>(read-only, hardcoded by ArchiveBox)</small>',
|
||||
key,
|
||||
)
|
||||
elif key in merged_config:
|
||||
section_header = format_html(
|
||||
'data / ArchiveBox.conf [{}] <b><code style="color: lightgray">{}</code></b>',
|
||||
find_config_section(key),
|
||||
key,
|
||||
)
|
||||
else:
|
||||
section_header = format_html(
|
||||
'[DYNAMIC CONFIG] <b><code style="color: lightgray">{}</code></b> <small>(read-only, calculated at runtime)</small>',
|
||||
key,
|
||||
)
|
||||
|
||||
definition_url, definition_label = get_config_definition_link(key)
|
||||
redacted_message = (
|
||||
mark_safe(
|
||||
'<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>',
|
||||
)
|
||||
if is_redacted
|
||||
else ""
|
||||
)
|
||||
default_command_value = val.strip("'") if (val := find_config_default(key)) else str(final_value).strip("'")
|
||||
machine_config_link = (
|
||||
format_html('<br/><a href="{}">→ Edit <code>{}</code> in Machine.config for this server</a>', machine_admin_url, key)
|
||||
if machine_admin_url
|
||||
else ""
|
||||
)
|
||||
machine_config_tip = (
|
||||
format_html(
|
||||
'<br/><b>Tip:</b> To override <code>{}</code> on this machine, <a href="{}">edit the Machine.config field</a> and add:<br/><code>{}</code>',
|
||||
key,
|
||||
machine_admin_url,
|
||||
f'{{"{key}": "your_value_here"}}',
|
||||
)
|
||||
if machine_admin_url and key not in CONSTANTS_CONFIG
|
||||
else ""
|
||||
)
|
||||
|
||||
section_data = cast(
|
||||
SectionData,
|
||||
{
|
||||
"name": section_header,
|
||||
"description": None,
|
||||
"fields": {
|
||||
"Key": key,
|
||||
"Type": find_config_type(key),
|
||||
"Value": final_value,
|
||||
"Currently read from": config_source,
|
||||
},
|
||||
"help_texts": {
|
||||
"Key": format_html(
|
||||
"""
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{}">Documentation</a>
|
||||
<span style="display: {}">
|
||||
Aliases: {}
|
||||
</span>
|
||||
""",
|
||||
key.lower(),
|
||||
"inline" if aliases else "none",
|
||||
", ".join(aliases),
|
||||
),
|
||||
"Type": format_html(
|
||||
"""
|
||||
<a href="{}" target="_blank" rel="noopener noreferrer">
|
||||
See full definition in <code>{}</code>...
|
||||
</a>
|
||||
""",
|
||||
definition_url,
|
||||
definition_label,
|
||||
),
|
||||
"Value": format_html(
|
||||
"""
|
||||
{}
|
||||
<br/><hr/><br/>
|
||||
<b>Configuration Sources (highest priority first):</b><br/><br/>
|
||||
{}
|
||||
<br/><br/>
|
||||
<p style="display: {}">
|
||||
<i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
|
||||
<br/><br/>
|
||||
<code>archivebox config --set {}="{}"</code>
|
||||
</p>
|
||||
""",
|
||||
redacted_message,
|
||||
sources_html,
|
||||
"block" if key in merged_config and key not in CONSTANTS_CONFIG else "none",
|
||||
key,
|
||||
default_command_value,
|
||||
),
|
||||
"Currently read from": format_html(
|
||||
"""
|
||||
The value shown in the "Value" field comes from the <b>{}</b> source.
|
||||
<br/><br/>
|
||||
Priority order (highest to lowest):
|
||||
<ol>
|
||||
<li><b style="color: purple">Machine</b> - Machine-specific overrides
|
||||
{}
|
||||
</li>
|
||||
<li><b style="color: blue">Environment</b> - process defaults from environment variables</li>
|
||||
<li><b style="color: green">File</b> - data/ArchiveBox.conf</li>
|
||||
<li><b style="color: gray">Plugin Default</b> - Default value from plugin config.json</li>
|
||||
<li><b style="color: gray">Default</b> - Default value from code</li>
|
||||
</ol>
|
||||
{}
|
||||
""",
|
||||
config_source,
|
||||
machine_config_link,
|
||||
machine_config_tip,
|
||||
),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
return ItemContext(
|
||||
slug=key,
|
||||
title=key,
|
||||
data=[section_data],
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
from pathlib import Path
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.routes_util import (
|
||||
get_snapshot_lookup_key,
|
||||
)
|
||||
|
||||
|
||||
def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
|
||||
target = archivefile or ""
|
||||
if target == "index.html":
|
||||
target = ""
|
||||
fullpath = Path(snapshot.output_dir) / target
|
||||
if fullpath.is_file():
|
||||
target = str(Path(target).parent)
|
||||
if target == ".":
|
||||
target = ""
|
||||
return target
|
||||
|
||||
|
||||
def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
|
||||
lookup = get_snapshot_lookup_key(snapshot_ref)
|
||||
if not lookup:
|
||||
return None
|
||||
|
||||
snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by")
|
||||
|
||||
if len(lookup) == 12 and "-" not in lookup:
|
||||
return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first()
|
||||
|
||||
try:
|
||||
return snapshots.get(pk=lookup)
|
||||
except Snapshot.DoesNotExist:
|
||||
try:
|
||||
return snapshots.get(id__startswith=lookup)
|
||||
except Snapshot.DoesNotExist:
|
||||
return None
|
||||
except Snapshot.MultipleObjectsReturned:
|
||||
return snapshots.filter(id__startswith=lookup).first()
|
||||
@@ -0,0 +1,292 @@
|
||||
from typing import ClassVar
|
||||
|
||||
from django.core.paginator import InvalidPage
|
||||
from django.db.models import Case, IntegerField, Value, When
|
||||
from django.http import Http404, HttpResponse
|
||||
from django.shortcuts import redirect
|
||||
from django.views import View
|
||||
from django.views.generic.list import ListView
|
||||
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.common import (
|
||||
get_request_config,
|
||||
)
|
||||
from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_PRIVATE,
|
||||
PERMISSIONS_PUBLIC,
|
||||
PERMISSIONS_UNLISTED,
|
||||
public_snapshots_queryset,
|
||||
)
|
||||
from archivebox.core.routes_util import (
|
||||
build_web_url,
|
||||
)
|
||||
from archivebox.misc.paginators import AcceleratedPaginator
|
||||
from archivebox.search.config import (
|
||||
get_search_mode,
|
||||
get_search_mode_backend,
|
||||
get_search_mode_base,
|
||||
get_search_mode_options,
|
||||
)
|
||||
from archivebox.search.views import get_cached_public_search_state
|
||||
|
||||
from .replay_auth import _admin_login_redirect_or_forbidden
|
||||
|
||||
|
||||
class HomepageView(View):
|
||||
def get(self, request):
|
||||
request_config = get_request_config(request)
|
||||
if not request_config.BASE_URL:
|
||||
return _admin_login_redirect_or_forbidden(request)
|
||||
|
||||
if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
|
||||
return redirect("/admin/core/snapshot/")
|
||||
|
||||
if request_config.PUBLIC_INDEX:
|
||||
return redirect("/public")
|
||||
|
||||
return _admin_login_redirect_or_forbidden(request)
|
||||
|
||||
|
||||
class PublicIndexView(ListView):
|
||||
template_name = "public_index.html"
|
||||
model = Snapshot
|
||||
ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
|
||||
paginator_class = AcceleratedPaginator
|
||||
public_page_scan_chunk_size = 50
|
||||
|
||||
def get_paginate_by(self, queryset):
|
||||
runtime_config = self.__dict__.get("runtime_config")
|
||||
if runtime_config is None:
|
||||
self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
|
||||
return runtime_config.SNAPSHOTS_PER_PAGE
|
||||
|
||||
def _base_public_snapshot_fields(self) -> tuple[str, ...]:
|
||||
return (
|
||||
"id",
|
||||
"created_at",
|
||||
"modified_at",
|
||||
"url",
|
||||
"timestamp",
|
||||
"bookmarked_at",
|
||||
"title",
|
||||
"downloaded_at",
|
||||
"status",
|
||||
"output_size",
|
||||
"permissions",
|
||||
)
|
||||
|
||||
def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
|
||||
target_count = page_number * page_size
|
||||
public_snapshots: list[Snapshot] = []
|
||||
scanned = 0
|
||||
chunk_size = max(self.public_page_scan_chunk_size, page_size)
|
||||
ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
|
||||
|
||||
while len(public_snapshots) < target_count:
|
||||
chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
|
||||
if not chunk:
|
||||
break
|
||||
scanned += len(chunk)
|
||||
public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
|
||||
|
||||
start = (page_number - 1) * page_size
|
||||
return public_snapshots[start:target_count]
|
||||
|
||||
def paginate_queryset(self, queryset, page_size):
|
||||
if self.request.GET.get("q", default="").strip():
|
||||
return super().paginate_queryset(queryset, page_size)
|
||||
|
||||
public_count = self.get_exact_public_snapshot_count()
|
||||
paginator = self.get_paginator(range(public_count), page_size)
|
||||
page_kwarg = self.kwargs.get(self.page_kwarg)
|
||||
page_query = self.request.GET.get(self.page_kwarg)
|
||||
page_number = page_kwarg or page_query or 1
|
||||
|
||||
try:
|
||||
page = paginator.page(page_number)
|
||||
except InvalidPage as err:
|
||||
raise Http404(f"Invalid page ({page_number}): {err}") from err
|
||||
|
||||
object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
|
||||
page.object_list = object_list
|
||||
return paginator, page, object_list, page.has_other_pages()
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
runtime_config = self.__dict__.get("runtime_config")
|
||||
if runtime_config is None:
|
||||
self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
|
||||
search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
|
||||
search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
|
||||
query = self.request.GET.get("q", default="").strip()
|
||||
public_search_state = self.__dict__.get("public_search_state")
|
||||
public_search_pending = bool(query and (public_search_state is None or not public_search_state.get("done")))
|
||||
context = {
|
||||
**super().get_context_data(**kwargs),
|
||||
"VERSION": VERSION,
|
||||
"CONFIG": runtime_config,
|
||||
"COMMIT_HASH": runtime_config.COMMIT_HASH,
|
||||
"FOOTER_INFO": runtime_config.FOOTER_INFO,
|
||||
"WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
|
||||
"search_mode": search_mode,
|
||||
"search_mode_options": get_search_mode_options(config=runtime_config),
|
||||
"public_search_stream_pending": public_search_pending,
|
||||
}
|
||||
context["show_search_index_hint"] = bool(
|
||||
query
|
||||
and not public_search_pending
|
||||
and get_search_mode_base(search_mode, config=runtime_config) == "deep"
|
||||
and search_mode_backend
|
||||
and context["paginator"].count == 0,
|
||||
)
|
||||
snapshots = list(context.get("object_list") or ())
|
||||
icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
|
||||
tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
|
||||
preview_paths_by_snapshot: dict[str, list[tuple[int, int, str]]] = {str(snapshot.id): [] for snapshot in snapshots}
|
||||
favicon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
|
||||
progress_by_snapshot: dict[str, dict[str, int]] = {
|
||||
str(snapshot.id): {
|
||||
"total": 0,
|
||||
"succeeded": 0,
|
||||
"failed": 0,
|
||||
"running": 0,
|
||||
"skipped": 0,
|
||||
"noresults": 0,
|
||||
}
|
||||
for snapshot in snapshots
|
||||
}
|
||||
if icons_by_snapshot:
|
||||
for snapshot_id, tag_name in (
|
||||
SnapshotTag.objects.filter(snapshot_id__in=icons_by_snapshot.keys())
|
||||
.order_by("tag__name")
|
||||
.values_list("snapshot_id", "tag__name")
|
||||
.iterator(chunk_size=1000)
|
||||
):
|
||||
tag_names_by_snapshot[str(snapshot_id)].append(tag_name)
|
||||
|
||||
preview_plugin_order = {
|
||||
"screenshot": 0,
|
||||
"chrome_extension_screenshot": 1,
|
||||
}
|
||||
preview_candidates = {
|
||||
"screenshot": ("screenshot.png",),
|
||||
"chrome_extension_screenshot": ("screenshot-1.png", "screenshot.png"),
|
||||
}
|
||||
|
||||
def result_output_path(result: ArchiveResult, filename: str) -> str | None:
|
||||
output_files = result.output_files or {}
|
||||
file_info = output_files.get(filename)
|
||||
if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
|
||||
return None
|
||||
if file_info.get("root_relative"):
|
||||
return filename
|
||||
return f"{result.plugin}/{filename}"
|
||||
|
||||
for snapshot_id, plugin, status in (
|
||||
ArchiveResult.objects.filter(
|
||||
snapshot_id__in=icons_by_snapshot.keys(),
|
||||
)
|
||||
.exclude(plugin="")
|
||||
.values_list("snapshot_id", "plugin", "status")
|
||||
.iterator(chunk_size=1000)
|
||||
):
|
||||
snapshot_key = str(snapshot_id)
|
||||
progress = progress_by_snapshot[snapshot_key]
|
||||
progress["total"] += 1
|
||||
if status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
icons_by_snapshot[snapshot_key].add(plugin)
|
||||
progress["succeeded"] += 1
|
||||
elif status == ArchiveResult.StatusChoices.FAILED:
|
||||
progress["failed"] += 1
|
||||
elif status == ArchiveResult.StatusChoices.STARTED:
|
||||
progress["running"] += 1
|
||||
elif status == ArchiveResult.StatusChoices.SKIPPED:
|
||||
progress["skipped"] += 1
|
||||
elif status == ArchiveResult.StatusChoices.NORESULTS:
|
||||
progress["noresults"] += 1
|
||||
|
||||
for result in (
|
||||
ArchiveResult.objects.filter(
|
||||
snapshot_id__in=icons_by_snapshot.keys(),
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
plugin__in=(*preview_candidates, "favicon"),
|
||||
)
|
||||
.only("snapshot_id", "plugin", "output_files")
|
||||
.iterator(chunk_size=1000)
|
||||
):
|
||||
snapshot_key = str(result.snapshot_id)
|
||||
if result.plugin in preview_candidates:
|
||||
plugin_rank = preview_plugin_order[result.plugin]
|
||||
for filename_rank, filename in enumerate(preview_candidates[result.plugin]):
|
||||
output_path = result_output_path(result, filename)
|
||||
if output_path:
|
||||
preview_paths_by_snapshot[snapshot_key].append((plugin_rank, filename_rank, output_path))
|
||||
elif result.plugin == "favicon":
|
||||
output_path = result_output_path(result, "favicon.ico")
|
||||
if output_path:
|
||||
favicon_paths_by_snapshot[snapshot_key].append(output_path)
|
||||
|
||||
for snapshot in snapshots:
|
||||
snapshot._icons_compact = True
|
||||
snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
|
||||
snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
|
||||
snapshot.num_outputs_cached = snapshot._icons_progress_stats.get("succeeded", 0)
|
||||
snapshot._tags_str_cached = ",".join(tag_names_by_snapshot.get(str(snapshot.id), []))
|
||||
snapshot._public_preview_paths = [
|
||||
output_path for _plugin_rank, _filename_rank, output_path in sorted(preview_paths_by_snapshot.get(str(snapshot.id), []))
|
||||
]
|
||||
snapshot._public_favicon_paths = favicon_paths_by_snapshot.get(str(snapshot.id), [])
|
||||
snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
|
||||
context["object_list"] = snapshots
|
||||
return context
|
||||
|
||||
def get_exact_public_snapshot_count(self) -> int:
|
||||
hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
|
||||
hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
|
||||
return Snapshot.objects.count() - hidden_count
|
||||
|
||||
def get_queryset(self, **kwargs):
|
||||
qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
|
||||
query = self.request.GET.get("q", default="").strip()
|
||||
|
||||
if not query:
|
||||
return qs
|
||||
|
||||
cached_state = get_cached_public_search_state(self.request)
|
||||
self.public_search_state = cached_state
|
||||
if cached_state is not None:
|
||||
cached_ids = cached_state.get("ids") or []
|
||||
if not cached_ids:
|
||||
return qs.none()
|
||||
search_rank = Case(
|
||||
*(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)),
|
||||
output_field=IntegerField(),
|
||||
)
|
||||
return qs.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank", *self.ordering)
|
||||
|
||||
return qs.none()
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
if self.request.user.is_authenticated:
|
||||
return redirect("/admin/core/snapshot/")
|
||||
if get_request_config(self.request).PUBLIC_INDEX:
|
||||
response = super().get(*args, **kwargs)
|
||||
return response
|
||||
else:
|
||||
return _admin_login_redirect_or_forbidden(self.request)
|
||||
|
||||
|
||||
class HealthCheckView(View):
|
||||
"""
|
||||
A Django view that renders plain text "OK" for service discovery tools
|
||||
"""
|
||||
|
||||
def get(self, request):
|
||||
"""
|
||||
Handle a GET request
|
||||
"""
|
||||
response = HttpResponse("OK", content_type="text/plain", status=200)
|
||||
response["Access-Control-Allow-Origin"] = "*"
|
||||
response["Access-Control-Expose-Headers"] = "X-ArchiveBox-Health"
|
||||
response["X-ArchiveBox-Health"] = "OK"
|
||||
return response
|
||||
@@ -0,0 +1,838 @@
|
||||
import posixpath
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
|
||||
from django import template
|
||||
from django.db.models import Q
|
||||
from django.http import Http404, HttpRequest, HttpResponse
|
||||
from django.shortcuts import redirect, render
|
||||
from django.utils.html import format_html
|
||||
from django.utils.safestring import mark_safe
|
||||
from django.views import View
|
||||
|
||||
from archivebox.config.common import (
|
||||
get_request_config,
|
||||
)
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.core.permissions import (
|
||||
can_view_snapshot,
|
||||
direct_snapshots_queryset,
|
||||
)
|
||||
from archivebox.core.routes_util import (
|
||||
build_admin_url,
|
||||
build_snapshot_url,
|
||||
build_web_url,
|
||||
get_snapshot_host,
|
||||
host_matches,
|
||||
)
|
||||
from archivebox.misc.serve_static import serve_static_with_byterange_support
|
||||
from archivebox.misc.util import (
|
||||
base_url,
|
||||
filter_queryset_by_uuid_substring,
|
||||
without_fragment,
|
||||
)
|
||||
from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
|
||||
from archivebox.progressmonitor.views import live_progress_view
|
||||
|
||||
from .lookup import _files_index_target, _find_snapshot_by_ref
|
||||
from .replay_auth import _has_replay_cookie, _private_snapshot_auth_redirect, _replay_auth_response
|
||||
|
||||
|
||||
class SnapshotView(View):
|
||||
# render static html index from filesystem archive/<timestamp>/index.html
|
||||
|
||||
@staticmethod
|
||||
def find_snapshots_for_url(path: str, *, allow_fallback: bool = True):
|
||||
"""Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.
|
||||
|
||||
Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
|
||||
"""
|
||||
|
||||
def _fragmentless_url_query(url: str) -> Q:
|
||||
from archivebox.misc.db import is_postgres
|
||||
|
||||
canonical = without_fragment(url)
|
||||
if not is_postgres():
|
||||
# Use a range comparison (url >= 'canonical#' AND url < 'canonical#\U0010ffff')
|
||||
# instead of LIKE/__startswith — SQLite's case-insensitive LIKE bypasses the
|
||||
# url index and forces a full-table scan over ~1M rows (~250ms). The range
|
||||
# form lets SQLite use a MULTI-INDEX OR and stays under 1ms.
|
||||
return Q(url=canonical) | (Q(url__gte=f"{canonical}#") & Q(url__lt=f"{canonical}#\U0010ffff"))
|
||||
# On postgres the range trick is unsafe: linguistic (ICU/libc) collations
|
||||
# don't compare '#'-suffixed strings bytewise, so the range can miss rows.
|
||||
# startswith compiles to LIKE 'prefix%' with wildcards escaped, which is
|
||||
# correct under any collation and uses the url pattern-ops index.
|
||||
return Q(url=canonical) | Q(url__startswith=f"{canonical}#")
|
||||
|
||||
normalized = without_fragment(path)
|
||||
if path.startswith(("http://", "https://")):
|
||||
# exact url match (indexed) — fastest path
|
||||
qs = Snapshot.objects.filter(_fragmentless_url_query(path))
|
||||
if not allow_fallback or qs.exists():
|
||||
return qs
|
||||
normalized = normalized.split("://", 1)[1]
|
||||
|
||||
# try exact match on full url (without scheme)
|
||||
qs = Snapshot.objects.filter(
|
||||
_fragmentless_url_query("http://" + normalized) | _fragmentless_url_query("https://" + normalized),
|
||||
)
|
||||
if qs.exists():
|
||||
return qs
|
||||
|
||||
# fall back to match on exact base_url
|
||||
base = base_url(normalized)
|
||||
qs = Snapshot.objects.filter(
|
||||
_fragmentless_url_query("http://" + base) | _fragmentless_url_query("https://" + base),
|
||||
)
|
||||
if qs.exists():
|
||||
return qs
|
||||
|
||||
# fall back to matching base_url as prefix
|
||||
return Snapshot.objects.filter(Q(url__startswith="http://" + base) | Q(url__startswith="https://" + base))
|
||||
|
||||
@staticmethod
|
||||
def find_snapshots_for_id(slug: str):
|
||||
"""Return a queryset of snapshots matching a (possibly truncated) UUID via prefix or suffix.
|
||||
|
||||
Strips non-hex characters from ``slug`` (so input with or without hyphens both work).
|
||||
Requires at least 8 hex chars — shorter inputs return an empty queryset to avoid
|
||||
scanning the entire snapshots table on too-broad matches.
|
||||
"""
|
||||
return filter_queryset_by_uuid_substring(Snapshot.objects.all(), slug)
|
||||
|
||||
@staticmethod
|
||||
def render_live_index(request, snapshot):
|
||||
return render(
|
||||
template_name="core/snapshot.html",
|
||||
request=request,
|
||||
context=snapshot.get_html_details_context(request=request),
|
||||
)
|
||||
|
||||
def get(self, request, path):
|
||||
snapshot = None
|
||||
|
||||
try:
|
||||
slug, archivefile = path.split("/", 1)
|
||||
except (IndexError, ValueError):
|
||||
slug, archivefile = path.split("/", 1)[0], "index.html"
|
||||
|
||||
# slug is a timestamp
|
||||
if slug.replace(".", "").isdigit():
|
||||
# missing trailing slash -> redirect to index
|
||||
if "/" not in path:
|
||||
return redirect(f"{path}/index.html")
|
||||
|
||||
try:
|
||||
try:
|
||||
snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
|
||||
if not can_view_snapshot(request, snapshot):
|
||||
return _private_snapshot_auth_redirect(request, snapshot, archivefile or "index.html")
|
||||
canonical_base = snapshot.url_path
|
||||
if canonical_base != snapshot.legacy_archive_path:
|
||||
target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
|
||||
query = request.META.get("QUERY_STRING")
|
||||
if query:
|
||||
target_path = f"{target_path}?{query}"
|
||||
return redirect(target_path)
|
||||
|
||||
if request.GET.get("files"):
|
||||
target_path = _files_index_target(snapshot, archivefile)
|
||||
response = serve_static_with_byterange_support(
|
||||
request,
|
||||
target_path,
|
||||
document_root=snapshot.output_dir,
|
||||
show_indexes=True,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
elif archivefile == "index.html":
|
||||
# if they requested snapshot index, serve live rendered template instead of static html
|
||||
response = self.render_live_index(request, snapshot)
|
||||
else:
|
||||
target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
|
||||
query = request.META.get("QUERY_STRING")
|
||||
if query:
|
||||
target = f"{target}?{query}"
|
||||
return redirect(target)
|
||||
response["Link"] = f'<{snapshot.url}>; rel="canonical"'
|
||||
return response
|
||||
except Snapshot.DoesNotExist:
|
||||
if Snapshot.objects.filter(timestamp__startswith=slug).exists():
|
||||
raise Snapshot.MultipleObjectsReturned
|
||||
else:
|
||||
raise
|
||||
except Snapshot.DoesNotExist:
|
||||
# Snapshot does not exist
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
(
|
||||
"<center><br/><br/><br/>"
|
||||
"No Snapshot directories match the given timestamp/ID: <code>{}</code><br/><br/>"
|
||||
'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
|
||||
"</center>"
|
||||
),
|
||||
slug,
|
||||
path,
|
||||
),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
except Snapshot.MultipleObjectsReturned:
|
||||
snapshot_hrefs = mark_safe("<br/>").join(
|
||||
format_html(
|
||||
'{} <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
|
||||
snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
snap.archive_path,
|
||||
snap.timestamp,
|
||||
snap.url,
|
||||
snap.title_stripped[:64] or "",
|
||||
)
|
||||
for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug))
|
||||
.only("url", "timestamp", "title", "bookmarked_at")
|
||||
.order_by("-bookmarked_at")
|
||||
)
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
("Multiple Snapshots match the given timestamp/ID <code>{}</code><br/><pre>"),
|
||||
slug,
|
||||
)
|
||||
+ snapshot_hrefs
|
||||
+ mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
except Http404:
|
||||
assert snapshot # (Snapshot.DoesNotExist is already handled above)
|
||||
|
||||
# Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
"""
|
||||
<html><head>
|
||||
<title>Snapshot Not Found</title>
|
||||
</head><body>
|
||||
<center><br/><br/><br/>
|
||||
Snapshot <a href="/{}/index.html" target="_top"><b><code>[{}]</code></b></a>: <a href="{}" target="_blank" rel="noreferrer">{}</a><br/>
|
||||
was queued on {}, but no files have been saved yet in:<br/><b><a href="/{}/" target="_top"><code>{}</code></a><code>/{}</code></b><br/><br/>
|
||||
It's possible {} during the last capture on {},<br/>or that the archiving process has not completed yet.<br/>
|
||||
<pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>
|
||||
<code style="user-select: all; color: #333">archivebox update -t timestamp {}</code></pre><br/><br/>
|
||||
<div class="text-align: left; width: 100%; max-width: 400px">
|
||||
<i><b>Next steps:</i></b><br/>
|
||||
- list all the <a href="/{}/" target="_top">Snapshot files <code>.*</code></a><br/>
|
||||
- view the <a href="/{}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>
|
||||
- go to the <a href="/admin/core/snapshot/{}/change/" target="_top">Snapshot admin</a> to edit<br/>
|
||||
- go to the <a href="/admin/core/snapshot/?id__exact={}" target="_top">Snapshot actions</a> to re-archive<br/>
|
||||
- or return to <a href="/" target="_top">the main index...</a></div>
|
||||
</center>
|
||||
</body></html>
|
||||
""",
|
||||
snapshot.archive_path,
|
||||
snapshot.timestamp,
|
||||
snapshot.url,
|
||||
snapshot.url,
|
||||
str(snapshot.bookmarked_at).split(".")[0],
|
||||
snapshot.archive_path,
|
||||
snapshot.timestamp,
|
||||
archivefile if str(archivefile) != "None" else "",
|
||||
f"the {archivefile} resource could not be fetched"
|
||||
if str(archivefile) != "None"
|
||||
else "the original site was not available",
|
||||
str(snapshot.bookmarked_at).split(".")[0],
|
||||
snapshot.timestamp,
|
||||
snapshot.archive_path,
|
||||
snapshot.archive_path,
|
||||
snapshot.pk,
|
||||
snapshot.id,
|
||||
),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
|
||||
# slug is either a URL or a (possibly truncated) snapshot UUID
|
||||
def _resolve_snapshots_for_slug(slug: str):
|
||||
# full URLs go straight to the url-only path (fast, indexed)
|
||||
if "://" in slug:
|
||||
return SnapshotView.find_snapshots_for_url(slug)
|
||||
# short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
|
||||
id_qs = SnapshotView.find_snapshots_for_id(slug)
|
||||
if id_qs.exists():
|
||||
return id_qs
|
||||
return SnapshotView.find_snapshots_for_url(slug)
|
||||
|
||||
snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
|
||||
try:
|
||||
if "://" in path:
|
||||
snapshot = snapshots.order_by("-bookmarked_at").first()
|
||||
if snapshot is None:
|
||||
raise Snapshot.DoesNotExist
|
||||
else:
|
||||
snapshot = snapshots.get()
|
||||
except Snapshot.DoesNotExist:
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
(
|
||||
"<center><br/><br/><br/>"
|
||||
"No Snapshots match the given url: <code>{}</code><br/><br/><br/>"
|
||||
'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
|
||||
'+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
|
||||
"</center>"
|
||||
),
|
||||
base_url(path),
|
||||
path if "://" in path else f"https://{path}",
|
||||
path,
|
||||
),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
except Snapshot.MultipleObjectsReturned:
|
||||
snapshot_hrefs = mark_safe("<br/>").join(
|
||||
format_html(
|
||||
'{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
|
||||
snap.bookmarked_at.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
str(snap.id)[:8],
|
||||
snap.archive_path,
|
||||
snap.timestamp,
|
||||
snap.url,
|
||||
snap.title_stripped[:64] or "",
|
||||
)
|
||||
for snap in snapshots.only("url", "timestamp", "title", "bookmarked_at").order_by("-bookmarked_at")
|
||||
)
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
("Multiple Snapshots match the given URL <code>{}</code><br/><pre>"),
|
||||
base_url(path),
|
||||
)
|
||||
+ snapshot_hrefs
|
||||
+ mark_safe('</pre><br/>Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
|
||||
target_path = build_snapshot_url(str(snapshot.id), "index.html", request=request)
|
||||
query = request.META.get("QUERY_STRING")
|
||||
if query:
|
||||
target_path = f"{target_path}?{query}"
|
||||
return redirect(target_path)
|
||||
|
||||
|
||||
class SnapshotPathView(View):
|
||||
"""Serve snapshots by the new URL scheme: /<username>/<YYYYMMDD>/<domain>/<uuid>/..."""
|
||||
|
||||
def get(
|
||||
self,
|
||||
request,
|
||||
username: str,
|
||||
date: str | None = None,
|
||||
domain: str | None = None,
|
||||
snapshot_id: str | None = None,
|
||||
path: str = "",
|
||||
url: str | None = None,
|
||||
):
|
||||
if username == "system":
|
||||
return redirect(request.path.replace("/system/", "/web/", 1))
|
||||
|
||||
if date and domain and domain == date:
|
||||
raise Http404
|
||||
|
||||
requested_url = url
|
||||
if not requested_url and domain and domain.startswith(("http://", "https://")):
|
||||
requested_url = domain
|
||||
|
||||
snapshot = None
|
||||
snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by"))
|
||||
if snapshot_id:
|
||||
snapshot = _find_snapshot_by_ref(snapshot_id)
|
||||
if snapshot and not can_view_snapshot(request, snapshot):
|
||||
return _private_snapshot_auth_redirect(request, snapshot, path or "index.html")
|
||||
else:
|
||||
# fuzzy lookup by date + domain/url (most recent)
|
||||
username_lookup = "system" if username == "web" else username
|
||||
if requested_url:
|
||||
qs = direct_snapshots_queryset(
|
||||
request,
|
||||
SnapshotView.find_snapshots_for_url(requested_url)
|
||||
.select_related("crawl", "crawl__created_by")
|
||||
.filter(
|
||||
crawl__created_by__username=username_lookup,
|
||||
),
|
||||
)
|
||||
else:
|
||||
qs = snapshots_qs.filter(crawl__created_by__username=username_lookup)
|
||||
|
||||
if date:
|
||||
try:
|
||||
if len(date) == 4:
|
||||
qs = qs.filter(bookmarked_at__year=int(date))
|
||||
elif len(date) == 6:
|
||||
qs = qs.filter(bookmarked_at__year=int(date[:4]), bookmarked_at__month=int(date[4:6]))
|
||||
elif len(date) == 8:
|
||||
qs = qs.filter(
|
||||
bookmarked_at__year=int(date[:4]),
|
||||
bookmarked_at__month=int(date[4:6]),
|
||||
bookmarked_at__day=int(date[6:8]),
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if requested_url:
|
||||
snapshot = qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
|
||||
else:
|
||||
requested_domain = domain or ""
|
||||
if requested_domain.startswith(("http://", "https://")):
|
||||
requested_domain = Snapshot.extract_domain_from_url(requested_domain)
|
||||
else:
|
||||
requested_domain = Snapshot.extract_domain_from_url(f"https://{requested_domain}")
|
||||
|
||||
# Prefer exact domain matches
|
||||
matches = [
|
||||
s for s in qs.order_by("-bookmarked_at", "-created_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain
|
||||
]
|
||||
snapshot = matches[0] if matches else qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
|
||||
|
||||
if not snapshot:
|
||||
return HttpResponse(
|
||||
format_html(
|
||||
(
|
||||
"<center><br/><br/><br/>"
|
||||
"No Snapshots match the given id or url: <code>{}</code><br/><br/><br/>"
|
||||
'Return to the <a href="/" target="_top">Main Index</a>'
|
||||
"</center>"
|
||||
),
|
||||
snapshot_id or requested_url or domain,
|
||||
),
|
||||
content_type="text/html",
|
||||
status=404,
|
||||
)
|
||||
|
||||
canonical_base = snapshot.url_path
|
||||
if date:
|
||||
requested_base = f"{username}/{date}/{domain or url or ''}"
|
||||
else:
|
||||
requested_base = f"{username}/{domain or url or ''}"
|
||||
if snapshot_id:
|
||||
requested_base = f"{requested_base}/{snapshot_id}"
|
||||
if canonical_base != requested_base:
|
||||
target = f"/{canonical_base}/{path or 'index.html'}"
|
||||
query = request.META.get("QUERY_STRING")
|
||||
if query:
|
||||
target = f"{target}?{query}"
|
||||
return redirect(target)
|
||||
|
||||
archivefile = path or "index.html"
|
||||
if archivefile != "index.html" and not request.GET.get("files"):
|
||||
target = build_snapshot_url(str(snapshot.id), archivefile, request=request)
|
||||
query = request.META.get("QUERY_STRING")
|
||||
if query:
|
||||
target = f"{target}?{query}"
|
||||
return redirect(target)
|
||||
|
||||
if request.GET.get("files"):
|
||||
target_path = _files_index_target(snapshot, archivefile)
|
||||
return serve_static_with_byterange_support(
|
||||
request,
|
||||
target_path,
|
||||
document_root=snapshot.output_dir,
|
||||
show_indexes=True,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
|
||||
if archivefile == "index.html":
|
||||
return SnapshotView.render_live_index(request, snapshot)
|
||||
|
||||
return serve_static_with_byterange_support(
|
||||
request,
|
||||
archivefile,
|
||||
document_root=snapshot.output_dir,
|
||||
show_indexes=True,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
|
||||
|
||||
def _safe_archive_relpath(path: str) -> str | None:
|
||||
if not path:
|
||||
return ""
|
||||
cleaned = posixpath.normpath(path)
|
||||
cleaned = cleaned.lstrip("/")
|
||||
if cleaned.startswith("..") or "/../" in f"/{cleaned}/":
|
||||
return None
|
||||
return cleaned
|
||||
|
||||
|
||||
def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, ArchiveResult | None]:
|
||||
"""Resolve plugin-relative output paths through ArchiveResult.output_files."""
|
||||
parts = Path(rel_path).parts
|
||||
if len(parts) < 2:
|
||||
return rel_path, None
|
||||
|
||||
plugin = parts[0]
|
||||
plugin_relpath = posixpath.join(*parts[1:])
|
||||
results = list(
|
||||
ArchiveResult.objects.filter(
|
||||
snapshot=snapshot,
|
||||
plugin=plugin,
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
).only("plugin", "output_files"),
|
||||
)
|
||||
if not results:
|
||||
return rel_path, None
|
||||
|
||||
for result in results:
|
||||
output_files = result.output_files or {}
|
||||
for candidate in (plugin_relpath, rel_path):
|
||||
file_info = output_files.get(candidate)
|
||||
if not isinstance(file_info, dict):
|
||||
continue
|
||||
if file_info.get("root_relative"):
|
||||
return candidate, result
|
||||
return rel_path, result
|
||||
|
||||
return rel_path, results[0]
|
||||
|
||||
|
||||
def _plugin_full_preview_response(
|
||||
request: HttpRequest,
|
||||
snapshot: Snapshot,
|
||||
rel_path: str,
|
||||
result: ArchiveResult | None,
|
||||
) -> HttpResponse | None:
|
||||
"""Render an explicit plugin full template as a trusted preview wrapper."""
|
||||
if not request.GET.get("preview"):
|
||||
return None
|
||||
|
||||
path_parts = Path(rel_path).parts
|
||||
plugin = get_plugin_name(result.plugin) if result else (path_parts[0] if len(path_parts) > 1 else "")
|
||||
if not plugin:
|
||||
return None
|
||||
|
||||
# ReplayWeb.page needs plugin-owned WACZ inspection and service-worker
|
||||
# context, so it remains the one narrow preview exception below.
|
||||
if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path):
|
||||
return None
|
||||
|
||||
template_str = get_plugin_template(plugin, "full", fallback=False)
|
||||
if not template_str:
|
||||
return None
|
||||
|
||||
raw_query = request.GET.copy()
|
||||
raw_query.pop("preview", None)
|
||||
output_url = request.path
|
||||
if raw_query:
|
||||
output_url = f"{output_url}?{raw_query.urlencode()}"
|
||||
|
||||
rendered = (
|
||||
template.Engine(debug=False)
|
||||
.from_string(template_str)
|
||||
.render(
|
||||
template.Context(
|
||||
{
|
||||
"result": result,
|
||||
"snapshot": snapshot,
|
||||
"output_path": output_url,
|
||||
"output_path_raw": rel_path,
|
||||
"plugin": plugin,
|
||||
"preview_base": f"{request.path.rsplit('/', 1)[0]}/",
|
||||
},
|
||||
),
|
||||
)
|
||||
)
|
||||
response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
|
||||
response.headers["Content-Disposition"] = f'inline; filename="{Path(rel_path).stem}.html"'
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE
|
||||
response.headers["Referrer-Policy"] = "no-referrer"
|
||||
# Trusted viewers live on the snapshot origin, but the collection UI lives
|
||||
# on web.* (and can be embedded by admin.*). Permit only those configured
|
||||
# origins, not arbitrary sites or other snapshots. This does not grant the
|
||||
# archived document access to the parent frame or its admin session cookies.
|
||||
viewer_origins = " ".join(dict.fromkeys((build_web_url(request=request), build_admin_url(request=request))))
|
||||
response.headers["Content-Security-Policy"] = (
|
||||
"default-src 'self' data: blob:; "
|
||||
"script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; "
|
||||
"style-src 'unsafe-inline' data: blob: 'self'; "
|
||||
"connect-src 'self' data: blob:; "
|
||||
"img-src 'self' data: blob:; "
|
||||
"media-src 'self' data: blob:; "
|
||||
"font-src 'self' data: blob:; "
|
||||
"frame-src 'self' data: blob:; "
|
||||
"worker-src 'self' blob:; "
|
||||
"object-src 'none'; "
|
||||
"base-uri 'none'; "
|
||||
"form-action 'none'; "
|
||||
f"frame-ancestors 'self' {viewer_origins};"
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _visible_response_snapshots_for_domain(request: HttpRequest, domain: str) -> list[Snapshot]:
|
||||
if not domain:
|
||||
return []
|
||||
|
||||
requested_domain = domain.split(":", 1)[0].lower()
|
||||
roots = (f"http://{requested_domain}", f"https://{requested_domain}")
|
||||
domain_query = Q(url__in=roots)
|
||||
|
||||
from archivebox.misc.db import is_postgres
|
||||
|
||||
postgres = is_postgres()
|
||||
for root in roots:
|
||||
for separator in ("/", "?", "#"):
|
||||
prefix = f"{root}{separator}"
|
||||
if postgres:
|
||||
domain_query |= Q(url__startswith=prefix)
|
||||
else:
|
||||
domain_query |= Q(url__gte=prefix, url__lt=f"{prefix}\U0010ffff")
|
||||
|
||||
candidates = (
|
||||
Snapshot.objects.filter(domain_query)
|
||||
.select_related("crawl", "crawl__created_by")
|
||||
.order_by("-bookmarked_at", "-created_at", "-timestamp")
|
||||
)
|
||||
return [snapshot for snapshot in candidates if can_view_snapshot(request, snapshot) or _has_replay_cookie(request, snapshot)]
|
||||
|
||||
|
||||
def _latest_response_match(
|
||||
snapshots: list[Snapshot],
|
||||
domain: str,
|
||||
rel_path: str,
|
||||
) -> tuple[Snapshot, Path, Path] | None:
|
||||
if not domain or not rel_path:
|
||||
return None
|
||||
requested_domain = domain.split(":", 1)[0].lower()
|
||||
rel_to_root = Path(rel_path)
|
||||
for snapshot in snapshots:
|
||||
responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
|
||||
if (responses_root / rel_to_root).exists():
|
||||
return snapshot, responses_root, rel_to_root
|
||||
return None
|
||||
|
||||
|
||||
def _latest_responses_root(snapshots: list[Snapshot], domain: str) -> tuple[Snapshot, Path] | None:
|
||||
if not domain:
|
||||
return None
|
||||
requested_domain = domain.split(":", 1)[0].lower()
|
||||
for snapshot in snapshots:
|
||||
responses_root = Path(snapshot.output_dir) / "responses" / requested_domain
|
||||
if responses_root.is_dir():
|
||||
return snapshot, responses_root
|
||||
return None
|
||||
|
||||
|
||||
def _original_request_url(domain: str, path: str = "", query_string: str = "") -> str:
|
||||
normalized_domain = (domain or "").split(":", 1)[0].lower()
|
||||
normalized_path = (path or "").lstrip("/")
|
||||
if normalized_path in ("", "index.html"):
|
||||
normalized_path = ""
|
||||
target = f"https://{normalized_domain}"
|
||||
if normalized_path:
|
||||
target = f"{target}/{normalized_path}"
|
||||
if query_string:
|
||||
target = f"{target}?{query_string}"
|
||||
return target
|
||||
|
||||
|
||||
def _serve_responses_path(request, responses_root: Path, rel_path: str, show_indexes: bool):
|
||||
candidates: list[str] = []
|
||||
rel_path = rel_path or ""
|
||||
if rel_path.endswith("/"):
|
||||
rel_path = f"{rel_path}index.html"
|
||||
if "." not in Path(rel_path).name:
|
||||
candidates.append(f"{rel_path.rstrip('/')}/index.html")
|
||||
candidates.append(rel_path)
|
||||
|
||||
for candidate in candidates:
|
||||
try:
|
||||
return serve_static_with_byterange_support(
|
||||
request,
|
||||
candidate,
|
||||
document_root=str(responses_root),
|
||||
show_indexes=show_indexes,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
except Http404:
|
||||
pass
|
||||
|
||||
if rel_path.endswith("index.html"):
|
||||
rel_dir = rel_path[: -len("index.html")]
|
||||
try:
|
||||
return serve_static_with_byterange_support(
|
||||
request,
|
||||
rel_dir,
|
||||
document_root=str(responses_root),
|
||||
show_indexes=True,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
except Http404:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
|
||||
rel_path = path or ""
|
||||
request_config = get_request_config(
|
||||
request,
|
||||
resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
|
||||
)
|
||||
request.archivebox_config = request_config
|
||||
request.archivebox_snapshot_url = snapshot.url
|
||||
snapshot._runtime_config = request_config
|
||||
|
||||
if rel_path.startswith("replay/") or rel_path == "replay":
|
||||
response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
|
||||
if response is not None:
|
||||
return response
|
||||
|
||||
if rel_path == "progress.json":
|
||||
# Host routing forwards every snap-* path to SnapshotHostView, so we forward
|
||||
# /progress.json on through to the same view used everywhere else. The caller
|
||||
# passes snapshot_id explicitly in the query string — we don't read it from the
|
||||
# subdomain (this keeps the endpoint identical across all security modes).
|
||||
return live_progress_view(request)
|
||||
|
||||
is_directory_request = bool(path) and path.endswith("/")
|
||||
show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
|
||||
if not show_indexes and (not rel_path or rel_path == "index.html"):
|
||||
return SnapshotView.render_live_index(request, snapshot)
|
||||
|
||||
if not rel_path or rel_path.endswith("/"):
|
||||
if show_indexes:
|
||||
rel_path = rel_path.rstrip("/")
|
||||
else:
|
||||
rel_path = f"{rel_path}index.html"
|
||||
rel_path = _safe_archive_relpath(rel_path)
|
||||
if rel_path is None:
|
||||
raise Http404
|
||||
|
||||
rel_path, archive_result = _resolve_archiveresult_relpath(snapshot, rel_path)
|
||||
|
||||
plugin_preview = _plugin_full_preview_response(request, snapshot, rel_path, archive_result)
|
||||
if plugin_preview is not None:
|
||||
return plugin_preview
|
||||
|
||||
try:
|
||||
return serve_static_with_byterange_support(
|
||||
request,
|
||||
rel_path,
|
||||
document_root=snapshot.output_dir,
|
||||
show_indexes=show_indexes,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
except Http404:
|
||||
pass
|
||||
|
||||
host = urlparse(snapshot.url).hostname or snapshot.domain
|
||||
responses_root = Path(snapshot.output_dir) / "responses" / host
|
||||
if responses_root.exists():
|
||||
response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
|
||||
if response is not None:
|
||||
return response
|
||||
|
||||
raise Http404
|
||||
|
||||
|
||||
def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
|
||||
request_config = get_request_config(request, resolve_plugins=False)
|
||||
request.archivebox_config = request_config
|
||||
requested_root_index = path in ("", "index.html") or path.endswith("/")
|
||||
rel_path = path or ""
|
||||
if not rel_path or rel_path.endswith("/"):
|
||||
rel_path = f"{rel_path}index.html"
|
||||
rel_path = _safe_archive_relpath(rel_path)
|
||||
if rel_path is None:
|
||||
raise Http404
|
||||
|
||||
domain = domain.split(":", 1)[0].lower()
|
||||
snapshots = _visible_response_snapshots_for_domain(request, domain)
|
||||
match = _latest_response_match(snapshots, domain, rel_path)
|
||||
if not match and "." not in Path(rel_path).name:
|
||||
index_path = f"{rel_path.rstrip('/')}/index.html"
|
||||
match = _latest_response_match(snapshots, domain, index_path)
|
||||
if not match and "." not in Path(rel_path).name:
|
||||
html_path = f"{rel_path}.html"
|
||||
match = _latest_response_match(snapshots, domain, html_path)
|
||||
|
||||
root_match = (match[0], match[1]) if match else _latest_responses_root(snapshots, domain)
|
||||
responses_root = root_match[1] if root_match else None
|
||||
if request_config.USES_SUBDOMAIN_ROUTING:
|
||||
snapshot = root_match[0] if root_match else (snapshots[0] if requested_root_index and snapshots else None)
|
||||
if snapshot:
|
||||
snapshot_path = f"responses/{domain}/{match[2]}" if match else path
|
||||
target = build_snapshot_url(str(snapshot.id), snapshot_path, request=request, config=request_config)
|
||||
if request.META.get("QUERY_STRING"):
|
||||
target = f"{target}?{request.META['QUERY_STRING']}"
|
||||
return redirect(target)
|
||||
|
||||
show_indexes = bool(request.GET.get("files"))
|
||||
if match:
|
||||
_snapshot, responses_root, rel_to_root = match
|
||||
response = _serve_responses_path(request, responses_root, str(rel_to_root), show_indexes)
|
||||
if response is not None:
|
||||
return response
|
||||
|
||||
if responses_root:
|
||||
response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
|
||||
if response is not None:
|
||||
return response
|
||||
|
||||
if requested_root_index and not show_indexes:
|
||||
if snapshots:
|
||||
return SnapshotView.render_live_index(request, snapshots[0])
|
||||
|
||||
if request_config.PUBLIC_ADD_VIEW or request.user.is_authenticated:
|
||||
target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", ""))
|
||||
return redirect(build_web_url(f"/web/{quote(target_url, safe=':/')}"))
|
||||
|
||||
raise Http404
|
||||
|
||||
|
||||
class SnapshotHostView(View):
|
||||
"""Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
|
||||
|
||||
def get(self, request, snapshot_id: str, path: str = ""):
|
||||
request_config = get_request_config(request)
|
||||
snapshot = _find_snapshot_by_ref(snapshot_id)
|
||||
|
||||
if not snapshot:
|
||||
raise Http404
|
||||
if path == "_auth":
|
||||
return _replay_auth_response(request, snapshot)
|
||||
if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
|
||||
return _private_snapshot_auth_redirect(request, snapshot, path)
|
||||
|
||||
canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
|
||||
if not host_matches(request.get_host(), canonical_host):
|
||||
target = build_snapshot_url(str(snapshot.id), path, request=request, config=request_config)
|
||||
if request.META.get("QUERY_STRING"):
|
||||
target = f"{target}?{request.META['QUERY_STRING']}"
|
||||
return redirect(target)
|
||||
|
||||
return _serve_snapshot_replay(request, snapshot, path)
|
||||
|
||||
|
||||
class SnapshotReplayView(View):
|
||||
"""Serve snapshot directory contents on a one-domain replay path."""
|
||||
|
||||
def get(self, request, snapshot_id: str, path: str = ""):
|
||||
snapshot = _find_snapshot_by_ref(snapshot_id)
|
||||
if not snapshot:
|
||||
raise Http404
|
||||
if path == "_auth":
|
||||
return _replay_auth_response(request, snapshot)
|
||||
if not can_view_snapshot(request, snapshot) and not _has_replay_cookie(request, snapshot):
|
||||
return _private_snapshot_auth_redirect(request, snapshot, path)
|
||||
|
||||
return _serve_snapshot_replay(request, snapshot, path)
|
||||
|
||||
|
||||
class OriginalDomainHostView(View):
|
||||
"""Serve responses from the most recent snapshot when using <domain>.<listen_host>/<path>."""
|
||||
|
||||
def get(self, request, domain: str, path: str = ""):
|
||||
return _serve_original_domain_replay(request, domain, path)
|
||||
|
||||
|
||||
class OriginalDomainReplayView(View):
|
||||
"""Serve original-domain replay content on a one-domain replay path."""
|
||||
|
||||
def get(self, request, domain: str, path: str = ""):
|
||||
return _serve_original_domain_replay(request, domain, path)
|
||||
@@ -0,0 +1,148 @@
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
|
||||
from django.contrib.sessions.models import Session
|
||||
from django.core import signing
|
||||
from django.http import Http404, HttpRequest, HttpResponseForbidden
|
||||
from django.shortcuts import redirect
|
||||
from django.views import View
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import (
|
||||
get_request_config,
|
||||
)
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.permissions import (
|
||||
is_admin_user,
|
||||
)
|
||||
from archivebox.core.routes_util import (
|
||||
build_admin_url,
|
||||
build_snapshot_url,
|
||||
)
|
||||
|
||||
from .lookup import _find_snapshot_by_ref
|
||||
|
||||
REPLAY_AUTH_SALT = "archivebox.private-snapshot-replay"
|
||||
REPLAY_COOKIE_PREFIX = f"archivebox_replay_{CONSTANTS.COLLECTION_ID}_"
|
||||
REPLAY_GRANT_MAX_AGE = 60
|
||||
|
||||
|
||||
def _admin_login_redirect_or_forbidden(request: HttpRequest):
|
||||
if get_request_config(request).CONTROL_PLANE_ENABLED:
|
||||
return redirect(f"/admin/login/?next={request.path}")
|
||||
return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
|
||||
|
||||
|
||||
def _replay_cookie_name(snapshot: Snapshot) -> str:
|
||||
return f"{REPLAY_COOKIE_PREFIX}{str(snapshot.id).replace('-', '')[-12:]}"
|
||||
|
||||
|
||||
def _clean_replay_next(path: str | None) -> str:
|
||||
"""Only allow same-snap relative replay paths; grants must never redirect off-host."""
|
||||
path = f"/{(path or 'index.html').lstrip('/')}"
|
||||
parsed = urlparse(path)
|
||||
if parsed.scheme or parsed.netloc or path.startswith("//"):
|
||||
return "/index.html"
|
||||
return path
|
||||
|
||||
|
||||
def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool:
|
||||
"""A replay cookie is not its own auth source; it must point at a live admin session.
|
||||
|
||||
Replayed pages can execute hostile JS, so admin cookies stay host-only on admin.*.
|
||||
The snap host gets only this host-only HttpOnly cookie, and every request checks
|
||||
that the original Django session still exists and still belongs to an active staff user.
|
||||
Logout, session expiry, user deletion/deactivation, or password auth-hash rotation all
|
||||
make the replay cookie inert without needing admin.* to delete a cookie on snap-*.
|
||||
"""
|
||||
if payload.get("snapshot_id") != str(snapshot.id):
|
||||
return False
|
||||
try:
|
||||
session = Session.objects.get(session_key=str(payload.get("session_key") or ""))
|
||||
session_data = session.get_decoded()
|
||||
user_id = str(session_data.get(SESSION_KEY) or "")
|
||||
auth_hash = str(session_data.get(HASH_SESSION_KEY) or "")
|
||||
user = get_user_model().objects.get(pk=user_id)
|
||||
except (Session.DoesNotExist, get_user_model().DoesNotExist, KeyError, TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
str(payload.get("user_id")) == user_id
|
||||
and str(payload.get("auth_hash") or "") == auth_hash
|
||||
and user.is_active
|
||||
and user.is_staff
|
||||
and auth_hash == user.get_session_auth_hash()
|
||||
)
|
||||
|
||||
|
||||
def _has_replay_cookie(request: HttpRequest, snapshot: Snapshot) -> bool:
|
||||
value = request.COOKIES.get(_replay_cookie_name(snapshot))
|
||||
if not value:
|
||||
return False
|
||||
try:
|
||||
payload = signing.loads(value, salt=REPLAY_AUTH_SALT, max_age=settings.SESSION_COOKIE_AGE)
|
||||
except signing.BadSignature:
|
||||
return False
|
||||
return isinstance(payload, dict) and _replay_payload_is_valid(payload, snapshot)
|
||||
|
||||
|
||||
def _private_snapshot_auth_redirect(request: HttpRequest, snapshot: Snapshot, path: str = "", *, preserve_query: bool = True):
|
||||
next_path = _clean_replay_next(path or "index.html")
|
||||
if preserve_query and request.META.get("QUERY_STRING"):
|
||||
next_path = f"{next_path}?{request.META['QUERY_STRING']}"
|
||||
target = build_admin_url(
|
||||
f"/admin/core/snapshot/replay-auth/?snapshot={snapshot.id}&next={quote(next_path, safe='')}",
|
||||
request=request,
|
||||
)
|
||||
return redirect(target)
|
||||
|
||||
|
||||
def _replay_auth_response(request: HttpRequest, snapshot: Snapshot):
|
||||
try:
|
||||
payload = signing.loads(str(request.GET.get("grant") or ""), salt=REPLAY_AUTH_SALT, max_age=REPLAY_GRANT_MAX_AGE)
|
||||
except signing.BadSignature:
|
||||
return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
|
||||
|
||||
if not isinstance(payload, dict) or not _replay_payload_is_valid(payload, snapshot):
|
||||
return _private_snapshot_auth_redirect(request, snapshot, "index.html", preserve_query=False)
|
||||
|
||||
cookie_value = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
|
||||
response = redirect(_clean_replay_next(request.GET.get("next")))
|
||||
response.set_cookie(
|
||||
_replay_cookie_name(snapshot),
|
||||
cookie_value,
|
||||
max_age=settings.SESSION_COOKIE_AGE,
|
||||
secure=request.is_secure(),
|
||||
httponly=True,
|
||||
samesite="Lax",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
class SnapshotReplayAuthView(View):
|
||||
"""Admin-only handoff that lets a snap host mint its own replay cookie.
|
||||
|
||||
admin.* cannot set a host-only cookie for snap-* (browsers forbid that), and
|
||||
widening the real Django session cookie to *.archivebox.localhost would let XSS
|
||||
in replayed pages hit the admin UI. Instead admin.* proves the user is logged in
|
||||
with a short URL grant, then snap-* validates it and sets a snap-host-only cookie.
|
||||
"""
|
||||
|
||||
def get(self, request: HttpRequest):
|
||||
if not is_admin_user(request):
|
||||
return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={quote(request.get_full_path(), safe='')}")
|
||||
|
||||
snapshot = _find_snapshot_by_ref(str(request.GET.get("snapshot") or ""))
|
||||
if not snapshot:
|
||||
raise Http404
|
||||
|
||||
payload = {
|
||||
"snapshot_id": str(snapshot.id),
|
||||
"user_id": str(request.user.pk),
|
||||
"session_key": request.session.session_key,
|
||||
"auth_hash": request.user.get_session_auth_hash(),
|
||||
}
|
||||
grant = signing.dumps(payload, salt=REPLAY_AUTH_SALT)
|
||||
next_path = _clean_replay_next(request.GET.get("next"))
|
||||
target = build_snapshot_url(str(snapshot.id), "_auth", request=request, config=get_request_config(request))
|
||||
return redirect(f"{target}?grant={quote(grant, safe='')}&next={quote(next_path, safe='')}")
|
||||
+61
-302
@@ -6,6 +6,7 @@ import hashlib
|
||||
from django import forms
|
||||
from django.db.models.manager import BaseManager
|
||||
from django.db.models.query import QuerySet
|
||||
from django.template.loader import render_to_string
|
||||
from django.utils.html import escape
|
||||
from django.utils.safestring import mark_safe
|
||||
|
||||
@@ -19,11 +20,7 @@ class TagEditorWidget(forms.Widget):
|
||||
- Uses AJAX for autocomplete and tag creation
|
||||
"""
|
||||
|
||||
template_name = "" # We render manually
|
||||
|
||||
class Media:
|
||||
css = {"all": []}
|
||||
js = []
|
||||
template_name = "admin/widgets/tags.html"
|
||||
|
||||
def __init__(self, attrs=None, snapshot_id=None):
|
||||
self.snapshot_id = snapshot_id
|
||||
@@ -40,16 +37,6 @@ class TagEditorWidget(forms.Widget):
|
||||
normalized = f"t_{normalized}"
|
||||
return normalized
|
||||
|
||||
def _json_for_inline_script(self, value):
|
||||
"""Serialize JSON so it cannot close the surrounding inline <script> tag."""
|
||||
return json.dumps(value).translate(
|
||||
{
|
||||
ord(">"): "\\u003E",
|
||||
ord("<"): "\\u003C",
|
||||
ord("&"): "\\u0026",
|
||||
},
|
||||
)
|
||||
|
||||
def _tag_style(self, value):
|
||||
"""Compute a stable pastel color style for a tag value."""
|
||||
tag = (value or "").strip().lower()
|
||||
@@ -106,293 +93,10 @@ class TagEditorWidget(forms.Widget):
|
||||
widget_id_raw = attrs.get("id", name) if attrs else name
|
||||
widget_id = self._normalize_id(widget_id_raw)
|
||||
|
||||
# Build pills HTML
|
||||
pills_html = ""
|
||||
for tag in tags:
|
||||
pills_html += f'''
|
||||
<span class="tag-pill" data-tag="{self._escape(tag)}" style="{self._tag_style(tag)}">
|
||||
{self._escape(tag)}
|
||||
<button type="button" class="tag-remove-btn" data-tag-name="{self._escape(tag)}">×</button>
|
||||
</span>
|
||||
'''
|
||||
|
||||
tags_json = self._json_for_inline_script(tags)
|
||||
|
||||
# Build the widget HTML
|
||||
html = f'''
|
||||
<div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
|
||||
<div id="{widget_id}_pills" class="tag-pills">
|
||||
{pills_html}
|
||||
</div>
|
||||
<input type="text"
|
||||
id="{widget_id}_input"
|
||||
class="tag-inline-input"
|
||||
list="{widget_id}_datalist"
|
||||
placeholder="Add tag..."
|
||||
autocomplete="off"
|
||||
onkeydown="handleTagKeydown_{widget_id}(event)"
|
||||
onkeypress="if(event.key==='Enter' || event.keyCode===13 || event.key===' ' || event.code==='Space' || event.key==='Spacebar'){{event.preventDefault(); event.stopPropagation();}}"
|
||||
oninput="fetchTagAutocomplete_{widget_id}(this.value)"
|
||||
>
|
||||
<datalist id="{widget_id}_datalist"></datalist>
|
||||
<input type="hidden" name="{name}" id="{widget_id}" value="{self._escape(",".join(tags))}">
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {{
|
||||
var currentTags_{widget_id} = {tags_json};
|
||||
var autocompleteTimeout_{widget_id} = null;
|
||||
|
||||
window.focusTagInput_{widget_id} = function(event) {{
|
||||
if (event.target.classList.contains('tag-remove-btn')) return;
|
||||
document.getElementById('{widget_id}_input').focus();
|
||||
}};
|
||||
|
||||
window.updateHiddenInput_{widget_id} = function() {{
|
||||
var hiddenInput = document.getElementById('{widget_id}');
|
||||
if (!hiddenInput) {{
|
||||
return;
|
||||
}}
|
||||
hiddenInput.value = currentTags_{widget_id}.join(',');
|
||||
hiddenInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
hiddenInput.dispatchEvent(new Event('change', {{ bubbles: true }}));
|
||||
}};
|
||||
|
||||
function normalizeTags_{widget_id}(value) {{
|
||||
var rawTags = Array.isArray(value) ? value : String(value || '').split(',');
|
||||
var seen = {{}};
|
||||
return rawTags
|
||||
.map(function(tag) {{ return String(tag || '').trim(); }})
|
||||
.filter(function(tag) {{
|
||||
if (!tag) return false;
|
||||
var normalized = tag.toLowerCase();
|
||||
if (seen[normalized]) return false;
|
||||
seen[normalized] = true;
|
||||
return true;
|
||||
}})
|
||||
.sort(function(a, b) {{
|
||||
return a.toLowerCase().localeCompare(b.toLowerCase());
|
||||
}});
|
||||
}}
|
||||
|
||||
window.setTags_{widget_id} = function(value, options) {{
|
||||
currentTags_{widget_id} = normalizeTags_{widget_id}(value);
|
||||
rebuildPills_{widget_id}();
|
||||
if (!(options && options.skipHiddenUpdate)) {{
|
||||
updateHiddenInput_{widget_id}();
|
||||
}}
|
||||
}};
|
||||
|
||||
window.syncTagEditorFromHidden_{widget_id} = function() {{
|
||||
var hiddenInput = document.getElementById('{widget_id}');
|
||||
if (!hiddenInput) {{
|
||||
return;
|
||||
}}
|
||||
setTags_{widget_id}(hiddenInput.value, {{ skipHiddenUpdate: true }});
|
||||
}};
|
||||
|
||||
function computeTagStyle_{widget_id}(tagName) {{
|
||||
var hash = 0;
|
||||
var name = String(tagName || '').toLowerCase();
|
||||
for (var i = 0; i < name.length; i++) {{
|
||||
hash = (hash * 31 + name.charCodeAt(i)) % 360;
|
||||
}}
|
||||
var bg = 'hsl(' + hash + ', 70%, 92%)';
|
||||
var border = 'hsl(' + hash + ', 60%, 82%)';
|
||||
var fg = 'hsl(' + hash + ', 35%, 28%)';
|
||||
return {{ bg: bg, border: border, fg: fg }};
|
||||
}}
|
||||
|
||||
function applyTagStyle_{widget_id}(el, tagName) {{
|
||||
var colors = computeTagStyle_{widget_id}(tagName);
|
||||
el.style.setProperty('--tag-bg', colors.bg);
|
||||
el.style.setProperty('--tag-border', colors.border);
|
||||
el.style.setProperty('--tag-fg', colors.fg);
|
||||
}}
|
||||
|
||||
function getApiKey() {{
|
||||
return (window.ARCHIVEBOX_API_KEY || '').trim();
|
||||
}}
|
||||
|
||||
function buildApiUrl(path) {{
|
||||
var apiKey = getApiKey();
|
||||
if (!apiKey) return path;
|
||||
var sep = path.indexOf('?') !== -1 ? '&' : '?';
|
||||
return path + sep + 'api_key=' + encodeURIComponent(apiKey);
|
||||
}}
|
||||
|
||||
function buildApiHeaders() {{
|
||||
var headers = {{
|
||||
'Content-Type': 'application/json',
|
||||
}};
|
||||
var apiKey = getApiKey();
|
||||
if (apiKey) headers['X-ArchiveBox-API-Key'] = apiKey;
|
||||
var csrfToken = getCSRFToken();
|
||||
if (csrfToken) headers['X-CSRFToken'] = csrfToken;
|
||||
return headers;
|
||||
}}
|
||||
|
||||
window.addTag_{widget_id} = function(tagName) {{
|
||||
tagName = tagName.trim();
|
||||
if (!tagName) return;
|
||||
|
||||
// Check if tag already exists (case-insensitive)
|
||||
var exists = currentTags_{widget_id}.some(function(t) {{
|
||||
return t.toLowerCase() === tagName.toLowerCase();
|
||||
}});
|
||||
if (exists) {{
|
||||
document.getElementById('{widget_id}_input').value = '';
|
||||
return;
|
||||
}}
|
||||
|
||||
// Add to current tags
|
||||
currentTags_{widget_id}.push(tagName);
|
||||
currentTags_{widget_id} = normalizeTags_{widget_id}(currentTags_{widget_id});
|
||||
|
||||
// Rebuild pills
|
||||
rebuildPills_{widget_id}();
|
||||
updateHiddenInput_{widget_id}();
|
||||
|
||||
// Clear input
|
||||
document.getElementById('{widget_id}_input').value = '';
|
||||
|
||||
// Create tag via API if it doesn't exist (fire and forget)
|
||||
fetch(buildApiUrl('/api/v1/core/tags/create/'), {{
|
||||
method: 'POST',
|
||||
headers: buildApiHeaders(),
|
||||
body: JSON.stringify({{ name: tagName }})
|
||||
}}).catch(function(err) {{
|
||||
console.log('Tag creation note:', err);
|
||||
}});
|
||||
}};
|
||||
|
||||
window.removeTag_{widget_id} = function(tagName) {{
|
||||
currentTags_{widget_id} = currentTags_{widget_id}.filter(function(t) {{
|
||||
return t.toLowerCase() !== tagName.toLowerCase();
|
||||
}});
|
||||
rebuildPills_{widget_id}();
|
||||
updateHiddenInput_{widget_id}();
|
||||
}};
|
||||
|
||||
window.rebuildPills_{widget_id} = function() {{
|
||||
var container = document.getElementById('{widget_id}_pills');
|
||||
container.innerHTML = '';
|
||||
currentTags_{widget_id}.forEach(function(tag) {{
|
||||
var pill = document.createElement('span');
|
||||
pill.className = 'tag-pill';
|
||||
pill.setAttribute('data-tag', tag);
|
||||
applyTagStyle_{widget_id}(pill, tag);
|
||||
|
||||
var tagText = document.createTextNode(tag);
|
||||
pill.appendChild(tagText);
|
||||
|
||||
var removeBtn = document.createElement('button');
|
||||
removeBtn.type = 'button';
|
||||
removeBtn.className = 'tag-remove-btn';
|
||||
removeBtn.setAttribute('data-tag-name', tag);
|
||||
removeBtn.innerHTML = '×';
|
||||
pill.appendChild(removeBtn);
|
||||
|
||||
container.appendChild(pill);
|
||||
}});
|
||||
}};
|
||||
|
||||
// Add event delegation for remove buttons
|
||||
document.getElementById('{widget_id}_pills').addEventListener('click', function(event) {{
|
||||
if (event.target.classList.contains('tag-remove-btn')) {{
|
||||
var tagName = event.target.getAttribute('data-tag-name');
|
||||
if (tagName) {{
|
||||
removeTag_{widget_id}(tagName);
|
||||
}}
|
||||
}}
|
||||
}});
|
||||
|
||||
document.getElementById('{widget_id}').addEventListener('change', function() {{
|
||||
syncTagEditorFromHidden_{widget_id}();
|
||||
}});
|
||||
|
||||
document.getElementById('{widget_id}').addEventListener('archivebox:sync-tags', function() {{
|
||||
syncTagEditorFromHidden_{widget_id}();
|
||||
}});
|
||||
|
||||
window.handleTagKeydown_{widget_id} = function(event) {{
|
||||
var input = event.target;
|
||||
var value = input.value.trim();
|
||||
var isSpace = event.key === ' ' || event.code === 'Space' || event.key === 'Spacebar';
|
||||
var isEnter = event.key === 'Enter' || event.keyCode === 13;
|
||||
var isComma = event.key === ',';
|
||||
|
||||
if (isEnter || isSpace || isComma) {{
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
if (value) {{
|
||||
// Treat commas and whitespace as tag boundaries.
|
||||
value.split(/[\\s,]+/).forEach(function(tag) {{
|
||||
addTag_{widget_id}(tag.trim());
|
||||
}});
|
||||
}}
|
||||
return false;
|
||||
}} else if (event.key === 'Backspace' && !value && currentTags_{widget_id}.length > 0) {{
|
||||
// Remove last tag on backspace when input is empty
|
||||
var lastTag = currentTags_{widget_id}.pop();
|
||||
rebuildPills_{widget_id}();
|
||||
updateHiddenInput_{widget_id}();
|
||||
}}
|
||||
}};
|
||||
|
||||
window.fetchTagAutocomplete_{widget_id} = function(query) {{
|
||||
if (autocompleteTimeout_{widget_id}) {{
|
||||
clearTimeout(autocompleteTimeout_{widget_id});
|
||||
}}
|
||||
|
||||
autocompleteTimeout_{widget_id} = setTimeout(function() {{
|
||||
if (!query || query.length < 1) {{
|
||||
document.getElementById('{widget_id}_datalist').innerHTML = '';
|
||||
return;
|
||||
}}
|
||||
|
||||
fetch(buildApiUrl('/api/v1/core/tags/autocomplete/?q=' + encodeURIComponent(query)))
|
||||
.then(function(response) {{ return response.json(); }})
|
||||
.then(function(data) {{
|
||||
var datalist = document.getElementById('{widget_id}_datalist');
|
||||
datalist.innerHTML = '';
|
||||
(data.tags || []).forEach(function(tag) {{
|
||||
var option = document.createElement('option');
|
||||
option.value = tag.name;
|
||||
datalist.appendChild(option);
|
||||
}});
|
||||
}})
|
||||
.catch(function(err) {{
|
||||
console.log('Autocomplete error:', err);
|
||||
}});
|
||||
}}, 150);
|
||||
}};
|
||||
|
||||
function escapeHtml(text) {{
|
||||
var div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}}
|
||||
|
||||
function getCSRFToken() {{
|
||||
var cookies = document.cookie.split(';');
|
||||
for (var i = 0; i < cookies.length; i++) {{
|
||||
var cookie = cookies[i].trim();
|
||||
if (cookie.startsWith('csrftoken=')) {{
|
||||
return cookie.substring('csrftoken='.length);
|
||||
}}
|
||||
}}
|
||||
// Fallback to hidden input
|
||||
var input = document.querySelector('input[name="csrfmiddlewaretoken"]');
|
||||
return input ? input.value : '';
|
||||
}}
|
||||
|
||||
syncTagEditorFromHidden_{widget_id}();
|
||||
}})();
|
||||
</script>
|
||||
'''
|
||||
|
||||
return mark_safe(html)
|
||||
return render_to_string(
|
||||
self.template_name,
|
||||
{"name": name, "widget_id": widget_id, "tags_id": f"{widget_id}_tags", "tags": tags},
|
||||
)
|
||||
|
||||
|
||||
class URLFiltersWidget(forms.Widget):
|
||||
@@ -748,3 +452,58 @@ class InlineTagEditorWidget(TagEditorWidget):
|
||||
'''
|
||||
|
||||
return mark_safe(html)
|
||||
|
||||
|
||||
def render_copy_block(text: str, *, multiline: bool = False, copy_text: str | None = None):
|
||||
return render_to_string(
|
||||
"admin/widgets/copy_block.html",
|
||||
{
|
||||
"text": text,
|
||||
"copy_text": text if copy_text is None else copy_text,
|
||||
"element": "pre" if multiline else "code",
|
||||
"multiline": multiline,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def render_permissions_badge(permissions, *, url: str, object_name: str):
|
||||
from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_META, normalize_permissions
|
||||
|
||||
permissions = normalize_permissions(permissions)
|
||||
icon, label, fg, bg = PERMISSIONS_META[permissions]
|
||||
return render_to_string(
|
||||
"admin/widgets/permissions_badge.html",
|
||||
{
|
||||
"permissions": permissions,
|
||||
"url": url,
|
||||
"object_name": object_name,
|
||||
"icon": icon,
|
||||
"label": label,
|
||||
"fg": fg,
|
||||
"bg": bg,
|
||||
"choices": [(value, *PERMISSIONS_META[value]) for value, _label in PERMISSIONS_CHOICES],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def render_snapshot_progress(stats: dict, *, successful_plugins=None):
|
||||
"""Render the same hook counts and progress bar on snapshot cards and rows."""
|
||||
from archivebox.plugins.discovery import get_plugin_icon
|
||||
|
||||
counts = {name: int(stats.get(name) or 0) for name in ("total", "succeeded", "failed", "skipped", "noresults", "running")}
|
||||
total = counts["total"]
|
||||
completed = sum(counts[name] for name in ("succeeded", "failed", "skipped", "noresults"))
|
||||
plugins = sorted(successful_plugins or ())
|
||||
return render_to_string(
|
||||
"admin/widgets/snapshot_progress.html",
|
||||
{
|
||||
**counts,
|
||||
"completed": completed,
|
||||
"percent": int(completed / total * 100) if total else 0,
|
||||
"succeeded_percent": int(counts["succeeded"] / total * 100) if total else 0,
|
||||
"failed_percent": int((counts["succeeded"] + counts["failed"]) / total * 100) if total else 0,
|
||||
"show_plugins": successful_plugins is not None,
|
||||
"plugins": [(name, mark_safe(icon)) for name in plugins[:8] if str(icon := get_plugin_icon(name)).strip()],
|
||||
"remaining_plugins": max(len(plugins) - 8, 0),
|
||||
},
|
||||
)
|
||||
+36
-722
@@ -1,37 +1,27 @@
|
||||
__package__ = "archivebox.crawls"
|
||||
|
||||
import json
|
||||
from copy import copy
|
||||
from typing import ClassVar
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
from django import forms
|
||||
from django.contrib import admin, messages
|
||||
from django.core.paginator import Paginator
|
||||
from django.db.models import Count, F, Q
|
||||
from django.db.models import Count, Q
|
||||
from django.http import HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed, JsonResponse
|
||||
from django.shortcuts import get_object_or_404, redirect
|
||||
from django.template.loader import render_to_string
|
||||
from django.template.response import TemplateResponse
|
||||
from django.urls import path, reverse
|
||||
from django.utils import timezone
|
||||
from django.utils.html import escape, format_html, format_html_join
|
||||
from django.utils.html import format_html, format_html_join
|
||||
from django.utils.safestring import mark_safe
|
||||
from django_object_actions import action
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.core.widgets import render_permissions_badge
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_CHOICES,
|
||||
PERMISSIONS_META,
|
||||
PERMISSIONS_PRIVATE,
|
||||
PERMISSIONS_PUBLIC,
|
||||
PERMISSIONS_UNLISTED,
|
||||
PERMISSIONS_VALUES,
|
||||
normalize_permissions,
|
||||
)
|
||||
from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
|
||||
from archivebox.crawls.models import Crawl, CrawlSchedule
|
||||
from archivebox.crawls.forms import CrawlAdminForm
|
||||
from archivebox.misc.paginators import AcceleratedPaginator
|
||||
from archivebox.progressmonitor.views import progress_endpoint
|
||||
from archivebox.workers.models import RETRY_AT_MAX
|
||||
@@ -51,554 +41,6 @@ class MaxDepthListFilter(admin.SimpleListFilter):
|
||||
return queryset
|
||||
|
||||
|
||||
def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, prefix="snapshots"):
|
||||
"""Render a nice inline list view of snapshots with status, title, URL, and progress."""
|
||||
|
||||
query_param = f"{prefix}_q"
|
||||
status_param = f"{prefix}_status"
|
||||
page_param = f"{prefix}_page"
|
||||
query = (request.GET.get(query_param, "") if request is not None else "").strip()
|
||||
status_filter = (request.GET.get(status_param, "") if request is not None else "").strip()
|
||||
valid_statuses = {choice[0] for choice in Snapshot.StatusChoices.choices}
|
||||
|
||||
filtered_qs = snapshots_qs
|
||||
if query:
|
||||
from archivebox.misc.util import filter_queryset_by_uuid_substring
|
||||
|
||||
id_match_pks = list(filter_queryset_by_uuid_substring(Snapshot.objects.all(), query).values_list("pk", flat=True)[:100])
|
||||
filtered_qs = filtered_qs.filter(Q(pk__in=id_match_pks) | Q(url__icontains=query) | Q(title__icontains=query))
|
||||
if status_filter in valid_statuses:
|
||||
filtered_qs = filtered_qs.filter(status=status_filter)
|
||||
|
||||
# Keep ArchiveResult counters as scalar subqueries so the paginated
|
||||
# Snapshot queryset does not become a join+GROUP BY over every result row.
|
||||
snapshots_qs = filtered_qs.order_by("-created_at").annotate(
|
||||
total_results=ArchiveResult.snapshot_count_expr(),
|
||||
succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
|
||||
failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
|
||||
started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
|
||||
skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
|
||||
snapshot_permissions=F("permissions"),
|
||||
)
|
||||
|
||||
page_number = request.GET.get(page_param, 1) if request is not None else 1
|
||||
paginator = Paginator(snapshots_qs, page_size)
|
||||
page_obj = paginator.get_page(page_number)
|
||||
snapshots = page_obj.object_list
|
||||
total_count = paginator.count
|
||||
|
||||
def querystring(**updates):
|
||||
if request is None:
|
||||
return "#"
|
||||
params = request.GET.copy()
|
||||
for key, value in updates.items():
|
||||
if value in (None, ""):
|
||||
params.pop(key, None)
|
||||
else:
|
||||
params[key] = str(value)
|
||||
return f"?{params.urlencode()}" if params else "?"
|
||||
|
||||
preserved_inputs = ""
|
||||
if request is not None:
|
||||
managed_params = {query_param, status_param, page_param}
|
||||
preserved_inputs = "".join(
|
||||
f'<input type="hidden" name="{escape(key)}" value="{escape(value)}">'
|
||||
for key, values in request.GET.lists()
|
||||
if key not in managed_params
|
||||
for value in values
|
||||
)
|
||||
|
||||
status_options = "".join(
|
||||
f'<option value="{escape(value)}"{" selected" if status_filter == value else ""}>{escape(label)}</option>'
|
||||
for value, label in Snapshot.StatusChoices.choices
|
||||
)
|
||||
|
||||
controls = f"""
|
||||
<div class="crawl-snapshots-toolbar" style="display: flex; gap: 10px; align-items: center; justify-content: space-between; flex-wrap: wrap; padding: 10px 12px; background: #f8fafc; border-bottom: 1px solid #e2e8f0;">
|
||||
<form method="get" style="display: flex; gap: 8px; align-items: center; flex: 1 1 540px; margin: 0;">
|
||||
{preserved_inputs}
|
||||
<input type="search" name="{query_param}" value="{escape(query)}" placeholder="Filter snapshots by title, URL, or ID"
|
||||
style="min-width: 260px; flex: 1 1 360px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
|
||||
<select name="{status_param}" style="max-width: 170px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
|
||||
<option value="">All statuses</option>
|
||||
{status_options}
|
||||
</select>
|
||||
<input type="hidden" name="{page_param}" value="1">
|
||||
<button type="submit" class="button" style="padding: 7px 12px;">Filter</button>
|
||||
{f'<a href="{querystring(**{query_param: None, status_param: None, page_param: None})}" style="font-size: 12px; color: #64748b;">Clear</a>' if query or status_filter else ""}
|
||||
</form>
|
||||
<div style="font-size: 12px; color: #64748b; white-space: nowrap;">
|
||||
{page_obj.start_index() if total_count else 0}-{page_obj.end_index() if total_count else 0} of {total_count}
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
if not snapshots:
|
||||
return mark_safe(f"""
|
||||
<div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
|
||||
{controls}
|
||||
<div style="color: #666; font-style: italic; padding: 12px;">No Snapshots found.</div>
|
||||
</div>
|
||||
""")
|
||||
|
||||
# Status colors matching Django admin and progress monitor
|
||||
status_colors = {
|
||||
"queued": ("#6c757d", "#f8f9fa"), # gray
|
||||
"started": ("#856404", "#fff3cd"), # amber
|
||||
"paused": ("#1d4ed8", "#dbeafe"), # blue
|
||||
"sealed": ("#155724", "#d4edda"), # green
|
||||
"failed": ("#721c24", "#f8d7da"), # red
|
||||
}
|
||||
|
||||
rows = []
|
||||
for snapshot in snapshots:
|
||||
status = snapshot.status or "queued"
|
||||
color, bg = status_colors.get(status, ("#6c757d", "#f8f9fa"))
|
||||
permissions = snapshot.snapshot_permissions
|
||||
permission_icon = {
|
||||
PERMISSIONS_PUBLIC: "👁",
|
||||
PERMISSIONS_UNLISTED: "🔗",
|
||||
PERMISSIONS_PRIVATE: "🔒",
|
||||
}[permissions]
|
||||
permission_fg, permission_bg = {
|
||||
PERMISSIONS_PUBLIC: ("#047857", "#d1fae5"),
|
||||
PERMISSIONS_UNLISTED: ("#1d4ed8", "#dbeafe"),
|
||||
PERMISSIONS_PRIVATE: ("#991b1b", "#fee2e2"),
|
||||
}[permissions]
|
||||
|
||||
# Calculate progress
|
||||
total = snapshot.total_results
|
||||
succeeded = snapshot.succeeded_results
|
||||
failed = snapshot.failed_results
|
||||
running = snapshot.started_results
|
||||
skipped = snapshot.skipped_results
|
||||
done = succeeded + failed + skipped
|
||||
pending = max(total - done - running, 0)
|
||||
progress_pct = int((done / total) * 100) if total > 0 else 0
|
||||
progress_text = f"{done}/{total}" if total > 0 else "-"
|
||||
progress_title = f"{succeeded} succeeded, {failed} failed, {running} running, {pending} pending, {skipped} skipped"
|
||||
progress_color = "#28a745"
|
||||
if failed:
|
||||
progress_color = "#dc3545"
|
||||
elif running:
|
||||
progress_color = "#17a2b8"
|
||||
elif pending:
|
||||
progress_color = "#ffc107"
|
||||
|
||||
# Truncate title and URL
|
||||
snapshot_title = snapshot.title or "Untitled"
|
||||
title = snapshot_title[:60]
|
||||
if len(snapshot_title) > 60:
|
||||
title += "..."
|
||||
url_display = snapshot.url[:50]
|
||||
if len(snapshot.url) > 50:
|
||||
url_display += "..."
|
||||
delete_button = ""
|
||||
exclude_button = ""
|
||||
if crawl is not None:
|
||||
delete_url = reverse("admin:crawls_crawl_snapshot_delete", args=[crawl.pk, snapshot.pk])
|
||||
exclude_url = reverse("admin:crawls_crawl_snapshot_exclude_domain", args=[crawl.pk, snapshot.pk])
|
||||
delete_button = f'''
|
||||
<button type="button"
|
||||
class="crawl-snapshots-action"
|
||||
data-post-url="{escape(delete_url)}"
|
||||
data-confirm="Delete this snapshot from the crawl?"
|
||||
title="Delete this snapshot from the crawl and remove its URL from the crawl queue."
|
||||
aria-label="Delete snapshot"
|
||||
style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">🗑</button>
|
||||
'''
|
||||
exclude_button = f'''
|
||||
<button type="button"
|
||||
class="crawl-snapshots-action"
|
||||
data-post-url="{escape(exclude_url)}"
|
||||
data-confirm="Exclude this domain from the crawl? This removes matching queued URLs, deletes pending matching snapshots, and blocks future matches."
|
||||
title="Exclude this domain from this crawl. This removes matching URLs from the crawl queue, deletes pending matching snapshots, and blocks future matches."
|
||||
aria-label="Exclude domain from crawl"
|
||||
style="border: 1px solid #ddd; background: #fff; color: #666; border-radius: 4px; width: 28px; height: 28px; cursor: pointer;">⊘</button>
|
||||
'''
|
||||
|
||||
# Format date
|
||||
date_str = snapshot.created_at.strftime("%Y-%m-%d %H:%M") if snapshot.created_at else "-"
|
||||
|
||||
rows.append(f'''
|
||||
<tr style="border-bottom: 1px solid #eee;">
|
||||
<td style="padding: 6px 8px; white-space: nowrap;">
|
||||
<span style="display: inline-block; padding: 2px 8px; border-radius: 10px;
|
||||
font-size: 11px; font-weight: 500; text-transform: uppercase;
|
||||
color: {color}; background: {bg};">{status}</span>
|
||||
</td>
|
||||
<td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
|
||||
<span title="{permissions}" style="display:inline-flex; align-items:center; justify-content:center; width:22px; height:22px; border-radius:999px; font-size:12px; color:{permission_fg}; background:{permission_bg};">{permission_icon}</span>
|
||||
</td>
|
||||
<td style="padding: 6px 8px; white-space: nowrap;">
|
||||
<a href="/{snapshot.archive_path}/" style="text-decoration: none;">
|
||||
<img src="/{snapshot.archive_path}/favicon.ico"
|
||||
style="width: 16px; height: 16px; vertical-align: middle; margin-right: 4px;"
|
||||
onerror="this.style.display='none'"/>
|
||||
</a>
|
||||
</td>
|
||||
<td style="padding: 6px 8px; max-width: 300px;">
|
||||
<a href="{snapshot.admin_change_url}" style="color: #417690; text-decoration: none; font-weight: 500;"
|
||||
title="{escape(snapshot_title)}">{escape(title)}</a>
|
||||
</td>
|
||||
<td style="padding: 6px 8px; max-width: 250px;">
|
||||
<a href="{escape(snapshot.url)}" target="_blank"
|
||||
style="color: #666; text-decoration: none; font-family: monospace; font-size: 11px;"
|
||||
title="{escape(snapshot.url)}">{escape(url_display)}</a>
|
||||
</td>
|
||||
<td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
|
||||
<div style="display: inline-flex; align-items: center; gap: 6px;" title="{escape(progress_title)}">
|
||||
<div style="width: 60px; height: 6px; background: #eee; border-radius: 3px; overflow: hidden;">
|
||||
<div style="width: {progress_pct}%; height: 100%;
|
||||
background: {progress_color};
|
||||
transition: width 0.3s;"></div>
|
||||
</div>
|
||||
<a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id}"
|
||||
style="font-size: 11px; color: #417690; min-width: 35px; text-decoration: none;"
|
||||
title="View archive results">{progress_text}</a>
|
||||
</div>
|
||||
</td>
|
||||
<td style="padding: 6px 8px; white-space: nowrap; color: #888; font-size: 11px;">
|
||||
{date_str}
|
||||
</td>
|
||||
{f'<td style="padding: 6px 8px; white-space: nowrap; text-align: right;"><div style="display: inline-flex; gap: 6px;">{exclude_button}{delete_button}</div></td>' if crawl is not None else ""}
|
||||
</tr>
|
||||
''')
|
||||
|
||||
pagination = ""
|
||||
if paginator.num_pages > 1:
|
||||
pagination = f"""
|
||||
<div style="display: flex; gap: 10px; align-items: center; justify-content: center; padding: 10px 12px; background: #f8fafc; border-top: 1px solid #e2e8f0; font-size: 12px;">
|
||||
{"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.previous_page_number()}) + "'>Previous</a>" if page_obj.has_previous() else "<span style='color:#94a3b8;'>Previous</span>"}
|
||||
<span style="color: #64748b;">Page {page_obj.number} of {paginator.num_pages}</span>
|
||||
{"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.next_page_number()}) + "'>Next</a>" if page_obj.has_next() else "<span style='color:#94a3b8;'>Next</span>"}
|
||||
</div>
|
||||
"""
|
||||
|
||||
return mark_safe(f"""
|
||||
<div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
|
||||
{controls}
|
||||
<table style="width: 100%; border-collapse: collapse; font-size: 13px;">
|
||||
<thead>
|
||||
<tr style="background: #f5f5f5; border-bottom: 2px solid #ddd;">
|
||||
<th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Status</th>
|
||||
<th style="padding: 8px 4px; text-align: center; font-weight: 600; color: #333; width: 22px;">🔒</th>
|
||||
<th style="padding: 8px; text-align: left; font-weight: 600; color: #333; width: 24px;"></th>
|
||||
<th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Title</th>
|
||||
<th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">URL</th>
|
||||
<th style="padding: 8px; text-align: center; font-weight: 600; color: #333;">Progress</th>
|
||||
<th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Created</th>
|
||||
{
|
||||
'<th style="padding: 8px; text-align: right; font-weight: 600; color: #333;">Actions</th>' if crawl is not None else ""
|
||||
}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{"".join(rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
{pagination}
|
||||
</div>
|
||||
{
|
||||
'''
|
||||
<script>
|
||||
(function() {
|
||||
if (window.__archiveboxCrawlSnapshotActionsBound) {
|
||||
return;
|
||||
}
|
||||
window.__archiveboxCrawlSnapshotActionsBound = true;
|
||||
|
||||
function getCookie(name) {
|
||||
var cookieValue = null;
|
||||
if (!document.cookie) {
|
||||
return cookieValue;
|
||||
}
|
||||
var cookies = document.cookie.split(';');
|
||||
for (var i = 0; i < cookies.length; i++) {
|
||||
var cookie = cookies[i].trim();
|
||||
if (cookie.substring(0, name.length + 1) === (name + '=')) {
|
||||
cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
|
||||
break;
|
||||
}
|
||||
}
|
||||
return cookieValue;
|
||||
}
|
||||
|
||||
document.addEventListener('click', function(event) {
|
||||
var button = event.target.closest('.crawl-snapshots-action');
|
||||
if (!button) {
|
||||
return;
|
||||
}
|
||||
event.preventDefault();
|
||||
|
||||
var confirmMessage = button.getAttribute('data-confirm');
|
||||
if (confirmMessage && !window.confirm(confirmMessage)) {
|
||||
return;
|
||||
}
|
||||
|
||||
button.disabled = true;
|
||||
|
||||
fetch(button.getAttribute('data-post-url'), {
|
||||
method: 'POST',
|
||||
credentials: 'same-origin',
|
||||
headers: {
|
||||
'X-CSRFToken': getCookie('csrftoken') || '',
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
}
|
||||
}).then(function(response) {
|
||||
return response.json().then(function(data) {
|
||||
if (!response.ok) {
|
||||
throw new Error(data.error || 'Request failed');
|
||||
}
|
||||
return data;
|
||||
});
|
||||
}).then(function() {
|
||||
window.location.reload();
|
||||
}).catch(function(error) {
|
||||
button.disabled = false;
|
||||
window.alert(error.message || 'Request failed');
|
||||
});
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
'''
|
||||
if crawl is not None
|
||||
else ""
|
||||
}
|
||||
""")
|
||||
|
||||
|
||||
class URLFiltersField(forms.Field):
|
||||
widget = URLFiltersWidget(source_selector="#id_urls")
|
||||
|
||||
def to_python(self, value):
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False, "only_new": False}
|
||||
|
||||
|
||||
class CrawlAdminForm(forms.ModelForm):
|
||||
"""Custom form for Crawl admin to render urls field as textarea."""
|
||||
|
||||
tags_editor = forms.CharField(
|
||||
label="Tags",
|
||||
required=False,
|
||||
widget=TagEditorWidget(),
|
||||
help_text="Type tag names and press Enter or Space to add. Click × to remove.",
|
||||
)
|
||||
url_filters = URLFiltersField(
|
||||
label="URL Filters",
|
||||
required=False,
|
||||
help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Crawl
|
||||
fields = "__all__"
|
||||
widgets: ClassVar[dict[str, forms.Widget]] = {
|
||||
"urls": forms.Textarea(
|
||||
attrs={
|
||||
"rows": 8,
|
||||
"style": "width: 100%; font-family: monospace; font-size: 13px;",
|
||||
"placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
|
||||
},
|
||||
),
|
||||
"notes": forms.Textarea(
|
||||
attrs={
|
||||
"rows": 1,
|
||||
"style": "width: 100%; min-height: 0; resize: vertical;",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
|
||||
if self.instance and self.instance.pk:
|
||||
self.initial["tags_editor"] = self.instance.tags_str
|
||||
effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
|
||||
derived_filter_toggles = self.derive_filter_toggles(
|
||||
self.instance.urls if self.instance and self.instance.pk else "",
|
||||
config.get("URL_ALLOWLIST", ""),
|
||||
)
|
||||
self.initial["url_filters"] = {
|
||||
"allowlist": config.get("URL_ALLOWLIST", ""),
|
||||
"denylist": config.get("URL_DENYLIST", ""),
|
||||
"same_domain_only": derived_filter_toggles["same_domain_only"],
|
||||
"subpaths_only": derived_filter_toggles["subpaths_only"],
|
||||
"only_new": effective_only_new,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def extract_url_line(line):
|
||||
line = str(line or "").strip()
|
||||
if not line or line.startswith("#"):
|
||||
return ""
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
return str(json.loads(line).get("url", "")).strip()
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return ""
|
||||
return line
|
||||
|
||||
@staticmethod
|
||||
def regex_escape(text):
|
||||
escaped = ""
|
||||
for char in str(text or ""):
|
||||
escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
|
||||
return escaped
|
||||
|
||||
@classmethod
|
||||
def generated_host_allowlist(cls, urls):
|
||||
seen = set()
|
||||
domains = []
|
||||
for raw_line in str(urls or "").splitlines():
|
||||
url = cls.extract_url_line(raw_line)
|
||||
if not url:
|
||||
continue
|
||||
parsed = urlparse(url)
|
||||
domain = (parsed.hostname or "").lower()
|
||||
if not domain or domain in seen:
|
||||
continue
|
||||
seen.add(domain)
|
||||
domains.append(domain)
|
||||
if not domains:
|
||||
return ""
|
||||
return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
|
||||
|
||||
@staticmethod
|
||||
def subpath_prefix(pathname):
|
||||
path = str(pathname or "/")
|
||||
while "//" in path:
|
||||
path = path.replace("//", "/")
|
||||
if not path or path == "/":
|
||||
return "/"
|
||||
if path.endswith("/"):
|
||||
return path
|
||||
last_slash = path.rfind("/")
|
||||
last_part = path[last_slash + 1 :]
|
||||
if "." in last_part:
|
||||
return path[: last_slash + 1] or "/"
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def parsed_host_and_port(parsed):
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host:
|
||||
return ""
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
port = None
|
||||
return f"{host}:{port}" if port is not None else host
|
||||
|
||||
@classmethod
|
||||
def generated_subpath_allowlist(cls, urls):
|
||||
seen = set()
|
||||
paths = []
|
||||
for raw_line in str(urls or "").splitlines():
|
||||
url = cls.extract_url_line(raw_line)
|
||||
if not url:
|
||||
continue
|
||||
parsed = urlparse(url)
|
||||
domain = (parsed.hostname or "").lower()
|
||||
if domain:
|
||||
seen.add(domain)
|
||||
host = cls.parsed_host_and_port(parsed)
|
||||
path = cls.subpath_prefix(parsed.path)
|
||||
path_key = f"{host}{path}"
|
||||
if not host or path_key in seen:
|
||||
continue
|
||||
seen.add(path_key)
|
||||
paths.append((host, path))
|
||||
if not paths:
|
||||
return ""
|
||||
patterns = []
|
||||
for host, path in paths:
|
||||
if path == "/":
|
||||
patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
|
||||
elif path.endswith("/"):
|
||||
patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
|
||||
else:
|
||||
patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
|
||||
return "\n".join(patterns)
|
||||
|
||||
@classmethod
|
||||
def derive_filter_toggles(cls, urls, allowlist):
|
||||
normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
|
||||
if not normalized_allowlist:
|
||||
return {"same_domain_only": False, "subpaths_only": False}
|
||||
if normalized_allowlist == cls.generated_subpath_allowlist(urls):
|
||||
return {"same_domain_only": True, "subpaths_only": True}
|
||||
if normalized_allowlist == cls.generated_host_allowlist(urls):
|
||||
return {"same_domain_only": True, "subpaths_only": False}
|
||||
return {"same_domain_only": False, "subpaths_only": False}
|
||||
|
||||
@staticmethod
|
||||
def effective_only_new(crawl=None):
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
if crawl is not None:
|
||||
return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
|
||||
return bool(get_config(resolve_plugins=False).ONLY_NEW)
|
||||
|
||||
@staticmethod
|
||||
def inherited_only_new(crawl):
|
||||
crawl_without_only_new = copy(crawl)
|
||||
config = dict(crawl.config or {})
|
||||
config.pop("ONLY_NEW", None)
|
||||
crawl_without_only_new.config = config
|
||||
return CrawlAdminForm.effective_only_new(crawl_without_only_new)
|
||||
|
||||
def clean_tags_editor(self):
|
||||
tags_str = self.cleaned_data.get("tags_editor", "")
|
||||
tag_names = []
|
||||
seen = set()
|
||||
for raw_name in tags_str.split(","):
|
||||
name = raw_name.strip()
|
||||
if not name:
|
||||
continue
|
||||
lowered = name.lower()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
tag_names.append(name)
|
||||
return ",".join(tag_names)
|
||||
|
||||
def clean_url_filters(self):
|
||||
value = self.cleaned_data.get("url_filters") or {}
|
||||
return {
|
||||
"allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
|
||||
"denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
|
||||
"same_domain_only": bool(value.get("same_domain_only")),
|
||||
"subpaths_only": bool(value.get("subpaths_only")),
|
||||
"only_new": bool(value.get("only_new")),
|
||||
}
|
||||
|
||||
def save(self, commit=True):
|
||||
instance = super().save(commit=False)
|
||||
instance.tags_str = self.cleaned_data.get("tags_editor", "")
|
||||
if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
|
||||
url_filters = self.cleaned_data.get("url_filters") or {}
|
||||
instance.set_url_filters(
|
||||
url_filters.get("allowlist", ""),
|
||||
url_filters.get("denylist", ""),
|
||||
)
|
||||
config = dict(instance.config or {})
|
||||
only_new = bool(url_filters.get("only_new"))
|
||||
inherited_only_new = self.inherited_only_new(instance)
|
||||
if only_new != inherited_only_new:
|
||||
config["ONLY_NEW"] = only_new
|
||||
else:
|
||||
config.pop("ONLY_NEW", None)
|
||||
instance.config = config
|
||||
if commit:
|
||||
instance.save()
|
||||
instance.apply_crawl_config_filters()
|
||||
self._save_m2m()
|
||||
return instance
|
||||
|
||||
|
||||
class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
form = CrawlAdminForm
|
||||
change_form_template = "admin/crawls/crawl/change_form.html"
|
||||
@@ -646,13 +88,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
readonly_fields = ("created_at", "modified_at", "stop_reason_display")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"URLs",
|
||||
{
|
||||
"fields": ("urls", "url_filters"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
card_fieldset("URLs", ("urls", "url_filters"), wide=True),
|
||||
(
|
||||
"Overview",
|
||||
{
|
||||
@@ -674,13 +110,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
),
|
||||
)
|
||||
add_fieldsets = (
|
||||
(
|
||||
"URLs",
|
||||
{
|
||||
"fields": ("urls", "url_filters"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
card_fieldset("URLs", ("urls", "url_filters"), wide=True),
|
||||
(
|
||||
"Overview",
|
||||
{
|
||||
@@ -1024,45 +454,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
|
||||
@admin.display(description="👁", ordering="permissions")
|
||||
def permissions_badge(self, obj):
|
||||
permissions = normalize_permissions(obj.permissions)
|
||||
icon, label, fg, bg = PERMISSIONS_META[permissions]
|
||||
menu_items = format_html_join(
|
||||
"",
|
||||
(
|
||||
'<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
|
||||
'<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
|
||||
"<span>{}</span>"
|
||||
"</button>"
|
||||
),
|
||||
(
|
||||
(
|
||||
" is-active" if choice_value == permissions else "",
|
||||
choice_value,
|
||||
choice_fg,
|
||||
choice_bg,
|
||||
choice_icon,
|
||||
choice_label,
|
||||
)
|
||||
for choice_value, choice_label in PERMISSIONS_CHOICES
|
||||
for choice_icon, _choice_title, choice_fg, choice_bg in [PERMISSIONS_META[choice_value]]
|
||||
),
|
||||
)
|
||||
return format_html(
|
||||
'<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
|
||||
'<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change crawl permissions: {}" aria-expanded="false">'
|
||||
'<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
|
||||
"</button>"
|
||||
'<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
|
||||
"</span>",
|
||||
permissions = obj.permissions
|
||||
return render_permissions_badge(
|
||||
permissions,
|
||||
reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
|
||||
permissions,
|
||||
label,
|
||||
label,
|
||||
fg,
|
||||
bg,
|
||||
icon,
|
||||
menu_items,
|
||||
url=reverse(f"{self.admin_site.name}:crawls_crawl_set_permissions", args=[obj.pk]),
|
||||
object_name="crawl",
|
||||
)
|
||||
|
||||
@admin.display(description="Pause")
|
||||
@@ -1089,36 +485,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
|
||||
@admin.display(description="Snapshots")
|
||||
def snapshots_changelist(self, obj):
|
||||
request = self.request
|
||||
snapshot_changelist = reverse("admin:core_snapshot_changelist")
|
||||
scoped_params = {"crawl_id": str(obj.pk)}
|
||||
full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
|
||||
|
||||
snapshot_admin = self.admin_site._registry[Snapshot]
|
||||
changelist_request = copy(request)
|
||||
changelist_request.method = "GET"
|
||||
changelist_request.path = snapshot_changelist
|
||||
changelist_request.GET = request.GET.copy()
|
||||
changelist_request.GET.update(
|
||||
{
|
||||
**scoped_params,
|
||||
"_embedded": "crawl",
|
||||
"per_page": "200",
|
||||
},
|
||||
return self.admin_site._registry[Snapshot].render_embedded_changelist(
|
||||
self.request,
|
||||
filters={"crawl_id": str(obj.pk)},
|
||||
title="Snapshots in this crawl",
|
||||
)
|
||||
changelist_request.POST = request.POST.copy()
|
||||
changelist_request.POST.clear()
|
||||
|
||||
response = snapshot_admin.changelist_view(
|
||||
changelist_request,
|
||||
extra_context={"embedded_changelist": True},
|
||||
)
|
||||
context = {
|
||||
**response.context_data,
|
||||
"snapshot_changelist_url": full_url,
|
||||
"crawl": obj,
|
||||
}
|
||||
return mark_safe(render_to_string("admin/crawls/crawl/snapshots_changelist.html", context, request=request))
|
||||
|
||||
def delete_snapshot_view(self, request: HttpRequest, object_id: str, snapshot_id: str):
|
||||
if request.method != "POST":
|
||||
@@ -1178,83 +549,25 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
first_url = next((line.strip() for line in (obj.urls or "").splitlines() if line.strip() and not line.strip().startswith("#")), "")
|
||||
return first_url[:80] + "..." if len(first_url) > 80 else first_url
|
||||
|
||||
@admin.display(description="URLs")
|
||||
def urls_editor(self, obj):
|
||||
"""Editor for crawl URLs."""
|
||||
widget_id = f"crawl_urls_{obj.pk}"
|
||||
|
||||
# Escape for safe HTML embedding
|
||||
escaped_urls = (obj.urls or "").replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||
|
||||
# Count lines for auto-expand logic
|
||||
line_count = len((obj.urls or "").split("\n"))
|
||||
uri_rows = min(max(3, line_count), 10)
|
||||
|
||||
html = f'''
|
||||
<div id="{widget_id}_container" style="max-width: 900px;">
|
||||
<!-- URLs input -->
|
||||
<div style="margin-bottom: 12px;">
|
||||
<label style="font-weight: bold; display: block; margin-bottom: 4px;">URLs (one per line):</label>
|
||||
<textarea id="{widget_id}_urls"
|
||||
style="width: 100%; font-family: monospace; font-size: 13px;
|
||||
padding: 8px; border: 1px solid #ccc; border-radius: 4px;
|
||||
resize: vertical;"
|
||||
rows="{uri_rows}"
|
||||
placeholder="https://example.com https://example2.com # Comments start with #"
|
||||
readonly>{escaped_urls}</textarea>
|
||||
<p style="color: #666; font-size: 12px; margin: 4px 0 0 0;">
|
||||
{line_count} URL{"s" if line_count != 1 else ""} · Note: URLs displayed here for reference only
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
'''
|
||||
return mark_safe(html)
|
||||
|
||||
|
||||
class CrawlScheduleAdmin(BaseModelAdmin):
|
||||
change_form_template = "admin/crawls/crawlschedule/change_form.html"
|
||||
|
||||
class Media:
|
||||
css: ClassVar[dict[str, tuple[str, ...]]] = {"all": ("admin/crawls/crawl_change.css",)}
|
||||
|
||||
list_display = ("id", "created_at", "created_by", "label", "notes", "template_str", "crawls", "num_crawls", "num_snapshots")
|
||||
sort_fields = ("id", "created_at", "created_by", "label", "notes", "template_str")
|
||||
search_fields = ("id", "created_by__username", "label", "notes", "schedule_id", "template_id", "template__urls")
|
||||
|
||||
readonly_fields = ("created_at", "modified_at", "crawls", "snapshots")
|
||||
readonly_fields = ("created_at", "modified_at", "crawls")
|
||||
autocomplete_fields = ("template", "created_by")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Schedule Info",
|
||||
{
|
||||
"fields": ("label", "notes"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Configuration",
|
||||
{
|
||||
"fields": ("is_enabled", "schedule", "template"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Metadata",
|
||||
{
|
||||
"fields": ("created_by", "created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Crawls",
|
||||
{
|
||||
"fields": ("crawls",),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Snapshots",
|
||||
{
|
||||
"fields": ("snapshots",),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
card_fieldset("Schedule Info", ("label", "notes")),
|
||||
card_fieldset("Configuration", ("is_enabled", "schedule", "template")),
|
||||
card_fieldset("Metadata", ("created_by", "created_at", "modified_at")),
|
||||
card_fieldset("Crawls", ("crawls",), wide=True),
|
||||
)
|
||||
|
||||
list_filter = ("created_by",)
|
||||
@@ -1276,6 +589,15 @@ class CrawlScheduleAdmin(BaseModelAdmin):
|
||||
|
||||
def change_view(self, request, object_id, form_url="", extra_context=None):
|
||||
self.request = request
|
||||
schedule = self.get_object(request, object_id)
|
||||
extra_context = {**(extra_context or {})}
|
||||
if schedule:
|
||||
extra_context["schedule_snapshots_changelist"] = self.admin_site._registry[Snapshot].render_embedded_changelist(
|
||||
request,
|
||||
filters={"crawl__schedule__id__exact": str(schedule.pk)},
|
||||
title="Snapshots in this schedule",
|
||||
default_search_mode="meta",
|
||||
)
|
||||
from django.db import connections, router
|
||||
|
||||
connection = connections[router.db_for_write(self.model)]
|
||||
@@ -1297,7 +619,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
|
||||
|
||||
def get_fieldsets(self, request, obj=None):
|
||||
if obj is None:
|
||||
return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] not in {"Crawls", "Snapshots"})
|
||||
return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] != "Crawls")
|
||||
return self.fieldsets
|
||||
|
||||
def save_model(self, request, obj, form, change):
|
||||
@@ -1330,14 +652,6 @@ class CrawlScheduleAdmin(BaseModelAdmin):
|
||||
((crawl.admin_change_url, crawl) for crawl in obj.crawl_set.all().order_by("-created_at")[:20]),
|
||||
) or mark_safe("<i>No Crawls yet...</i>")
|
||||
|
||||
def snapshots(self, obj):
|
||||
crawl_ids = obj.crawl_set.values_list("pk", flat=True)
|
||||
return render_snapshots_list(
|
||||
Snapshot.objects.filter(crawl_id__in=crawl_ids),
|
||||
request=self.request,
|
||||
prefix="schedule_snapshots",
|
||||
)
|
||||
|
||||
|
||||
def register_admin(admin_site):
|
||||
admin_site.register(Crawl, CrawlAdmin)
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Crawl form input parsing; lifecycle operations remain on Crawl."""
|
||||
|
||||
import json
|
||||
from copy import copy
|
||||
from typing import ClassVar
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django import forms
|
||||
|
||||
from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
|
||||
class URLFiltersField(forms.Field):
|
||||
widget = URLFiltersWidget(source_selector="#id_urls")
|
||||
|
||||
def to_python(self, value):
|
||||
value = value if isinstance(value, dict) else {}
|
||||
return {
|
||||
"allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
|
||||
"denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
|
||||
"same_domain_only": bool(value.get("same_domain_only")),
|
||||
"subpaths_only": bool(value.get("subpaths_only")),
|
||||
"only_new": bool(value.get("only_new")),
|
||||
}
|
||||
|
||||
|
||||
class CrawlAdminForm(forms.ModelForm):
|
||||
"""Custom form for Crawl admin to render urls field as textarea."""
|
||||
|
||||
tags_editor = forms.CharField(
|
||||
label="Tags",
|
||||
required=False,
|
||||
widget=TagEditorWidget(),
|
||||
help_text="Type tag names and press Enter or Space to add. Click × to remove.",
|
||||
)
|
||||
url_filters = URLFiltersField(
|
||||
label="URL Filters",
|
||||
required=False,
|
||||
help_text="Set URL_ALLOWLIST / URL_DENYLIST for this crawl.",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = Crawl
|
||||
fields = "__all__"
|
||||
widgets: ClassVar[dict[str, forms.Widget]] = {
|
||||
"urls": forms.Textarea(
|
||||
attrs={
|
||||
"rows": 8,
|
||||
"style": "width: 100%; font-family: monospace; font-size: 13px;",
|
||||
"placeholder": "https://example.com\nhttps://example2.com\n# Comments start with #",
|
||||
},
|
||||
),
|
||||
"notes": forms.Textarea(
|
||||
attrs={
|
||||
"rows": 1,
|
||||
"style": "width: 100%; min-height: 0; resize: vertical;",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
|
||||
if self.instance and self.instance.pk:
|
||||
self.initial["tags_editor"] = self.instance.tags_str
|
||||
effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
|
||||
derived_filter_toggles = self.derive_filter_toggles(
|
||||
self.instance.urls if self.instance and self.instance.pk else "",
|
||||
config.get("URL_ALLOWLIST", ""),
|
||||
)
|
||||
self.initial["url_filters"] = {
|
||||
"allowlist": config.get("URL_ALLOWLIST", ""),
|
||||
"denylist": config.get("URL_DENYLIST", ""),
|
||||
"same_domain_only": derived_filter_toggles["same_domain_only"],
|
||||
"subpaths_only": derived_filter_toggles["subpaths_only"],
|
||||
"only_new": effective_only_new,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def extract_url_line(line):
|
||||
line = str(line or "").strip()
|
||||
if not line or line.startswith("#"):
|
||||
return ""
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
return str(json.loads(line).get("url", "")).strip()
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return ""
|
||||
return line
|
||||
|
||||
@staticmethod
|
||||
def regex_escape(text):
|
||||
escaped = ""
|
||||
for char in str(text or ""):
|
||||
escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
|
||||
return escaped
|
||||
|
||||
@classmethod
|
||||
def generated_host_allowlist(cls, urls):
|
||||
seen = set()
|
||||
domains = []
|
||||
for raw_line in str(urls or "").splitlines():
|
||||
url = cls.extract_url_line(raw_line)
|
||||
if not url:
|
||||
continue
|
||||
parsed = urlparse(url)
|
||||
domain = (parsed.hostname or "").lower()
|
||||
if not domain or domain in seen:
|
||||
continue
|
||||
seen.add(domain)
|
||||
domains.append(domain)
|
||||
if not domains:
|
||||
return ""
|
||||
return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
|
||||
|
||||
@staticmethod
|
||||
def subpath_prefix(pathname):
|
||||
path = str(pathname or "/")
|
||||
while "//" in path:
|
||||
path = path.replace("//", "/")
|
||||
if not path or path == "/":
|
||||
return "/"
|
||||
if path.endswith("/"):
|
||||
return path
|
||||
last_slash = path.rfind("/")
|
||||
last_part = path[last_slash + 1 :]
|
||||
if "." in last_part:
|
||||
return path[: last_slash + 1] or "/"
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def parsed_host_and_port(parsed):
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host:
|
||||
return ""
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
port = None
|
||||
return f"{host}:{port}" if port is not None else host
|
||||
|
||||
@classmethod
|
||||
def generated_subpath_allowlist(cls, urls):
|
||||
seen = set()
|
||||
paths = []
|
||||
for raw_line in str(urls or "").splitlines():
|
||||
url = cls.extract_url_line(raw_line)
|
||||
if not url:
|
||||
continue
|
||||
parsed = urlparse(url)
|
||||
domain = (parsed.hostname or "").lower()
|
||||
if domain:
|
||||
seen.add(domain)
|
||||
host = cls.parsed_host_and_port(parsed)
|
||||
path = cls.subpath_prefix(parsed.path)
|
||||
path_key = f"{host}{path}"
|
||||
if not host or path_key in seen:
|
||||
continue
|
||||
seen.add(path_key)
|
||||
paths.append((host, path))
|
||||
if not paths:
|
||||
return ""
|
||||
patterns = []
|
||||
for host, path in paths:
|
||||
if path == "/":
|
||||
patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
|
||||
elif path.endswith("/"):
|
||||
patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
|
||||
else:
|
||||
patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
|
||||
return "\n".join(patterns)
|
||||
|
||||
@classmethod
|
||||
def derive_filter_toggles(cls, urls, allowlist):
|
||||
normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
|
||||
if not normalized_allowlist:
|
||||
return {"same_domain_only": False, "subpaths_only": False}
|
||||
if normalized_allowlist == cls.generated_subpath_allowlist(urls):
|
||||
return {"same_domain_only": True, "subpaths_only": True}
|
||||
if normalized_allowlist == cls.generated_host_allowlist(urls):
|
||||
return {"same_domain_only": True, "subpaths_only": False}
|
||||
return {"same_domain_only": False, "subpaths_only": False}
|
||||
|
||||
@staticmethod
|
||||
def effective_only_new(crawl=None):
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
if crawl is not None:
|
||||
return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
|
||||
return bool(get_config(resolve_plugins=False).ONLY_NEW)
|
||||
|
||||
@staticmethod
|
||||
def inherited_only_new(crawl):
|
||||
crawl_without_only_new = copy(crawl)
|
||||
config = dict(crawl.config or {})
|
||||
config.pop("ONLY_NEW", None)
|
||||
crawl_without_only_new.config = config
|
||||
return CrawlAdminForm.effective_only_new(crawl_without_only_new)
|
||||
|
||||
def clean_tags_editor(self):
|
||||
tags_str = self.cleaned_data.get("tags_editor", "")
|
||||
tag_names = []
|
||||
seen = set()
|
||||
for raw_name in tags_str.split(","):
|
||||
name = raw_name.strip()
|
||||
if not name:
|
||||
continue
|
||||
lowered = name.lower()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
tag_names.append(name)
|
||||
return ",".join(tag_names)
|
||||
|
||||
def save(self, commit=True):
|
||||
instance = super().save(commit=False)
|
||||
instance.tags_str = self.cleaned_data.get("tags_editor", "")
|
||||
if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
|
||||
url_filters = self.cleaned_data.get("url_filters") or {}
|
||||
instance.set_url_filters(
|
||||
url_filters.get("allowlist", ""),
|
||||
url_filters.get("denylist", ""),
|
||||
)
|
||||
config = dict(instance.config or {})
|
||||
only_new = bool(url_filters.get("only_new"))
|
||||
inherited_only_new = self.inherited_only_new(instance)
|
||||
if only_new != inherited_only_new:
|
||||
config["ONLY_NEW"] = only_new
|
||||
else:
|
||||
config.pop("ONLY_NEW", None)
|
||||
instance.config = config
|
||||
if commit:
|
||||
instance.save()
|
||||
instance.apply_crawl_config_filters()
|
||||
self._save_m2m()
|
||||
return instance
|
||||
@@ -1,6 +1,6 @@
|
||||
__package__ = "archivebox.crawls"
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
from collections.abc import Iterable, Mapping
|
||||
import uuid
|
||||
import json
|
||||
@@ -31,7 +31,7 @@ from archivebox.base_models.models import (
|
||||
ModelWithHealthStats,
|
||||
get_or_create_system_user_pk,
|
||||
)
|
||||
from archivebox.workers.models import ModelWithQueue
|
||||
from archivebox.workers.models import DefaultStatusChoices, ModelWithQueue
|
||||
from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
|
||||
from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
|
||||
|
||||
@@ -184,7 +184,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
|
||||
retry_at_field_name = "retry_at"
|
||||
state_field_name = "status"
|
||||
StatusChoices = ModelWithQueue.StatusChoices
|
||||
StatusChoices: ClassVar[type[DefaultStatusChoices]] = DefaultStatusChoices
|
||||
INITIAL_STATE = StatusChoices.QUEUED
|
||||
ACTIVE_STATE = StatusChoices.STARTED
|
||||
FINAL_STATES = (StatusChoices.SEALED,)
|
||||
|
||||
+44
-255
@@ -1,6 +1,5 @@
|
||||
__package__ = "archivebox.machine"
|
||||
|
||||
import json
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
|
||||
@@ -14,50 +13,13 @@ from django.utils.html import format_html
|
||||
from django.utils.safestring import mark_safe
|
||||
from django_object_actions import action
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.core.widgets import render_copy_block
|
||||
from archivebox.machine.env_util import env_to_dotenv_text
|
||||
from archivebox.machine.models import Binary, Machine, NetworkInterface, Process
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
|
||||
|
||||
def _render_copy_block(text: str, *, multiline: bool = False):
|
||||
if multiline:
|
||||
return format_html(
|
||||
"""
|
||||
<div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
|
||||
<button type="button"
|
||||
data-command="{}"
|
||||
onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
|
||||
style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
|
||||
Copy
|
||||
</button>
|
||||
<pre title="{}" style="display: block; width: 100%; max-width: 100%; overflow: auto; max-height: 300px; margin: 0; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; line-height: 1.45; white-space: pre-wrap; word-break: break-word; box-sizing: border-box;">{}</pre>
|
||||
</div>
|
||||
""",
|
||||
text,
|
||||
text,
|
||||
text,
|
||||
)
|
||||
return format_html(
|
||||
"""
|
||||
<div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
|
||||
<button type="button"
|
||||
data-command="{}"
|
||||
onclick="(function(btn){{var text=btn.dataset.command||''; if(navigator.clipboard&&navigator.clipboard.writeText){{navigator.clipboard.writeText(text);}} else {{var ta=document.createElement('textarea'); ta.value=text; document.body.appendChild(ta); ta.select(); document.execCommand('copy'); document.body.removeChild(ta);}}}})(this); return false;"
|
||||
style="position: absolute; top: 6px; right: 6px; z-index: 1; padding: 2px 8px; border: 0; border-radius: 4px; background: #e2e8f0; color: #334155; font-size: 11px; cursor: pointer;">
|
||||
Copy
|
||||
</button>
|
||||
<code title="{}" style="display: block; width: 100%; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding: 8px 56px 8px 8px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; font-size: 11px; box-sizing: border-box;">
|
||||
{}
|
||||
</code>
|
||||
</div>
|
||||
""",
|
||||
text,
|
||||
text,
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def _format_process_duration_seconds(started_at, ended_at) -> str:
|
||||
if not started_at:
|
||||
return "-"
|
||||
@@ -122,59 +84,27 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
readonly_fields = ("guid", "created_at", "modified_at", "ips")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Identity",
|
||||
{
|
||||
"fields": ("hostname", "guid", "ips"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Hardware",
|
||||
{
|
||||
"fields": ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Operating System",
|
||||
{
|
||||
"fields": ("os_platform", "os_family", "os_arch", "os_kernel", "os_release"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Statistics",
|
||||
{
|
||||
"fields": ("stats", "num_uses_succeeded", "num_uses_failed"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
card_fieldset("Identity", ("hostname", "guid", "ips")),
|
||||
card_fieldset("Hardware", ("hw_manufacturer", "hw_product", "hw_uuid", "hw_in_docker", "hw_in_vm")),
|
||||
card_fieldset("Operating System", ("os_platform", "os_family", "os_arch", "os_kernel", "os_release")),
|
||||
card_fieldset("Statistics", ("stats", "num_uses_succeeded", "num_uses_failed")),
|
||||
card_fieldset(
|
||||
"Configuration",
|
||||
{
|
||||
"fields": ("config",),
|
||||
"classes": ("card", "wide"),
|
||||
"description": mark_safe(
|
||||
'<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
|
||||
"border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
|
||||
'color:#7c2d12;font-size:12px;line-height:1.45;">'
|
||||
"<b>Heads up:</b> saving here also rewrites "
|
||||
"<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
|
||||
"kept in 1:1 sync, so any keys you remove here will be removed from the file "
|
||||
"too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
|
||||
"propagate back into this field on the next request."
|
||||
"</div>",
|
||||
),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
("config",),
|
||||
wide=True,
|
||||
description=mark_safe(
|
||||
'<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
|
||||
"border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
|
||||
'color:#7c2d12;font-size:12px;line-height:1.45;">'
|
||||
"<b>Heads up:</b> saving here also rewrites "
|
||||
"<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
|
||||
"kept in 1:1 sync, so any keys you remove here will be removed from the file "
|
||||
"too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
|
||||
"propagate back into this field on the next request."
|
||||
"</div>",
|
||||
),
|
||||
),
|
||||
card_fieldset("Timestamps", ("created_at", "modified_at")),
|
||||
)
|
||||
|
||||
list_filter = ("hw_in_docker", "hw_in_vm", "os_arch", "os_family", "os_platform")
|
||||
@@ -190,12 +120,6 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
", ".join(machine.networkinterface_set.values_list("ip_public", flat=True)),
|
||||
)
|
||||
|
||||
@admin.display(description="Health", ordering="health")
|
||||
def health_display(self, obj):
|
||||
h = obj.health
|
||||
color = "green" if h >= 80 else "orange" if h >= 50 else "red"
|
||||
return format_html('<span style="color: {};">{}</span>', color, h)
|
||||
|
||||
@admin.display(description="ID", ordering="id")
|
||||
def id_display(self, machine):
|
||||
# Highlight the row representing the machine that ``Machine.current()``
|
||||
@@ -275,41 +199,11 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
|
||||
readonly_fields = ("machine", "created_at", "modified_at", "mac_address", "ip_public", "ip_local", "dns_server")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Machine",
|
||||
{
|
||||
"fields": ("machine",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Network",
|
||||
{
|
||||
"fields": ("iface", "ip_public", "ip_local", "mac_address", "dns_server"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Location",
|
||||
{
|
||||
"fields": ("hostname", "isp", "city", "region", "country"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Usage",
|
||||
{
|
||||
"fields": ("num_uses_succeeded", "num_uses_failed"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Machine", ("machine",)),
|
||||
card_fieldset("Network", ("iface", "ip_public", "ip_local", "mac_address", "dns_server")),
|
||||
card_fieldset("Location", ("hostname", "isp", "city", "region", "country")),
|
||||
card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
|
||||
card_fieldset("Timestamps", ("created_at", "modified_at")),
|
||||
)
|
||||
|
||||
list_filter = ("isp", "country", "region")
|
||||
@@ -326,12 +220,6 @@ class NetworkInterfaceAdmin(BaseModelAdmin):
|
||||
iface.machine.hostname,
|
||||
)
|
||||
|
||||
@admin.display(description="Health", ordering="health")
|
||||
def health_display(self, obj):
|
||||
h = obj.health
|
||||
color = "green" if h >= 80 else "orange" if h >= 50 else "red"
|
||||
return format_html('<span style="color: {};">{}</span>', color, h)
|
||||
|
||||
|
||||
class BinaryAdmin(BaseModelAdmin):
|
||||
list_display = ("id", "created_at", "machine_info", "name", "binprovider", "version", "abspath", "sha256", "status", "health_display")
|
||||
@@ -341,48 +229,12 @@ class BinaryAdmin(BaseModelAdmin):
|
||||
readonly_fields = ("created_at", "modified_at", "output_dir")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Binary Info",
|
||||
{
|
||||
"fields": ("name", "binproviders", "binprovider", "overrides"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Location",
|
||||
{
|
||||
"fields": ("machine", "abspath"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Version",
|
||||
{
|
||||
"fields": ("version", "sha256"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"State",
|
||||
{
|
||||
"fields": ("status", "retry_at", "output_dir"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Usage",
|
||||
{
|
||||
"fields": ("num_uses_succeeded", "num_uses_failed"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Binary Info", ("name", "binproviders", "binprovider", "overrides")),
|
||||
card_fieldset("Location", ("machine", "abspath")),
|
||||
card_fieldset("Version", ("version", "sha256")),
|
||||
card_fieldset("State", ("status", "retry_at", "output_dir")),
|
||||
card_fieldset("Usage", ("num_uses_succeeded", "num_uses_failed")),
|
||||
card_fieldset("Timestamps", ("created_at", "modified_at")),
|
||||
)
|
||||
|
||||
list_filter = ("name", "binprovider", "status", "machine_id")
|
||||
@@ -399,12 +251,6 @@ class BinaryAdmin(BaseModelAdmin):
|
||||
binary.machine.hostname,
|
||||
)
|
||||
|
||||
@admin.display(description="Health", ordering="health")
|
||||
def health_display(self, obj):
|
||||
h = obj.health
|
||||
color = "green" if h >= 80 else "orange" if h >= 50 else "red"
|
||||
return format_html('<span style="color: {};">{}</span>', color, h)
|
||||
|
||||
|
||||
class ProcessAdmin(BaseModelAdmin):
|
||||
list_display = (
|
||||
@@ -463,34 +309,10 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
)
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Process Info",
|
||||
{
|
||||
"fields": ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Command",
|
||||
{
|
||||
"fields": ("cmd_display", "pwd", "env_display", "timeout"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Execution",
|
||||
{
|
||||
"fields": ("binary_link", "iface_link", "pid", "exit_code", "url"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timing",
|
||||
{
|
||||
"fields": ("started_at", "ended_at", "duration_display"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Process Info", ("machine", "archiveresult_link", "snapshot_link", "crawl_link", "status", "retry_at")),
|
||||
card_fieldset("Command", ("cmd_display", "pwd", "env_display", "timeout"), wide=True),
|
||||
card_fieldset("Execution", ("binary_link", "iface_link", "pid", "exit_code", "url")),
|
||||
card_fieldset("Timing", ("started_at", "ended_at", "duration_display")),
|
||||
(
|
||||
"Output",
|
||||
{
|
||||
@@ -498,13 +320,7 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
"classes": ("card", "wide", "collapse"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
card_fieldset("Timestamps", ("created_at", "modified_at")),
|
||||
)
|
||||
|
||||
list_filter = ("status", "exit_code", "machine_id")
|
||||
@@ -703,36 +519,9 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
@admin.display(description="Output", ordering="archiveresult__output_size")
|
||||
def output_summary(self, process):
|
||||
try:
|
||||
output_files = process.archiveresult.output_files or {}
|
||||
file_count, total_bytes = process.archiveresult.output_file_stats()
|
||||
except Process.archiveresult.RelatedObjectDoesNotExist:
|
||||
output_files = {}
|
||||
|
||||
if isinstance(output_files, str):
|
||||
try:
|
||||
output_files = json.loads(output_files)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
output_files = {}
|
||||
|
||||
file_count = 0
|
||||
total_bytes = 0
|
||||
|
||||
if isinstance(output_files, dict):
|
||||
file_count = len(output_files)
|
||||
items = output_files.values()
|
||||
elif isinstance(output_files, (list, tuple, set)):
|
||||
file_count = len(output_files)
|
||||
items = output_files
|
||||
else:
|
||||
items = ()
|
||||
|
||||
for metadata in items:
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
size = metadata.get("size", 0)
|
||||
try:
|
||||
total_bytes += int(size or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
file_count, total_bytes = 0, 0
|
||||
|
||||
file_label = "file" if file_count == 1 else "files"
|
||||
return format_html(
|
||||
@@ -750,26 +539,26 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
cmd = shlex.join(str(arg) for arg in process.cmd)
|
||||
else:
|
||||
cmd = str(process.cmd)
|
||||
return _render_copy_block(cmd)
|
||||
return render_copy_block(cmd)
|
||||
|
||||
@admin.display(description="Environment")
|
||||
def env_display(self, process):
|
||||
env_text = env_to_dotenv_text(process.env)
|
||||
if not env_text:
|
||||
return "-"
|
||||
return _render_copy_block(env_text, multiline=True)
|
||||
return render_copy_block(env_text, multiline=True)
|
||||
|
||||
@admin.display(description="Stdout")
|
||||
def stdout_display(self, process):
|
||||
if not process.stdout:
|
||||
return "-"
|
||||
return _render_copy_block(process.stdout, multiline=True)
|
||||
return render_copy_block(process.stdout, multiline=True)
|
||||
|
||||
@admin.display(description="Stderr")
|
||||
def stderr_display(self, process):
|
||||
if not process.stderr:
|
||||
return "-"
|
||||
return _render_copy_block(process.stderr, multiline=True)
|
||||
return render_copy_block(process.stderr, multiline=True)
|
||||
|
||||
@admin.display(description="ArchiveResult Output")
|
||||
def archiveresult_output_display(self, process):
|
||||
@@ -779,7 +568,7 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
return "-"
|
||||
if not output:
|
||||
return "-"
|
||||
return _render_copy_block(output, multiline=True)
|
||||
return render_copy_block(output, multiline=True)
|
||||
|
||||
|
||||
def register_admin(admin_site):
|
||||
|
||||
@@ -217,26 +217,27 @@ def get_isp_info(ip=None):
|
||||
}
|
||||
|
||||
|
||||
def get_local_interface(address: str) -> dict[str, str]:
|
||||
"""Describe the real host interface owning an IPv4 address."""
|
||||
for interface, addresses in psutil.net_if_addrs().items():
|
||||
if any(ip.address == address for ip in ip_addrs(addresses)):
|
||||
return {
|
||||
"iface": interface,
|
||||
"mac_address": next((mac.address for mac in mac_addrs(addresses)), ""),
|
||||
"ip_local": address,
|
||||
}
|
||||
raise ValueError(f"No host interface owns {address}")
|
||||
|
||||
|
||||
def get_host_network() -> dict[str, Any]:
|
||||
default_gateway_local_ip = get_local_ip()
|
||||
gateways = psutil.net_if_addrs()
|
||||
|
||||
for interface, ips in gateways.items():
|
||||
for local_ip in ip_addrs(ips):
|
||||
if default_gateway_local_ip == local_ip.address:
|
||||
mac_address = next(mac_addrs(ips)).address
|
||||
public_ip = get_public_ip()
|
||||
return {
|
||||
"hostname": max([socket.gethostname(), platform.node()], key=len),
|
||||
"iface": interface,
|
||||
"mac_address": mac_address,
|
||||
"ip_local": local_ip.address,
|
||||
"ip_public": public_ip,
|
||||
# "is_behind_nat": local_ip.address != public_ip,
|
||||
**get_isp_info(public_ip),
|
||||
}
|
||||
|
||||
raise Exception("Could not determine host network info")
|
||||
interface = get_local_interface(get_local_ip())
|
||||
public_ip = get_public_ip()
|
||||
return {
|
||||
"hostname": max([socket.gethostname(), platform.node()], key=len),
|
||||
**interface,
|
||||
"ip_public": public_ip,
|
||||
**get_isp_info(public_ip),
|
||||
}
|
||||
|
||||
|
||||
def get_os_info() -> dict[str, Any]:
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,50 @@
|
||||
"""Machine, interface, binary, and process persistence with process-local caches."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .binaries import Binary as Binary
|
||||
from .binaries import BinaryManager as BinaryManager
|
||||
from .binaries import _canonical_binary_name as _canonical_binary_name
|
||||
from .binaries import _find_existing_binary_for_reference as _find_existing_binary_for_reference
|
||||
from .constants import (
|
||||
BINARY_RECHECK_INTERVAL as BINARY_RECHECK_INTERVAL,
|
||||
)
|
||||
from .constants import (
|
||||
MACHINE_RECHECK_INTERVAL as MACHINE_RECHECK_INTERVAL,
|
||||
)
|
||||
from .constants import (
|
||||
NETWORK_INTERFACE_RECHECK_INTERVAL as NETWORK_INTERFACE_RECHECK_INTERVAL,
|
||||
)
|
||||
from .constants import (
|
||||
PID_REUSE_WINDOW as PID_REUSE_WINDOW,
|
||||
)
|
||||
from .constants import (
|
||||
PROCESS_PID_NAMESPACE_KEY as PROCESS_PID_NAMESPACE_KEY,
|
||||
)
|
||||
from .constants import (
|
||||
PROCESS_RECHECK_INTERVAL as PROCESS_RECHECK_INTERVAL,
|
||||
)
|
||||
from .constants import (
|
||||
PROCESS_TIMEOUT_GRACE as PROCESS_TIMEOUT_GRACE,
|
||||
)
|
||||
from .constants import (
|
||||
PSUTIL_AVAILABLE as PSUTIL_AVAILABLE,
|
||||
)
|
||||
from .constants import (
|
||||
START_TIME_TOLERANCE as START_TIME_TOLERANCE,
|
||||
)
|
||||
from .interfaces import NetworkInterface as NetworkInterface
|
||||
from .interfaces import NetworkInterfaceManager as NetworkInterfaceManager
|
||||
from .machines import Machine as Machine
|
||||
from .machines import MachineManager as MachineManager
|
||||
from .machines import _sanitize_machine_config as _sanitize_machine_config
|
||||
from .processes import Process as Process
|
||||
from .processes import ProcessManager as ProcessManager
|
||||
from .processes import _default_exit_code_for_unowned_process as _default_exit_code_for_unowned_process
|
||||
from .processes import _get_process_binary_env_keys as _get_process_binary_env_keys
|
||||
from .processes import get_current_pid_namespace as get_current_pid_namespace
|
||||
|
||||
_CURRENT_MACHINE: Machine | None = None
|
||||
_CURRENT_INTERFACE: NetworkInterface | None = None
|
||||
_CURRENT_BINARIES: dict[str, Binary] = {}
|
||||
_CURRENT_PROCESS: Process | None = None
|
||||
@@ -0,0 +1,425 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from django.db import models, transaction
|
||||
from django.utils import timezone
|
||||
from django.utils.functional import cached_property
|
||||
|
||||
from archivebox.base_models.models import ModelWithHealthStats
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import rprint
|
||||
from archivebox.machine import models as state
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
from archivebox.workers.models import ModelWithQueue
|
||||
|
||||
from .constants import BINARY_RECHECK_INTERVAL
|
||||
from .machines import Machine
|
||||
|
||||
|
||||
def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
|
||||
reference = str(reference or "").strip()
|
||||
if not reference:
|
||||
return None
|
||||
|
||||
qs = Binary.objects.filter(machine=machine)
|
||||
|
||||
direct_match = qs.filter(abspath=reference).order_by("-modified_at").first()
|
||||
if direct_match:
|
||||
return direct_match
|
||||
|
||||
ref_name = Path(reference).name
|
||||
if ref_name:
|
||||
named_match = qs.filter(name=ref_name).order_by("-modified_at").first()
|
||||
if named_match:
|
||||
return named_match
|
||||
|
||||
return qs.filter(name=reference).order_by("-modified_at").first()
|
||||
|
||||
|
||||
def _canonical_binary_name(name: Any) -> str:
|
||||
name = str(name or "").strip()
|
||||
if "/" in name or "\\" in name or name.startswith("~"):
|
||||
return Path(name).expanduser().name
|
||||
return name
|
||||
|
||||
|
||||
class BinaryManager(models.Manager):
|
||||
def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
|
||||
"""Get or create an Binary record from the database or cache."""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
cached = state._CURRENT_BINARIES.get(name)
|
||||
if cached and timezone.now() < cached.modified_at + timedelta(seconds=BINARY_RECHECK_INTERVAL):
|
||||
return cached
|
||||
state._CURRENT_BINARIES[name], _ = self.update_or_create(
|
||||
machine=Machine.current(),
|
||||
name=name,
|
||||
binprovider=binprovider,
|
||||
version=version,
|
||||
abspath=abspath,
|
||||
sha256=sha256,
|
||||
)
|
||||
return state._CURRENT_BINARIES[name]
|
||||
|
||||
def get_valid_binary(self, name: str, machine: Machine | None = None) -> Binary | None:
|
||||
"""Get a valid Binary for the given name on the current machine, or None if not found."""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
machine = machine or Machine.current()
|
||||
return (
|
||||
self.filter(
|
||||
machine=machine,
|
||||
name__iexact=name,
|
||||
)
|
||||
.exclude(abspath="")
|
||||
.exclude(abspath__isnull=True)
|
||||
.order_by("-modified_at")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
class Binary(ModelWithHealthStats, ModelWithQueue):
|
||||
"""
|
||||
Tracks a binary on a specific machine.
|
||||
|
||||
Simple queue lifecycle with 2 states:
|
||||
- queued: Binary needs to be installed
|
||||
- installed: Binary installed successfully (abspath, version, sha256 populated)
|
||||
|
||||
Installation is synchronous during queued→installed transition.
|
||||
If installation fails, Binary stays in queued with retry_at set for later retry.
|
||||
|
||||
BinaryService claims queued rows with a conditional update, then run()
|
||||
emits an abxpkg BinaryRequestEvent and persists the resolved installation.
|
||||
The database row is the only lifecycle state; there is intentionally no
|
||||
second in-memory state machine to reconcile after a worker interruption.
|
||||
"""
|
||||
|
||||
class StatusChoices(models.TextChoices):
|
||||
QUEUED = "queued", "Queued"
|
||||
INSTALLED = "installed", "Installed"
|
||||
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
machine = models.ForeignKey(Machine, on_delete=models.CASCADE, null=False)
|
||||
|
||||
# Binary metadata
|
||||
name = models.CharField(max_length=63, default="", null=False, blank=True, db_index=True)
|
||||
binproviders = models.CharField(
|
||||
max_length=127,
|
||||
default="env",
|
||||
null=False,
|
||||
blank=True,
|
||||
help_text="Comma-separated list of allowed providers: apt,brew,pip,npm,env",
|
||||
)
|
||||
overrides = models.JSONField(
|
||||
default=dict,
|
||||
blank=True,
|
||||
help_text="Provider-specific overrides: {'apt': {'install_args': ['pkg']}, ...}",
|
||||
)
|
||||
|
||||
# Installation results (populated after installation)
|
||||
binprovider = models.CharField(
|
||||
max_length=31,
|
||||
default="",
|
||||
null=False,
|
||||
blank=True,
|
||||
help_text="Provider that successfully installed this binary",
|
||||
)
|
||||
abspath = models.CharField(max_length=255, default="", null=False, blank=True)
|
||||
version = models.CharField(max_length=32, default="", null=False, blank=True)
|
||||
sha256 = models.CharField(max_length=64, default="", null=False, blank=True)
|
||||
|
||||
# Durable queue lifecycle fields
|
||||
status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
|
||||
retry_at = ModelWithQueue.RetryAtField(
|
||||
default=timezone.now,
|
||||
help_text="When to retry this binary installation",
|
||||
)
|
||||
|
||||
# Health stats
|
||||
num_uses_failed = models.PositiveIntegerField(default=0)
|
||||
num_uses_succeeded = models.PositiveIntegerField(default=0)
|
||||
|
||||
machine_id: uuid.UUID
|
||||
|
||||
INITIAL_STATE = StatusChoices.QUEUED
|
||||
ACTIVE_STATE = StatusChoices.QUEUED
|
||||
FINAL_STATES = (StatusChoices.INSTALLED,)
|
||||
FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
|
||||
active_state: str = StatusChoices.QUEUED
|
||||
warn_on_save_outside_runner = False
|
||||
|
||||
objects = BinaryManager() # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
|
||||
class Meta(ModelWithHealthStats.Meta, ModelWithQueue.Meta):
|
||||
app_label = "machine"
|
||||
verbose_name = "Binary"
|
||||
verbose_name_plural = "Binaries"
|
||||
unique_together = (("machine", "name", "abspath", "version", "sha256"),)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.name}@{self.binprovider}+{self.abspath}@{self.version}"
|
||||
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
"""A binary is valid if it has a resolved path and is marked installed."""
|
||||
return bool(self.abspath) and self.status == self.StatusChoices.INSTALLED
|
||||
|
||||
@property
|
||||
def can_install(self) -> bool:
|
||||
return bool(self.name and self.binproviders)
|
||||
|
||||
@cached_property
|
||||
def binary_info(self) -> dict:
|
||||
"""Return info about the binary."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"abspath": self.abspath,
|
||||
"version": self.version,
|
||||
"binprovider": self.binprovider,
|
||||
"is_valid": self.is_valid,
|
||||
}
|
||||
|
||||
@property
|
||||
def output_dir(self) -> Path:
|
||||
"""
|
||||
Get output directory for this binary's hook logs.
|
||||
Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
|
||||
"""
|
||||
return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""
|
||||
Convert Binary model instance to a JSON-serializable dict.
|
||||
"""
|
||||
from archivebox.config import VERSION
|
||||
|
||||
is_installed = bool(self.abspath and self.version)
|
||||
return {
|
||||
"type": "Binary" if is_installed else "BinaryRequest",
|
||||
"schema_version": VERSION,
|
||||
"id": str(self.id),
|
||||
"machine_id": str(self.machine_id),
|
||||
"name": self.name,
|
||||
"binproviders": self.binproviders,
|
||||
"overrides": self.overrides,
|
||||
"binprovider": self.binprovider,
|
||||
"abspath": self.abspath,
|
||||
"version": self.version,
|
||||
"sha256": self.sha256,
|
||||
"status": self.status,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
|
||||
"""
|
||||
Create/update Binary from JSON dict.
|
||||
|
||||
Handles two cases:
|
||||
1. From binaries.json: creates queued binary with name, binproviders, overrides
|
||||
2. From hook output: updates binary with abspath, version, sha256, binprovider
|
||||
|
||||
Args:
|
||||
record: JSON dict with 'name' and either:
|
||||
- 'binproviders', 'overrides' (from binaries.json)
|
||||
- 'abspath', 'version', 'sha256', 'binprovider' (from hook output)
|
||||
overrides: Not used
|
||||
|
||||
Returns:
|
||||
Binary instance or None
|
||||
"""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
name = _canonical_binary_name(record.get("name"))
|
||||
if not name:
|
||||
return None
|
||||
|
||||
abspath, version = record.get("abspath"), record.get("version")
|
||||
binproviders = record.get("binproviders")
|
||||
installed = bool(abspath and version) and (bool(binproviders) or "binproviders" not in record)
|
||||
if installed:
|
||||
defaults = {
|
||||
"abspath": abspath,
|
||||
"version": version,
|
||||
"sha256": record.get("sha256", ""),
|
||||
"binprovider": record.get("binprovider", "env"),
|
||||
"status": Binary.StatusChoices.INSTALLED,
|
||||
"retry_at": None,
|
||||
}
|
||||
if binproviders:
|
||||
defaults["binproviders"] = binproviders
|
||||
elif "binproviders" in record or ("overrides" in record and not abspath):
|
||||
binary_overrides = record.get("overrides", {})
|
||||
defaults = {
|
||||
"binproviders": record.get("binproviders", "env"),
|
||||
"overrides": binary_overrides if isinstance(binary_overrides, dict) else {},
|
||||
"status": Binary.StatusChoices.QUEUED,
|
||||
"retry_at": timezone.now(),
|
||||
}
|
||||
else:
|
||||
return None
|
||||
|
||||
binary, _ = Binary.objects.update_or_create(machine=Machine.current(), name=name, defaults=defaults)
|
||||
if installed:
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
binary.symlink_to_lib_bin_after_commit(get_config().ABXPKG_LIB_DIR / "bin")
|
||||
return binary
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Execute binary installation through the ArchiveBox binary runner.
|
||||
"""
|
||||
from archivebox.services.runner import run_binary
|
||||
|
||||
run_binary(str(self.id))
|
||||
|
||||
def install(self) -> bool:
|
||||
"""Run one synchronous installation attempt for a claimed Binary."""
|
||||
if self.status == self.StatusChoices.INSTALLED:
|
||||
return True
|
||||
if not self.can_install:
|
||||
return False
|
||||
|
||||
rprint(f"[cyan] 🔄 installing {self.name}[/cyan]", file=sys.stderr)
|
||||
self.run()
|
||||
self.refresh_from_db()
|
||||
if self.status != self.StatusChoices.INSTALLED:
|
||||
self.update_and_requeue(
|
||||
retry_at=timezone.now() + timedelta(seconds=300),
|
||||
status=self.StatusChoices.QUEUED,
|
||||
)
|
||||
self.increment_health_stats(success=False)
|
||||
raise RuntimeError(f"Binary {self.name} installation failed")
|
||||
|
||||
self.update_and_requeue(retry_at=None, status=self.StatusChoices.INSTALLED)
|
||||
self.increment_health_stats(success=True)
|
||||
return True
|
||||
|
||||
def advance_lifecycle(self) -> bool:
|
||||
"""Advance the explicit binary lifecycle after its queue row is claimed."""
|
||||
return self.install()
|
||||
|
||||
def install_claimed(self, *, lock_seconds: int = 600) -> bool:
|
||||
if not self.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
self.refresh_from_db()
|
||||
return self.advance_lifecycle()
|
||||
|
||||
def cleanup(self):
|
||||
"""
|
||||
Clean up background binary installation hooks.
|
||||
|
||||
Called after an installation attempt if needed (not typically used for binaries
|
||||
since installations are foreground, but included for consistency).
|
||||
"""
|
||||
|
||||
# Clean up .pid files from output directory
|
||||
output_dir = self.output_dir
|
||||
if output_dir.exists():
|
||||
for pid_file in output_dir.glob("**/*.pid"):
|
||||
pid_file.unlink(missing_ok=True)
|
||||
|
||||
def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
|
||||
"""
|
||||
Symlink this binary into a derived lib/bin directory for human-facing convenience.
|
||||
|
||||
After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
|
||||
we can optionally expose a flat convenience directory for shell users.
|
||||
ArchiveBox/abx-dl runtime lookup must use the provider-specific ABXPKG_LIB_DIR
|
||||
paths, not this indirection.
|
||||
|
||||
Args:
|
||||
lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)
|
||||
|
||||
Returns:
|
||||
Path to the created symlink, or None if symlinking failed
|
||||
|
||||
Example:
|
||||
>>> binary = Binary.objects.get(name='yt-dlp')
|
||||
>>> binary.symlink_to_lib_bin('/data/lib/arm64-darwin/bin')
|
||||
Path('/data/lib/arm64-darwin/bin/yt-dlp')
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if not self.abspath:
|
||||
return None
|
||||
|
||||
binary_abspath = Path(self.abspath).resolve()
|
||||
lib_bin_dir = Path(lib_bin_dir).resolve()
|
||||
binary_parts = binary_abspath.parts
|
||||
try:
|
||||
app_index = next(index for index, part in enumerate(binary_parts) if part.endswith(".app"))
|
||||
except StopIteration:
|
||||
app_index = -1
|
||||
|
||||
# Create the derived convenience bin dir if it doesn't exist.
|
||||
try:
|
||||
lib_bin_dir.mkdir(parents=True, exist_ok=True)
|
||||
except (OSError, PermissionError) as e:
|
||||
print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# Expose the canonical Binary.name in the convenience bin dir. Some providers point
|
||||
# abspath at implementation files like cli.js or manifest.json; those
|
||||
# are valid targets, but they are not user-facing binary names.
|
||||
binary_name = _canonical_binary_name(self.name) or binary_abspath.name
|
||||
symlink_path = lib_bin_dir / binary_name
|
||||
|
||||
if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"):
|
||||
if symlink_path.exists() or symlink_path.is_symlink():
|
||||
try:
|
||||
symlink_path.unlink()
|
||||
except (OSError, PermissionError) as e:
|
||||
print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
|
||||
return None
|
||||
return binary_abspath
|
||||
|
||||
# Remove existing symlink/file if it exists
|
||||
if symlink_path.exists() or symlink_path.is_symlink():
|
||||
try:
|
||||
# Check if it's already pointing to the right place
|
||||
if symlink_path.is_symlink() and symlink_path.resolve() == binary_abspath:
|
||||
# Already correctly symlinked, nothing to do
|
||||
return symlink_path
|
||||
|
||||
# Remove old symlink/file
|
||||
symlink_path.unlink()
|
||||
except (OSError, PermissionError) as e:
|
||||
print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# Create new symlink
|
||||
try:
|
||||
symlink_path.symlink_to(binary_abspath)
|
||||
return symlink_path
|
||||
except (OSError, PermissionError) as e:
|
||||
print(f"Failed to create symlink {symlink_path} -> {binary_abspath}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
def symlink_to_lib_bin_after_commit(self, lib_bin_dir: str | Path) -> None:
|
||||
"""
|
||||
Symlink after the current DB transaction commits.
|
||||
|
||||
Binary rows are projections of provider/hook state and are allowed to be
|
||||
updated directly, but filesystem writes must not run while an outer
|
||||
transaction is still open. Refetch after commit so the symlink points at
|
||||
the committed row, not a possibly-rolled-back in-memory value.
|
||||
"""
|
||||
binary_id = self.id
|
||||
lib_bin_path = Path(lib_bin_dir)
|
||||
|
||||
def create_symlink() -> None:
|
||||
binary = type(self).objects.filter(id=binary_id).first()
|
||||
if binary is not None:
|
||||
binary.symlink_to_lib_bin(lib_bin_path)
|
||||
|
||||
transaction.on_commit(create_symlink)
|
||||
@@ -0,0 +1,26 @@
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
_psutil: Any | None = None
|
||||
try:
|
||||
import psutil as _psutil_import
|
||||
|
||||
PSUTIL_AVAILABLE = True
|
||||
except ImportError:
|
||||
PSUTIL_AVAILABLE = False
|
||||
else:
|
||||
_psutil = _psutil_import
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import psutil
|
||||
else:
|
||||
psutil = cast(Any, _psutil)
|
||||
|
||||
MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
|
||||
NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
|
||||
BINARY_RECHECK_INTERVAL = 1 * 30 * 60
|
||||
PROCESS_RECHECK_INTERVAL = 60 # Re-validate every 60 seconds
|
||||
PID_REUSE_WINDOW = timedelta(hours=24) # Max age for considering a PID match valid
|
||||
PROCESS_TIMEOUT_GRACE = timedelta(seconds=30) # Extra margin before force-cleaning timed-out RUNNING rows
|
||||
START_TIME_TOLERANCE = 5.0 # Seconds tolerance for start time matching
|
||||
PROCESS_PID_NAMESPACE_KEY = "_ARCHIVEBOX_PID_NAMESPACE"
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
|
||||
from django.db import IntegrityError, models
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.base_models.models import ModelWithHealthStats
|
||||
from archivebox.machine import models as state
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
|
||||
from ..detect import get_host_network
|
||||
from .constants import NETWORK_INTERFACE_RECHECK_INTERVAL
|
||||
from .machines import Machine
|
||||
|
||||
|
||||
class NetworkInterfaceManager(models.Manager):
|
||||
def current(self) -> NetworkInterface:
|
||||
return NetworkInterface.current()
|
||||
|
||||
|
||||
class NetworkInterface(ModelWithHealthStats):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
|
||||
mac_address = models.CharField(max_length=17, default=None, null=False, editable=False)
|
||||
ip_public = models.GenericIPAddressField(default=None, null=False, editable=False)
|
||||
ip_local = models.GenericIPAddressField(default=None, null=False, editable=False)
|
||||
dns_server = models.GenericIPAddressField(default=None, null=False, editable=False)
|
||||
hostname = models.CharField(max_length=63, default="", null=False)
|
||||
iface = models.CharField(max_length=15, default="", null=False)
|
||||
isp = models.CharField(max_length=63, default="", null=False)
|
||||
city = models.CharField(max_length=63, default="", null=False)
|
||||
region = models.CharField(max_length=63, default="", null=False)
|
||||
country = models.CharField(max_length=63, default="", null=False)
|
||||
objects = NetworkInterfaceManager() # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
machine_id: uuid.UUID
|
||||
|
||||
class Meta(ModelWithHealthStats.Meta):
|
||||
app_label = "machine"
|
||||
unique_together = (("machine", "ip_public", "ip_local", "dns_server"),)
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["machine", "ip_public", "ip_local", "dns_server"],
|
||||
name="unique_network_interface_identity",
|
||||
),
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def current(cls, refresh: bool = False) -> NetworkInterface:
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
machine = Machine.current(refresh=refresh)
|
||||
if state._CURRENT_INTERFACE and state._CURRENT_INTERFACE.machine_id == machine.id:
|
||||
if not refresh:
|
||||
# Callers that pass refresh=False are asking for attribution to
|
||||
# the currently known interface, not for public-IP/ISP probing.
|
||||
# Maintenance paths create many short-lived services per run;
|
||||
# expiring this in-memory object by age forced every crawl to
|
||||
# hit external network APIs even though Process rows only need
|
||||
# a stable existing FK. Active downloading paths opt into live
|
||||
# detection with refresh=True.
|
||||
return state._CURRENT_INTERFACE
|
||||
if timezone.now() < state._CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
|
||||
return state._CURRENT_INTERFACE
|
||||
state._CURRENT_INTERFACE = None
|
||||
|
||||
if not refresh:
|
||||
state._CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
|
||||
if state._CURRENT_INTERFACE is not None:
|
||||
return state._CURRENT_INTERFACE
|
||||
|
||||
net_info = get_host_network()
|
||||
lookup = dict(
|
||||
machine=machine,
|
||||
ip_public=net_info.pop("ip_public"),
|
||||
ip_local=net_info.pop("ip_local"),
|
||||
dns_server=net_info.pop("dns_server"),
|
||||
)
|
||||
state._CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
|
||||
if state._CURRENT_INTERFACE is None:
|
||||
try:
|
||||
state._CURRENT_INTERFACE = cls.objects.create(**lookup, **net_info)
|
||||
except IntegrityError:
|
||||
state._CURRENT_INTERFACE = cls.objects.filter(**lookup).order_by("-modified_at", "-created_at").first()
|
||||
if state._CURRENT_INTERFACE is None:
|
||||
raise
|
||||
else:
|
||||
# Avoid update_or_create() here: command startup calls this before
|
||||
# leadership handoff, and SQLite should not take a write lock unless
|
||||
# the cached interface metadata is actually stale.
|
||||
if refresh or timezone.now() >= state._CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
|
||||
updates = ["modified_at"]
|
||||
for key, value in net_info.items():
|
||||
if state._CURRENT_INTERFACE.__dict__.get(key) != value:
|
||||
setattr(state._CURRENT_INTERFACE, key, value)
|
||||
updates.append(key)
|
||||
if len(updates) > 1:
|
||||
state._CURRENT_INTERFACE.save(update_fields=updates)
|
||||
return state._CURRENT_INTERFACE
|
||||
@@ -0,0 +1,281 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.base_models.models import ModelWithHealthStats, normalize_config_json_values
|
||||
from archivebox.machine import models as state
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
|
||||
from ..detect import get_host_guid, get_host_stats, get_os_info, get_vm_info
|
||||
from .constants import MACHINE_RECHECK_INTERVAL
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .interfaces import NetworkInterface
|
||||
|
||||
|
||||
def _sanitize_machine_config(config: dict[str, Any] | None, *, lib_dir: str | Path | None = None) -> dict[str, Any]:
|
||||
"""Validate ``Machine.config`` in place.
|
||||
|
||||
Drops stale ``*_BINARY`` overrides whose path no longer exists or whose
|
||||
path falls outside of ``ABXPKG_LIB_DIR`` (so a binary uninstall or a lib_dir
|
||||
move clears the override automatically). Non-``_BINARY`` keys
|
||||
(``BASE_URL``, ``SERVER_SECURITY_MODE``, plugin tunables, etc.) are
|
||||
pass-through — they're arbitrary config overrides and not ours to filter.
|
||||
"""
|
||||
if not isinstance(config, dict):
|
||||
return {}
|
||||
|
||||
sanitized = dict(config)
|
||||
active_lib_dir = Path(lib_dir).expanduser().absolute() if lib_dir else None
|
||||
for key, value in list(sanitized.items()):
|
||||
if not str(key).endswith("_BINARY"):
|
||||
continue
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
value = value.strip()
|
||||
if not value:
|
||||
sanitized.pop(key, None)
|
||||
continue
|
||||
if "/" in value or value.startswith("~"):
|
||||
try:
|
||||
path = Path(value).expanduser()
|
||||
if not path.exists():
|
||||
sanitized.pop(key, None)
|
||||
continue
|
||||
if active_lib_dir is not None:
|
||||
resolved_path = path.absolute()
|
||||
try:
|
||||
resolved_path.relative_to(active_lib_dir)
|
||||
except ValueError:
|
||||
sanitized.pop(key, None)
|
||||
except OSError:
|
||||
sanitized.pop(key, None)
|
||||
return sanitized
|
||||
|
||||
|
||||
class MachineManager(models.Manager):
|
||||
def current(self) -> Machine:
|
||||
return Machine.current()
|
||||
|
||||
|
||||
class Machine(ModelWithHealthStats):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
|
||||
hostname = models.CharField(max_length=63, default=None, null=False)
|
||||
hw_in_docker = models.BooleanField(default=False, null=False)
|
||||
hw_in_vm = models.BooleanField(default=False, null=False)
|
||||
hw_manufacturer = models.CharField(max_length=63, default=None, null=False)
|
||||
hw_product = models.CharField(max_length=63, default=None, null=False)
|
||||
hw_uuid = models.CharField(max_length=255, default=None, null=False)
|
||||
os_arch = models.CharField(max_length=15, default=None, null=False)
|
||||
os_family = models.CharField(max_length=15, default=None, null=False)
|
||||
os_platform = models.CharField(max_length=63, default=None, null=False)
|
||||
os_release = models.CharField(max_length=63, default=None, null=False)
|
||||
os_kernel = models.CharField(max_length=255, default=None, null=False)
|
||||
stats = models.JSONField(default=dict, null=True, blank=True)
|
||||
config = models.JSONField(
|
||||
default=dict,
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text="Machine-specific config overrides.",
|
||||
)
|
||||
num_uses_failed = models.PositiveIntegerField(default=0)
|
||||
num_uses_succeeded = models.PositiveIntegerField(default=0)
|
||||
|
||||
objects = MachineManager() # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
networkinterface_set: models.Manager[NetworkInterface]
|
||||
|
||||
class Meta(ModelWithHealthStats.Meta):
|
||||
app_label = "machine"
|
||||
|
||||
@classmethod
|
||||
def current(cls, refresh: bool = False) -> Machine:
|
||||
if refresh:
|
||||
state._CURRENT_MACHINE = None
|
||||
if state._CURRENT_MACHINE:
|
||||
if timezone.now() < state._CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
|
||||
# Reconciliation can save() and clear the shared cache; keep
|
||||
# this call's instance so we never return None after that save.
|
||||
machine = state._CURRENT_MACHINE
|
||||
# One-time-per-process reconciliation between ArchiveBox.conf
|
||||
# and Machine.config. Fast-path: bool check + early-return when
|
||||
# the sync has already run, so the cached-machine return path
|
||||
# stays sub-microsecond.
|
||||
try:
|
||||
from archivebox.config.collection import sync_machine_and_file
|
||||
|
||||
sync_machine_and_file(machine)
|
||||
except Exception:
|
||||
pass
|
||||
return machine
|
||||
else:
|
||||
state._CURRENT_MACHINE = None
|
||||
|
||||
host_guid = get_host_guid()
|
||||
# save() intentionally invalidates state._CURRENT_MACHINE so config edits
|
||||
# made through other instances become visible in this process. Host
|
||||
# refresh (after seven days), sanitation, and reconciliation can all
|
||||
# save here, so the working instance must not live only in that cache.
|
||||
# Otherwise a routine refresh leaves startup dereferencing None.
|
||||
try:
|
||||
machine = cls.objects.get(guid=host_guid)
|
||||
except cls.DoesNotExist:
|
||||
config = {}
|
||||
try:
|
||||
from archivebox.config.collection import _coerce_from_str_dict, _load_file_config_dict
|
||||
|
||||
file_config, _file_mtime = _load_file_config_dict()
|
||||
config = _coerce_from_str_dict(file_config)
|
||||
except Exception:
|
||||
config = {}
|
||||
machine = cls.objects.create(
|
||||
guid=host_guid,
|
||||
hostname=socket.gethostname(),
|
||||
config=config,
|
||||
**get_os_info(),
|
||||
**get_vm_info(),
|
||||
stats=get_host_stats(),
|
||||
)
|
||||
else:
|
||||
if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
|
||||
for key, value in {
|
||||
"hostname": socket.gethostname(),
|
||||
**get_os_info(),
|
||||
**get_vm_info(),
|
||||
"stats": get_host_stats(),
|
||||
}.items():
|
||||
setattr(machine, key, value)
|
||||
machine.save(
|
||||
update_fields=[
|
||||
"hostname",
|
||||
"hw_in_docker",
|
||||
"hw_in_vm",
|
||||
"hw_manufacturer",
|
||||
"hw_product",
|
||||
"hw_uuid",
|
||||
"os_arch",
|
||||
"os_family",
|
||||
"os_platform",
|
||||
"os_release",
|
||||
"os_kernel",
|
||||
"stats",
|
||||
"modified_at",
|
||||
],
|
||||
)
|
||||
machine = cls._sanitize_config(machine)
|
||||
# Same one-time sync as the cached-return path. Triggers here on the
|
||||
# very first ``Machine.current()`` call in a process before any
|
||||
# cached return can occur.
|
||||
try:
|
||||
from archivebox.config.collection import sync_machine_and_file
|
||||
|
||||
sync_machine_and_file(machine)
|
||||
except Exception:
|
||||
pass
|
||||
# Publish only after all save-capable steps; publishing earlier would
|
||||
# let those steps invalidate the cache we are in the middle of filling.
|
||||
state._CURRENT_MACHINE = machine
|
||||
return machine
|
||||
|
||||
@classmethod
|
||||
def _sanitize_config(cls, machine: Machine) -> Machine:
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
sanitized = _sanitize_machine_config(machine.config, lib_dir=get_config(include_machine=False).ABXPKG_LIB_DIR)
|
||||
current = machine.config or {}
|
||||
if sanitized != current:
|
||||
machine.config = sanitized
|
||||
machine.save(update_fields=["config", "modified_at"])
|
||||
return machine
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""
|
||||
Convert Machine model instance to a JSON-serializable dict.
|
||||
"""
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return {
|
||||
"type": "Machine",
|
||||
"schema_version": VERSION,
|
||||
"id": str(self.id),
|
||||
"guid": self.guid,
|
||||
"hostname": self.hostname,
|
||||
"hw_in_docker": self.hw_in_docker,
|
||||
"hw_in_vm": self.hw_in_vm,
|
||||
"hw_manufacturer": self.hw_manufacturer,
|
||||
"hw_product": self.hw_product,
|
||||
"hw_uuid": self.hw_uuid,
|
||||
"os_arch": self.os_arch,
|
||||
"os_family": self.os_family,
|
||||
"os_platform": self.os_platform,
|
||||
"os_kernel": self.os_kernel,
|
||||
"os_release": self.os_release,
|
||||
"stats": self.stats,
|
||||
"config": redact_sensitive_config(self.config),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_json(record: dict[str, Any], overrides: dict[str, Any] | None = None):
|
||||
"""
|
||||
Update Machine config from JSON dict.
|
||||
|
||||
Args:
|
||||
record: JSON dict with 'config': {key: value} patch
|
||||
overrides: Not used
|
||||
|
||||
Returns:
|
||||
Machine instance or None
|
||||
"""
|
||||
config_patch = _sanitize_machine_config(record.get("config"))
|
||||
if config_patch:
|
||||
machine = Machine.current()
|
||||
machine.config = _sanitize_machine_config(machine.config)
|
||||
machine.config.update(config_patch)
|
||||
machine.save(update_fields=["config"])
|
||||
return machine
|
||||
return None
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
normalized_config = normalize_config_json_values(self.config)
|
||||
if normalized_config != self.config:
|
||||
self.config = normalized_config
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if update_fields is not None:
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
|
||||
|
||||
# Drop the ``Machine.current()`` module-level cache on every save so
|
||||
# config edits (admin form, from_json, etc.) become live in the same
|
||||
# process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
|
||||
# The cache reads ``state._CURRENT_MACHINE.modified_at`` and that value
|
||||
# never moves forward on the cached object even when the row is
|
||||
# updated in the DB, so without this we'd keep serving stale
|
||||
# ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
|
||||
update_fields = kwargs.get("update_fields")
|
||||
super().save(*args, **kwargs)
|
||||
if state._CURRENT_MACHINE is not None and state._CURRENT_MACHINE.pk == self.pk:
|
||||
state._CURRENT_MACHINE = None
|
||||
|
||||
# Mirror Machine.config into ArchiveBox.conf so the two stores stay
|
||||
# 1:1. Skipped when ``update_fields`` is set and doesn't touch
|
||||
# ``config`` (binary autodetection + ``hostname``/``stats`` refreshes
|
||||
# save fields we don't need to disk-mirror, which keeps hot paths
|
||||
# zero-IO). Errors during mirroring are swallowed: the DB write
|
||||
# already succeeded and we don't want a config-file write hiccup to
|
||||
# turn a routine save into a 500.
|
||||
if update_fields is not None and "config" not in update_fields:
|
||||
return
|
||||
try:
|
||||
from archivebox.config.collection import mirror_machine_config_to_file
|
||||
|
||||
mirror_machine_config_to_file(self.config)
|
||||
except Exception:
|
||||
pass
|
||||
File diff suppressed because it is too large.
Load diff
@@ -406,3 +406,73 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
|
||||
f"ABXPKG_LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.",
|
||||
) from e
|
||||
return False
|
||||
|
||||
|
||||
MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES = 256 * 1024 * 1024
|
||||
|
||||
|
||||
MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES = MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES
|
||||
|
||||
|
||||
def _read_cgroup_limit(path: Path) -> int | None:
|
||||
try:
|
||||
raw_value = path.read_text().strip()
|
||||
return None if raw_value == "max" else int(raw_value)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def effective_available_memory_bytes() -> int:
|
||||
import psutil
|
||||
|
||||
available = psutil.virtual_memory().available + psutil.swap_memory().free
|
||||
|
||||
cgroup_root = Path("/sys/fs/cgroup")
|
||||
memory_max = _read_cgroup_limit(cgroup_root / "memory.max")
|
||||
memory_current = _read_cgroup_limit(cgroup_root / "memory.current")
|
||||
swap_max = _read_cgroup_limit(cgroup_root / "memory.swap.max")
|
||||
swap_current = _read_cgroup_limit(cgroup_root / "memory.swap.current")
|
||||
if memory_max is not None and memory_current is not None:
|
||||
cgroup_available = max(memory_max - memory_current, 0)
|
||||
if swap_max is not None and swap_current is not None:
|
||||
cgroup_available += max(swap_max - swap_current, 0)
|
||||
available = min(available, cgroup_available)
|
||||
|
||||
return available
|
||||
|
||||
|
||||
def _require_available_memory(operation: str, idle_message: str, available_bytes: int, required_bytes: int) -> None:
|
||||
from archivebox.misc.logging import STDERR
|
||||
|
||||
if available_bytes >= required_bytes:
|
||||
return
|
||||
|
||||
available_mib = available_bytes // (1024 * 1024)
|
||||
required_mib = required_bytes // (1024 * 1024)
|
||||
STDERR.print(f"[red][X] Not enough available memory to {operation} safely.[/red]")
|
||||
STDERR.print(
|
||||
f" Available RAM + swap: {available_mib} MiB; at least {required_mib} MiB must be free before this operation.",
|
||||
)
|
||||
STDERR.print(idle_message)
|
||||
STDERR.print(" Use a host/container with at least 1 GB RAM or configure swap, then run the same command again.")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def require_server_worker_memory(available_bytes: int | None = None) -> None:
|
||||
available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
|
||||
_require_available_memory(
|
||||
"start ArchiveBox",
|
||||
" No server, runner, or Sonic workers were started.",
|
||||
available_bytes,
|
||||
MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES,
|
||||
)
|
||||
|
||||
|
||||
def require_dependency_install_memory(available_bytes: int | None = None) -> None:
|
||||
available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes
|
||||
_require_available_memory(
|
||||
"install ArchiveBox dependencies",
|
||||
" No plugin dependency installers were started.",
|
||||
available_bytes,
|
||||
MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES,
|
||||
)
|
||||
+24
-1
@@ -350,7 +350,9 @@ def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
|
||||
for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]):
|
||||
try:
|
||||
open_files = proc.open_files()
|
||||
except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess):
|
||||
except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess, RuntimeError):
|
||||
# macOS libproc can report an uninspectable process as RuntimeError.
|
||||
# Optional holder diagnostics must not abort the database operation.
|
||||
continue
|
||||
for open_file in open_files:
|
||||
try:
|
||||
@@ -662,3 +664,24 @@ def apply_migrations(
|
||||
out1 = retry_sqlite_locks(migrate, label="applying migrations")
|
||||
|
||||
return [line.strip() for line in out1.readlines() if line.strip()]
|
||||
|
||||
|
||||
def uuid_ref_query(field_name: str, ref: str):
|
||||
from uuid import UUID
|
||||
from django.db.models import Q
|
||||
|
||||
raw_ref = str(ref or "").strip()
|
||||
query = Q(**{f"{field_name}__startswith": raw_ref})
|
||||
if raw_ref:
|
||||
query |= Q(**{f"{field_name}__icontains": raw_ref})
|
||||
try:
|
||||
parsed_uuid = UUID(raw_ref)
|
||||
except (TypeError, ValueError):
|
||||
normalized_ref = raw_ref.replace("-", "")
|
||||
if normalized_ref and normalized_ref != raw_ref:
|
||||
query |= Q(**{f"{field_name}__startswith": normalized_ref})
|
||||
query |= Q(**{f"{field_name}__icontains": normalized_ref})
|
||||
else:
|
||||
query |= Q(**{field_name: parsed_uuid})
|
||||
query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
|
||||
return query
|
||||
@@ -1,260 +0,0 @@
|
||||
# Bootable hashing helpers (file content + mtime/size cache, mime detection).
|
||||
# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
|
||||
|
||||
import hashlib
|
||||
import mimetypes
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
|
||||
"""Internal function to calculate file hash with cache key based on path, size and mtime."""
|
||||
sha256_hash = hashlib.sha256()
|
||||
|
||||
with open(filepath, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(chunk)
|
||||
|
||||
return sha256_hash.hexdigest()
|
||||
|
||||
|
||||
@lru_cache(maxsize=10)
|
||||
def hash_file(file_path: Path, pwd: Path | None = None) -> str:
|
||||
"""Calculate SHA256 hash of a file with caching based on path, size and mtime."""
|
||||
pwd = Path(pwd) if pwd else None
|
||||
file_path = Path(file_path)
|
||||
if not file_path.is_absolute():
|
||||
file_path = pwd / file_path if pwd else file_path.absolute()
|
||||
|
||||
abs_path = file_path.resolve()
|
||||
stat_info = abs_path.stat()
|
||||
|
||||
return _cached_file_hash(
|
||||
str(abs_path),
|
||||
stat_info.st_size,
|
||||
stat_info.st_mtime,
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=10)
|
||||
def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
|
||||
"""Calculate SHA256 hashes for all files and directories recursively."""
|
||||
pwd = Path(pwd) if pwd else None
|
||||
dir_path = Path(dir_path)
|
||||
if not dir_path.is_absolute():
|
||||
dir_path = pwd / dir_path if pwd else dir_path.absolute()
|
||||
|
||||
if not dir_path.is_dir():
|
||||
raise ValueError(f"Not a directory: {dir_path}")
|
||||
if max_depth < -1:
|
||||
raise ValueError(f"max_depth must be >= -1, got {max_depth}")
|
||||
|
||||
# Get all files recursively
|
||||
all_files = get_dir_entries(
|
||||
dir_path,
|
||||
pwd=pwd,
|
||||
recursive=True,
|
||||
include_files=True,
|
||||
include_dirs=False,
|
||||
filter_func=filter_func,
|
||||
)
|
||||
|
||||
hashes: dict[str, str] = {}
|
||||
hashable_summary = []
|
||||
|
||||
# Calculate hashes for all files
|
||||
for subfile in all_files:
|
||||
subfile_path = dir_path / subfile
|
||||
sha256_hash = hash_file(subfile_path)
|
||||
hashes[subfile] = sha256_hash
|
||||
hashable_summary.append(f"{sha256_hash} ./{subfile}")
|
||||
|
||||
# Calculate hashes for all directories
|
||||
subdirs = get_dir_entries(
|
||||
dir_path,
|
||||
pwd=pwd,
|
||||
recursive=True,
|
||||
include_files=False,
|
||||
include_dirs=True,
|
||||
include_hidden=False,
|
||||
filter_func=filter_func,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
|
||||
for subdir in subdirs:
|
||||
subdir_path = dir_path / subdir
|
||||
subdir_hashes = get_dir_hashes(
|
||||
subdir_path,
|
||||
filter_func=filter_func,
|
||||
max_depth=0,
|
||||
)
|
||||
hashes[subdir] = subdir_hashes["."]
|
||||
|
||||
# Filter results by max_depth
|
||||
if max_depth >= 0:
|
||||
hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
|
||||
|
||||
# Calculate root directory hash
|
||||
hashable_summary.sort()
|
||||
root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
|
||||
hashes["."] = root_sha256
|
||||
|
||||
return hashes
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def get_dir_entries(
|
||||
dir_path: Path,
|
||||
pwd: Path | None = None,
|
||||
recursive: bool = True,
|
||||
include_files: bool = True,
|
||||
include_dirs: bool = True,
|
||||
include_hidden: bool = False,
|
||||
filter_func: Callable | None = None,
|
||||
max_depth: int = -1,
|
||||
) -> tuple[str, ...]:
|
||||
"""Get filtered list of directory entries."""
|
||||
pwd = Path(pwd) if pwd else None
|
||||
dir_path = Path(dir_path)
|
||||
if not dir_path.is_absolute():
|
||||
dir_path = pwd / dir_path if pwd else dir_path.absolute()
|
||||
|
||||
results = []
|
||||
|
||||
def process_path(path: Path, depth: int):
|
||||
if not include_hidden and path.name.startswith("."):
|
||||
return False
|
||||
if max_depth >= 0 and depth > max_depth:
|
||||
return False
|
||||
if filter_func:
|
||||
info = {
|
||||
"abspath": str(path.absolute()),
|
||||
"relpath": str(path.relative_to(dir_path)),
|
||||
}
|
||||
if not filter_func(info):
|
||||
return False
|
||||
return True
|
||||
|
||||
for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
|
||||
current_depth = len(path.relative_to(dir_path).parts)
|
||||
|
||||
if path.is_file() and include_files and process_path(path, current_depth):
|
||||
results.append(str(path.relative_to(dir_path)))
|
||||
elif path.is_dir() and include_dirs and process_path(path, current_depth):
|
||||
results.append(str(path.relative_to(dir_path)))
|
||||
|
||||
if not recursive:
|
||||
break
|
||||
|
||||
return tuple(sorted(results)) # Make immutable for caching
|
||||
|
||||
|
||||
@lru_cache(maxsize=1024)
|
||||
def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
|
||||
"""Calculate sizes for all files and directories recursively."""
|
||||
sizes: dict[str, int] = {}
|
||||
hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
|
||||
dir_path = Path(dir_path)
|
||||
|
||||
for path_key in hashes:
|
||||
full_path = dir_path / path_key
|
||||
if full_path.is_file():
|
||||
sizes[path_key] = full_path.stat().st_size
|
||||
else:
|
||||
total = 0
|
||||
for file_path in full_path.rglob("*"):
|
||||
if file_path.is_file() and not file_path.name.startswith("."):
|
||||
total += file_path.stat().st_size
|
||||
sizes[path_key + "/"] = total
|
||||
|
||||
return sizes
|
||||
|
||||
|
||||
@lru_cache(maxsize=10)
|
||||
def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
|
||||
"""Get detailed information about directory contents including hashes and sizes."""
|
||||
pwd = Path(pwd) if pwd else None
|
||||
dir_path = Path(dir_path)
|
||||
if not dir_path.is_absolute():
|
||||
dir_path = pwd / dir_path if pwd else dir_path.absolute()
|
||||
|
||||
hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
|
||||
sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
|
||||
|
||||
num_total_subpaths = sum(1 for name in hashes if name != ".")
|
||||
details = {}
|
||||
|
||||
for filename, sha256_hash in sorted(hashes.items()):
|
||||
abs_path = (dir_path / filename).resolve()
|
||||
stat_info = abs_path.stat()
|
||||
num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
|
||||
is_dir = abs_path.is_dir()
|
||||
if is_dir:
|
||||
mime_type = "inode/directory"
|
||||
basename = abs_path.name
|
||||
extension = ""
|
||||
num_bytes = sizes[filename + "/"]
|
||||
if filename == ".":
|
||||
num_subpaths = num_total_subpaths
|
||||
else:
|
||||
filename += "/"
|
||||
num_subpaths = num_subpaths
|
||||
else: # is_file
|
||||
num_subpaths = None
|
||||
mime_type = mimetypes.guess_type(str(abs_path))[0]
|
||||
extension = abs_path.suffix
|
||||
basename = abs_path.name.rsplit(extension, 1)[0]
|
||||
num_bytes = sizes[filename]
|
||||
|
||||
details[filename] = {
|
||||
"basename": basename,
|
||||
"mime_type": mime_type,
|
||||
"extension": extension,
|
||||
"num_subpaths": num_subpaths,
|
||||
"num_bytes": num_bytes,
|
||||
"hash_sha256": sha256_hash,
|
||||
"created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
|
||||
"modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
|
||||
}
|
||||
|
||||
if filter_func and not filter_func(details[filename]):
|
||||
del details[filename]
|
||||
|
||||
return details
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
|
||||
dir_info = get_dir_info(Path("."), max_depth=6)
|
||||
with open(".hashes.json", "w") as f:
|
||||
json.dump(dir_info, f, indent=4)
|
||||
print("Wrote .hashes.json")
|
||||
|
||||
# Example output:
|
||||
# {
|
||||
# ".": {
|
||||
# "basename": "misc",
|
||||
# "mime_type": "inode/directory",
|
||||
# "extension": "",
|
||||
# "num_subpaths": 25,
|
||||
# "num_bytes": 214677,
|
||||
# "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
|
||||
# "created_at": "2024-12-04T00:08:38.537449",
|
||||
# "modified_at": "2024-12-04T00:08:38.537449"
|
||||
# },
|
||||
# "__init__.py": {
|
||||
# "basename": "__init__",
|
||||
# "mime_type": "text/x-python",
|
||||
# "extension": ".py",
|
||||
# "num_subpaths": null,
|
||||
# "num_bytes": 32,
|
||||
# "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
|
||||
# "created_at": "2024-10-08T00:51:41.001359",
|
||||
# "modified_at": "2024-10-08T00:51:41.001359"
|
||||
# },
|
||||
# ...
|
||||
# }
|
||||
@@ -32,7 +32,6 @@ import sys
|
||||
import json
|
||||
from typing import Any, TextIO
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Type constants for JSONL records
|
||||
@@ -120,19 +119,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
|
||||
yield record
|
||||
|
||||
|
||||
def read_file(path: Path) -> Iterator[dict[str, Any]]:
|
||||
"""
|
||||
Read JSONL or plain URLs from a file.
|
||||
|
||||
Yields parsed records as dicts.
|
||||
"""
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
record = parse_line(line)
|
||||
if record:
|
||||
yield record
|
||||
|
||||
|
||||
def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
|
||||
"""
|
||||
Read from CLI arguments if provided, otherwise from stdin.
|
||||
|
||||
+95
-285
@@ -5,133 +5,21 @@ __package__ = "archivebox"
|
||||
# references Django ORM types. For pre-bootstrap logging primitives use
|
||||
# misc/logging.py, which has no archivebox or Django dependencies.
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from math import log
|
||||
from multiprocessing import Process
|
||||
from pathlib import Path
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional, TYPE_CHECKING, cast
|
||||
from typing import Any, TYPE_CHECKING, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from archivebox.core.models import Snapshot
|
||||
pass
|
||||
|
||||
from rich import print
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.misc.util import enforce_types
|
||||
from archivebox.misc.logging import ANSI
|
||||
|
||||
|
||||
class TimedProgress:
|
||||
"""Show a progress bar and measure elapsed time until .end() is called"""
|
||||
|
||||
def __init__(self, seconds, prefix="", config=None, **config_kwargs):
|
||||
|
||||
config = config or get_config(**config_kwargs)
|
||||
self.SHOW_PROGRESS = config.SHOW_PROGRESS
|
||||
self.ANSI = config.ANSI
|
||||
self.TERM_WIDTH = config.TERM_WIDTH
|
||||
|
||||
if self.SHOW_PROGRESS:
|
||||
self.p = Process(target=progress_bar, args=(seconds, prefix, self.ANSI))
|
||||
self.p.start()
|
||||
|
||||
self.stats = {"start_ts": datetime.now(timezone.utc), "end_ts": None}
|
||||
|
||||
def end(self):
|
||||
"""immediately end progress, clear the progressbar line, and save end_ts"""
|
||||
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
self.stats["end_ts"] = end_ts
|
||||
|
||||
if self.SHOW_PROGRESS:
|
||||
# terminate if we havent already terminated
|
||||
try:
|
||||
# kill the progress bar subprocess
|
||||
try:
|
||||
self.p.close() # must be closed *before* its terminnated
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
print()
|
||||
raise
|
||||
except BaseException: # lgtm [py/catch-base-exception]
|
||||
pass
|
||||
self.p.terminate()
|
||||
time.sleep(0.1)
|
||||
# sometimes the timer doesn't terminate properly, then blocks at the join until
|
||||
# the full time has elapsed. sending a kill tries to avoid that.
|
||||
try:
|
||||
self.p.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# clear whole terminal line
|
||||
try:
|
||||
sys.stdout.write("\r{}{}\r".format((" " * self.TERM_WIDTH), self.ANSI["reset"]))
|
||||
except (OSError, BrokenPipeError):
|
||||
# ignore when the parent proc has stopped listening to our stdout
|
||||
pass
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
@enforce_types
|
||||
def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, config=None, **config_kwargs) -> None:
|
||||
"""show timer in the form of progress bar, with percentage and seconds remaining"""
|
||||
output_buf = sys.stdout or sys.__stdout__ or sys.stderr or sys.__stderr__
|
||||
chunk = "█" if output_buf and output_buf.encoding.upper() == "UTF-8" else "#"
|
||||
config = config or get_config(**config_kwargs)
|
||||
last_width = config.TERM_WIDTH
|
||||
chunks = last_width - len(prefix) - 20 # number of progress chunks to show (aka max bar width)
|
||||
try:
|
||||
for s in range(seconds * chunks):
|
||||
max_width = config.TERM_WIDTH
|
||||
if max_width < last_width:
|
||||
# when the terminal size is shrunk, we have to write a newline
|
||||
# otherwise the progress bar will keep wrapping incorrectly
|
||||
sys.stdout.write("\r\n")
|
||||
sys.stdout.flush()
|
||||
chunks = max_width - len(prefix) - 20
|
||||
pct_complete = s / chunks / seconds * 100
|
||||
log_pct = (log(pct_complete or 1, 10) / 2) * 100 # everyone likes faster progress bars ;)
|
||||
bar_width = round(log_pct / (100 / chunks))
|
||||
last_width = max_width
|
||||
|
||||
# ████████████████████ 0.9% (1/60sec)
|
||||
sys.stdout.write(
|
||||
"\r{}{}{}{} {}% ({}/{}sec)".format(
|
||||
prefix,
|
||||
ANSI["green" if pct_complete < 80 else "lightyellow"],
|
||||
(chunk * bar_width).ljust(chunks),
|
||||
ANSI["reset"],
|
||||
round(pct_complete, 1),
|
||||
round(s / chunks),
|
||||
seconds,
|
||||
),
|
||||
)
|
||||
sys.stdout.flush()
|
||||
time.sleep(1 / chunks)
|
||||
|
||||
# ██████████████████████████████████ 100.0% (60/60sec)
|
||||
sys.stdout.write(
|
||||
"\r{}{}{}{} {}% ({}/{}sec)".format(
|
||||
prefix,
|
||||
ANSI["red"],
|
||||
chunk * chunks,
|
||||
ANSI["reset"],
|
||||
100.0,
|
||||
seconds,
|
||||
seconds,
|
||||
),
|
||||
)
|
||||
sys.stdout.flush()
|
||||
except (KeyboardInterrupt, BrokenPipeError):
|
||||
print()
|
||||
|
||||
|
||||
def log_list_started(filter_patterns: list[str] | None, filter_type: str):
|
||||
@@ -236,152 +124,6 @@ def format_duration(seconds: float) -> str:
|
||||
return f"{hours}hr {minutes}min" if minutes else f"{hours}hr"
|
||||
|
||||
|
||||
@enforce_types
|
||||
def truncate_url(url: str, max_length: int = 60) -> str:
|
||||
"""Truncate URL to max_length, keeping domain and adding ellipsis."""
|
||||
if len(url) <= max_length:
|
||||
return url
|
||||
# Try to keep the domain and beginning of path
|
||||
if "://" in url:
|
||||
protocol, rest = url.split("://", 1)
|
||||
if "/" in rest:
|
||||
domain, path = rest.split("/", 1)
|
||||
available = max_length - len(protocol) - len(domain) - 6 # for "://", "/", "..."
|
||||
if available > 10:
|
||||
return f"{protocol}://{domain}/{path[:available]}..."
|
||||
# Fallback: just truncate
|
||||
return url[: max_length - 3] + "..."
|
||||
|
||||
|
||||
@enforce_types
|
||||
def log_worker_event(
|
||||
worker_type: str,
|
||||
event: str,
|
||||
indent_level: int = 0,
|
||||
pid: int | None = None,
|
||||
worker_id: str | None = None,
|
||||
url: str | None = None,
|
||||
plugin: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
error: Exception | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Log a worker event with structured metadata and indentation.
|
||||
|
||||
Args:
|
||||
worker_type: Type of worker (Orchestrator, CrawlWorker, SnapshotWorker)
|
||||
event: Event name (Starting, Completed, Failed, etc.)
|
||||
indent_level: Indentation level (0=Orchestrator, 1=CrawlWorker, 2=SnapshotWorker)
|
||||
pid: Process ID
|
||||
worker_id: Worker ID (UUID for workers)
|
||||
url: URL being processed (for SnapshotWorker)
|
||||
plugin: Plugin name (for hook processes)
|
||||
metadata: Dict of metadata to show in curly braces
|
||||
error: Exception if event is an error
|
||||
"""
|
||||
indent = " " * indent_level
|
||||
|
||||
from rich.markup import escape
|
||||
|
||||
# Build worker identifier (without URL/plugin)
|
||||
worker_parts = [worker_type]
|
||||
# Don't add pid/worker_id for DB operations (they happen in whatever process is running)
|
||||
if pid and worker_type != "DB":
|
||||
worker_parts.append(f"pid={pid}")
|
||||
if worker_id and worker_type in ("CrawlWorker", "Orchestrator") and worker_type != "DB":
|
||||
worker_parts.append(f"id={worker_id}")
|
||||
|
||||
# Build worker label parts for brackets (shown inside brackets)
|
||||
worker_label_base = worker_parts[0]
|
||||
worker_bracket_content = ", ".join(worker_parts[1:]) if len(worker_parts) > 1 else None
|
||||
|
||||
# Build URL/plugin display (shown AFTER the label, outside brackets)
|
||||
url_extractor_parts = []
|
||||
if url:
|
||||
url_extractor_parts.append(f"url: {escape(url)}")
|
||||
if plugin:
|
||||
url_extractor_parts.append(f"extractor: {escape(plugin)}")
|
||||
|
||||
url_extractor_str = " | ".join(url_extractor_parts) if url_extractor_parts else ""
|
||||
|
||||
# Build metadata string
|
||||
metadata_str = ""
|
||||
if metadata:
|
||||
# Format metadata nicely
|
||||
meta_parts = []
|
||||
for k, v in metadata.items():
|
||||
if isinstance(v, float):
|
||||
# Format floats nicely (durations, sizes)
|
||||
if "duration" in k.lower():
|
||||
meta_parts.append(f"{k}: {format_duration(v)}")
|
||||
elif "size" in k.lower():
|
||||
meta_parts.append(f"{k}: {printable_filesize(int(v))}")
|
||||
else:
|
||||
meta_parts.append(f"{k}: {v:.2f}")
|
||||
elif isinstance(v, int):
|
||||
# Format integers - check if it's a size
|
||||
if "size" in k.lower() or "bytes" in k.lower():
|
||||
meta_parts.append(f"{k}: {printable_filesize(v)}")
|
||||
else:
|
||||
meta_parts.append(f"{k}: {v}")
|
||||
elif isinstance(v, (list, tuple)):
|
||||
meta_parts.append(f"{k}: {len(v)}")
|
||||
else:
|
||||
meta_parts.append(f"{k}: {v}")
|
||||
metadata_str = " | ".join(meta_parts)
|
||||
|
||||
# Determine color based on event
|
||||
color = "white"
|
||||
if event in ("Starting...", "Started", "STARTED", "Started in background"):
|
||||
color = "green"
|
||||
elif event.startswith("Created"):
|
||||
color = "cyan" # DB creation events
|
||||
elif event in ("Completed", "COMPLETED", "All work complete"):
|
||||
color = "blue"
|
||||
elif event in ("Failed", "ERROR", "Failed to spawn worker"):
|
||||
color = "red"
|
||||
elif event in ("Shutting down", "SHUTDOWN"):
|
||||
color = "grey53"
|
||||
|
||||
# Build final message
|
||||
error_str = f" {type(error).__name__}: {error}" if error else ""
|
||||
from archivebox.misc.logging import CONSOLE, STDERR
|
||||
from rich.text import Text
|
||||
|
||||
# Create a Rich Text object for proper formatting
|
||||
# Text.append() treats content as literal (no markup parsing)
|
||||
text = Text()
|
||||
text.append(indent)
|
||||
text.append(worker_label_base, style=color)
|
||||
|
||||
# Add bracketed content if present (using Text.append to avoid markup issues)
|
||||
if worker_bracket_content:
|
||||
text.append("[", style=color)
|
||||
text.append(worker_bracket_content, style=color)
|
||||
text.append("]", style=color)
|
||||
|
||||
text.append(f" {event}{error_str}", style=color)
|
||||
|
||||
# Add URL/plugin info first (more important)
|
||||
if url_extractor_str:
|
||||
text.append(f" | {url_extractor_str}")
|
||||
|
||||
# Then add other metadata
|
||||
if metadata_str:
|
||||
text.append(f" | {metadata_str}")
|
||||
|
||||
# Stdout is reserved for JSONL records whenever commands are piped together.
|
||||
# Route worker/DB progress to stderr in non-TTY contexts so pipelines like
|
||||
# `archivebox snapshot list | archivebox run` keep stdout machine-readable.
|
||||
output_console = CONSOLE if sys.stdout.isatty() else STDERR
|
||||
output_console.print(text, soft_wrap=True)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def printable_folders(folders: dict[str, Optional["Snapshot"]], with_headers: bool = False) -> str:
|
||||
return "\n".join(f'{folder} {snapshot and snapshot.url} "{snapshot and snapshot.title}"' for folder, snapshot in folders.items())
|
||||
|
||||
|
||||
@enforce_types
|
||||
def printable_config(config: dict, prefix: str = "") -> str:
|
||||
return f"\n{prefix}".join(f"{key}={val}" for key, val in config.items() if not (isinstance(val, dict) or callable(val)))
|
||||
@@ -432,32 +174,100 @@ def printable_folder_status(name: str, folder: dict) -> str:
|
||||
)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def printable_dependency_version(name: str, dependency: dict) -> str:
|
||||
color, symbol, note, version = "red", "X", "invalid", "?"
|
||||
def _warn_background_cleanup(context: str, err: BaseException) -> None:
|
||||
from archivebox.misc.logging import STDERR
|
||||
|
||||
if dependency["enabled"]:
|
||||
if dependency["is_valid"]:
|
||||
color, symbol, note = "green", "√", "valid"
|
||||
STDERR.print(f"[yellow][!] {context}: {err!s}[/yellow]")
|
||||
|
||||
parsed_version_num = re.search(r"[\d\.]+", dependency["version"])
|
||||
if parsed_version_num:
|
||||
version = f"v{parsed_version_num[0]}"
|
||||
else:
|
||||
color, symbol, note, version = "lightyellow", "-", "disabled", "-"
|
||||
|
||||
path = pretty_path(dependency["path"])
|
||||
def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep_running=None):
|
||||
"""Tail multiple log files simultaneously, interleaving their output.
|
||||
|
||||
return " ".join(
|
||||
(
|
||||
ANSI[color],
|
||||
symbol,
|
||||
ANSI["reset"],
|
||||
name.ljust(21),
|
||||
version.ljust(14),
|
||||
ANSI[color],
|
||||
note.ljust(8),
|
||||
ANSI["reset"],
|
||||
path.ljust(76),
|
||||
),
|
||||
)
|
||||
Args:
|
||||
log_files: List of log file paths to tail
|
||||
follow: Whether to keep following (True) or just read existing content (False)
|
||||
proc: Optional subprocess.Popen object - stop tailing when this process exits
|
||||
"""
|
||||
import re
|
||||
from archivebox.config.common import rprint as print
|
||||
|
||||
# Convert relative paths to absolute paths
|
||||
log_paths = []
|
||||
for log_file in log_files:
|
||||
log_path = Path(log_file)
|
||||
if not log_path.is_absolute():
|
||||
log_path = CONSTANTS.DATA_DIR / log_path
|
||||
|
||||
# Create log file if it doesn't exist
|
||||
if not log_path.exists():
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_path.touch()
|
||||
|
||||
log_paths.append(log_path)
|
||||
|
||||
# Open all log files
|
||||
file_handles = []
|
||||
for log_path in log_paths:
|
||||
try:
|
||||
f = log_path.open()
|
||||
# Seek to end - only show NEW logs from now on, not old logs
|
||||
f.seek(0, 2) # Go to end
|
||||
|
||||
file_handles.append((log_path, f))
|
||||
print(f" [tailing {log_path.name}]")
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"Warning: Could not open {log_path}: {e}\n")
|
||||
|
||||
if not file_handles:
|
||||
sys.stderr.write("No log files could be opened\n")
|
||||
return
|
||||
|
||||
print()
|
||||
|
||||
# Log display needs a short refresh interval, but each ownership check
|
||||
# queries the database and validates OS processes. Check leadership once
|
||||
# per second even when workers continuously produce log output.
|
||||
next_ownership_check = 0.0
|
||||
try:
|
||||
while follow:
|
||||
now = time.monotonic()
|
||||
if keep_running is not None and now >= next_ownership_check:
|
||||
if not keep_running():
|
||||
print("\n[newer ArchiveBox process is now running the orchestrator and server]")
|
||||
return "transferred"
|
||||
next_ownership_check = time.monotonic() + 1.0
|
||||
|
||||
# Check if the monitored process has exited
|
||||
if proc is not None and proc.poll() is not None:
|
||||
print(f"\n[server process exited with code {proc.returncode}]")
|
||||
return "exited"
|
||||
|
||||
had_output = False
|
||||
# Read ALL available lines from all files (not just one per iteration)
|
||||
for log_path, f in file_handles:
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break # No more lines available in this file
|
||||
had_output = True
|
||||
# Strip ANSI codes if present (supervisord does this but just in case)
|
||||
line_clean = re.sub(r"\x1b\[[0-9;]*m", "", line.rstrip())
|
||||
if line_clean:
|
||||
print(line_clean)
|
||||
|
||||
# Small sleep to avoid busy-waiting (only when no output)
|
||||
if not had_output:
|
||||
time.sleep(0.05)
|
||||
|
||||
except (KeyboardInterrupt, BrokenPipeError, OSError):
|
||||
return "interrupted" # Let the caller handle the cleanup message
|
||||
except SystemExit:
|
||||
return "interrupted"
|
||||
finally:
|
||||
# Close all file handles
|
||||
for _, f in file_handles:
|
||||
try:
|
||||
f.close()
|
||||
except OSError as err:
|
||||
_warn_background_cleanup("Could not close worker log file", err)
|
||||
return "stopped"
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Transform captured text, Markdown, and images into replay documents.
|
||||
|
||||
HTTP caching, range requests, and replay security headers belong to serve_static;
|
||||
these helpers only render content and resolve references to captured responses.
|
||||
"""
|
||||
|
||||
import html
|
||||
import posixpath
|
||||
import re
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
|
||||
|
||||
|
||||
TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
|
||||
html {
|
||||
width: 100%;
|
||||
min-width: 100%;
|
||||
background: #fff;
|
||||
}
|
||||
body {
|
||||
box-sizing: border-box;
|
||||
width: min(100%, 72rem);
|
||||
max-width: none;
|
||||
min-height: 100vh;
|
||||
margin: 0 auto;
|
||||
padding: clamp(1rem, 3vw, 2rem);
|
||||
background: #fff;
|
||||
color: #111827;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.55;
|
||||
}
|
||||
body > * {
|
||||
max-width: 100%;
|
||||
}
|
||||
img:not([width]):not([height]) {
|
||||
max-width: min(100%, 12rem);
|
||||
max-height: 12rem;
|
||||
width: auto;
|
||||
height: auto;
|
||||
object-fit: contain;
|
||||
}
|
||||
a > img:not([width]):not([height]) {
|
||||
max-width: min(100%, 2.5rem);
|
||||
max-height: 2.5rem;
|
||||
}
|
||||
</style>"""
|
||||
|
||||
|
||||
MARKDOWN_INLINE_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+(?:\([^)]*\)[^)\s]*)*)\)")
|
||||
|
||||
|
||||
HTML_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", flags=re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def _extract_markdown_candidate(text: str) -> str:
|
||||
candidate = text
|
||||
body_match = HTML_BODY_RE.search(candidate)
|
||||
if body_match:
|
||||
candidate = body_match.group(1)
|
||||
candidate = re.sub(r"^\s*<p[^>]*>", "", candidate, flags=re.IGNORECASE)
|
||||
candidate = re.sub(r"</p>\s*$", "", candidate, flags=re.IGNORECASE)
|
||||
return candidate.strip()
|
||||
|
||||
|
||||
def _looks_like_markdown(text: str) -> bool:
|
||||
lower = text.lower()
|
||||
if "<html" in lower and "<head" in lower and "</body>" in lower:
|
||||
return False
|
||||
md_markers = 0
|
||||
md_markers += len(re.findall(r"^\s{0,3}#{1,6}\s+\S", text, flags=re.MULTILINE))
|
||||
md_markers += len(re.findall(r"^\s*[-*+]\s+\S", text, flags=re.MULTILINE))
|
||||
md_markers += len(re.findall(r"^\s*\d+\.\s+\S", text, flags=re.MULTILINE))
|
||||
md_markers += text.count("[TOC]")
|
||||
md_markers += len(MARKDOWN_INLINE_LINK_RE.findall(text))
|
||||
md_markers += text.count("\n---") + text.count("\n***")
|
||||
return md_markers >= 6
|
||||
|
||||
|
||||
def _render_text_preview_document(text: str, title: str) -> str:
|
||||
escaped_title = html.escape(title)
|
||||
escaped_text = html.escape(text)
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{escaped_title}</title>
|
||||
<style>
|
||||
:root {{
|
||||
color-scheme: dark;
|
||||
}}
|
||||
html, body {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: #111;
|
||||
color: #f3f3f3;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
|
||||
}}
|
||||
.archivebox-text-preview-header {{
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1;
|
||||
padding: 10px 14px;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
color: #bbb;
|
||||
background: rgba(17, 17, 17, 0.96);
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
backdrop-filter: blur(8px);
|
||||
}}
|
||||
.archivebox-text-preview {{
|
||||
margin: 0;
|
||||
padding: 14px;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
tab-size: 2;
|
||||
line-height: 1.45;
|
||||
font-size: 13px;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="archivebox-text-preview-header">{escaped_title}</div>
|
||||
<pre class="archivebox-text-preview">{escaped_text}</pre>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _render_image_preview_document(image_url: str, title: str) -> str:
|
||||
escaped_title = html.escape(title)
|
||||
escaped_url = html.escape(image_url, quote=True)
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{escaped_title}</title>
|
||||
<style>
|
||||
:root {{
|
||||
color-scheme: dark;
|
||||
}}
|
||||
html, body {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
width: 100%;
|
||||
min-height: 100%;
|
||||
background: #fff;
|
||||
}}
|
||||
body {{
|
||||
overflow: auto;
|
||||
}}
|
||||
.archivebox-image-preview {{
|
||||
width: 100%;
|
||||
min-width: 100%;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: flex-start;
|
||||
box-sizing: border-box;
|
||||
}}
|
||||
.archivebox-image-preview img {{
|
||||
display: block;
|
||||
width: auto;
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
margin: 0 auto;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="archivebox-image-preview">
|
||||
<img src="{escaped_url}" alt="{escaped_title}">
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
|
||||
raw_url = str(image_url or "").strip()
|
||||
if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
|
||||
return None
|
||||
|
||||
absolute_url = urljoin(page_url or "", raw_url)
|
||||
if not absolute_url.startswith(("http://", "https://")):
|
||||
return None
|
||||
|
||||
return quote(absolute_url, safe="").replace("%", "_")
|
||||
|
||||
|
||||
def _index_responses_paths_for_html_images(
|
||||
snapshot_root: Path,
|
||||
html_rel_path: str,
|
||||
encoded_urls: set[str],
|
||||
) -> dict[str, str]:
|
||||
responses_root = snapshot_root / "responses"
|
||||
if not encoded_urls or not responses_root.is_dir():
|
||||
return {}
|
||||
|
||||
best_matches: dict[str, str] = {}
|
||||
image_matches: set[str] = set()
|
||||
image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
|
||||
for candidate in responses_root.rglob("*"):
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
try:
|
||||
rel_path = candidate.relative_to(snapshot_root)
|
||||
except ValueError:
|
||||
continue
|
||||
relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
|
||||
candidate_name = candidate.name
|
||||
is_image = candidate.suffix.lower() in image_suffixes
|
||||
for encoded_url in encoded_urls:
|
||||
if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
|
||||
continue
|
||||
best_matches[encoded_url] = relative_match
|
||||
if is_image:
|
||||
# Preserve the old rglob behavior: the first image match wins,
|
||||
# otherwise the last matching response file is used.
|
||||
image_matches.add(encoded_url)
|
||||
return best_matches
|
||||
|
||||
|
||||
def _rewrite_html_image_sources_to_responses(
|
||||
html_text: str,
|
||||
snapshot_root: Path,
|
||||
html_rel_path: str,
|
||||
page_url: str | None,
|
||||
) -> tuple[str, int]:
|
||||
encoded_urls_by_src: dict[str, str | None] = {}
|
||||
for match in IMG_SRC_ATTR_RE.finditer(html_text):
|
||||
image_url = html.unescape(match.group(2))
|
||||
if image_url not in encoded_urls_by_src:
|
||||
encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
|
||||
|
||||
response_paths = _index_responses_paths_for_html_images(
|
||||
snapshot_root,
|
||||
html_rel_path,
|
||||
{encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
|
||||
)
|
||||
rewrites = 0
|
||||
|
||||
def replace_src(match: re.Match[str]) -> str:
|
||||
nonlocal rewrites
|
||||
image_url = html.unescape(match.group(2))
|
||||
encoded_url = encoded_urls_by_src.get(image_url)
|
||||
local_path = response_paths.get(encoded_url or "")
|
||||
if not local_path:
|
||||
return match.group(0)
|
||||
rewrites += 1
|
||||
return f"{match.group(1)}{html.escape(local_path, quote=True)}{match.group(3)}"
|
||||
|
||||
return IMG_SRC_ATTR_RE.sub(replace_src, html_text), rewrites
|
||||
|
||||
|
||||
def _rewrite_html_image_sources_for_request(
|
||||
request,
|
||||
html_text: str,
|
||||
document_root: Path | None,
|
||||
html_rel_path: str,
|
||||
) -> tuple[str, int]:
|
||||
if not document_root:
|
||||
return html_text, 0
|
||||
return _rewrite_html_image_sources_to_responses(
|
||||
html_text,
|
||||
document_root,
|
||||
html_rel_path,
|
||||
request.__dict__.get("archivebox_snapshot_url"),
|
||||
)
|
||||
|
||||
|
||||
def _apply_transformed_html_preview_style(html_text: str) -> str:
|
||||
if "archivebox-static-html-preview-style" in html_text:
|
||||
return html_text
|
||||
if re.search(r"</head\s*>", html_text, flags=re.IGNORECASE):
|
||||
return re.sub(r"</head\s*>", f"{TRANSFORMED_HTML_PREVIEW_STYLE}\\g<0>", html_text, count=1, flags=re.IGNORECASE)
|
||||
return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
|
||||
|
||||
|
||||
def _render_markdown(markdown_text: str) -> str:
|
||||
"""Render archived Markdown with stable heading links and optional [TOC]."""
|
||||
parser = MarkdownIt("commonmark").enable("table")
|
||||
tokens = parser.parse(markdown_text)
|
||||
headings = []
|
||||
for index, token in enumerate(tokens):
|
||||
if token.type == "heading_open":
|
||||
title = tokens[index + 1].content
|
||||
slug = re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-") or "section"
|
||||
token.attrSet("id", slug)
|
||||
headings.append(f'<li class="toc-level-{token.tag[1:]}"><a href="#{slug}">{html.escape(title)}</a></li>')
|
||||
toc = '<nav class="toc"><ul>' + "".join(headings) + "</ul></nav>"
|
||||
# Replace only a standalone TOC paragraph, leaving code and inline mentions intact.
|
||||
for index in range(len(tokens) - 2):
|
||||
opening, content, closing = tokens[index : index + 3]
|
||||
if opening.type == "paragraph_open" and content.content == "[TOC]" and closing.type == "paragraph_close":
|
||||
opening.hidden = closing.hidden = True
|
||||
content.type = "html_block"
|
||||
content.content = toc
|
||||
return parser.renderer.render(tokens, parser.options, {})
|
||||
|
||||
|
||||
def _render_markdown_document(markdown_text: str) -> str:
|
||||
body = _render_markdown(markdown_text)
|
||||
wrapped = (
|
||||
'<!doctype html><html><head><meta charset="utf-8">'
|
||||
'<meta name="viewport" content="width=device-width,initial-scale=1">'
|
||||
"<style>body{max-width:900px;margin:24px auto;padding:0 16px;"
|
||||
"font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;"
|
||||
"line-height:1.55;} img{max-width:100%;} pre{background:#f6f6f6;padding:12px;overflow:auto;}"
|
||||
".toc ul{list-style:none;padding-left:0;} .toc li{margin:4px 0;}</style>"
|
||||
"</head><body>"
|
||||
f"{body}"
|
||||
"</body></html>"
|
||||
)
|
||||
return wrapped
|
||||
+94
-639
@@ -1,21 +1,18 @@
|
||||
import asyncio
|
||||
import html
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import posixpath
|
||||
import queue
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zipfile
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlencode, urljoin
|
||||
from functools import partial
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
|
||||
from django.contrib.staticfiles import finders
|
||||
@@ -28,43 +25,10 @@ from django.utils.translation import gettext as _
|
||||
from django.views import static
|
||||
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.misc import replay_preview
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
|
||||
_HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
|
||||
IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
|
||||
TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
|
||||
html {
|
||||
width: 100%;
|
||||
min-width: 100%;
|
||||
background: #fff;
|
||||
}
|
||||
body {
|
||||
box-sizing: border-box;
|
||||
width: min(100%, 72rem);
|
||||
max-width: none;
|
||||
min-height: 100vh;
|
||||
margin: 0 auto;
|
||||
padding: clamp(1rem, 3vw, 2rem);
|
||||
background: #fff;
|
||||
color: #111827;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.55;
|
||||
}
|
||||
body > * {
|
||||
max-width: 100%;
|
||||
}
|
||||
img:not([width]):not([height]) {
|
||||
max-width: min(100%, 12rem);
|
||||
max-height: 12rem;
|
||||
width: auto;
|
||||
height: auto;
|
||||
object-fit: contain;
|
||||
}
|
||||
a > img:not([width]):not([height]) {
|
||||
max-width: min(100%, 2.5rem);
|
||||
max-height: 2.5rem;
|
||||
}
|
||||
</style>"""
|
||||
|
||||
|
||||
def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
|
||||
@@ -142,33 +106,25 @@ def _safe_zip_stem(name: str) -> str:
|
||||
return safe_name or "archivebox"
|
||||
|
||||
|
||||
class _StreamingQueueWriter:
|
||||
"""Expose a write-only file-like object so zipfile can stream into a queue."""
|
||||
class _ZipBuffer(io.RawIOBase):
|
||||
"""A non-seekable ZIP destination drained after each source block."""
|
||||
|
||||
def __init__(self, output_queue: queue.Queue[bytes | BaseException | object]) -> None:
|
||||
self.output_queue = output_queue
|
||||
def __init__(self):
|
||||
self.pending = bytearray()
|
||||
self.position = 0
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
if data:
|
||||
self.output_queue.put(data)
|
||||
self.position += len(data)
|
||||
def write(self, data):
|
||||
self.pending.extend(data)
|
||||
self.position += len(data)
|
||||
return len(data)
|
||||
|
||||
def tell(self) -> int:
|
||||
def tell(self):
|
||||
return self.position
|
||||
|
||||
def flush(self) -> None:
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
def writable(self) -> bool:
|
||||
return True
|
||||
|
||||
def seekable(self) -> bool:
|
||||
return False
|
||||
def drain(self):
|
||||
if self.pending:
|
||||
yield bytes(self.pending)
|
||||
self.pending.clear()
|
||||
|
||||
|
||||
def _iter_visible_files(root: Path):
|
||||
@@ -180,6 +136,42 @@ def _iter_visible_files(root: Path):
|
||||
yield Path(current_root) / filename
|
||||
|
||||
|
||||
def _iter_directory_zip(fullpath: Path, root_name: str):
|
||||
# Reading a block and yielding its compressed bytes in the same iterator
|
||||
# provides backpressure and closes all files when a download disconnects.
|
||||
with _ZipBuffer() as buffer:
|
||||
with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
||||
for entry in _iter_visible_files(fullpath):
|
||||
info = zipfile.ZipInfo.from_file(entry, (Path(root_name) / entry.relative_to(fullpath)).as_posix())
|
||||
info.compress_type = archive.compression
|
||||
info.compress_level = archive.compresslevel
|
||||
with entry.open("rb") as source, archive.open(info, "w") as destination:
|
||||
while chunk := source.read(64 * 1024):
|
||||
destination.write(chunk)
|
||||
yield from buffer.drain()
|
||||
yield from buffer.drain()
|
||||
yield from buffer.drain()
|
||||
|
||||
|
||||
async def _stream_async(stream):
|
||||
# Django ASGI buffers synchronous iterators. Advance off the event loop,
|
||||
# waiting for an in-flight read before closing its iterator on cancellation.
|
||||
iterator = iter(stream)
|
||||
try:
|
||||
while True:
|
||||
read = asyncio.create_task(asyncio.to_thread(next, iterator, None))
|
||||
try:
|
||||
chunk = await asyncio.shield(read)
|
||||
except asyncio.CancelledError:
|
||||
await read
|
||||
raise
|
||||
if chunk is None:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
iterator.close()
|
||||
|
||||
|
||||
def _build_directory_zip_response(
|
||||
fullpath: Path,
|
||||
path: str,
|
||||
@@ -189,74 +181,11 @@ def _build_directory_zip_response(
|
||||
config=None,
|
||||
) -> StreamingHttpResponse:
|
||||
root_name = _safe_zip_stem(fullpath.name or Path(path).name or "archivebox")
|
||||
sentinel = object()
|
||||
output_queue: queue.Queue[bytes | BaseException | object] = queue.Queue(maxsize=8)
|
||||
initial_chunk_target = 64 * 1024
|
||||
initial_chunk_wait = 0.05
|
||||
stream = _iter_directory_zip(fullpath, root_name)
|
||||
|
||||
def build_zip() -> None:
|
||||
# zipfile wants a write-only file object. Feed those bytes straight into
|
||||
# a queue so the response can stream them out as soon as they are ready.
|
||||
writer = _StreamingQueueWriter(output_queue)
|
||||
try:
|
||||
with zipfile.ZipFile(writer, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zip_file:
|
||||
for entry in _iter_visible_files(fullpath):
|
||||
rel_parts = entry.relative_to(fullpath).parts
|
||||
arcname = Path(root_name, *rel_parts).as_posix()
|
||||
zip_file.write(entry, arcname)
|
||||
except (OSError, RuntimeError, TypeError, ValueError, zipfile.BadZipFile) as err:
|
||||
output_queue.put(err)
|
||||
finally:
|
||||
output_queue.put(sentinel)
|
||||
|
||||
threading.Thread(target=build_zip, name=f"zip-stream-{root_name}", daemon=True).start()
|
||||
|
||||
def iter_zip_chunks():
|
||||
# Emit a meaningful first chunk quickly so browsers show the download
|
||||
# immediately instead of waiting on dozens of tiny ZIP header writes.
|
||||
first_chunk = bytearray()
|
||||
initial_deadline = time.monotonic() + initial_chunk_wait
|
||||
|
||||
while True:
|
||||
timeout = max(initial_deadline - time.monotonic(), 0) if len(first_chunk) < initial_chunk_target else None
|
||||
try:
|
||||
chunk = output_queue.get(timeout=timeout) if timeout is not None else output_queue.get()
|
||||
except queue.Empty:
|
||||
if first_chunk:
|
||||
yield bytes(first_chunk)
|
||||
first_chunk.clear()
|
||||
continue
|
||||
chunk = output_queue.get()
|
||||
|
||||
if chunk is sentinel:
|
||||
if first_chunk:
|
||||
yield bytes(first_chunk)
|
||||
break
|
||||
if isinstance(chunk, BaseException):
|
||||
raise chunk
|
||||
if len(first_chunk) < initial_chunk_target:
|
||||
first_chunk.extend(chunk)
|
||||
if len(first_chunk) >= initial_chunk_target or time.monotonic() >= initial_deadline:
|
||||
yield bytes(first_chunk)
|
||||
first_chunk.clear()
|
||||
continue
|
||||
yield chunk
|
||||
|
||||
async def stream_zip_async():
|
||||
# Django ASGI buffers sync StreamingHttpResponse iterators by consuming
|
||||
# them into a list. Drive the same sync iterator from a worker thread so
|
||||
# Daphne can send each chunk as it arrives instead of buffering the ZIP.
|
||||
iterator = iter(iter_zip_chunks())
|
||||
while True:
|
||||
chunk = await asyncio.to_thread(next, iterator, None)
|
||||
if chunk is None:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
response = StreamingHttpResponse(
|
||||
stream_zip_async() if use_async_stream else iter_zip_chunks(),
|
||||
content_type="application/zip",
|
||||
)
|
||||
response = StreamingHttpResponse(stream, content_type="application/zip")
|
||||
if use_async_stream:
|
||||
response.streaming_content = _stream_async(stream)
|
||||
response.headers["Content-Disposition"] = f'attachment; filename="{root_name}.zip"'
|
||||
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
|
||||
response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
|
||||
@@ -270,18 +199,6 @@ def _build_directory_zip_response(
|
||||
)
|
||||
|
||||
|
||||
async def _stream_ranged_file_async(ranged_file: "RangedFileReader"):
|
||||
iterator = iter(ranged_file)
|
||||
try:
|
||||
while True:
|
||||
chunk = await asyncio.to_thread(next, iterator, None)
|
||||
if chunk is None:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
ranged_file.close()
|
||||
|
||||
|
||||
def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
|
||||
try:
|
||||
template = loader.select_template(
|
||||
@@ -356,17 +273,6 @@ mimetypes.add_type("image/svg+xml", ".svg")
|
||||
mimetypes.add_type("multipart/related", ".mhtml")
|
||||
mimetypes.add_type("multipart/related", ".mht")
|
||||
|
||||
try:
|
||||
_markdown = importlib.import_module("markdown").markdown
|
||||
except ImportError:
|
||||
_markdown: Callable[..., str] | None = None
|
||||
|
||||
MARKDOWN_INLINE_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)\s]+(?:\([^)]*\)[^)\s]*)*)\)")
|
||||
MARKDOWN_INLINE_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
|
||||
MARKDOWN_BOLD_RE = re.compile(r"\*\*([^*]+)\*\*")
|
||||
MARKDOWN_ITALIC_RE = re.compile(r"(?<!\*)\*([^*]+)\*(?!\*)")
|
||||
HTML_TAG_RE = re.compile(r"<[A-Za-z][^>]*>")
|
||||
HTML_BODY_RE = re.compile(r"<body[^>]*>(.*)</body>", flags=re.IGNORECASE | re.DOTALL)
|
||||
RISKY_REPLAY_MIMETYPES = {
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
@@ -380,231 +286,6 @@ RISKY_REPLAY_MARKERS = (
|
||||
)
|
||||
|
||||
|
||||
def _extract_markdown_candidate(text: str) -> str:
|
||||
candidate = text
|
||||
body_match = HTML_BODY_RE.search(candidate)
|
||||
if body_match:
|
||||
candidate = body_match.group(1)
|
||||
candidate = re.sub(r"^\s*<p[^>]*>", "", candidate, flags=re.IGNORECASE)
|
||||
candidate = re.sub(r"</p>\s*$", "", candidate, flags=re.IGNORECASE)
|
||||
return candidate.strip()
|
||||
|
||||
|
||||
def _looks_like_markdown(text: str) -> bool:
|
||||
lower = text.lower()
|
||||
if "<html" in lower and "<head" in lower and "</body>" in lower:
|
||||
return False
|
||||
md_markers = 0
|
||||
md_markers += len(re.findall(r"^\s{0,3}#{1,6}\s+\S", text, flags=re.MULTILINE))
|
||||
md_markers += len(re.findall(r"^\s*[-*+]\s+\S", text, flags=re.MULTILINE))
|
||||
md_markers += len(re.findall(r"^\s*\d+\.\s+\S", text, flags=re.MULTILINE))
|
||||
md_markers += text.count("[TOC]")
|
||||
md_markers += len(MARKDOWN_INLINE_LINK_RE.findall(text))
|
||||
md_markers += text.count("\n---") + text.count("\n***")
|
||||
return md_markers >= 6
|
||||
|
||||
|
||||
def _render_text_preview_document(text: str, title: str) -> str:
|
||||
escaped_title = html.escape(title)
|
||||
escaped_text = html.escape(text)
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{escaped_title}</title>
|
||||
<style>
|
||||
:root {{
|
||||
color-scheme: dark;
|
||||
}}
|
||||
html, body {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: #111;
|
||||
color: #f3f3f3;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
|
||||
}}
|
||||
.archivebox-text-preview-header {{
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1;
|
||||
padding: 10px 14px;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
color: #bbb;
|
||||
background: rgba(17, 17, 17, 0.96);
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
backdrop-filter: blur(8px);
|
||||
}}
|
||||
.archivebox-text-preview {{
|
||||
margin: 0;
|
||||
padding: 14px;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
tab-size: 2;
|
||||
line-height: 1.45;
|
||||
font-size: 13px;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="archivebox-text-preview-header">{escaped_title}</div>
|
||||
<pre class="archivebox-text-preview">{escaped_text}</pre>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _render_image_preview_document(image_url: str, title: str) -> str:
|
||||
escaped_title = html.escape(title)
|
||||
escaped_url = html.escape(image_url, quote=True)
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{escaped_title}</title>
|
||||
<style>
|
||||
:root {{
|
||||
color-scheme: dark;
|
||||
}}
|
||||
html, body {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
width: 100%;
|
||||
min-height: 100%;
|
||||
background: #fff;
|
||||
}}
|
||||
body {{
|
||||
overflow: auto;
|
||||
}}
|
||||
.archivebox-image-preview {{
|
||||
width: 100%;
|
||||
min-width: 100%;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: flex-start;
|
||||
box-sizing: border-box;
|
||||
}}
|
||||
.archivebox-image-preview img {{
|
||||
display: block;
|
||||
width: auto;
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
margin: 0 auto;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="archivebox-image-preview">
|
||||
<img src="{escaped_url}" alt="{escaped_title}">
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _encoded_responses_image_url(image_url: str, page_url: str | None) -> str | None:
|
||||
raw_url = str(image_url or "").strip()
|
||||
if not raw_url or raw_url.startswith(("#", "data:", "blob:", "about:", "javascript:")):
|
||||
return None
|
||||
|
||||
absolute_url = urljoin(page_url or "", raw_url)
|
||||
if not absolute_url.startswith(("http://", "https://")):
|
||||
return None
|
||||
|
||||
return quote(absolute_url, safe="").replace("%", "_")
|
||||
|
||||
|
||||
def _index_responses_paths_for_html_images(
|
||||
snapshot_root: Path,
|
||||
html_rel_path: str,
|
||||
encoded_urls: set[str],
|
||||
) -> dict[str, str]:
|
||||
responses_root = snapshot_root / "responses"
|
||||
if not encoded_urls or not responses_root.is_dir():
|
||||
return {}
|
||||
|
||||
best_matches: dict[str, str] = {}
|
||||
image_matches: set[str] = set()
|
||||
image_suffixes = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"}
|
||||
for candidate in responses_root.rglob("*"):
|
||||
if not candidate.is_file():
|
||||
continue
|
||||
try:
|
||||
rel_path = candidate.relative_to(snapshot_root)
|
||||
except ValueError:
|
||||
continue
|
||||
relative_match = posixpath.relpath(rel_path.as_posix(), start=posixpath.dirname(html_rel_path) or ".")
|
||||
candidate_name = candidate.name
|
||||
is_image = candidate.suffix.lower() in image_suffixes
|
||||
for encoded_url in encoded_urls:
|
||||
if encoded_url in image_matches or f"__GET__{encoded_url}" not in candidate_name:
|
||||
continue
|
||||
best_matches[encoded_url] = relative_match
|
||||
if is_image:
|
||||
# Preserve the old rglob behavior: the first image match wins,
|
||||
# otherwise the last matching response file is used.
|
||||
image_matches.add(encoded_url)
|
||||
return best_matches
|
||||
|
||||
|
||||
def _rewrite_html_image_sources_to_responses(
|
||||
html_text: str,
|
||||
snapshot_root: Path,
|
||||
html_rel_path: str,
|
||||
page_url: str | None,
|
||||
) -> tuple[str, int]:
|
||||
encoded_urls_by_src: dict[str, str | None] = {}
|
||||
for match in IMG_SRC_ATTR_RE.finditer(html_text):
|
||||
image_url = html.unescape(match.group(2))
|
||||
if image_url not in encoded_urls_by_src:
|
||||
encoded_urls_by_src[image_url] = _encoded_responses_image_url(image_url, page_url)
|
||||
|
||||
response_paths = _index_responses_paths_for_html_images(
|
||||
snapshot_root,
|
||||
html_rel_path,
|
||||
{encoded_url for encoded_url in encoded_urls_by_src.values() if encoded_url},
|
||||
)
|
||||
rewrites = 0
|
||||
|
||||
def replace_src(match: re.Match[str]) -> str:
|
||||
nonlocal rewrites
|
||||
image_url = html.unescape(match.group(2))
|
||||
encoded_url = encoded_urls_by_src.get(image_url)
|
||||
local_path = response_paths.get(encoded_url or "")
|
||||
if not local_path:
|
||||
return match.group(0)
|
||||
rewrites += 1
|
||||
return f"{match.group(1)}{html.escape(local_path, quote=True)}{match.group(3)}"
|
||||
|
||||
return IMG_SRC_ATTR_RE.sub(replace_src, html_text), rewrites
|
||||
|
||||
|
||||
def _rewrite_html_image_sources_for_request(
|
||||
request,
|
||||
html_text: str,
|
||||
document_root: Path | None,
|
||||
html_rel_path: str,
|
||||
) -> tuple[str, int]:
|
||||
if not document_root:
|
||||
return html_text, 0
|
||||
return _rewrite_html_image_sources_to_responses(
|
||||
html_text,
|
||||
document_root,
|
||||
html_rel_path,
|
||||
request.__dict__.get("archivebox_snapshot_url"),
|
||||
)
|
||||
|
||||
|
||||
def _apply_transformed_html_preview_style(html_text: str) -> str:
|
||||
if "archivebox-static-html-preview-style" in html_text:
|
||||
return html_text
|
||||
if re.search(r"</head\s*>", html_text, flags=re.IGNORECASE):
|
||||
return re.sub(r"</head\s*>", f"{TRANSFORMED_HTML_PREVIEW_STYLE}\\g<0>", html_text, count=1, flags=re.IGNORECASE)
|
||||
return f"{TRANSFORMED_HTML_PREVIEW_STYLE}\n{html_text}"
|
||||
|
||||
|
||||
def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat_result, encoding: str | None, config) -> None:
|
||||
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
|
||||
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
|
||||
@@ -613,177 +294,6 @@ def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat
|
||||
response.headers["Content-Encoding"] = encoding
|
||||
|
||||
|
||||
def _render_markdown_fallback(text: str) -> str:
|
||||
if _markdown is not None and not HTML_TAG_RE.search(text):
|
||||
try:
|
||||
return _markdown(
|
||||
text,
|
||||
extensions=["extra", "toc", "sane_lists"],
|
||||
output_format="html",
|
||||
)
|
||||
except (ImportError, RuntimeError, ValueError):
|
||||
pass
|
||||
|
||||
lines = text.splitlines()
|
||||
headings = []
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
slug = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
|
||||
return slug or "section"
|
||||
|
||||
for raw_line in lines:
|
||||
heading_match = re.match(r"^\s{0,3}(#{1,6})\s+(.*)$", raw_line)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
content = heading_match.group(2).strip()
|
||||
headings.append((level, content, slugify(content)))
|
||||
|
||||
html_lines = []
|
||||
in_code = False
|
||||
in_ul = False
|
||||
in_ol = False
|
||||
in_blockquote = False
|
||||
|
||||
def render_inline(markup: str) -> str:
|
||||
content = MARKDOWN_INLINE_IMAGE_RE.sub(r'<img alt="\1" src="\2">', markup)
|
||||
content = MARKDOWN_INLINE_LINK_RE.sub(r'<a href="\2">\1</a>', content)
|
||||
content = MARKDOWN_BOLD_RE.sub(r"<strong>\1</strong>", content)
|
||||
content = MARKDOWN_ITALIC_RE.sub(r"<em>\1</em>", content)
|
||||
return content
|
||||
|
||||
def close_lists():
|
||||
nonlocal in_ul, in_ol
|
||||
if in_ul:
|
||||
html_lines.append("</ul>")
|
||||
in_ul = False
|
||||
if in_ol:
|
||||
html_lines.append("</ol>")
|
||||
in_ol = False
|
||||
|
||||
for raw_line in lines:
|
||||
line = raw_line.rstrip("\n")
|
||||
stripped = line.strip()
|
||||
|
||||
if stripped.startswith("```"):
|
||||
if in_code:
|
||||
html_lines.append("</code></pre>")
|
||||
in_code = False
|
||||
else:
|
||||
close_lists()
|
||||
if in_blockquote:
|
||||
html_lines.append("</blockquote>")
|
||||
in_blockquote = False
|
||||
html_lines.append("<pre><code>")
|
||||
in_code = True
|
||||
continue
|
||||
|
||||
if in_code:
|
||||
html_lines.append(html.escape(line))
|
||||
continue
|
||||
|
||||
if not stripped:
|
||||
close_lists()
|
||||
if in_blockquote:
|
||||
html_lines.append("</blockquote>")
|
||||
in_blockquote = False
|
||||
html_lines.append("<br/>")
|
||||
continue
|
||||
|
||||
heading_match = re.match(r"^\s*((?:<[^>]+>\s*)*)(#{1,6})\s+(.*)$", line)
|
||||
if heading_match:
|
||||
close_lists()
|
||||
if in_blockquote:
|
||||
html_lines.append("</blockquote>")
|
||||
in_blockquote = False
|
||||
leading_tags = heading_match.group(1).strip()
|
||||
level = len(heading_match.group(2))
|
||||
content = heading_match.group(3).strip()
|
||||
if leading_tags:
|
||||
html_lines.append(leading_tags)
|
||||
html_lines.append(f'<h{level} id="{slugify(content)}">{render_inline(content)}</h{level}>')
|
||||
continue
|
||||
|
||||
if stripped in ("---", "***"):
|
||||
close_lists()
|
||||
html_lines.append("<hr/>")
|
||||
continue
|
||||
|
||||
if stripped.startswith("> "):
|
||||
if not in_blockquote:
|
||||
close_lists()
|
||||
html_lines.append("<blockquote>")
|
||||
in_blockquote = True
|
||||
content = stripped[2:]
|
||||
html_lines.append(render_inline(content))
|
||||
continue
|
||||
else:
|
||||
if in_blockquote:
|
||||
html_lines.append("</blockquote>")
|
||||
in_blockquote = False
|
||||
|
||||
ul_match = re.match(r"^\s*[-*+]\s+(.*)$", line)
|
||||
if ul_match:
|
||||
if in_ol:
|
||||
html_lines.append("</ol>")
|
||||
in_ol = False
|
||||
if not in_ul:
|
||||
html_lines.append("<ul>")
|
||||
in_ul = True
|
||||
html_lines.append(f"<li>{render_inline(ul_match.group(1))}</li>")
|
||||
continue
|
||||
|
||||
ol_match = re.match(r"^\s*\d+\.\s+(.*)$", line)
|
||||
if ol_match:
|
||||
if in_ul:
|
||||
html_lines.append("</ul>")
|
||||
in_ul = False
|
||||
if not in_ol:
|
||||
html_lines.append("<ol>")
|
||||
in_ol = True
|
||||
html_lines.append(f"<li>{render_inline(ol_match.group(1))}</li>")
|
||||
continue
|
||||
|
||||
close_lists()
|
||||
|
||||
# Inline conversions (leave raw HTML intact)
|
||||
if stripped == "[TOC]":
|
||||
toc_items = []
|
||||
for level, title, slug in headings:
|
||||
toc_items.append(
|
||||
f'<li class="toc-level-{level}"><a href="#{slug}">{title}</a></li>',
|
||||
)
|
||||
html_lines.append(
|
||||
'<nav class="toc"><ul>' + "".join(toc_items) + "</ul></nav>",
|
||||
)
|
||||
continue
|
||||
|
||||
html_lines.append(f"<p>{render_inline(line)}</p>")
|
||||
|
||||
close_lists()
|
||||
if in_blockquote:
|
||||
html_lines.append("</blockquote>")
|
||||
if in_code:
|
||||
html_lines.append("</code></pre>")
|
||||
|
||||
return "\n".join(html_lines)
|
||||
|
||||
|
||||
def _render_markdown_document(markdown_text: str) -> str:
|
||||
body = _render_markdown_fallback(markdown_text)
|
||||
wrapped = (
|
||||
'<!doctype html><html><head><meta charset="utf-8">'
|
||||
'<meta name="viewport" content="width=device-width,initial-scale=1">'
|
||||
"<style>body{max-width:900px;margin:24px auto;padding:0 16px;"
|
||||
"font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;"
|
||||
"line-height:1.55;} img{max-width:100%;} pre{background:#f6f6f6;padding:12px;overflow:auto;}"
|
||||
".toc ul{list-style:none;padding-left:0;} .toc li{margin:4px 0;}</style>"
|
||||
"</head><body>"
|
||||
f"{body}"
|
||||
"</body></html>"
|
||||
)
|
||||
return wrapped
|
||||
|
||||
|
||||
def _content_type_base(content_type: str) -> str:
|
||||
return (content_type or "").split(";", 1)[0].strip().lower()
|
||||
|
||||
@@ -859,6 +369,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
if config is None:
|
||||
config = get_config(resolve_plugins=False)
|
||||
fullpath, path = _resolve_archive_path(document_root, path)
|
||||
replay_response = partial(_apply_archive_replay_headers, fullpath=fullpath, is_archive_replay=is_archive_replay, config=config)
|
||||
if os.access(fullpath, os.R_OK) and fullpath.is_dir():
|
||||
if request.GET.get("download") == "zip" and show_indexes:
|
||||
return _build_directory_zip_response(
|
||||
@@ -872,13 +383,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
response = _render_directory_index(request, path, fullpath)
|
||||
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
|
||||
response.headers["Last-Modified"] = http_date(fullpath.stat().st_mtime)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type="text/html",
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
return replay_response(response, content_type="text/html")
|
||||
raise Http404(_("Directory indexes are not allowed here."))
|
||||
if not os.access(fullpath, os.R_OK):
|
||||
raise Http404(_("“%(path)s” does not exist") % {"path": fullpath})
|
||||
@@ -901,13 +406,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
not_modified.headers["ETag"] = etag
|
||||
not_modified.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
|
||||
not_modified.headers["Last-Modified"] = http_date(statobj.st_mtime)
|
||||
return _apply_archive_replay_headers(
|
||||
not_modified,
|
||||
fullpath=fullpath,
|
||||
content_type="",
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
return replay_response(not_modified, content_type="")
|
||||
|
||||
content_type, encoding = mimetypes.guess_type(str(fullpath))
|
||||
preserve_plain_text = fullpath.suffix.lower() in {".log", ".sh"}
|
||||
@@ -940,13 +439,12 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
request.META.get("HTTP_IF_MODIFIED_SINCE"),
|
||||
statobj.st_mtime,
|
||||
):
|
||||
return _apply_archive_replay_headers(
|
||||
HttpResponseNotModified(),
|
||||
fullpath=fullpath,
|
||||
content_type=content_type,
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
return replay_response(HttpResponseNotModified(), content_type=content_type)
|
||||
|
||||
def transformed_response(body: str, response_type: str):
|
||||
response = HttpResponse(body, content_type=response_type)
|
||||
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
|
||||
return replay_response(response, content_type=response_type)
|
||||
|
||||
# Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
|
||||
if preview_as_text_html:
|
||||
@@ -954,16 +452,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
max_preview_size = 10 * 1024 * 1024
|
||||
if statobj.st_size <= max_preview_size:
|
||||
decoded = fullpath.read_text(encoding="utf-8", errors="replace")
|
||||
wrapped = _render_text_preview_document(decoded, fullpath.name)
|
||||
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
|
||||
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type="text/html; charset=utf-8",
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
wrapped = replay_preview._render_text_preview_document(decoded, fullpath.name)
|
||||
return transformed_response(wrapped, "text/html; charset=utf-8")
|
||||
except (OSError, UnicodeDecodeError, ValueError):
|
||||
preview_as_text_html = False
|
||||
|
||||
@@ -974,16 +464,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
raw_image_url = request.path
|
||||
if preview_query:
|
||||
raw_image_url = f"{raw_image_url}?{urlencode(list(preview_query.lists()), doseq=True)}"
|
||||
wrapped = _render_image_preview_document(raw_image_url, fullpath.name)
|
||||
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
|
||||
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type="text/html; charset=utf-8",
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
wrapped = replay_preview._render_image_preview_document(raw_image_url, fullpath.name)
|
||||
return transformed_response(wrapped, "text/html; charset=utf-8")
|
||||
except (OSError, ValueError):
|
||||
preview_as_image_html = False
|
||||
|
||||
@@ -1011,13 +493,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
response = HttpResponse(body, content_type=preview_content_type)
|
||||
for key, value in headers.items():
|
||||
response.headers[key] = value
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type=preview_content_type,
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
return replay_response(response, content_type=preview_content_type)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
preview_as_archivewebpage_html = False
|
||||
|
||||
@@ -1035,52 +511,37 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
decoded = html.unescape(decoded)
|
||||
rewritten_html, rewritten_count = ("", 0)
|
||||
if content_type.startswith("text/html") and document_root:
|
||||
rewritten_html, rewritten_count = _rewrite_html_image_sources_for_request(request, decoded, document_root, rel_path)
|
||||
markdown_candidate = _extract_markdown_candidate(decoded)
|
||||
if _looks_like_markdown(markdown_candidate):
|
||||
wrapped = _render_markdown_document(markdown_candidate)
|
||||
wrapped, _rewrite_count = _rewrite_html_image_sources_for_request(request, wrapped, document_root, rel_path)
|
||||
wrapped = _apply_transformed_html_preview_style(wrapped)
|
||||
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
|
||||
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type="text/html; charset=utf-8",
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
rewritten_html, rewritten_count = replay_preview._rewrite_html_image_sources_for_request(
|
||||
request,
|
||||
decoded,
|
||||
document_root,
|
||||
rel_path,
|
||||
)
|
||||
markdown_candidate = replay_preview._extract_markdown_candidate(decoded)
|
||||
if replay_preview._looks_like_markdown(markdown_candidate):
|
||||
wrapped = replay_preview._render_markdown_document(markdown_candidate)
|
||||
wrapped, _rewrite_count = replay_preview._rewrite_html_image_sources_for_request(
|
||||
request,
|
||||
wrapped,
|
||||
document_root,
|
||||
rel_path,
|
||||
)
|
||||
wrapped = replay_preview._apply_transformed_html_preview_style(wrapped)
|
||||
return transformed_response(wrapped, "text/html; charset=utf-8")
|
||||
if rewritten_count:
|
||||
rewritten_html = _apply_transformed_html_preview_style(rewritten_html)
|
||||
response = HttpResponse(rewritten_html, content_type=content_type)
|
||||
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type=content_type,
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
rewritten_html = replay_preview._apply_transformed_html_preview_style(rewritten_html)
|
||||
return transformed_response(rewritten_html, content_type)
|
||||
if escaped_count and escaped_count > tag_count * 2:
|
||||
decoded = _apply_transformed_html_preview_style(decoded)
|
||||
response = HttpResponse(decoded, content_type=content_type)
|
||||
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type=content_type,
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
decoded = replay_preview._apply_transformed_html_preview_style(decoded)
|
||||
return transformed_response(decoded, content_type)
|
||||
except (OSError, UnicodeDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
# setup response object
|
||||
ranged_file = RangedFileReader(fullpath.open("rb"))
|
||||
response = StreamingHttpResponse(
|
||||
_stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file,
|
||||
content_type=content_type,
|
||||
)
|
||||
response = StreamingHttpResponse(ranged_file, content_type=content_type)
|
||||
if _is_asgi_request(request):
|
||||
response.streaming_content = _stream_async(ranged_file)
|
||||
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
|
||||
if etag:
|
||||
response.headers["ETag"] = etag
|
||||
@@ -1118,13 +579,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
response.status_code = 206
|
||||
if encoding:
|
||||
response.headers["Content-Encoding"] = encoding
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type=content_type,
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
return replay_response(response, content_type=content_type)
|
||||
|
||||
|
||||
def serve_static(request, path, **kwargs):
|
||||
|
||||
@@ -2,94 +2,11 @@ from typing import Any, cast
|
||||
from collections.abc import Callable
|
||||
|
||||
import json
|
||||
import ast
|
||||
import inspect
|
||||
import toml
|
||||
import re
|
||||
import configparser
|
||||
|
||||
from pathlib import Path, PosixPath
|
||||
|
||||
from pydantic.json_schema import GenerateJsonSchema
|
||||
from pydantic_core import to_jsonable_python
|
||||
|
||||
JSONValue = str | bool | int | None | list["JSONValue"]
|
||||
|
||||
TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
|
||||
|
||||
|
||||
def load_ini_value(val: str) -> JSONValue:
|
||||
"""Convert lax INI values into strict TOML-compliant (JSON) values"""
|
||||
if val.lower() in ("true", "yes", "1"):
|
||||
return True
|
||||
if val.lower() in ("false", "no", "0"):
|
||||
return False
|
||||
if val.isdigit():
|
||||
return int(val)
|
||||
|
||||
try:
|
||||
return ast.literal_eval(val)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
return json.loads(val)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return val
|
||||
|
||||
|
||||
def convert(ini_str: str) -> str:
|
||||
"""Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
setattr(config, "optionxform", str) # capitalize key names
|
||||
config.read_string(ini_str)
|
||||
|
||||
# Initialize an empty dictionary to store the TOML representation
|
||||
toml_dict = {}
|
||||
|
||||
# Iterate over each section in the INI configuration
|
||||
for section in config.sections():
|
||||
toml_dict[section] = {}
|
||||
|
||||
# Iterate over each key-value pair in the section
|
||||
for key, value in config.items(section):
|
||||
parsed_value = load_ini_value(value)
|
||||
|
||||
# Convert the parsed value to its TOML-compatible JSON representation
|
||||
toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
|
||||
|
||||
# Build the TOML string
|
||||
toml_str = TOML_HEADER
|
||||
for section, items in toml_dict.items():
|
||||
toml_str += f"[{section}]\n"
|
||||
for key, value in items.items():
|
||||
toml_str += f"{key} = {value}\n"
|
||||
toml_str += "\n"
|
||||
|
||||
return toml_str.strip()
|
||||
|
||||
|
||||
class JSONSchemaWithLambdas(GenerateJsonSchema):
|
||||
"""
|
||||
Encode lambda functions in default values properly.
|
||||
Usage:
|
||||
>>> json.dumps(value, encoder=JSONSchemaWithLambdas())
|
||||
"""
|
||||
|
||||
def encode_default(self, dft: Any) -> Any:
|
||||
config = self._config
|
||||
if isinstance(dft, Callable):
|
||||
return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
|
||||
return to_jsonable_python(
|
||||
dft,
|
||||
timedelta_mode=config.ser_json_timedelta,
|
||||
bytes_mode=config.ser_json_bytes,
|
||||
serialize_unknown=True,
|
||||
)
|
||||
|
||||
|
||||
def better_toml_dump_str(val: Any) -> str:
|
||||
try:
|
||||
|
||||
@@ -5,7 +5,7 @@ import shutil
|
||||
from django.contrib import admin, messages
|
||||
from django.utils.html import format_html, format_html_join
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.base_models.admin import card_fieldset, BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.personas.forms import PersonaAdminForm
|
||||
from archivebox.personas.importers import discover_local_browser_profiles
|
||||
from archivebox.personas.models import Persona
|
||||
@@ -30,28 +30,20 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
"classes": ("card", "persona-card-primary"),
|
||||
},
|
||||
),
|
||||
(
|
||||
card_fieldset(
|
||||
"Browser Import",
|
||||
{
|
||||
"fields": (
|
||||
"import_mode",
|
||||
"import_discovered_profile",
|
||||
"import_source",
|
||||
"import_profile_name",
|
||||
"import_copy_profile",
|
||||
"import_extract_cookies",
|
||||
"import_capture_storage",
|
||||
),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Advanced",
|
||||
{
|
||||
"fields": ("config",),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
(
|
||||
"import_mode",
|
||||
"import_discovered_profile",
|
||||
"import_source",
|
||||
"import_profile_name",
|
||||
"import_copy_profile",
|
||||
"import_extract_cookies",
|
||||
"import_capture_storage",
|
||||
),
|
||||
wide=True,
|
||||
),
|
||||
card_fieldset("Advanced", ("config",), wide=True),
|
||||
)
|
||||
|
||||
change_fieldsets = (
|
||||
@@ -65,13 +57,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
),
|
||||
add_fieldsets[1],
|
||||
add_fieldsets[2],
|
||||
(
|
||||
"Artifacts",
|
||||
{
|
||||
"fields": ("persona_paths", "import_artifact_status"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
card_fieldset("Artifacts", ("persona_paths", "import_artifact_status"), wide=True),
|
||||
)
|
||||
|
||||
@admin.display(description="Chrome Profile")
|
||||
|
||||
@@ -216,116 +216,51 @@ class PersonaImportResult:
|
||||
return self.profile_copied or self.cookies_imported or self.storage_captured or self.user_agent_imported
|
||||
|
||||
|
||||
def get_chrome_user_data_dir() -> Path | None:
|
||||
"""Get the default Chrome user data directory for the current platform."""
|
||||
system = platform.system()
|
||||
home = Path.home()
|
||||
|
||||
if system == "Darwin":
|
||||
candidates = [
|
||||
home / "Library" / "Application Support" / "Google" / "Chrome",
|
||||
]
|
||||
elif system == "Linux":
|
||||
candidates = [
|
||||
home / ".config" / "google-chrome",
|
||||
home / ".config" / "chrome",
|
||||
]
|
||||
elif system == "Windows":
|
||||
local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
|
||||
candidates = [
|
||||
local_app_data / "Google" / "Chrome" / "User Data",
|
||||
]
|
||||
else:
|
||||
candidates = []
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.exists() and _list_profile_names(candidate):
|
||||
return candidate
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_chromium_user_data_dir() -> Path | None:
|
||||
"""Select Chromium's own profile even when Google Chrome is installed."""
|
||||
home = Path.home()
|
||||
if platform.system() == "Darwin":
|
||||
candidates = [home / "Library/Application Support/Chromium"]
|
||||
elif platform.system() == "Windows":
|
||||
candidates = [Path(os.environ.get("LOCALAPPDATA", home / "AppData/Local")) / "Chromium/User Data"]
|
||||
else:
|
||||
candidates = [
|
||||
Path(os.environ.get("XDG_CONFIG_HOME", home / ".config")) / "chromium",
|
||||
home / "snap/chromium/common/chromium",
|
||||
]
|
||||
return next((path for path in candidates if path.is_dir() and _list_profile_names(path)), None)
|
||||
|
||||
|
||||
def get_brave_user_data_dir() -> Path | None:
|
||||
"""Get the default Brave user data directory for the current platform."""
|
||||
system = platform.system()
|
||||
home = Path.home()
|
||||
|
||||
if system == "Darwin":
|
||||
candidates = [
|
||||
home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
|
||||
]
|
||||
elif system == "Linux":
|
||||
candidates = [
|
||||
home / ".config" / "BraveSoftware" / "Brave-Browser",
|
||||
]
|
||||
elif system == "Windows":
|
||||
local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
|
||||
candidates = [
|
||||
local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
|
||||
]
|
||||
else:
|
||||
candidates = []
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.exists() and _list_profile_names(candidate):
|
||||
return candidate
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_edge_user_data_dir() -> Path | None:
|
||||
"""Get the default Edge user data directory for the current platform."""
|
||||
system = platform.system()
|
||||
home = Path.home()
|
||||
|
||||
if system == "Darwin":
|
||||
candidates = [
|
||||
home / "Library" / "Application Support" / "Microsoft Edge",
|
||||
]
|
||||
elif system == "Linux":
|
||||
candidates = [
|
||||
home / ".config" / "microsoft-edge",
|
||||
home / ".config" / "microsoft-edge-beta",
|
||||
home / ".config" / "microsoft-edge-dev",
|
||||
]
|
||||
elif system == "Windows":
|
||||
local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
|
||||
candidates = [
|
||||
local_app_data / "Microsoft" / "Edge" / "User Data",
|
||||
]
|
||||
else:
|
||||
candidates = []
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.exists() and _list_profile_names(candidate):
|
||||
return candidate
|
||||
|
||||
return None
|
||||
|
||||
|
||||
BROWSER_PROFILE_FINDERS = {
|
||||
"chrome": get_chrome_user_data_dir,
|
||||
"chromium": get_chromium_user_data_dir,
|
||||
"brave": get_brave_user_data_dir,
|
||||
"edge": get_edge_user_data_dir,
|
||||
# Relative profile roots in lookup order. Chromium additionally supports XDG
|
||||
# and snap; the other browsers retain their own conventional config locations.
|
||||
BROWSER_PROFILE_PATHS = {
|
||||
"chrome": {
|
||||
"Darwin": ("Google/Chrome",),
|
||||
"Linux": ("google-chrome", "chrome"),
|
||||
"Windows": ("Google/Chrome/User Data",),
|
||||
},
|
||||
"chromium": {
|
||||
"Darwin": ("Chromium",),
|
||||
"Linux": ("chromium",),
|
||||
"Windows": ("Chromium/User Data",),
|
||||
},
|
||||
"brave": {
|
||||
"Darwin": ("BraveSoftware/Brave-Browser",),
|
||||
"Linux": ("BraveSoftware/Brave-Browser",),
|
||||
"Windows": ("BraveSoftware/Brave-Browser/User Data",),
|
||||
},
|
||||
"edge": {
|
||||
"Darwin": ("Microsoft Edge",),
|
||||
"Linux": ("microsoft-edge", "microsoft-edge-beta", "microsoft-edge-dev"),
|
||||
"Windows": ("Microsoft/Edge/User Data",),
|
||||
},
|
||||
}
|
||||
CHROMIUM_BROWSERS = tuple(BROWSER_PROFILE_PATHS)
|
||||
|
||||
CHROMIUM_BROWSERS = tuple(BROWSER_PROFILE_FINDERS.keys())
|
||||
|
||||
def get_browser_user_data_dir(browser: str) -> Path | None:
|
||||
"""Find a browser's first installed profile root on the current platform."""
|
||||
home = Path.home()
|
||||
system = platform.system()
|
||||
# Chromium historically uses Linux locations on other Unix platforms too.
|
||||
if browser == "chromium" and system not in ("Darwin", "Windows"):
|
||||
system = "Linux"
|
||||
roots = {
|
||||
"Darwin": home / "Library/Application Support",
|
||||
"Linux": Path(os.environ.get("XDG_CONFIG_HOME", home / ".config")) if browser == "chromium" else home / ".config",
|
||||
"Windows": Path(os.environ.get("LOCALAPPDATA", home / "AppData/Local")),
|
||||
}
|
||||
if system not in roots:
|
||||
return None
|
||||
candidates = [roots[system] / relative for relative in BROWSER_PROFILE_PATHS[browser].get(system, ())]
|
||||
if browser == "chromium" and system == "Linux":
|
||||
candidates.append(home / "snap/chromium/common/chromium")
|
||||
return next((path for path in candidates if path.is_dir() and _list_profile_names(path)), None)
|
||||
|
||||
|
||||
NETSCAPE_COOKIE_HEADER = [
|
||||
@@ -356,8 +291,8 @@ def validate_persona_name(name: str) -> tuple[bool, str]:
|
||||
def discover_local_browser_profiles() -> list[PersonaImportSource]:
|
||||
discovered: list[PersonaImportSource] = []
|
||||
|
||||
for browser, finder in BROWSER_PROFILE_FINDERS.items():
|
||||
user_data_dir = finder()
|
||||
for browser in CHROMIUM_BROWSERS:
|
||||
user_data_dir = get_browser_user_data_dir(browser)
|
||||
if not user_data_dir:
|
||||
continue
|
||||
|
||||
@@ -430,11 +365,11 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
|
||||
|
||||
def resolve_browser_import_source(browser: str, profile_dir: str | None = None) -> PersonaImportSource:
|
||||
browser = browser.lower().strip()
|
||||
if browser not in BROWSER_PROFILE_FINDERS:
|
||||
supported = ", ".join(BROWSER_PROFILE_FINDERS)
|
||||
if browser not in CHROMIUM_BROWSERS:
|
||||
supported = ", ".join(CHROMIUM_BROWSERS)
|
||||
raise ValueError(f"Unknown browser: {browser}. Supported browsers: {supported}")
|
||||
|
||||
user_data_dir = BROWSER_PROFILE_FINDERS[browser]()
|
||||
user_data_dir = get_browser_user_data_dir(browser)
|
||||
if not user_data_dir:
|
||||
raise ValueError(
|
||||
f"Could not find {browser} profile directory. Use --source /path/to/browser-data. "
|
||||
|
||||
@@ -131,6 +131,16 @@ class Persona(ModelWithConfig):
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""Portable CLI record, shared by create, list, and update."""
|
||||
return {
|
||||
"id": str(self.id),
|
||||
"name": self.name,
|
||||
"path": str(self.path),
|
||||
"CHROME_USER_DATA_DIR": self.CHROME_USER_DATA_DIR,
|
||||
"COOKIES_FILE": self.COOKIES_FILE,
|
||||
}
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
"""Path to persona directory under PERSONAS_DIR."""
|
||||
@@ -289,87 +299,54 @@ class Persona(ModelWithConfig):
|
||||
|
||||
def prepare_runtime_for_crawl(self, crawl, chrome_binary: str = "") -> dict[str, str]:
|
||||
self.ensure_dirs()
|
||||
|
||||
template_dir = Path(self.CHROME_USER_DATA_DIR)
|
||||
runtime_root = self.runtime_root_for_crawl(crawl)
|
||||
runtime_profile_dir = self.runtime_profile_dir_for_crawl(crawl)
|
||||
runtime_downloads_dir = self.runtime_downloads_dir_for_crawl(crawl)
|
||||
|
||||
with self.lock_runtime_for_crawl():
|
||||
if runtime_root.exists():
|
||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||
if template_dir.exists() and any(template_dir.iterdir()):
|
||||
self.copy_chrome_profile(template_dir, runtime_profile_dir)
|
||||
else:
|
||||
runtime_profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for filename in ("cookies.txt", "auth.json"):
|
||||
source = self.path / filename
|
||||
if source.is_file():
|
||||
shutil.copy2(source, runtime_root / filename)
|
||||
runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.cleanup_chrome_profile(runtime_profile_dir)
|
||||
|
||||
(runtime_root / "persona_name.txt").write_text(self.name)
|
||||
(runtime_root / "template_dir.txt").write_text(str(template_dir))
|
||||
if chrome_binary:
|
||||
(runtime_root / "chrome_binary.txt").write_text(chrome_binary)
|
||||
|
||||
return {
|
||||
# Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
|
||||
# PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
|
||||
# per-crawl runtime root here so CHROME_ISOLATION=crawl never
|
||||
# leaks or reuses the template profile while keeping Chrome path
|
||||
# derivation centralized in the Chrome plugin helpers.
|
||||
"PERSONAS_DIR": str(runtime_root.parent),
|
||||
"ACTIVE_PERSONA": self.name,
|
||||
**{
|
||||
key: str(runtime_root / filename)
|
||||
for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
|
||||
if (runtime_root / filename).is_file()
|
||||
},
|
||||
}
|
||||
return self._prepare_runtime(
|
||||
self.runtime_root_for_crawl(crawl),
|
||||
Path(self.CHROME_USER_DATA_DIR),
|
||||
(self.path,),
|
||||
chrome_binary,
|
||||
)
|
||||
|
||||
def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
|
||||
crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
|
||||
template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR)
|
||||
runtime_root = self.runtime_root_for_snapshot(snapshot)
|
||||
runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot)
|
||||
runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot)
|
||||
crawl_profile = self.runtime_profile_dir_for_crawl(snapshot.crawl)
|
||||
return self._prepare_runtime(
|
||||
self.runtime_root_for_snapshot(snapshot),
|
||||
crawl_profile if crawl_profile.exists() else Path(self.CHROME_USER_DATA_DIR),
|
||||
(self.runtime_root_for_crawl(snapshot.crawl), self.path),
|
||||
chrome_binary,
|
||||
)
|
||||
|
||||
if runtime_root.exists():
|
||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||
def _prepare_runtime(
|
||||
self,
|
||||
runtime_root: Path,
|
||||
template_dir: Path,
|
||||
auth_sources: tuple[Path, ...],
|
||||
chrome_binary: str,
|
||||
) -> dict[str, str]:
|
||||
"""Clone one isolated profile, taking each auth file from its first available source."""
|
||||
profile_dir = runtime_root / "chrome_profile"
|
||||
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||
if template_dir.exists() and any(template_dir.iterdir()):
|
||||
self.copy_chrome_profile(template_dir, runtime_profile_dir)
|
||||
self.copy_chrome_profile(template_dir, profile_dir)
|
||||
else:
|
||||
runtime_profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for filename in ("cookies.txt", "auth.json"):
|
||||
source = self.runtime_root_for_crawl(snapshot.crawl) / filename
|
||||
if not source.is_file():
|
||||
source = self.path / filename
|
||||
if source.is_file():
|
||||
shutil.copy2(source, runtime_root / filename)
|
||||
runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.cleanup_chrome_profile(runtime_profile_dir)
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
overrides = {"PERSONAS_DIR": str(runtime_root.parent), "ACTIVE_PERSONA": self.name}
|
||||
for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json")):
|
||||
source = next((root / filename for root in auth_sources if (root / filename).is_file()), None)
|
||||
if source is not None:
|
||||
destination = runtime_root / filename
|
||||
shutil.copy2(source, destination)
|
||||
overrides[key] = str(destination)
|
||||
(runtime_root / "chrome_downloads").mkdir(parents=True, exist_ok=True)
|
||||
self.cleanup_chrome_profile(profile_dir)
|
||||
(runtime_root / "persona_name.txt").write_text(self.name)
|
||||
(runtime_root / "template_dir.txt").write_text(str(template_dir))
|
||||
if chrome_binary:
|
||||
(runtime_root / "chrome_binary.txt").write_text(chrome_binary)
|
||||
|
||||
return {
|
||||
# See prepare_runtime_for_crawl(): snapshot isolation changes the
|
||||
# persona root, not individual CHROME_* config keys, so standalone
|
||||
# Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
|
||||
"PERSONAS_DIR": str(runtime_root.parent),
|
||||
"ACTIVE_PERSONA": self.name,
|
||||
**{
|
||||
key: str(runtime_root / filename)
|
||||
for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
|
||||
if (runtime_root / filename).is_file()
|
||||
},
|
||||
}
|
||||
# Chrome hooks derive their directories from this isolated persona root.
|
||||
return overrides
|
||||
|
||||
def cleanup_runtime_for_crawl(self, crawl) -> None:
|
||||
shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)
|
||||
|
||||
@@ -55,10 +55,6 @@ def get_plugins() -> list[str]:
|
||||
return sorted(get_plugin_catalog())
|
||||
|
||||
|
||||
def get_plugin_models():
|
||||
return get_plugin_catalog().plugins
|
||||
|
||||
|
||||
def get_plugin_name(plugin: str) -> str:
|
||||
"""
|
||||
Get the base plugin name without numeric prefix.
|
||||
|
||||
+22
-35
@@ -87,23 +87,14 @@ def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> An
|
||||
return False
|
||||
raise forms.ValidationError("Must be true or false.")
|
||||
|
||||
if "integer" in schema_types:
|
||||
value = int(str(raw_value).strip())
|
||||
if "integer" in schema_types or "number" in schema_types:
|
||||
convert = int if "integer" in schema_types else float
|
||||
value = convert(str(raw_value).strip())
|
||||
minimum = schema.get("minimum")
|
||||
maximum = schema.get("maximum")
|
||||
if minimum is not None and value < int(minimum):
|
||||
if minimum is not None and value < convert(minimum):
|
||||
raise forms.ValidationError(f"Must be at least {minimum}.")
|
||||
if maximum is not None and value > int(maximum):
|
||||
raise forms.ValidationError(f"Must be at most {maximum}.")
|
||||
return value
|
||||
|
||||
if "number" in schema_types:
|
||||
value = float(str(raw_value).strip())
|
||||
minimum = schema.get("minimum")
|
||||
maximum = schema.get("maximum")
|
||||
if minimum is not None and value < float(minimum):
|
||||
raise forms.ValidationError(f"Must be at least {minimum}.")
|
||||
if maximum is not None and value > float(maximum):
|
||||
if maximum is not None and value > convert(maximum):
|
||||
raise forms.ValidationError(f"Must be at most {maximum}.")
|
||||
return value
|
||||
|
||||
@@ -148,28 +139,24 @@ class PluginConfigFormMixin:
|
||||
self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
|
||||
grouped_plugins = catalog.groups()
|
||||
|
||||
group_specs = []
|
||||
for category, field_name, title in PLUGIN_GROUPS:
|
||||
plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []))
|
||||
group_specs.append((field_name, title, "", "", "", plugin_names))
|
||||
if field_name in self.fields:
|
||||
get_choice_field(self, field_name).choices = [
|
||||
(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
|
||||
]
|
||||
binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
|
||||
self.plugin_groups = [
|
||||
{
|
||||
"field_name": field_name,
|
||||
"title": title,
|
||||
"note": note,
|
||||
"dom_id": dom_id,
|
||||
"select_all_group": select_all_group,
|
||||
"show_selectors": field_name in self.fields,
|
||||
"plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
|
||||
}
|
||||
for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
|
||||
if any(plugin in all_plugins for plugin in plugin_names)
|
||||
]
|
||||
self.plugin_groups = []
|
||||
for category, field_name, title in PLUGIN_GROUPS:
|
||||
plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []) if plugin.name in all_plugins)
|
||||
if field_name in self.fields:
|
||||
get_choice_field(self, field_name).choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names]
|
||||
if plugin_names:
|
||||
self.plugin_groups.append(
|
||||
{
|
||||
"field_name": field_name,
|
||||
"title": title,
|
||||
"note": "",
|
||||
"dom_id": "",
|
||||
"select_all_group": "",
|
||||
"show_selectors": field_name in self.fields,
|
||||
"plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
|
||||
},
|
||||
)
|
||||
|
||||
def _build_plugin_cards(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Render the monitor's crawl, snapshot, and hook payloads from loaded data."""
|
||||
|
||||
from pathlib import Path
|
||||
from archivebox.config.constants import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.routes_util import build_snapshot_url, build_web_url
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
|
||||
|
||||
def render_crawl(report, crawl):
|
||||
crawl_id = str(crawl["id"])
|
||||
crawl_snapshot_counts = report.snapshot_counts_by_crawl.get(crawl_id, {})
|
||||
total_snapshots = sum(crawl_snapshot_counts.values())
|
||||
completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
|
||||
started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
|
||||
pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
|
||||
cancelled_snapshots = report.cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
|
||||
|
||||
# Count URLs in the crawl (for when snapshots haven't been created yet)
|
||||
urls_count = 0
|
||||
if crawl["urls"]:
|
||||
urls_count = len([u for u in crawl["urls"].split("\n") if u.strip() and not u.startswith("#")])
|
||||
|
||||
# Calculate crawl progress
|
||||
crawl_progress = int((completed_snapshots / total_snapshots) * 100) if total_snapshots > 0 else 0
|
||||
crawl_run_started_at = crawl["created_at"]
|
||||
crawl_setup_plugins = [
|
||||
payload
|
||||
for payload, proc_started_at in report.process_records_by_crawl.get(crawl_id, [])
|
||||
if is_current_run_timestamp(proc_started_at, crawl_run_started_at)
|
||||
]
|
||||
crawl_setup_total = len(crawl_setup_plugins)
|
||||
crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
|
||||
crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
|
||||
crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
|
||||
crawl_screencast_url = screencast_frame_url(report, crawl_id, report.active_crawl_objects[crawl_id].output_dir)
|
||||
crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
|
||||
|
||||
active_snapshots_for_crawl = [
|
||||
render_snapshot(report, snapshot, crawl_id) for snapshot in report.displayed_snapshots_by_crawl.get(crawl_id, [])
|
||||
]
|
||||
|
||||
# Check if crawl can start (for debugging stuck crawls)
|
||||
can_start = bool(crawl["urls"])
|
||||
urls_preview = crawl["urls"][:60] if crawl["urls"] else None
|
||||
crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
|
||||
persona_details = report.persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
|
||||
persona_name = persona_details["name"] if persona_details else "Default"
|
||||
persona_details = persona_details or report.persona_details_by_name.get(persona_name)
|
||||
crawl_output_size = report.crawl_output_sizes_by_crawl.get(crawl_id, 0)
|
||||
avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
|
||||
crawl_obj = report.active_crawl_objects[crawl_id]
|
||||
effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
|
||||
max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
|
||||
crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
|
||||
crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
|
||||
snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
|
||||
|
||||
# Check if retry_at is in the future (would prevent worker from claiming)
|
||||
retry_at_future = crawl["retry_at"] > report.now if crawl["retry_at"] else False
|
||||
is_paused = crawl_obj.is_paused
|
||||
seconds_until_retry = (
|
||||
0 if is_paused else int((crawl["retry_at"] - report.now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
|
||||
)
|
||||
crawl_worker_state = (
|
||||
"running"
|
||||
if report.crawl_process_pids.get(crawl_id)
|
||||
or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl)
|
||||
else "waiting"
|
||||
)
|
||||
if is_paused:
|
||||
crawl_worker_state = "paused"
|
||||
elif crawl["status"] == Crawl.StatusChoices.STARTED and crawl_worker_state == "waiting" and (started_snapshots or pending_snapshots):
|
||||
crawl_worker_state = "waiting" if report.orchestrator_running else "crashed"
|
||||
|
||||
return {
|
||||
"id": crawl_id,
|
||||
"label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60],
|
||||
"status": crawl["status"],
|
||||
"is_paused": is_paused,
|
||||
"started": crawl["created_at"].isoformat() if crawl["created_at"] else None,
|
||||
"progress": crawl_progress,
|
||||
"created_by": crawl["created_by__username"],
|
||||
"persona": persona_name,
|
||||
"persona_admin_url": persona_details["admin_url"] if persona_details else None,
|
||||
"max_depth": crawl["max_depth"],
|
||||
"max_urls": max_urls,
|
||||
"max_crawl_size": crawl_max_size,
|
||||
"crawl_timeout": crawl_timeout,
|
||||
"max_snapshot_size": snapshot_max_size,
|
||||
"max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited",
|
||||
"crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited",
|
||||
"max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
|
||||
"crawl_output_size": crawl_output_size,
|
||||
"avg_snapshot_size": avg_snapshot_size,
|
||||
"crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
|
||||
"avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
|
||||
"tags": crawl_tags,
|
||||
"urls_count": urls_count,
|
||||
"total_snapshots": total_snapshots,
|
||||
"completed_snapshots": completed_snapshots,
|
||||
"started_snapshots": started_snapshots,
|
||||
"failed_snapshots": 0,
|
||||
"pending_snapshots": pending_snapshots,
|
||||
"cancelled_snapshots": cancelled_snapshots,
|
||||
"setup_plugins": crawl_setup_plugins,
|
||||
"setup_total_plugins": crawl_setup_total,
|
||||
"setup_completed_plugins": crawl_setup_completed,
|
||||
"setup_failed_plugins": crawl_setup_failed,
|
||||
"setup_pending_plugins": crawl_setup_pending,
|
||||
"screencast_url": crawl_screencast_url,
|
||||
"screencast_link": crawl_screencast_link,
|
||||
"active_snapshots": active_snapshots_for_crawl,
|
||||
"queued_snapshots_hidden": report.queued_snapshot_overflow_by_crawl.get(crawl_id, 0),
|
||||
"can_start": can_start,
|
||||
"urls_preview": urls_preview,
|
||||
"retry_at_future": retry_at_future,
|
||||
"seconds_until_retry": seconds_until_retry,
|
||||
"worker_pid": report.crawl_process_pids.get(crawl_id),
|
||||
"worker_state": crawl_worker_state,
|
||||
}
|
||||
|
||||
|
||||
def render_snapshot(report, snapshot, crawl_id):
|
||||
snapshot_run_started_at = snapshot.get("downloaded_at") or snapshot.get("created_at")
|
||||
snapshot_process_started_at = snapshot.get("created_at")
|
||||
# Get archive results only for displayed active snapshots. Large crawls can
|
||||
# contain thousands of sealed snapshots, and prefetching all their results
|
||||
# makes the progress endpoint compete with the runner.
|
||||
snapshot_results = [
|
||||
ar
|
||||
for ar in report.archiveresults_by_snapshot.get(str(snapshot["id"]), [])
|
||||
if archiveresult_matches_current_run(ar, snapshot_run_started_at)
|
||||
]
|
||||
if snapshot["status"] == Snapshot.StatusChoices.QUEUED:
|
||||
snapshot_results = []
|
||||
|
||||
plugin_progress_values: list[int] = []
|
||||
all_plugins: list[dict[str, object]] = []
|
||||
seen_plugin_keys: set[str] = set()
|
||||
snapshot_title = (
|
||||
str(snapshot["title"] or "")
|
||||
if snapshot["status"] == Snapshot.StatusChoices.QUEUED
|
||||
else Snapshot._normalize_title_candidate(snapshot["title"], snapshot_url=snapshot["url"])
|
||||
)
|
||||
snapshot_favicon_url = ""
|
||||
snapshot_preview_url = ""
|
||||
snapshot_preview_link = ""
|
||||
snapshot_screencast_url = ""
|
||||
snapshot_screencast_link = ""
|
||||
snapshot_fallback_urls: list[str] = []
|
||||
result_by_plugin = {result.plugin: result for result in snapshot_results}
|
||||
title_result = result_by_plugin.get("title")
|
||||
if not snapshot_title and title_result is not None and title_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
snapshot_title = Snapshot._normalize_title_candidate(title_result.output_str, snapshot_url=snapshot["url"])
|
||||
favicon_result = result_by_plugin.get("favicon")
|
||||
if favicon_result is not None and favicon_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
favicon_path = (
|
||||
favicon_result.embed_path_db(
|
||||
favicon_result.output_files if isinstance(favicon_result.output_files, dict) else {},
|
||||
check_filesystem=False,
|
||||
)
|
||||
or "favicon/favicon.ico"
|
||||
)
|
||||
snapshot_favicon_url = snapshot_output_url(report, snapshot, favicon_path)
|
||||
screenshot_result = result_by_plugin.get("screenshot")
|
||||
if screenshot_result is not None and screenshot_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
snapshot_preview_link = snapshot_view_url(report, snapshot)
|
||||
screenshot_path = (
|
||||
screenshot_result.embed_path_db(
|
||||
screenshot_result.output_files if isinstance(screenshot_result.output_files, dict) else {},
|
||||
check_filesystem=False,
|
||||
)
|
||||
or "screenshot/screenshot.png"
|
||||
)
|
||||
snapshot_preview_url = snapshot_output_url(report, snapshot, screenshot_path)
|
||||
snapshot_preview_link = snapshot_view_url(report, snapshot, screenshot_path)
|
||||
if snapshot_favicon_url:
|
||||
snapshot_fallback_urls.append(snapshot_favicon_url)
|
||||
elif snapshot_favicon_url:
|
||||
snapshot_preview_url = snapshot_favicon_url
|
||||
|
||||
if snapshot["status"] == Snapshot.StatusChoices.STARTED:
|
||||
snapshot_screencast_url = screencast_frame_url(report, crawl_id, report.active_crawl_objects[crawl_id].output_dir)
|
||||
snapshot_screencast_link = snapshot_view_url(report, snapshot) if snapshot_screencast_url else ""
|
||||
|
||||
def plugin_sort_key(ar):
|
||||
status_order = {
|
||||
ArchiveResult.StatusChoices.STARTED: 0,
|
||||
ArchiveResult.StatusChoices.QUEUED: 1,
|
||||
ArchiveResult.StatusChoices.SUCCEEDED: 2,
|
||||
ArchiveResult.StatusChoices.NORESULTS: 3,
|
||||
ArchiveResult.StatusChoices.FAILED: 4,
|
||||
}
|
||||
return (status_order.get(ar.status, 5), ar.plugin, ar.hook_name or "")
|
||||
|
||||
for ar in sorted(snapshot_results, key=plugin_sort_key):
|
||||
status = ar.status
|
||||
process = ar.process_record
|
||||
progress_value = 0
|
||||
if status in (
|
||||
ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
ArchiveResult.StatusChoices.FAILED,
|
||||
ArchiveResult.StatusChoices.SKIPPED,
|
||||
ArchiveResult.StatusChoices.NORESULTS,
|
||||
):
|
||||
progress_value = 100
|
||||
elif status == ArchiveResult.StatusChoices.STARTED:
|
||||
started_at = ar.start_ts or (process.started_at if process else None)
|
||||
timeout = process.timeout if process else 120
|
||||
if started_at and timeout:
|
||||
elapsed = max(0.0, (report.now - started_at).total_seconds())
|
||||
progress_value = int(min(99, max(1, (elapsed / float(timeout)) * 100)))
|
||||
else:
|
||||
progress_value = 1
|
||||
else:
|
||||
progress_value = 0
|
||||
|
||||
plugin_progress_values.append(progress_value)
|
||||
plugin, label, phase, hook_name = hook_details(ar.hook_name or ar.plugin, plugin=ar.plugin)
|
||||
|
||||
plugin_payload = {
|
||||
"id": str(ar.id),
|
||||
"plugin": ar.plugin,
|
||||
"label": label,
|
||||
"hook_name": hook_name,
|
||||
"phase": phase,
|
||||
"status": status,
|
||||
"process_id": str(process.id) if process else None,
|
||||
"admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
|
||||
}
|
||||
output_path = ar.embed_path_db(ar.output_files if isinstance(ar.output_files, dict) else {}, check_filesystem=False)
|
||||
if output_path:
|
||||
plugin_payload["output_path"] = output_path
|
||||
plugin_payload["output_url"] = snapshot_view_url(report, snapshot, output_path)
|
||||
if status == ArchiveResult.StatusChoices.STARTED and process:
|
||||
plugin_payload["pid"] = process.pid
|
||||
if status == ArchiveResult.StatusChoices.STARTED:
|
||||
plugin_payload["progress"] = progress_value
|
||||
plugin_payload["timeout"] = process.timeout if process else 120
|
||||
plugin_payload["source"] = "archiveresult"
|
||||
all_plugins.append(plugin_payload)
|
||||
seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
|
||||
|
||||
for proc_payload, proc_started_at in report.process_records_by_snapshot.get(str(snapshot["id"]), []):
|
||||
if not is_current_run_timestamp(proc_started_at, snapshot_process_started_at):
|
||||
continue
|
||||
proc_key = str(proc_payload.get("process_id") or f"{proc_payload.get('plugin')}:{proc_payload.get('hook_name')}")
|
||||
if proc_key in seen_plugin_keys:
|
||||
continue
|
||||
seen_plugin_keys.add(proc_key)
|
||||
all_plugins.append(proc_payload)
|
||||
|
||||
proc_status = proc_payload.get("status")
|
||||
if proc_status in ("succeeded", "failed", "skipped"):
|
||||
plugin_progress_values.append(100)
|
||||
elif proc_status == "started":
|
||||
plugin_progress_values.append(1)
|
||||
else:
|
||||
plugin_progress_values.append(0)
|
||||
|
||||
total_plugins = len(all_plugins)
|
||||
completed_plugins = sum(1 for item in all_plugins if item.get("status") == "succeeded")
|
||||
failed_plugins = sum(1 for item in all_plugins if item.get("status") == "failed")
|
||||
pending_plugins = sum(1 for item in all_plugins if item.get("status") == "queued")
|
||||
|
||||
snapshot_progress = int(sum(plugin_progress_values) / len(plugin_progress_values)) if plugin_progress_values else 0
|
||||
worker_state = "running" if report.snapshot_process_pids.get(str(snapshot["id"])) else "waiting"
|
||||
if (
|
||||
snapshot["status"] == Snapshot.StatusChoices.STARTED
|
||||
and worker_state == "waiting"
|
||||
and not all_plugins
|
||||
and snapshot["modified_at"]
|
||||
and (report.now - snapshot["modified_at"]).total_seconds() > 30
|
||||
):
|
||||
worker_state = "waiting" if report.orchestrator_running else "crashed"
|
||||
|
||||
if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not report.snapshot_process_pids.get(str(snapshot["id"])):
|
||||
compact_snapshot = [
|
||||
str(snapshot["id"]),
|
||||
snapshot_display_url(snapshot["url"]),
|
||||
]
|
||||
if snapshot_title:
|
||||
compact_snapshot.append(snapshot_title)
|
||||
return compact_snapshot
|
||||
|
||||
snapshot_payload = {
|
||||
"id": str(snapshot["id"]),
|
||||
"url": snapshot_display_url(snapshot["url"]),
|
||||
"title": snapshot_title,
|
||||
"status": snapshot["status"],
|
||||
"worker_state": worker_state,
|
||||
}
|
||||
if snapshot["status"] != Snapshot.StatusChoices.QUEUED or all_plugins or report.snapshot_process_pids.get(str(snapshot["id"])):
|
||||
snapshot_payload.update(
|
||||
{
|
||||
"view_url": snapshot_view_url(report, snapshot),
|
||||
"started": (snapshot["downloaded_at"] or snapshot["created_at"]).isoformat()
|
||||
if (snapshot["downloaded_at"] or snapshot["created_at"])
|
||||
else None,
|
||||
"progress": snapshot_progress,
|
||||
"total_plugins": total_plugins,
|
||||
"completed_plugins": completed_plugins,
|
||||
"failed_plugins": failed_plugins,
|
||||
"pending_plugins": pending_plugins,
|
||||
"all_plugins": all_plugins,
|
||||
},
|
||||
)
|
||||
if snapshot_favicon_url:
|
||||
snapshot_payload["favicon_url"] = snapshot_favicon_url
|
||||
if snapshot_preview_url:
|
||||
snapshot_payload["preview_url"] = snapshot_preview_url
|
||||
snapshot_payload["preview_link"] = snapshot_preview_link
|
||||
if snapshot_screencast_url:
|
||||
snapshot_payload["screencast_url"] = snapshot_screencast_url
|
||||
snapshot_payload["screencast_link"] = snapshot_screencast_link
|
||||
if snapshot_fallback_urls:
|
||||
snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls
|
||||
if report.snapshot_process_pids.get(str(snapshot["id"])):
|
||||
snapshot_payload["worker_pid"] = report.snapshot_process_pids[str(snapshot["id"])]
|
||||
|
||||
return snapshot_payload
|
||||
|
||||
|
||||
def is_current_run_timestamp(event_ts, run_started_at) -> bool:
|
||||
if run_started_at is None:
|
||||
return True
|
||||
if event_ts is None:
|
||||
return False
|
||||
return event_ts >= run_started_at
|
||||
|
||||
|
||||
def archiveresult_matches_current_run(ar, run_started_at) -> bool:
|
||||
if run_started_at is None:
|
||||
return True
|
||||
if ar.status in (
|
||||
ArchiveResult.StatusChoices.QUEUED,
|
||||
ArchiveResult.StatusChoices.STARTED,
|
||||
ArchiveResult.StatusChoices.BACKOFF,
|
||||
):
|
||||
return True
|
||||
event_ts = ar.end_ts or ar.start_ts or ar.modified_at or ar.created_at
|
||||
return is_current_run_timestamp(event_ts, run_started_at)
|
||||
|
||||
|
||||
def hook_details(hook_name: str, plugin: str = "setup") -> tuple[str, str, str, str]:
|
||||
normalized_hook_name = Path(hook_name).name if hook_name else ""
|
||||
if not normalized_hook_name:
|
||||
return (plugin, plugin, "unknown", "")
|
||||
|
||||
phase = "unknown"
|
||||
if normalized_hook_name == "InstallEvent":
|
||||
phase = "install"
|
||||
elif normalized_hook_name.startswith("on_CrawlSetup__"):
|
||||
phase = "crawl"
|
||||
elif normalized_hook_name.startswith("on_Snapshot__"):
|
||||
phase = "snapshot"
|
||||
|
||||
label = normalized_hook_name
|
||||
if "__" in normalized_hook_name:
|
||||
label = normalized_hook_name.split("__", 1)[1]
|
||||
label = label.rsplit(".", 1)[0]
|
||||
if len(label) > 3 and label[:2].isdigit() and label[2] == "_":
|
||||
label = label[3:]
|
||||
label = label.replace("_", " ").strip() or plugin
|
||||
|
||||
return (plugin, label, phase, normalized_hook_name)
|
||||
|
||||
|
||||
def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]:
|
||||
hook_path = ""
|
||||
if isinstance(cmd, list) and cmd:
|
||||
hook_path = next(
|
||||
(arg for arg in cmd if isinstance(arg, str) and Path(arg).name.startswith("on_") and "__" in Path(arg).name),
|
||||
"",
|
||||
)
|
||||
if not hook_path and isinstance(cmd[0], str):
|
||||
hook_path = cmd[0]
|
||||
|
||||
if not hook_path:
|
||||
return ("", "setup", "unknown", "")
|
||||
|
||||
return hook_details(Path(hook_path).name, plugin=Path(hook_path).parent.name or "setup")
|
||||
|
||||
|
||||
def snapshot_output_url(report, snapshot, output_path: str) -> str:
|
||||
return build_snapshot_url(str(snapshot["id"]), output_path, request=report.request, config=report.request_config)
|
||||
|
||||
|
||||
def snapshot_archive_path(report, snapshot) -> str:
|
||||
if snapshot["fs_version"] in ("0.7.0", "0.8.0"):
|
||||
return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{snapshot['timestamp']}"
|
||||
crawl = report.crawls_by_id.get(str(snapshot["crawl_id"]))
|
||||
username = "web"
|
||||
if crawl is not None and crawl["created_by_id"]:
|
||||
username = crawl["created_by__username"]
|
||||
if username == "system":
|
||||
username = "web"
|
||||
date_base = snapshot["bookmarked_at"] or snapshot["created_at"]
|
||||
date_str = date_base.strftime("%Y%m%d") if date_base else "unknown"
|
||||
domain = Snapshot.extract_domain_from_url(snapshot["url"])
|
||||
return f"{username}/{date_str}/{domain}/{snapshot['id']}"
|
||||
|
||||
|
||||
def snapshot_view_url(report, snapshot, output_path: str = "") -> str:
|
||||
anchor = f"#{output_path}" if output_path else ""
|
||||
return build_web_url(
|
||||
f"/{snapshot_archive_path(report, snapshot)}/index.html{anchor}",
|
||||
request=report.request,
|
||||
config=report.request_config,
|
||||
)
|
||||
|
||||
|
||||
def snapshot_display_url(url: str) -> str:
|
||||
url = str(url or "")
|
||||
return url if len(url) <= 96 else f"{url[:93]}..."
|
||||
|
||||
|
||||
def screencast_frame_url(report, crawl_id: str, crawl_dir: Path) -> str:
|
||||
frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
|
||||
try:
|
||||
frame_stat = frame_path.stat()
|
||||
except OSError:
|
||||
return ""
|
||||
if frame_stat.st_size <= 0:
|
||||
return ""
|
||||
if report.now.timestamp() - frame_stat.st_mtime > 15:
|
||||
return ""
|
||||
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
|
||||
return f"{report.api_base}{rel}" if report.api_base else rel
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Assemble one live-progress report from scoped, bounded database queries."""
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from pathlib import Path
|
||||
|
||||
from abx_dl.events import PROCESS_EXIT_SKIPPED
|
||||
from django.db.models import CharField, Count, Q, Sum
|
||||
from django.db.models.functions import Cast
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.core.routes_util import get_api_base_url
|
||||
from archivebox.plugins.discovery import discover_plugin_configs
|
||||
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Machine, Process
|
||||
|
||||
from .presentation import render_crawl, process_label
|
||||
from datetime import timedelta
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
|
||||
plugin_configs = discover_plugin_configs()
|
||||
indexing_plugin_names = frozenset(
|
||||
plugin_name
|
||||
for plugin_name, plugin_config in plugin_configs.items()
|
||||
if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
|
||||
)
|
||||
download_plugin_names = frozenset(
|
||||
plugin_name
|
||||
for plugin_name, plugin_config in plugin_configs.items()
|
||||
if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
|
||||
)
|
||||
return download_plugin_names, indexing_plugin_names
|
||||
|
||||
|
||||
class ProgressReport:
|
||||
"""Per-request query results shared by status, process, and preview rendering."""
|
||||
|
||||
def __init__(self, request, *, scoped_snapshot, crawl_id_filter, is_admin):
|
||||
self.request = request
|
||||
self.scoped_snapshot = scoped_snapshot
|
||||
self.crawl_id_filter = crawl_id_filter
|
||||
self.is_admin = is_admin
|
||||
self.request_config = self.request.archivebox_config
|
||||
self.now = timezone.now()
|
||||
self.crawl_scope = Crawl.objects.all()
|
||||
self.snapshot_scope = Snapshot.objects.all()
|
||||
self.archiveresult_scope = ArchiveResult.objects.all()
|
||||
if self.is_admin and not self.request.user.is_superuser:
|
||||
self.crawl_scope = self.crawl_scope.filter(created_by=self.request.user)
|
||||
self.snapshot_scope = self.snapshot_scope.filter(crawl__created_by=self.request.user)
|
||||
self.archiveresult_scope = self.archiveresult_scope.filter(snapshot__crawl__created_by=self.request.user)
|
||||
if self.scoped_snapshot is not None:
|
||||
self.snapshot_scope = Snapshot.objects.filter(id=self.scoped_snapshot.id)
|
||||
self.crawl_scope = Crawl.objects.filter(id=self.scoped_snapshot.crawl_id)
|
||||
self.archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=self.scoped_snapshot.id)
|
||||
elif self.crawl_id_filter:
|
||||
self.snapshot_scope = self.snapshot_scope.filter(crawl_id=self.crawl_id_filter)
|
||||
self.crawl_scope = self.crawl_scope.filter(id=self.crawl_id_filter)
|
||||
self.archiveresult_scope = self.archiveresult_scope.filter(snapshot__crawl_id=self.crawl_id_filter)
|
||||
|
||||
self.api_base = get_api_base_url(request=self.request, config=self.request_config) if self.scoped_snapshot is not None else ""
|
||||
|
||||
def load_orchestrator(self):
|
||||
self.machine_id = Machine.current().id
|
||||
orchestrator_proc = (
|
||||
Process.objects.filter(
|
||||
machine_id=self.machine_id,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
.only("id", "pid", "started_at", "machine_id", "process_type", "status")
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
if self.machine_id is not None
|
||||
else None
|
||||
)
|
||||
runner_worker = None
|
||||
orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
|
||||
if not orchestrator_proc_running:
|
||||
try:
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
|
||||
|
||||
supervisor = get_existing_supervisord_process(quiet=True)
|
||||
runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
|
||||
except (OSError, RuntimeError, TimeoutError):
|
||||
runner_worker = None
|
||||
|
||||
runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
|
||||
runner_worker_pid = runner_worker.get("pid") if runner_worker else None
|
||||
self.orchestrator_running = orchestrator_proc_running or runner_worker_running
|
||||
self.orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
|
||||
|
||||
def load_counts(self):
|
||||
downloads, indexing = _live_progress_plugin_names()
|
||||
download_scope = self.archiveresult_scope.filter(
|
||||
plugin__in=downloads,
|
||||
snapshot__status__in=Snapshot.RUNNABLE_STATES,
|
||||
snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
|
||||
)
|
||||
self.counts = {}
|
||||
for label, model, scope in (
|
||||
("crawls", Crawl, self.crawl_scope),
|
||||
("snapshots", Snapshot, self.snapshot_scope),
|
||||
("archiveresults", ArchiveResult, self.archiveresult_scope),
|
||||
("downloads", ArchiveResult, download_scope),
|
||||
("indexing", ArchiveResult, self.archiveresult_scope.filter(plugin__in=indexing)),
|
||||
):
|
||||
counts = model.status_counts(scope, ("queued", "started"))
|
||||
self.counts[f"{label}_queued"] = counts.get("queued", 0)
|
||||
self.counts[f"{label}_active"] = counts.get("started", 0)
|
||||
self.counts["crawls_recent"] = self.crawl_scope.filter(created_at__gte=self.now - timedelta(days=1)).count()
|
||||
|
||||
def load_crawls(self):
|
||||
# Build hierarchical active crawls with nested snapshots and archive results
|
||||
max_active_crawls = 10
|
||||
max_queued_crawls = 10
|
||||
self.max_started_snapshots_per_crawl = 50
|
||||
self.max_queued_snapshots_per_crawl = 50
|
||||
|
||||
active_crawl_fields = (
|
||||
"id",
|
||||
"created_at",
|
||||
"created_by_id",
|
||||
"modified_at",
|
||||
"urls",
|
||||
"config",
|
||||
"max_depth",
|
||||
"tags_str",
|
||||
"persona_id",
|
||||
"status",
|
||||
"retry_at",
|
||||
"label",
|
||||
"created_by__id",
|
||||
"created_by__username",
|
||||
)
|
||||
started_crawls = list(
|
||||
self.crawl_scope.filter(status=Crawl.StatusChoices.STARTED)
|
||||
.values(*active_crawl_fields)
|
||||
.order_by("-modified_at")[:max_active_crawls],
|
||||
)
|
||||
paused_crawls = list(
|
||||
self.crawl_scope.filter(
|
||||
Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=self.now - timedelta(hours=12))
|
||||
| Q(
|
||||
status=Crawl.StatusChoices.PAUSED,
|
||||
snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
|
||||
snapshot_set__retry_at__lte=self.now,
|
||||
)
|
||||
| Q(
|
||||
status=Crawl.StatusChoices.PAUSED,
|
||||
snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
|
||||
),
|
||||
)
|
||||
.values(*active_crawl_fields)
|
||||
.distinct()
|
||||
.order_by("-modified_at")[:max_active_crawls],
|
||||
)
|
||||
queued_crawls = list(
|
||||
self.crawl_scope.filter(status=Crawl.StatusChoices.QUEUED)
|
||||
.values(*active_crawl_fields)
|
||||
.order_by("-modified_at")[:max_queued_crawls],
|
||||
)
|
||||
self.queued_crawls_hidden = max(self.counts["crawls_queued"] - len(queued_crawls), 0)
|
||||
self.active_crawls_list = started_crawls + paused_crawls + queued_crawls
|
||||
for crawl in self.active_crawls_list:
|
||||
crawl["id"] = str(crawl["id"])
|
||||
if crawl["persona_id"]:
|
||||
crawl["persona_id"] = str(crawl["persona_id"])
|
||||
self.persona_details_by_id: dict[str, dict[str, str]] = {}
|
||||
self.persona_details_by_name: dict[str, dict[str, str]] = {}
|
||||
persona_ids = {crawl["persona_id"] for crawl in self.active_crawls_list if crawl["persona_id"]}
|
||||
persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in self.active_crawls_list) else set()
|
||||
if persona_ids or persona_names:
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
|
||||
persona_details = {
|
||||
"name": persona.name,
|
||||
"admin_url": f"/admin/personas/persona/{persona.pk}/change/",
|
||||
}
|
||||
self.persona_details_by_id[str(persona.id)] = persona_details
|
||||
self.persona_details_by_name[persona.name] = persona_details
|
||||
self.active_crawl_ids = [crawl["id"] for crawl in self.active_crawls_list]
|
||||
self.active_crawl_objects = {}
|
||||
if self.active_crawl_ids:
|
||||
for crawl_obj in Crawl.objects.filter(id__in=self.active_crawl_ids).select_related("created_by", "persona"):
|
||||
crawl_obj._runtime_config = self.request_config
|
||||
self.active_crawl_objects[str(crawl_obj.id)] = crawl_obj
|
||||
self.snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in self.active_crawl_ids}
|
||||
self.cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in self.active_crawl_ids}
|
||||
self.crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in self.active_crawl_ids}
|
||||
self.queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in self.active_crawl_ids}
|
||||
self.active_snapshot_scope = self.snapshot_scope.filter(crawl_id__in=self.active_crawl_ids)
|
||||
if self.active_crawl_ids:
|
||||
for row in self.active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
|
||||
self.snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
|
||||
|
||||
for row in (
|
||||
self.active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
|
||||
.values("crawl_id")
|
||||
.annotate(count=Count("id"))
|
||||
):
|
||||
self.cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
|
||||
|
||||
for row in (
|
||||
self.active_snapshot_scope.filter(
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
)
|
||||
.values("crawl_id")
|
||||
.annotate(size=Sum("output_size"))
|
||||
):
|
||||
self.crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
|
||||
|
||||
def load_snapshots(self):
|
||||
self.crawl_process_pids: dict[str, int] = {}
|
||||
self.snapshot_process_pids: dict[str, int] = {}
|
||||
self.process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
|
||||
self.process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
|
||||
self.seen_process_records: set[str] = set()
|
||||
self.crawls_by_id = {str(crawl["id"]): crawl for crawl in self.active_crawls_list}
|
||||
started_snapshot_fields = (
|
||||
"id_str",
|
||||
"created_at",
|
||||
"modified_at",
|
||||
"url",
|
||||
"timestamp",
|
||||
"bookmarked_at",
|
||||
"crawl_id_str",
|
||||
"title",
|
||||
"downloaded_at",
|
||||
"fs_version",
|
||||
"status",
|
||||
)
|
||||
queued_snapshot_fields = (
|
||||
"id_str",
|
||||
"url",
|
||||
"crawl_id_str",
|
||||
"title",
|
||||
"status",
|
||||
)
|
||||
snapshots = []
|
||||
for crawl_id in self.active_crawl_ids:
|
||||
crawl_snapshot_scope = self.active_snapshot_scope.filter(crawl_id=crawl_id)
|
||||
snapshots.extend(
|
||||
crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED)
|
||||
.annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
|
||||
.values(*started_snapshot_fields)
|
||||
.order_by("-modified_at")[: self.max_started_snapshots_per_crawl],
|
||||
)
|
||||
queued_snapshots = list(
|
||||
crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
|
||||
.annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
|
||||
.values(
|
||||
*queued_snapshot_fields,
|
||||
)
|
||||
.order_by("modified_at")[: self.max_queued_snapshots_per_crawl],
|
||||
)
|
||||
self.queued_snapshot_overflow_by_crawl[str(crawl_id)] = max(
|
||||
self.snapshot_counts_by_crawl.get(str(crawl_id), {}).get(Snapshot.StatusChoices.QUEUED, 0) - len(queued_snapshots),
|
||||
0,
|
||||
)
|
||||
snapshots.extend(queued_snapshots)
|
||||
|
||||
for snapshot in snapshots:
|
||||
# Process.pwd points at Snapshot.output_dir, which uses CompactUUID
|
||||
# hex path components. Keep progress IDs compact too so process rows
|
||||
# can be matched without carrying dashed/undashed variants.
|
||||
snapshot["id"] = str(snapshot.pop("id_str")).replace("-", "")
|
||||
snapshot["crawl_id"] = str(snapshot.pop("crawl_id_str")).replace("-", "")
|
||||
self.snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
|
||||
self.displayed_snapshots_by_crawl: dict[str, list[dict[str, Any]]] = {str(crawl_id): [] for crawl_id in self.active_crawl_ids}
|
||||
for snapshot in snapshots:
|
||||
crawl_snapshots = self.displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
|
||||
crawl_snapshots.append(snapshot)
|
||||
displayed_snapshot_ids = [
|
||||
snapshot["id"] for crawl_snapshots in self.displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots
|
||||
]
|
||||
detailed_snapshot_ids = [snapshot["id"] for snapshot in snapshots if snapshot["status"] != Snapshot.StatusChoices.QUEUED]
|
||||
process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "ended_at", "modified_at")
|
||||
if self.active_crawl_ids or displayed_snapshot_ids:
|
||||
process_scope = Process.objects.filter(
|
||||
machine_id=self.machine_id,
|
||||
process_type__in=[
|
||||
Process.TypeChoices.HOOK,
|
||||
Process.TypeChoices.BINARY,
|
||||
],
|
||||
)
|
||||
self.running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields)
|
||||
self.recent_processes = (
|
||||
process_scope.filter(modified_at__gte=self.now - timedelta(minutes=10))
|
||||
.values(*process_value_fields)
|
||||
.order_by("-modified_at")
|
||||
)
|
||||
else:
|
||||
self.running_processes = Process.objects.none().values(*process_value_fields)
|
||||
self.recent_processes = Process.objects.none().values(*process_value_fields)
|
||||
|
||||
self.archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in detailed_snapshot_ids}
|
||||
if detailed_snapshot_ids:
|
||||
displayed_archiveresults = (
|
||||
self.archiveresult_scope.filter(snapshot_id__in=detailed_snapshot_ids)
|
||||
.select_related("process")
|
||||
.only(
|
||||
"id",
|
||||
"snapshot_id",
|
||||
"plugin",
|
||||
"hook_name",
|
||||
"status",
|
||||
"output_str",
|
||||
"output_files",
|
||||
"output_size",
|
||||
"start_ts",
|
||||
"end_ts",
|
||||
"created_at",
|
||||
"modified_at",
|
||||
"process_id",
|
||||
"process__id",
|
||||
"process__pid",
|
||||
"process__started_at",
|
||||
"process__timeout",
|
||||
)
|
||||
.order_by("snapshot_id", "start_ts", "created_at")
|
||||
)
|
||||
for archiveresult in displayed_archiveresults:
|
||||
self.archiveresults_by_snapshot.setdefault(str(archiveresult.snapshot_id), []).append(archiveresult)
|
||||
|
||||
def load_processes(self):
|
||||
self.running_worker_ids: set[str] = set()
|
||||
for proc in self.running_processes:
|
||||
if not proc["pwd"]:
|
||||
continue
|
||||
scope = self.process_scope_ids(Path(proc["pwd"]))
|
||||
if scope is None:
|
||||
continue
|
||||
crawl_id, snapshot_id = scope
|
||||
self.running_worker_ids.add(str(proc["id"]))
|
||||
_plugin, _label, phase, _hook_name = process_label(proc["cmd"])
|
||||
if crawl_id and proc["pid"]:
|
||||
self.crawl_process_pids.setdefault(crawl_id, proc["pid"])
|
||||
if phase == "snapshot" and snapshot_id and proc["pid"]:
|
||||
self.snapshot_process_pids.setdefault(snapshot_id, proc["pid"])
|
||||
|
||||
for proc in self.recent_processes:
|
||||
if not proc["pwd"]:
|
||||
continue
|
||||
scope = self.process_scope_ids(Path(proc["pwd"]))
|
||||
if scope is None:
|
||||
continue
|
||||
crawl_id, snapshot_id = scope
|
||||
|
||||
plugin, label, phase, hook_name = process_label(proc["cmd"])
|
||||
|
||||
record_scope = str(snapshot_id) if phase == "snapshot" and snapshot_id else str(crawl_id)
|
||||
proc_key = f"{record_scope}:{plugin}:{label}:{proc['status']}:{proc['exit_code']}"
|
||||
if proc_key in self.seen_process_records:
|
||||
continue
|
||||
self.seen_process_records.add(proc_key)
|
||||
|
||||
status = (
|
||||
"started"
|
||||
if proc["status"] == Process.StatusChoices.RUNNING
|
||||
else (
|
||||
"skipped"
|
||||
if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0))
|
||||
else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded")
|
||||
)
|
||||
)
|
||||
payload: dict[str, object] = {
|
||||
"id": str(proc["id"]),
|
||||
"plugin": plugin,
|
||||
"label": label,
|
||||
"hook_name": hook_name,
|
||||
"status": status,
|
||||
"phase": phase,
|
||||
"source": "process",
|
||||
"process_id": str(proc["id"]),
|
||||
}
|
||||
if status == "started" and proc["pid"]:
|
||||
payload["pid"] = proc["pid"]
|
||||
proc_started_at = proc["started_at"] or proc["modified_at"]
|
||||
proc_run_at = (
|
||||
proc_started_at
|
||||
if proc["status"] == Process.StatusChoices.RUNNING
|
||||
else (proc["ended_at"] or proc["modified_at"] or proc_started_at)
|
||||
)
|
||||
if phase == "snapshot" and snapshot_id:
|
||||
self.process_records_by_snapshot.setdefault(snapshot_id, []).append((payload, proc_run_at))
|
||||
elif crawl_id:
|
||||
self.process_records_by_crawl.setdefault(crawl_id, []).append((payload, proc_run_at))
|
||||
|
||||
def payload(self):
|
||||
self.load_orchestrator()
|
||||
self.load_counts()
|
||||
self.load_crawls()
|
||||
self.load_snapshots()
|
||||
self.load_processes()
|
||||
return {
|
||||
"is_admin": self.is_admin,
|
||||
"scope": {
|
||||
"snapshot_id": str(self.scoped_snapshot.id) if self.scoped_snapshot is not None else "",
|
||||
"crawl_id": self.crawl_id_filter,
|
||||
},
|
||||
"orchestrator_running": self.orchestrator_running,
|
||||
"orchestrator_pid": self.orchestrator_pid,
|
||||
"total_workers": len(self.running_worker_ids),
|
||||
**self.counts,
|
||||
"active_crawls": [render_crawl(self, crawl) for crawl in self.active_crawls_list],
|
||||
"queued_crawls_hidden": self.queued_crawls_hidden,
|
||||
"server_time": timezone.now().isoformat(),
|
||||
}
|
||||
|
||||
def process_scope_ids(self, proc_pwd: Path) -> tuple[str, str] | None:
|
||||
"""Match a process directory to the displayed snapshot or crawl IDs."""
|
||||
parts = [part.replace("-", "") for part in reversed(proc_pwd.parts)]
|
||||
for part in parts:
|
||||
if snapshot := self.snapshots_by_id.get(part):
|
||||
return str(snapshot["crawl_id"]), str(snapshot["id"])
|
||||
for part in parts:
|
||||
if crawl := self.crawls_by_id.get(part):
|
||||
return str(crawl["id"]), ""
|
||||
return None
|
||||
@@ -0,0 +1,984 @@
|
||||
/* Progress Monitor Container */
|
||||
#progress-monitor {
|
||||
background: linear-gradient(135deg, #0d1117 0%, #161b22 100%);
|
||||
color: #c9d1d9;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
|
||||
font-size: 12px;
|
||||
border-bottom: 1px solid #30363d;
|
||||
position: relative;
|
||||
z-index: 100;
|
||||
}
|
||||
#progress-monitor.hidden {
|
||||
display: none;
|
||||
}
|
||||
#progress-monitor .tree-container {
|
||||
max-height: 350px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
#progress-monitor .progress-content {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 12px;
|
||||
padding: 12px 16px;
|
||||
}
|
||||
|
||||
/* Header Bar */
|
||||
#progress-monitor .header-bar {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 8px 16px;
|
||||
background: rgba(0,0,0,0.2);
|
||||
border-bottom: 1px solid #30363d;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
}
|
||||
#progress-monitor.collapsed .header-bar {
|
||||
cursor: pointer;
|
||||
}
|
||||
#progress-monitor .header-left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
min-width: 0;
|
||||
}
|
||||
#progress-monitor .header-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
/* Orchestrator Status */
|
||||
#progress-monitor .orchestrator-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
#progress-monitor .status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#progress-monitor .status-dot.running {
|
||||
background: #238636;
|
||||
box-shadow: 0 0 8px #238636;
|
||||
animation: pulse 2s infinite;
|
||||
}
|
||||
#progress-monitor .status-dot.idle {
|
||||
background: #7ee787;
|
||||
box-shadow: 0 0 4px #7ee787;
|
||||
animation: idle-pulse 5s infinite;
|
||||
}
|
||||
#progress-monitor .status-dot.stopped {
|
||||
background: #d29922;
|
||||
box-shadow: 0 0 4px #d29922;
|
||||
}
|
||||
#progress-monitor .status-dot.error {
|
||||
background: #f85149;
|
||||
box-shadow: 0 0 8px #f85149;
|
||||
}
|
||||
#progress-monitor .status-dot.flash {
|
||||
animation: flash 0.3s ease-out;
|
||||
}
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; box-shadow: 0 0 8px #238636; }
|
||||
50% { opacity: 0.6; box-shadow: 0 0 4px #238636; }
|
||||
}
|
||||
@keyframes idle-pulse {
|
||||
0%, 100% { opacity: 1; box-shadow: 0 0 4px #7ee787; }
|
||||
50% { opacity: 0.65; box-shadow: 0 0 2px #7ee787; }
|
||||
}
|
||||
@keyframes flash {
|
||||
0% { transform: scale(1.5); }
|
||||
100% { transform: scale(1); }
|
||||
}
|
||||
|
||||
/* Stats */
|
||||
#progress-monitor .stats {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
}
|
||||
#progress-monitor .stat {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
#progress-monitor .stat-label {
|
||||
color: #8b949e;
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
#progress-monitor .stat-value {
|
||||
font-weight: 600;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
#progress-monitor .stat-value.compact {
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
#progress-monitor .stat-value.success { color: #3fb950; }
|
||||
#progress-monitor .stat-value.error { color: #f85149; }
|
||||
#progress-monitor .stat-value.warning { color: #d29922; }
|
||||
#progress-monitor .stat-value.info { color: #58a6ff; }
|
||||
|
||||
/* Toggle Button */
|
||||
#progress-monitor .toggle-btn {
|
||||
background: transparent;
|
||||
border: 1px solid #30363d;
|
||||
color: #8b949e;
|
||||
cursor: pointer;
|
||||
padding: 4px 8px;
|
||||
border-radius: 6px;
|
||||
font-size: 11px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
#progress-monitor .toggle-btn:hover {
|
||||
background: #21262d;
|
||||
color: #c9d1d9;
|
||||
border-color: #8b949e;
|
||||
}
|
||||
#progress-monitor .crawl-action-btn,
|
||||
#progress-monitor .cancel-item-btn {
|
||||
background: transparent;
|
||||
border: 1px solid #30363d;
|
||||
cursor: pointer;
|
||||
padding: 2px 6px;
|
||||
border-radius: 6px;
|
||||
font-size: 11px;
|
||||
line-height: 1;
|
||||
transition: all 0.2s;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#progress-monitor .pause-item-btn {
|
||||
color: #d29922;
|
||||
}
|
||||
#progress-monitor .pause-item-btn:hover {
|
||||
background: rgba(210, 153, 34, 0.12);
|
||||
border-color: #d29922;
|
||||
color: #f0b72f;
|
||||
}
|
||||
#progress-monitor .cancel-item-btn {
|
||||
color: #f85149;
|
||||
}
|
||||
#progress-monitor .cancel-item-btn:hover {
|
||||
background: rgba(248, 81, 73, 0.12);
|
||||
border-color: #f85149;
|
||||
color: #ff7b72;
|
||||
}
|
||||
#progress-monitor .crawl-action-btn.is-busy,
|
||||
#progress-monitor .cancel-item-btn.is-busy {
|
||||
opacity: 0.6;
|
||||
cursor: wait;
|
||||
border-color: #6e7681;
|
||||
color: #6e7681;
|
||||
}
|
||||
#progress-monitor .progress-overflow-note {
|
||||
padding: 8px 12px;
|
||||
color: #8b949e;
|
||||
font-size: 11px;
|
||||
text-align: center;
|
||||
background: rgba(33, 38, 45, 0.55);
|
||||
border-top: 1px solid #30363d;
|
||||
}
|
||||
|
||||
/* Tree Container */
|
||||
#progress-monitor .tree-container {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
padding: 0;
|
||||
}
|
||||
#progress-monitor.collapsed .progress-content {
|
||||
display: none;
|
||||
}
|
||||
/* Hide admin-only controls when viewer is not staff, or when the monitor is
|
||||
embedded on a non-admin host (snap-* subdomain) where cross-origin POSTs
|
||||
to /api would violate the subdomain isolation model. */
|
||||
#progress-monitor.is-guest .crawl-action-btn,
|
||||
#progress-monitor.is-guest .cancel-item-btn,
|
||||
#progress-monitor.is-guest .pause-item-btn,
|
||||
#progress-monitor[data-progress-scope="snapshot"] .crawl-action-btn,
|
||||
#progress-monitor[data-progress-scope="snapshot"] .cancel-item-btn,
|
||||
#progress-monitor[data-progress-scope="snapshot"] .pause-item-btn {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
/* Chrome Screencast */
|
||||
#progress-monitor .screencast-panel {
|
||||
display: none;
|
||||
flex: 0 0 336px;
|
||||
width: 336px;
|
||||
position: sticky;
|
||||
top: 49px;
|
||||
border: 1px solid rgba(88, 166, 255, 0.5);
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
background: #0d1117;
|
||||
box-shadow: 0 12px 34px rgba(1, 4, 9, 0.4), 0 0 0 1px rgba(88, 166, 255, 0.12);
|
||||
}
|
||||
#progress-monitor .screencast-panel.visible {
|
||||
display: block;
|
||||
}
|
||||
#progress-monitor .screencast-frame {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 310px;
|
||||
background: #010409;
|
||||
color: #6e7681;
|
||||
text-decoration: none;
|
||||
overflow: hidden;
|
||||
}
|
||||
#progress-monitor .screencast-frame img {
|
||||
display: block;
|
||||
width: 120%;
|
||||
height: 100%;
|
||||
margin-left: -10%;
|
||||
object-fit: cover;
|
||||
object-position: top center;
|
||||
background: #ffffff;
|
||||
opacity: 1 !important;
|
||||
}
|
||||
#progress-monitor .screencast-placeholder {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
text-align: center;
|
||||
color: #8b949e;
|
||||
background: #010409;
|
||||
}
|
||||
#progress-monitor .screencast-placeholder-icon {
|
||||
color: #c9d1d9;
|
||||
font-size: 32px;
|
||||
font-weight: 700;
|
||||
line-height: 1;
|
||||
}
|
||||
#progress-monitor .screencast-placeholder.launching .screencast-placeholder-icon {
|
||||
color: #d29922;
|
||||
}
|
||||
#progress-monitor .screencast-placeholder-title {
|
||||
color: #f0f6fc;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
line-height: 1.25;
|
||||
}
|
||||
#progress-monitor .screencast-placeholder-subtitle {
|
||||
max-width: 260px;
|
||||
color: #8b949e;
|
||||
font-size: 11px;
|
||||
line-height: 1.35;
|
||||
}
|
||||
#progress-monitor .screencast-caption {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 10px;
|
||||
border-top: 1px solid #21262d;
|
||||
background: rgba(13, 17, 23, 0.96);
|
||||
color: inherit;
|
||||
text-decoration: none;
|
||||
}
|
||||
#progress-monitor .screencast-dot {
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
flex: 0 0 7px;
|
||||
background: #3fb950;
|
||||
box-shadow: 0 0 8px rgba(63, 185, 80, 0.9);
|
||||
}
|
||||
#progress-monitor .screencast-dot.not-launched {
|
||||
background: #6e7681;
|
||||
box-shadow: none;
|
||||
}
|
||||
#progress-monitor .screencast-dot.launching {
|
||||
background: #d29922;
|
||||
box-shadow: 0 0 8px rgba(210, 153, 34, 0.8);
|
||||
}
|
||||
#progress-monitor .screencast-text {
|
||||
min-width: 0;
|
||||
}
|
||||
#progress-monitor .screencast-title {
|
||||
display: block;
|
||||
color: #f0f6fc;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
line-height: 1.25;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor .screencast-url {
|
||||
display: block;
|
||||
color: #8b949e;
|
||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
||||
font-size: 10px;
|
||||
line-height: 1.25;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
/* Idle Message */
|
||||
#progress-monitor .idle-message {
|
||||
color: #8b949e;
|
||||
font-style: italic;
|
||||
padding: 8px 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Crawl Item */
|
||||
#progress-monitor .crawl-item {
|
||||
background: #161b22;
|
||||
border: 1px solid #30363d;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 9px;
|
||||
overflow: hidden;
|
||||
}
|
||||
#progress-monitor .crawl-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
padding: 8px 12px;
|
||||
background: rgba(0,0,0,0.2);
|
||||
}
|
||||
#progress-monitor .crawl-header:hover {
|
||||
background: rgba(88, 166, 255, 0.1);
|
||||
}
|
||||
#progress-monitor .crawl-header-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
cursor: pointer;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
#progress-monitor .crawl-header-link,
|
||||
#progress-monitor .crawl-header-link * {
|
||||
text-decoration: none !important;
|
||||
}
|
||||
#progress-monitor a.crawl-header-link:visited {
|
||||
color: inherit;
|
||||
}
|
||||
#progress-monitor a.crawl-header-link:link,
|
||||
#progress-monitor a.crawl-header-link:visited,
|
||||
#progress-monitor a.crawl-header-link:hover,
|
||||
#progress-monitor a.crawl-header-link:active {
|
||||
text-decoration: none !important;
|
||||
}
|
||||
#progress-monitor .crawl-info {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
#progress-monitor .crawl-label {
|
||||
font-weight: 600;
|
||||
color: #f0f6fc;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor .crawl-badges {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px;
|
||||
margin-top: 5px;
|
||||
}
|
||||
#progress-monitor .crawl-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
max-width: 240px;
|
||||
padding: 2px 7px;
|
||||
border-radius: 999px;
|
||||
font-size: 11px;
|
||||
line-height: 1.35;
|
||||
color: #c9d1d9;
|
||||
background: rgba(110, 118, 129, 0.16);
|
||||
border: 1px solid rgba(110, 118, 129, 0.22);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor a.crawl-badge {
|
||||
cursor: pointer;
|
||||
text-decoration: none !important;
|
||||
}
|
||||
#progress-monitor a.crawl-badge:hover {
|
||||
border-color: currentColor;
|
||||
filter: brightness(1.14);
|
||||
}
|
||||
#progress-monitor .crawl-badge strong {
|
||||
color: #8b949e;
|
||||
font-weight: 600;
|
||||
}
|
||||
#progress-monitor .crawl-title-link {
|
||||
display: inline-block;
|
||||
max-width: 100%;
|
||||
color: inherit;
|
||||
text-decoration: none !important;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor .crawl-title-link:hover {
|
||||
color: #79c0ff;
|
||||
}
|
||||
#progress-monitor .crawl-badge.persona {
|
||||
color: #d2a8ff;
|
||||
background: rgba(163, 113, 247, 0.16);
|
||||
border-color: rgba(163, 113, 247, 0.28);
|
||||
}
|
||||
#progress-monitor .crawl-badge.limit {
|
||||
color: #79c0ff;
|
||||
background: rgba(88, 166, 255, 0.13);
|
||||
border-color: rgba(88, 166, 255, 0.24);
|
||||
}
|
||||
#progress-monitor .crawl-badge.size {
|
||||
color: #ffa657;
|
||||
background: rgba(240, 136, 62, 0.13);
|
||||
border-color: rgba(240, 136, 62, 0.24);
|
||||
}
|
||||
#progress-monitor .crawl-badge.count {
|
||||
color: #7ee787;
|
||||
background: rgba(63, 185, 80, 0.13);
|
||||
border-color: rgba(63, 185, 80, 0.24);
|
||||
}
|
||||
#progress-monitor .crawl-badge.tag {
|
||||
color: #a5d6ff;
|
||||
background: rgba(31, 111, 235, 0.16);
|
||||
border-color: rgba(31, 111, 235, 0.28);
|
||||
}
|
||||
#progress-monitor .crawl-stats {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
/* Progress Bar */
|
||||
#progress-monitor .progress-bar-container {
|
||||
height: 4px;
|
||||
background: #21262d;
|
||||
border-radius: 2px;
|
||||
overflow: hidden;
|
||||
position: relative;
|
||||
}
|
||||
#progress-monitor .progress-bar {
|
||||
height: 100%;
|
||||
border-radius: 2px;
|
||||
transition: width 0.5s ease-out;
|
||||
position: relative;
|
||||
}
|
||||
#progress-monitor .progress-bar.crawl {
|
||||
background: linear-gradient(90deg, #238636 0%, #3fb950 100%);
|
||||
}
|
||||
#progress-monitor .progress-bar.snapshot {
|
||||
background: linear-gradient(90deg, #1f6feb 0%, #58a6ff 100%);
|
||||
}
|
||||
#progress-monitor .progress-bar.extractor {
|
||||
background: linear-gradient(90deg, #8957e5 0%, #a371f7 100%);
|
||||
}
|
||||
#progress-monitor .progress-bar.indeterminate {
|
||||
background: linear-gradient(90deg, transparent 0%, #58a6ff 50%, transparent 100%);
|
||||
animation: indeterminate 1.5s infinite linear;
|
||||
width: 30% !important;
|
||||
}
|
||||
@keyframes indeterminate {
|
||||
0% { transform: translateX(-100%); }
|
||||
100% { transform: translateX(400%); }
|
||||
}
|
||||
|
||||
/* Crawl Body */
|
||||
#progress-monitor .crawl-body {
|
||||
padding: 0 14px 14px;
|
||||
}
|
||||
#progress-monitor .crawl-progress {
|
||||
padding: 10px 14px;
|
||||
border-bottom: 1px solid #21262d;
|
||||
}
|
||||
|
||||
/* Snapshot List */
|
||||
#progress-monitor .snapshot-list {
|
||||
margin-top: 8px;
|
||||
}
|
||||
#progress-monitor .snapshot-item {
|
||||
background: #0d1117;
|
||||
border: 1px solid #21262d;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
#progress-monitor .snapshot-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
padding: 8px 12px;
|
||||
}
|
||||
#progress-monitor .snapshot-header:hover {
|
||||
background: rgba(88, 166, 255, 0.05);
|
||||
}
|
||||
#progress-monitor .snapshot-header-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
cursor: pointer;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
#progress-monitor a.snapshot-header-link:visited {
|
||||
color: inherit;
|
||||
}
|
||||
#progress-monitor .snapshot-icon {
|
||||
font-size: 14px;
|
||||
width: 18px;
|
||||
text-align: center;
|
||||
color: #58a6ff;
|
||||
}
|
||||
#progress-monitor .snapshot-preview {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 84px;
|
||||
height: 52px;
|
||||
flex: 0 0 84px;
|
||||
border-radius: 6px;
|
||||
border: 1px solid #30363d;
|
||||
background: #161b22;
|
||||
color: #6e7681;
|
||||
overflow: hidden;
|
||||
text-decoration: none;
|
||||
}
|
||||
#progress-monitor .snapshot-preview:hover {
|
||||
border-color: #58a6ff;
|
||||
box-shadow: 0 0 0 1px rgba(88, 166, 255, 0.2);
|
||||
}
|
||||
#progress-monitor .snapshot-preview img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
object-position: top center;
|
||||
display: block;
|
||||
}
|
||||
#progress-monitor .snapshot-preview.placeholder {
|
||||
font-size: 20px;
|
||||
}
|
||||
#progress-monitor .snapshot-info {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
#progress-monitor .snapshot-title-line {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
min-width: 0;
|
||||
}
|
||||
#progress-monitor .snapshot-favicon {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 3px;
|
||||
object-fit: contain;
|
||||
background: #fff;
|
||||
flex: 0 0 16px;
|
||||
}
|
||||
#progress-monitor .snapshot-title {
|
||||
color: #f0f6fc;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
line-height: 1.3;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor .snapshot-url {
|
||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
||||
font-size: 11px;
|
||||
color: #c9d1d9;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor .snapshot-meta {
|
||||
font-size: 10px;
|
||||
color: #8b949e;
|
||||
margin-top: 2px;
|
||||
}
|
||||
#progress-monitor .snapshot-progress {
|
||||
padding: 0 12px 8px;
|
||||
}
|
||||
|
||||
/* Extractor List - Compact Badge Layout */
|
||||
#progress-monitor .extractor-list {
|
||||
padding: 8px 12px;
|
||||
background: rgba(0,0,0,0.2);
|
||||
border-top: 1px solid #21262d;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px;
|
||||
}
|
||||
#progress-monitor .extractor-badge {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
padding: 3px 8px;
|
||||
border-radius: 4px;
|
||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
||||
font-size: 10px;
|
||||
background: #21262d;
|
||||
overflow: hidden;
|
||||
white-space: nowrap;
|
||||
color: inherit;
|
||||
text-decoration: none;
|
||||
}
|
||||
#progress-monitor a.extractor-badge:hover {
|
||||
background: #30363d;
|
||||
color: #f0f6fc;
|
||||
}
|
||||
#progress-monitor .extractor-badge .progress-fill {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
bottom: 0;
|
||||
z-index: 0;
|
||||
transition: width 0.3s ease-out;
|
||||
}
|
||||
#progress-monitor .extractor-badge .badge-content {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
#progress-monitor .extractor-badge.queued {
|
||||
color: #8b949e;
|
||||
}
|
||||
#progress-monitor .extractor-badge.queued .progress-fill {
|
||||
background: rgba(110, 118, 129, 0.2);
|
||||
width: 0%;
|
||||
}
|
||||
#progress-monitor .extractor-badge.started {
|
||||
color: #d29922;
|
||||
}
|
||||
#progress-monitor .extractor-badge.started .progress-fill {
|
||||
background: rgba(210, 153, 34, 0.3);
|
||||
animation: progress-pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
@keyframes progress-pulse {
|
||||
0%, 100% { opacity: 0.5; }
|
||||
50% { opacity: 1; }
|
||||
}
|
||||
#progress-monitor .extractor-badge.succeeded {
|
||||
color: #3fb950;
|
||||
}
|
||||
#progress-monitor .extractor-badge.succeeded .progress-fill {
|
||||
background: rgba(63, 185, 80, 0.25);
|
||||
width: 100%;
|
||||
}
|
||||
#progress-monitor .extractor-badge.failed {
|
||||
color: #f85149;
|
||||
}
|
||||
#progress-monitor .extractor-badge.failed .progress-fill {
|
||||
background: rgba(248, 81, 73, 0.25);
|
||||
width: 100%;
|
||||
}
|
||||
#progress-monitor .extractor-badge.backoff {
|
||||
color: #b8860b;
|
||||
}
|
||||
#progress-monitor .extractor-badge.backoff .progress-fill {
|
||||
background: rgba(210, 153, 34, 0.2);
|
||||
width: 30%;
|
||||
}
|
||||
#progress-monitor .extractor-badge.skipped {
|
||||
color: #6e7681;
|
||||
}
|
||||
#progress-monitor .extractor-badge.skipped .progress-fill {
|
||||
background: rgba(110, 118, 129, 0.15);
|
||||
width: 100%;
|
||||
}
|
||||
#progress-monitor .extractor-badge .badge-icon {
|
||||
font-size: 10px;
|
||||
}
|
||||
/* Status Badge */
|
||||
#progress-monitor .status-badge {
|
||||
font-size: 10px;
|
||||
padding: 2px 6px;
|
||||
border-radius: 10px;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.3px;
|
||||
}
|
||||
#progress-monitor .status-badge.queued {
|
||||
background: #21262d;
|
||||
color: #8b949e;
|
||||
}
|
||||
#progress-monitor .status-badge.started {
|
||||
background: rgba(210, 153, 34, 0.2);
|
||||
color: #d29922;
|
||||
}
|
||||
#progress-monitor .status-badge.paused {
|
||||
background: rgba(88, 166, 255, 0.18);
|
||||
color: #58a6ff;
|
||||
}
|
||||
#progress-monitor .status-badge.sealed,
|
||||
#progress-monitor .status-badge.succeeded {
|
||||
background: rgba(63, 185, 80, 0.2);
|
||||
color: #3fb950;
|
||||
}
|
||||
#progress-monitor .status-badge.failed {
|
||||
background: rgba(248, 81, 73, 0.2);
|
||||
color: #f85149;
|
||||
}
|
||||
#progress-monitor .status-badge.backoff {
|
||||
background: rgba(210, 153, 34, 0.15);
|
||||
color: #b8860b;
|
||||
}
|
||||
#progress-monitor .status-badge.unknown {
|
||||
background: #21262d;
|
||||
color: #6e7681;
|
||||
}
|
||||
#progress-monitor .duration-badge {
|
||||
flex-shrink: 0;
|
||||
min-width: 28px;
|
||||
padding: 2px 6px;
|
||||
border-radius: 10px;
|
||||
background: rgba(88, 166, 255, 0.12);
|
||||
color: #a5d6ff;
|
||||
border: 1px solid rgba(88, 166, 255, 0.2);
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
line-height: 1.2;
|
||||
text-align: center;
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
#progress-monitor .pid-label {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
padding: 2px 6px;
|
||||
border-radius: 999px;
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
color: #8b949e;
|
||||
background: rgba(148, 163, 184, 0.12);
|
||||
border: 1px solid rgba(148, 163, 184, 0.2);
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
|
||||
letter-spacing: 0.2px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
#progress-monitor .pid-label.compact {
|
||||
padding: 1px 5px;
|
||||
font-size: 9px;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
#progress-monitor .header-bar {
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
}
|
||||
#progress-monitor .header-left {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px 14px;
|
||||
}
|
||||
#progress-monitor .stats {
|
||||
flex-wrap: wrap;
|
||||
gap: 8px 12px;
|
||||
}
|
||||
#progress-monitor .tree-container {
|
||||
max-height: 310px;
|
||||
}
|
||||
#progress-monitor .progress-content {
|
||||
flex-direction: column;
|
||||
padding: 10px 8px;
|
||||
}
|
||||
#progress-monitor .screencast-panel {
|
||||
position: static;
|
||||
order: -1;
|
||||
width: min(100%, 336px);
|
||||
flex-basis: auto;
|
||||
align-self: center;
|
||||
}
|
||||
#progress-monitor .crawl-header {
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
padding: 8px;
|
||||
}
|
||||
#progress-monitor .crawl-header-link {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto auto auto;
|
||||
align-items: start;
|
||||
gap: 8px;
|
||||
width: 100%;
|
||||
}
|
||||
#progress-monitor .crawl-info {
|
||||
grid-column: 1 / -1;
|
||||
width: 100%;
|
||||
}
|
||||
#progress-monitor .crawl-label {
|
||||
max-width: 100%;
|
||||
}
|
||||
#progress-monitor .crawl-badges,
|
||||
#progress-monitor .crawl-stats {
|
||||
justify-content: flex-start;
|
||||
min-width: 0;
|
||||
}
|
||||
#progress-monitor .crawl-badge {
|
||||
max-width: min(100%, 260px);
|
||||
}
|
||||
#progress-monitor .crawl-stats {
|
||||
grid-column: 1 / 2;
|
||||
}
|
||||
#progress-monitor .crawl-body {
|
||||
padding: 0 8px 10px;
|
||||
}
|
||||
#progress-monitor .crawl-progress {
|
||||
padding: 8px;
|
||||
}
|
||||
#progress-monitor .snapshot-header {
|
||||
align-items: flex-start;
|
||||
padding: 8px;
|
||||
}
|
||||
#progress-monitor .snapshot-header-link {
|
||||
min-width: 0;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 520px) {
|
||||
#progress-monitor {
|
||||
font-size: 11px;
|
||||
}
|
||||
#progress-monitor .header-bar {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
align-items: start;
|
||||
padding: 7px 8px;
|
||||
}
|
||||
#progress-monitor .header-left {
|
||||
gap: 6px 10px;
|
||||
}
|
||||
#progress-monitor .header-right {
|
||||
justify-self: end;
|
||||
}
|
||||
#progress-monitor .toggle-btn {
|
||||
white-space: nowrap;
|
||||
}
|
||||
#progress-monitor .orchestrator-status {
|
||||
max-width: 100%;
|
||||
}
|
||||
#progress-monitor .stats {
|
||||
flex: 1 0 100%;
|
||||
gap: 6px 10px;
|
||||
}
|
||||
#progress-monitor .stat-label {
|
||||
font-size: 9px;
|
||||
}
|
||||
#progress-monitor .tree-container {
|
||||
max-height: 300px;
|
||||
}
|
||||
#progress-monitor .progress-content {
|
||||
padding: 8px;
|
||||
}
|
||||
#progress-monitor .screencast-panel {
|
||||
width: 100%;
|
||||
align-self: stretch;
|
||||
}
|
||||
#progress-monitor .screencast-frame {
|
||||
height: 300px;
|
||||
}
|
||||
#progress-monitor .crawl-header {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
padding: 8px;
|
||||
}
|
||||
#progress-monitor .crawl-header-link {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
#progress-monitor .crawl-stats {
|
||||
grid-column: auto;
|
||||
}
|
||||
#progress-monitor .crawl-title-link {
|
||||
white-space: normal;
|
||||
line-height: 1.35;
|
||||
}
|
||||
#progress-monitor .crawl-badges {
|
||||
gap: 4px;
|
||||
}
|
||||
#progress-monitor .crawl-badge {
|
||||
max-width: 100%;
|
||||
min-width: 0;
|
||||
font-size: 10px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
#progress-monitor .crawl-stats {
|
||||
gap: 4px;
|
||||
}
|
||||
#progress-monitor .status-badge,
|
||||
#progress-monitor .duration-badge,
|
||||
#progress-monitor .pid-label {
|
||||
justify-self: start;
|
||||
}
|
||||
#progress-monitor .snapshot-header-link {
|
||||
display: grid;
|
||||
grid-template-columns: 56px minmax(0, 1fr);
|
||||
gap: 8px;
|
||||
}
|
||||
#progress-monitor .snapshot-preview {
|
||||
width: 56px;
|
||||
height: 42px;
|
||||
flex-basis: 56px;
|
||||
}
|
||||
#progress-monitor .snapshot-title-line {
|
||||
align-items: flex-start;
|
||||
}
|
||||
#progress-monitor .snapshot-title,
|
||||
#progress-monitor .snapshot-url {
|
||||
white-space: normal;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
#progress-monitor .extractor-list {
|
||||
padding: 7px 8px;
|
||||
gap: 4px;
|
||||
}
|
||||
#progress-monitor .extractor-badge {
|
||||
max-width: 100%;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
#progress-monitor .crawl-notice {
|
||||
padding: 8px 14px;
|
||||
font-size: 11px;
|
||||
color: var(--notice-color);
|
||||
border-top: 1px solid var(--notice-border, var(--notice-color));
|
||||
background: var(--notice-background);
|
||||
}
|
||||
#progress-monitor .crawl-notice.error {
|
||||
--notice-color: #f85149;
|
||||
--notice-background: rgba(248, 81, 73, 0.1);
|
||||
}
|
||||
#progress-monitor .crawl-notice.warning {
|
||||
--notice-color: #d29922;
|
||||
--notice-background: rgba(210, 153, 34, 0.1);
|
||||
}
|
||||
#progress-monitor .crawl-notice.info {
|
||||
--notice-color: #58a6ff;
|
||||
--notice-background: rgba(88, 166, 255, 0.1);
|
||||
}
|
||||
#progress-monitor .crawl-notice.cancelled {
|
||||
--notice-color: #8b949e;
|
||||
--notice-border: #6e7681;
|
||||
--notice-background: rgba(139, 148, 158, 0.1);
|
||||
}
|
||||
@@ -0,0 +1,860 @@
|
||||
(function() {
|
||||
const monitor = document.getElementById('progress-monitor');
|
||||
const collapseBtn = document.getElementById('progress-collapse');
|
||||
const treeContainer = document.getElementById('tree-container');
|
||||
const crawlTree = document.getElementById('crawl-tree');
|
||||
const idleMessage = document.getElementById('idle-message');
|
||||
const screencastPanel = document.getElementById('screencast-panel');
|
||||
|
||||
let pollInterval = null;
|
||||
let pollDelayMs = 1000;
|
||||
let idleTicks = 0;
|
||||
let isCollapsed = monitor.dataset.autoExpand !== '1';
|
||||
const snapshotMedia = new Map();
|
||||
|
||||
function stableSnapshotMedia(snapshot) {
|
||||
// The progress endpoint can briefly omit media URLs while ArchiveResult
|
||||
// rows settle. Once this page has seen a snapshot image URL, keep using
|
||||
// it so the preview/favicon branch does not flicker back to placeholder.
|
||||
const snapshotId = String(snapshot.id || '');
|
||||
const media = snapshotMedia.get(snapshotId) || {};
|
||||
if (snapshot.favicon_url) media.favicon_url ||= snapshot.favicon_url;
|
||||
if (snapshot.preview_url) {
|
||||
media.preview_url ||= snapshot.preview_url;
|
||||
media.preview_link ||= snapshot.preview_link;
|
||||
media.preview_fallbacks ||= snapshot.preview_fallbacks || [];
|
||||
}
|
||||
if (snapshotId) snapshotMedia.set(snapshotId, media);
|
||||
return media;
|
||||
}
|
||||
function preserveStableMediaNodes(nextRoot) {
|
||||
// Snapshot preview/favicon files are immutable once visible. Keep the
|
||||
// loaded DOM nodes across poll renders so the browser never reloads
|
||||
// them while progress text/badges continue updating around them.
|
||||
const currentCards = new Map(
|
||||
Array.from(crawlTree.querySelectorAll('.snapshot-item[data-snapshot-id]')).map(card => [card.dataset.snapshotId, card])
|
||||
);
|
||||
nextRoot.querySelectorAll('.snapshot-item[data-snapshot-id]').forEach(nextCard => {
|
||||
const currentCard = currentCards.get(nextCard.dataset.snapshotId);
|
||||
if (!currentCard) return;
|
||||
currentCard.querySelectorAll('[data-stable-media]').forEach(currentNode => {
|
||||
const nextNode = nextCard.querySelector(`[data-stable-media="${currentNode.dataset.stableMedia}"]`);
|
||||
if (nextNode) nextNode.replaceWith(currentNode);
|
||||
});
|
||||
});
|
||||
}
|
||||
function replaceCrawlTree(html) {
|
||||
const template = document.createElement('template');
|
||||
template.innerHTML = html;
|
||||
preserveStableMediaNodes(template.content);
|
||||
crawlTree.replaceChildren(...template.content.childNodes);
|
||||
}
|
||||
window.nextPreviewFallback = function(img) {
|
||||
const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean);
|
||||
if (fallbacks.length > 0) {
|
||||
img.src = fallbacks.shift();
|
||||
img.dataset.fallbacks = fallbacks.join(',');
|
||||
} else {
|
||||
const preview = img.closest('.snapshot-preview');
|
||||
if (preview) {
|
||||
preview.removeAttribute('data-stable-media');
|
||||
preview.classList.add('placeholder');
|
||||
preview.innerHTML = '<span>▤</span>';
|
||||
}
|
||||
}
|
||||
};
|
||||
function formatUrl(url) {
|
||||
if (!url) return '(no URL)';
|
||||
try {
|
||||
const u = new URL(url);
|
||||
return u.hostname + u.pathname.substring(0, 30) + (u.pathname.length > 30 ? '...' : '');
|
||||
} catch {
|
||||
return String(url).substring(0, 50) + (String(url).length > 50 ? '...' : '');
|
||||
}
|
||||
}
|
||||
|
||||
function getPluginIcon(plugin) {
|
||||
const icons = {
|
||||
'screenshot': '▧',
|
||||
'chrome_mhtml': '▧',
|
||||
'favicon': '◆',
|
||||
'dom': '</>',
|
||||
'pdf': 'PDF',
|
||||
'title': 'T',
|
||||
'headers': '{}',
|
||||
'singlefile': '▣',
|
||||
'readability': 'R',
|
||||
'mercury': 'M',
|
||||
'wget': '↓',
|
||||
'media': '▶',
|
||||
};
|
||||
return icons[plugin] || '•';
|
||||
}
|
||||
|
||||
function formatDuration(seconds) {
|
||||
seconds = Math.max(0, Math.floor(seconds || 0));
|
||||
const hours = Math.floor(seconds / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
const secs = seconds % 60;
|
||||
if (hours > 0) return `${hours}hr ${minutes}m ${secs}s`;
|
||||
if (minutes > 0) return `${minutes}m ${secs}s`;
|
||||
return `${secs}s`;
|
||||
}
|
||||
|
||||
function durationText(startedAt) {
|
||||
const startedMs = Date.parse(startedAt || '');
|
||||
if (!Number.isFinite(startedMs)) return '';
|
||||
return formatDuration((Date.now() - startedMs) / 1000);
|
||||
}
|
||||
|
||||
function renderDurationBadge(startedAt) {
|
||||
const text = durationText(startedAt);
|
||||
if (!text) return '';
|
||||
return `<span class="duration-badge" data-started-at="${ArchiveBoxUI.escapeHtml(startedAt)}" title="Duration">${ArchiveBoxUI.escapeHtml(text)}</span>`;
|
||||
}
|
||||
|
||||
function updateDurationBadges() {
|
||||
document.querySelectorAll('#progress-monitor .duration-badge[data-started-at]').forEach((badge) => {
|
||||
const text = durationText(badge.dataset.startedAt);
|
||||
if (text) badge.textContent = text;
|
||||
});
|
||||
}
|
||||
|
||||
function activeScreencastTarget(data) {
|
||||
const hookText = (item) => `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase();
|
||||
let fallback = null;
|
||||
for (const crawl of data.active_crawls || []) {
|
||||
for (const snapshot of crawl.active_snapshots || []) {
|
||||
if (!Array.isArray(snapshot) && snapshot.screencast_url) {
|
||||
return {type: 'frame', crawl, snapshot};
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const crawl of data.active_crawls || []) {
|
||||
if (fallback) continue;
|
||||
|
||||
const setupPlugins = crawl.setup_plugins || [];
|
||||
const browserReady = setupPlugins.some((item) => (
|
||||
item.status === 'succeeded' && hookText(item).includes('chrome wait')
|
||||
));
|
||||
if (!browserReady && setupPlugins.some((item) => item.status === 'started' && hookText(item).includes('chrome launch'))) {
|
||||
fallback = {
|
||||
type: 'placeholder',
|
||||
crawl,
|
||||
state: 'launching',
|
||||
icon: '▣',
|
||||
title: 'Launching browser',
|
||||
subtitle: 'Chrome is starting and publishing its CDP session.',
|
||||
};
|
||||
continue;
|
||||
}
|
||||
const openingSnapshot = (crawl.active_snapshots || []).find((snapshot) => {
|
||||
if (Array.isArray(snapshot)) return false;
|
||||
const plugins = snapshot.all_plugins || [];
|
||||
const tabReady = plugins.some((item) => (
|
||||
item.status === 'succeeded' && hookText(item).includes('chrome wait')
|
||||
));
|
||||
return !tabReady && plugins.some((item) => (
|
||||
item.status === 'started' && hookText(item).includes('chrome tab')
|
||||
));
|
||||
});
|
||||
if (openingSnapshot) {
|
||||
fallback = {
|
||||
type: 'placeholder',
|
||||
crawl,
|
||||
snapshot: openingSnapshot,
|
||||
state: 'launching',
|
||||
icon: '▣',
|
||||
title: 'Opening browser tab',
|
||||
subtitle: 'The snapshot tab is being attached for capture.',
|
||||
};
|
||||
continue;
|
||||
}
|
||||
if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => hookText(item).includes('chrome launch'))) {
|
||||
fallback = {
|
||||
type: 'placeholder',
|
||||
crawl,
|
||||
state: 'not-launched',
|
||||
icon: '▯',
|
||||
title: 'Browser not launched yet',
|
||||
subtitle: 'Waiting for the crawl setup hooks to start Chrome.',
|
||||
};
|
||||
}
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function updateScreencastPanel(data) {
|
||||
const hasWork = (data.active_crawls || []).length > 0 ||
|
||||
(data.crawls_queued || 0) > 0 ||
|
||||
(data.crawls_active || 0) > 0 ||
|
||||
(data.snapshots_queued || 0) > 0 ||
|
||||
(data.snapshots_active || 0) > 0;
|
||||
let target = activeScreencastTarget(data);
|
||||
if (!target) {
|
||||
const currentFrame = screencastPanel.querySelector('.screencast-frame img');
|
||||
if (
|
||||
hasWork &&
|
||||
screencastPanel.dataset.mode === 'frame' &&
|
||||
currentFrame?.getAttribute('src') &&
|
||||
currentFrame.complete &&
|
||||
currentFrame.naturalWidth > 0
|
||||
) {
|
||||
screencastPanel.classList.add('visible');
|
||||
currentFrame.style.opacity = '1';
|
||||
return;
|
||||
}
|
||||
if (!hasWork) {
|
||||
screencastPanel.classList.remove('visible');
|
||||
screencastPanel.dataset.mode = '';
|
||||
screencastPanel.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
target = {
|
||||
type: 'placeholder',
|
||||
state: 'not-launched',
|
||||
icon: '▯',
|
||||
title: 'Waiting for browser',
|
||||
subtitle: 'Live preview will appear whenever Chrome is active.',
|
||||
};
|
||||
}
|
||||
if (target.type === 'placeholder') {
|
||||
const currentFrame = screencastPanel.querySelector('.screencast-frame img');
|
||||
if (
|
||||
screencastPanel.dataset.mode === 'frame' &&
|
||||
currentFrame?.getAttribute('src') &&
|
||||
currentFrame.complete &&
|
||||
currentFrame.naturalWidth > 0
|
||||
) {
|
||||
screencastPanel.classList.add('visible');
|
||||
currentFrame.style.opacity = '1';
|
||||
return;
|
||||
}
|
||||
const crawlUrl = target.crawl?.id ? `/admin/crawls/crawl/${target.crawl.id}/change/` : '/admin/';
|
||||
const placeholderKey = [target.state, target.title, target.subtitle, crawlUrl].join('|');
|
||||
if (screencastPanel.dataset.mode === 'placeholder' && screencastPanel.dataset.key === placeholderKey) {
|
||||
return;
|
||||
}
|
||||
screencastPanel.dataset.mode = 'placeholder';
|
||||
screencastPanel.dataset.key = placeholderKey;
|
||||
screencastPanel.classList.add('visible');
|
||||
screencastPanel.innerHTML = `
|
||||
<a class="screencast-frame" href="${ArchiveBoxUI.escapeHtml(crawlUrl)}" title="Open crawl admin">
|
||||
<span class="screencast-placeholder ${ArchiveBoxUI.escapeHtml(target.state)}">
|
||||
<span class="screencast-placeholder-icon">${ArchiveBoxUI.escapeHtml(target.icon)}</span>
|
||||
<span class="screencast-placeholder-title">${ArchiveBoxUI.escapeHtml(target.title)}</span>
|
||||
<span class="screencast-placeholder-subtitle">${ArchiveBoxUI.escapeHtml(target.subtitle)}</span>
|
||||
</span>
|
||||
</a>
|
||||
<a class="screencast-caption" href="${ArchiveBoxUI.escapeHtml(crawlUrl)}" title="Open crawl admin">
|
||||
<span class="screencast-dot ${ArchiveBoxUI.escapeHtml(target.state)}"></span>
|
||||
<span class="screencast-text">
|
||||
<span class="screencast-title">${ArchiveBoxUI.escapeHtml(target.title)}</span>
|
||||
<span class="screencast-url">Crawl setup</span>
|
||||
</span>
|
||||
</a>
|
||||
`;
|
||||
return;
|
||||
}
|
||||
const snapshot = target.snapshot;
|
||||
const adminUrl = snapshot.admin_url || `/admin/core/snapshot/${snapshot.id || 'unknown'}/change/`;
|
||||
const linkUrl = snapshot.screencast_link || snapshot.view_url || adminUrl;
|
||||
const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
|
||||
const urlText = snapshot.full_url || snapshot.url || '';
|
||||
const newUrl = snapshot.screencast_url;
|
||||
const snapshotKey = `${target.crawl?.id || ''}|${snapshot.id || ''}`;
|
||||
screencastPanel.dataset.desiredFrameKey = snapshotKey;
|
||||
|
||||
if (screencastPanel.dataset.mode !== 'frame' && !screencastPanel.querySelector('.screencast-frame img')?.getAttribute('src')) {
|
||||
if (screencastPanel.dataset.pendingFrameSrc) return;
|
||||
screencastPanel.dataset.pendingFrameSrc = newUrl;
|
||||
screencastPanel.dataset.pendingFrameKey = snapshotKey;
|
||||
const firstFrame = new Image();
|
||||
firstFrame.decoding = 'async';
|
||||
firstFrame.onload = () => {
|
||||
(firstFrame.decode ? firstFrame.decode() : Promise.resolve()).catch(() => {}).finally(() => {
|
||||
if (screencastPanel.dataset.pendingFrameSrc !== newUrl) return;
|
||||
delete screencastPanel.dataset.pendingFrameSrc;
|
||||
delete screencastPanel.dataset.pendingFrameKey;
|
||||
if (screencastPanel.dataset.desiredFrameKey !== snapshotKey) return;
|
||||
screencastPanel.dataset.mode = 'frame';
|
||||
screencastPanel.dataset.key = '';
|
||||
screencastPanel.dataset.currentFrameSrc = newUrl;
|
||||
screencastPanel.innerHTML = `
|
||||
<a class="screencast-frame" href="${ArchiveBoxUI.escapeHtml(linkUrl)}" title="Open active snapshot">
|
||||
<img src="${ArchiveBoxUI.escapeHtml(newUrl)}" alt="" decoding="async" loading="eager">
|
||||
</a>
|
||||
<a class="screencast-caption" href="${ArchiveBoxUI.escapeHtml(adminUrl)}" title="Open snapshot admin">
|
||||
<span class="screencast-dot"></span>
|
||||
<span class="screencast-text">
|
||||
<span class="screencast-title">${ArchiveBoxUI.escapeHtml(titleText)}</span>
|
||||
<span class="screencast-url">${ArchiveBoxUI.escapeHtml(urlText)}</span>
|
||||
</span>
|
||||
</a>
|
||||
`;
|
||||
screencastPanel.classList.add('visible');
|
||||
});
|
||||
};
|
||||
firstFrame.onerror = () => {
|
||||
if (screencastPanel.dataset.pendingFrameSrc === newUrl) {
|
||||
delete screencastPanel.dataset.pendingFrameSrc;
|
||||
delete screencastPanel.dataset.pendingFrameKey;
|
||||
}
|
||||
};
|
||||
firstFrame.src = newUrl;
|
||||
return;
|
||||
}
|
||||
|
||||
if (screencastPanel.dataset.mode !== 'frame') {
|
||||
screencastPanel.dataset.mode = 'frame';
|
||||
screencastPanel.dataset.key = '';
|
||||
screencastPanel.innerHTML = `
|
||||
<a class="screencast-frame" href="" title="Open active snapshot">
|
||||
<img src="" alt="" decoding="async" loading="eager">
|
||||
</a>
|
||||
<a class="screencast-caption" href="" title="Open snapshot admin">
|
||||
<span class="screencast-dot"></span>
|
||||
<span class="screencast-text">
|
||||
<span class="screencast-title"></span>
|
||||
<span class="screencast-url"></span>
|
||||
</span>
|
||||
</a>
|
||||
`;
|
||||
}
|
||||
const frameLink = screencastPanel.querySelector('.screencast-frame');
|
||||
const captionLink = screencastPanel.querySelector('.screencast-caption');
|
||||
const img = screencastPanel.querySelector('.screencast-frame img');
|
||||
const titleEl = screencastPanel.querySelector('.screencast-title');
|
||||
const urlEl = screencastPanel.querySelector('.screencast-url');
|
||||
frameLink.href = linkUrl;
|
||||
captionLink.href = adminUrl;
|
||||
titleEl.textContent = titleText;
|
||||
urlEl.textContent = urlText;
|
||||
if (img.getAttribute('src')) {
|
||||
img.style.opacity = '1';
|
||||
screencastPanel.classList.add('visible');
|
||||
}
|
||||
if (screencastPanel.dataset.currentFrameSrc !== newUrl && screencastPanel.dataset.pendingFrameSrc !== newUrl && !screencastPanel.dataset.pendingFrameSrc) {
|
||||
screencastPanel.dataset.pendingFrameSrc = newUrl;
|
||||
screencastPanel.dataset.pendingFrameKey = snapshotKey;
|
||||
const nextFrame = new Image();
|
||||
nextFrame.decoding = 'async';
|
||||
nextFrame.onload = () => {
|
||||
(nextFrame.decode ? nextFrame.decode() : Promise.resolve()).catch(() => {}).finally(() => {
|
||||
if (screencastPanel.dataset.pendingFrameSrc !== newUrl) return;
|
||||
delete screencastPanel.dataset.pendingFrameSrc;
|
||||
delete screencastPanel.dataset.pendingFrameKey;
|
||||
if (screencastPanel.dataset.desiredFrameKey !== snapshotKey) return;
|
||||
const currentImg = screencastPanel.querySelector('.screencast-frame img');
|
||||
if (!currentImg) return;
|
||||
nextFrame.alt = '';
|
||||
nextFrame.loading = 'eager';
|
||||
nextFrame.style.opacity = '1';
|
||||
currentImg.replaceWith(nextFrame);
|
||||
screencastPanel.dataset.currentFrameSrc = newUrl;
|
||||
screencastPanel.classList.add('visible');
|
||||
});
|
||||
};
|
||||
nextFrame.onerror = () => {
|
||||
if (screencastPanel.dataset.pendingFrameSrc === newUrl) {
|
||||
delete screencastPanel.dataset.pendingFrameSrc;
|
||||
delete screencastPanel.dataset.pendingFrameKey;
|
||||
if (!screencastPanel.querySelector('.screencast-frame img')?.getAttribute('src')) {
|
||||
screencastPanel.classList.remove('visible');
|
||||
}
|
||||
}
|
||||
};
|
||||
nextFrame.src = newUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function renderExtractor(extractor) {
|
||||
const statusIcons = {started: '▶', succeeded: '✓', failed: '!', backoff: 'wait', skipped: 'skip', noresults: '∅'};
|
||||
const icon = statusIcons[extractor.status] || getPluginIcon(extractor.plugin);
|
||||
const progress = typeof extractor.progress === 'number'
|
||||
? Math.max(0, Math.min(100, extractor.progress))
|
||||
: null;
|
||||
const progressStyle = progress !== null ? ` style="width: ${progress}%;"` : '';
|
||||
const pidHtml = extractor.status === 'started' && extractor.pid ? `<span class="pid-label compact">pid ${extractor.pid}</span>` : '';
|
||||
const href = extractor.output_url || extractor.admin_url || '';
|
||||
const tag = href ? 'a' : 'span';
|
||||
const hrefAttr = href ? ` href="${ArchiveBoxUI.escapeHtml(href)}"` : '';
|
||||
const title = extractor.output_path
|
||||
? `${extractor.plugin || 'output'}: ${extractor.output_path}`
|
||||
: `${extractor.plugin || 'hook'}${extractor.hook_name ? `: ${extractor.hook_name}` : ''}`;
|
||||
|
||||
return `
|
||||
<${tag} class="extractor-badge ${extractor.status || 'queued'}"${hrefAttr} title="${ArchiveBoxUI.escapeHtml(title)}">
|
||||
<span class="progress-fill"${progressStyle}></span>
|
||||
<span class="badge-content">
|
||||
<span class="badge-icon">${icon}</span>
|
||||
<span>${ArchiveBoxUI.escapeHtml(extractor.label || extractor.plugin || 'unknown')}</span>
|
||||
${pidHtml}
|
||||
</span>
|
||||
</${tag}>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderExtractors(extractors = []) {
|
||||
if (!extractors.length) return '';
|
||||
const sorted = [...extractors].sort((a, b) => (a.plugin || '').localeCompare(b.plugin || ''));
|
||||
return `<div class="extractor-list">${sorted.map(renderExtractor).join('')}</div>`;
|
||||
}
|
||||
|
||||
function renderSnapshot(snapshot) {
|
||||
if (Array.isArray(snapshot)) {
|
||||
snapshot = {
|
||||
id: snapshot[0],
|
||||
url: snapshot[1],
|
||||
title: snapshot[2],
|
||||
status: snapshot[3] || 'queued',
|
||||
};
|
||||
}
|
||||
const statusIcon = snapshot.status === 'started' ? '▤' : '▢';
|
||||
const adminUrl = snapshot.admin_url || `/admin/core/snapshot/${snapshot.id || 'unknown'}/change/`;
|
||||
const canCancel = snapshot.status === 'queued';
|
||||
const cancelBtn = canCancel
|
||||
? `<button class="cancel-item-btn" data-cancel-type="snapshot" data-snapshot-id="${snapshot.id}" data-label="✕" title="Cancel snapshot">✕</button>`
|
||||
: '';
|
||||
const snapshotPidHtml = snapshot.worker_pid ? `<span class="pid-label compact">pid ${snapshot.worker_pid}</span>` : '';
|
||||
const snapshotDurationHtml = renderDurationBadge(snapshot.started);
|
||||
const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
|
||||
const urlText = snapshot.full_url || snapshot.url || '';
|
||||
const media = stableSnapshotMedia(snapshot);
|
||||
const faviconHtml = media.favicon_url
|
||||
? `<img class="snapshot-favicon" data-stable-media="favicon" src="${ArchiveBoxUI.escapeHtml(media.favicon_url)}" alt="" decoding="async" loading="lazy" onerror="this.remove()">`
|
||||
: '';
|
||||
const previewHtml = media.preview_url
|
||||
? `<a class="snapshot-preview" data-stable-media="preview" href="${ArchiveBoxUI.escapeHtml(media.preview_link || snapshot.view_url || adminUrl)}" title="Open snapshot output"><img src="${ArchiveBoxUI.escapeHtml(media.preview_url)}" alt="" decoding="async" loading="lazy" data-fallbacks="${ArchiveBoxUI.escapeHtml((media.preview_fallbacks || []).join(','))}" onerror="nextPreviewFallback(this)"></a>`
|
||||
: `<a class="snapshot-preview placeholder" href="${ArchiveBoxUI.escapeHtml(snapshot.view_url || adminUrl)}" title="Open snapshot"><span>${statusIcon}</span></a>`;
|
||||
|
||||
const extractorHtml = renderExtractors(snapshot.all_plugins);
|
||||
|
||||
const hasProcessEntries = (snapshot.all_plugins || []).some(extractor => extractor.source === 'process');
|
||||
const hasArchiveResults = (snapshot.all_plugins || []).some(extractor => extractor.source === 'archiveresult');
|
||||
const processOnly = hasProcessEntries && !hasArchiveResults;
|
||||
const runningProcessCount = (snapshot.all_plugins || []).filter(extractor => extractor.source === 'process' && extractor.status === 'started').length;
|
||||
const failedProcessCount = (snapshot.all_plugins || []).filter(extractor => extractor.source === 'process' && extractor.status === 'failed').length;
|
||||
const snapshotMeta = (snapshot.total_plugins || 0) > 0
|
||||
? processOnly
|
||||
? runningProcessCount > 0
|
||||
? `Running ${runningProcessCount}/${snapshot.total_plugins || 0} setup hooks`
|
||||
: failedProcessCount > 0
|
||||
? `${failedProcessCount} setup hook${failedProcessCount === 1 ? '' : 's'} failed`
|
||||
: `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} setup hooks`
|
||||
: hasProcessEntries
|
||||
? `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} tasks${(snapshot.failed_plugins || 0) > 0 ? ` <span style="color:#f85149">(${snapshot.failed_plugins} failed)</span>` : ''}${runningProcessCount > 0 ? ` <span style="color:#d29922">(${runningProcessCount} hooks running)</span>` : ''}`
|
||||
: `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} extractors${(snapshot.failed_plugins || 0) > 0 ? ` <span style="color:#f85149">(${snapshot.failed_plugins} failed)</span>` : ''}`
|
||||
: snapshot.worker_state === 'crashed'
|
||||
? '<span style="color:#f85149">Worker stopped before extractors started</span>'
|
||||
: snapshot.worker_state === 'stalled'
|
||||
? '<span style="color:#d29922">Waiting for runner to resume</span>'
|
||||
: snapshot.worker_state === 'cancelled'
|
||||
? '<span style="color:#8b949e">Cancelled before completion</span>'
|
||||
: 'Waiting for extractors...';
|
||||
|
||||
return `
|
||||
<div class="snapshot-item" data-snapshot-id="${ArchiveBoxUI.escapeHtml(snapshot.id || '')}">
|
||||
<div class="snapshot-header">
|
||||
${previewHtml}
|
||||
<a class="snapshot-header-link" href="${adminUrl}">
|
||||
<div class="snapshot-info">
|
||||
<div class="snapshot-title-line">
|
||||
${faviconHtml}
|
||||
<span class="snapshot-title">${ArchiveBoxUI.escapeHtml(titleText)}</span>
|
||||
</div>
|
||||
<div class="snapshot-url">${ArchiveBoxUI.escapeHtml(urlText)}</div>
|
||||
<div class="snapshot-meta">
|
||||
${snapshotMeta}
|
||||
</div>
|
||||
</div>
|
||||
${snapshotPidHtml}
|
||||
${snapshotDurationHtml}
|
||||
<span class="status-badge ${snapshot.status || 'unknown'}">${snapshot.status || 'unknown'}</span>
|
||||
</a>
|
||||
${cancelBtn}
|
||||
</div>
|
||||
<div class="snapshot-progress">
|
||||
<div class="progress-bar-container">
|
||||
<div class="progress-bar snapshot ${((processOnly && runningProcessCount > 0) || (snapshot.status === 'started' && (snapshot.progress || 0) === 0)) ? 'indeterminate' : ''}"
|
||||
style="width: ${snapshot.progress || 0}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
${extractorHtml}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderCrawl(crawl) {
|
||||
const adminUrl = `/admin/crawls/crawl/${crawl.id || 'unknown'}/change/`;
|
||||
const adminFieldUrl = (fieldName) => `${adminUrl}#id_${fieldName}`;
|
||||
const crawlBadge = (className, label, value, href, title) => {
|
||||
const tag = href ? 'a' : 'span';
|
||||
const hrefAttr = href ? ` href="${ArchiveBoxUI.escapeHtml(href)}"` : '';
|
||||
const titleAttr = title ? ` title="${ArchiveBoxUI.escapeHtml(title)}"` : '';
|
||||
return `<${tag} class="crawl-badge ${className}"${hrefAttr}${titleAttr}><strong>${ArchiveBoxUI.escapeHtml(label)}</strong>${ArchiveBoxUI.escapeHtml(value)}</${tag}>`;
|
||||
};
|
||||
const crawlId = (crawl.id || 'unknown').toString();
|
||||
const crawlShortId = crawlId === 'unknown' ? 'unknown' : crawlId.slice(-8);
|
||||
const startedDate = crawl.started ? crawl.started.slice(0, 10) : 'unknown date';
|
||||
const canCancel = crawl.status === 'queued' || crawl.status === 'started' || crawl.status === 'paused';
|
||||
const canPause = (crawl.status === 'queued' || crawl.status === 'started') && !crawl.is_paused;
|
||||
const canResume = crawl.status === 'paused' || crawl.is_paused;
|
||||
const pauseBtn = canPause
|
||||
? `<button class="crawl-action-btn pause-item-btn" data-crawl-action="pause" data-crawl-id="${crawl.id}" data-label="Ⅱ" title="Pause crawl">Ⅱ</button>`
|
||||
: canResume
|
||||
? `<button class="crawl-action-btn pause-item-btn" data-crawl-action="resume" data-crawl-id="${crawl.id}" data-label="▶" title="Resume crawl">▶</button>`
|
||||
: '';
|
||||
const cancelBtn = canCancel
|
||||
? `<button class="cancel-item-btn" data-cancel-type="crawl" data-crawl-id="${crawl.id}" data-label="✕" title="Cancel crawl">✕</button>`
|
||||
: '';
|
||||
const crawlPidHtml = crawl.worker_pid ? `<span class="pid-label compact">pid ${crawl.worker_pid}</span>` : '';
|
||||
const crawlDurationHtml = renderDurationBadge(crawl.started);
|
||||
|
||||
const snapshotsHtml = (crawl.active_snapshots || []).map(renderSnapshot).join('');
|
||||
const queuedSnapshotsHidden = Number(crawl.queued_snapshots_hidden || 0);
|
||||
const queuedSnapshotsNote = queuedSnapshotsHidden > 0
|
||||
? `<div class="progress-overflow-note">${queuedSnapshotsHidden} more queued snapshot${queuedSnapshotsHidden === 1 ? '' : 's'} not shown</div>`
|
||||
: '';
|
||||
let setupHtml = '';
|
||||
if (crawl.setup_plugins && crawl.setup_plugins.length > 0) {
|
||||
setupHtml = `
|
||||
<div class="snapshot-item">
|
||||
<div class="snapshot-header">
|
||||
<div class="snapshot-header-link">
|
||||
<span class="snapshot-icon">⚙</span>
|
||||
<div class="snapshot-info">
|
||||
<div class="snapshot-url">Crawl Setup</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
${renderExtractors(crawl.setup_plugins)}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
const urlPreview = crawl.urls_preview ? ` (${ArchiveBoxUI.escapeHtml(crawl.urls_preview)})` : '';
|
||||
const warnings = [
|
||||
[crawl.status === 'queued' && !crawl.can_start, 'error',
|
||||
`Crawl cannot start: ${crawl.urls_preview ? 'unknown error' : 'no URLs'}`],
|
||||
[crawl.is_paused, 'warning',
|
||||
'Crawl is paused. Resume it to continue processing queued snapshots.'],
|
||||
[crawl.status === 'queued' && crawl.retry_at_future, 'info',
|
||||
`Trying in ${crawl.seconds_until_retry || 0}s...${urlPreview}`],
|
||||
[crawl.status === 'queued' && crawl.total_snapshots === 0, 'warning',
|
||||
`Waiting for the runner to pick up...${urlPreview}`],
|
||||
[crawl.status === 'started' && crawl.worker_state === 'crashed', 'error',
|
||||
`Runner stopped with ${crawl.started_snapshots || 0} active and ${crawl.pending_snapshots || 0} pending snapshots. It will resume when the runner starts again.`],
|
||||
[crawl.worker_state === 'cancelled', 'cancelled',
|
||||
`Crawl was cancelled. ${crawl.cancelled_snapshots || 0} snapshot${(crawl.cancelled_snapshots || 0) === 1 ? '' : 's'} stopped before completion.`],
|
||||
];
|
||||
const warning = warnings.find(([matches]) => matches);
|
||||
const warningHtml = warning ? `<div class="crawl-notice ${warning[1]}">${warning[2]}</div>` : '';
|
||||
|
||||
// Show crawl-scale limits and sealed output sizes from DB metadata.
|
||||
const currentUrlCount = Math.max(crawl.total_snapshots || 0, crawl.urls_count || 0);
|
||||
const maxUrlsText = (crawl.max_urls || 0) > 0 ? crawl.max_urls : 'unlimited';
|
||||
const urlLimitText = `${currentUrlCount} / ${maxUrlsText}`;
|
||||
const crawlSizeLimitText = `${crawl.crawl_output_size_display || '0 B'} / ${crawl.max_crawl_size_display || 'unlimited'}`;
|
||||
const crawlTimeoutText = (crawl.crawl_timeout || 0) > 0 ? `${crawl.crawl_timeout}s` : 'unlimited';
|
||||
const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`;
|
||||
const crawlBadges = [
|
||||
crawlBadge('persona', 'Persona Config', crawl.persona || 'Default', crawl.persona_admin_url, 'Edit persona config'),
|
||||
crawlBadge('limit', 'depth', crawl.max_depth || 0, adminFieldUrl('max_depth'), 'Edit crawl depth'),
|
||||
crawlBadge('limit', 'urls', urlLimitText, adminFieldUrl('max_urls'), 'Edit max URLs'),
|
||||
crawlBadge('size', 'crawl size', crawlSizeLimitText, adminFieldUrl('crawl_max_size'), 'Edit max crawl size'),
|
||||
crawlBadge('limit', 'time', crawlTimeoutText, adminFieldUrl('crawl_timeout'), 'Edit max crawl time'),
|
||||
crawlBadge('size', 'avg snap', snapshotSizeLimitText, adminFieldUrl('snapshot_max_size'), 'Edit max snapshot size'),
|
||||
...(crawl.tags || []).map(tag => `<a class="crawl-badge tag" href="${ArchiveBoxUI.escapeHtml(adminFieldUrl('tags_editor'))}" title="Edit crawl tags">#${ArchiveBoxUI.escapeHtml(tag)}</a>`),
|
||||
].join('');
|
||||
const statsHtml = [
|
||||
`<span class="crawl-badge count"><strong>done</strong>${crawl.completed_snapshots || 0}</span>`,
|
||||
`<span class="crawl-badge size"><strong>active</strong>${crawl.started_snapshots || 0}</span>`,
|
||||
`<span class="crawl-badge"><strong>pending</strong>${crawl.pending_snapshots || 0}</span>`,
|
||||
(crawl.cancelled_snapshots || 0) > 0 ? `<span class="crawl-badge"><strong>cancelled</strong>${crawl.cancelled_snapshots}</span>` : '',
|
||||
].join('');
|
||||
|
||||
return `
|
||||
<div class="crawl-item" data-crawl-id="${crawl.id || 'unknown'}">
|
||||
<div class="crawl-header">
|
||||
<div class="crawl-header-link">
|
||||
<div class="crawl-info">
|
||||
<a class="crawl-label crawl-title-link" href="${ArchiveBoxUI.escapeHtml(adminUrl)}">Crawl #${ArchiveBoxUI.escapeHtml(crawlShortId)} ${ArchiveBoxUI.escapeHtml(startedDate)} started by ${ArchiveBoxUI.escapeHtml(crawl.created_by || 'unknown')}</a>
|
||||
<div class="crawl-badges">${crawlBadges}</div>
|
||||
</div>
|
||||
<div class="crawl-stats">
|
||||
${statsHtml}
|
||||
</div>
|
||||
${crawlPidHtml}
|
||||
${crawlDurationHtml}
|
||||
<span class="status-badge ${crawl.status || 'unknown'}">${crawl.is_paused ? 'paused' : (crawl.status || 'unknown')}</span>
|
||||
</div>
|
||||
${pauseBtn}
|
||||
${cancelBtn}
|
||||
</div>
|
||||
<div class="crawl-progress">
|
||||
<div class="progress-bar-container">
|
||||
<div class="progress-bar crawl ${crawl.status === 'started' && (crawl.progress || 0) === 0 ? 'indeterminate' : ''}"
|
||||
style="width: ${crawl.progress || 0}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
${warningHtml}
|
||||
<div class="crawl-body">
|
||||
<div class="snapshot-list">
|
||||
${setupHtml}
|
||||
${snapshotsHtml}
|
||||
${queuedSnapshotsNote}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
function setOrchestratorState(state, label) {
|
||||
const dot = document.getElementById('orchestrator-dot');
|
||||
dot.classList.remove('stopped', 'idle', 'running', 'error');
|
||||
dot.classList.add(state);
|
||||
document.getElementById('orchestrator-text').textContent = label;
|
||||
return dot;
|
||||
}
|
||||
|
||||
function updateProgress(data) {
|
||||
idleMessage.style.color = '';
|
||||
function setCount(id, value) {
|
||||
const el = document.getElementById(id);
|
||||
if (el) el.textContent = Number(value || 0).toLocaleString();
|
||||
}
|
||||
function setActiveCount(id, value) {
|
||||
setCount(id, value);
|
||||
const segment = document.getElementById(`${id}-segment`);
|
||||
if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none';
|
||||
}
|
||||
|
||||
// Calculate if there's activity
|
||||
const hasActivity = data.active_crawls.length > 0 ||
|
||||
data.crawls_queued > 0 || data.crawls_active > 0 ||
|
||||
data.snapshots_queued > 0 || data.snapshots_active > 0 ||
|
||||
data.archiveresults_queued > 0 || data.archiveresults_active > 0;
|
||||
if (hasActivity) {
|
||||
idleTicks = 0;
|
||||
if (pollDelayMs !== 1000) {
|
||||
setPollingDelay(1000);
|
||||
}
|
||||
} else {
|
||||
idleTicks += 1;
|
||||
if (idleTicks > 5 && pollDelayMs !== 10000) {
|
||||
setPollingDelay(10000);
|
||||
}
|
||||
}
|
||||
|
||||
// Update orchestrator status - show "Running" only when there are active workers.
|
||||
const pidEl = document.getElementById('orchestrator-pid');
|
||||
const hasWorkers = data.total_workers > 0;
|
||||
let dot = null;
|
||||
|
||||
if (hasWorkers) {
|
||||
dot = setOrchestratorState('running', 'Running');
|
||||
} else if (!data.orchestrator_running) {
|
||||
dot = setOrchestratorState('stopped', 'Runner stopped');
|
||||
} else if (hasActivity) {
|
||||
dot = setOrchestratorState('idle', 'Idle');
|
||||
} else {
|
||||
dot = setOrchestratorState('idle', 'Idle');
|
||||
}
|
||||
|
||||
if (data.orchestrator_pid) {
|
||||
pidEl.textContent = `pid ${data.orchestrator_pid}`;
|
||||
pidEl.style.display = 'inline-flex';
|
||||
} else {
|
||||
pidEl.textContent = '';
|
||||
pidEl.style.display = 'none';
|
||||
}
|
||||
|
||||
// Pulse the dot to show we got fresh data
|
||||
dot.classList.add('flash');
|
||||
setTimeout(() => dot.classList.remove('flash'), 300);
|
||||
|
||||
[
|
||||
['crawls-queued', data.crawls_queued],
|
||||
['snapshots-queued', data.snapshots_queued],
|
||||
['downloads-queued', data.downloads_queued],
|
||||
['indexing-queued', data.indexing_queued],
|
||||
].forEach(([id, value]) => setCount(id, value));
|
||||
[
|
||||
['crawls-active', data.crawls_active],
|
||||
['snapshots-active', data.snapshots_active],
|
||||
['downloads-active', data.downloads_active],
|
||||
['indexing-active', data.indexing_active],
|
||||
].forEach(([id, value]) => setActiveCount(id, value));
|
||||
updateScreencastPanel(data);
|
||||
|
||||
// Render crawl tree
|
||||
if (data.active_crawls.length > 0) {
|
||||
idleMessage.style.display = 'none';
|
||||
const queuedCrawlsHidden = Number(data.queued_crawls_hidden || 0);
|
||||
const queuedCrawlsNote = queuedCrawlsHidden > 0
|
||||
? `<div class="progress-overflow-note">${queuedCrawlsHidden} more queued crawl${queuedCrawlsHidden === 1 ? '' : 's'} not shown</div>`
|
||||
: '';
|
||||
replaceCrawlTree(data.active_crawls.map(c => renderCrawl(c)).join('') + queuedCrawlsNote);
|
||||
} else if (hasActivity) {
|
||||
idleMessage.style.display = 'none';
|
||||
replaceCrawlTree(`
|
||||
<div class="idle-message">
|
||||
${data.snapshots_active || 0} snapshots processing, ${data.archiveresults_active || 0} extractors running
|
||||
</div>
|
||||
`);
|
||||
} else {
|
||||
idleMessage.style.display = '';
|
||||
// Build the URL for recent crawls (last 24 hours)
|
||||
var yesterday = new Date(Date.now() - 24*60*60*1000).toISOString().split('T')[0];
|
||||
var recentUrl = '/admin/crawls/crawl/?created_at__gte=' + yesterday + '&o=-1';
|
||||
idleMessage.innerHTML = `No active crawls (${data.crawls_queued || 0} pending, ${data.crawls_active || 0} started, <a href="${recentUrl}" style="color: #58a6ff;">${data.crawls_recent || 0} recent</a>)`;
|
||||
replaceCrawlTree('');
|
||||
}
|
||||
|
||||
updateDurationBadges();
|
||||
}
|
||||
|
||||
const progressEndpoint = monitor.dataset.progressEndpoint || '/progress.json';
|
||||
|
||||
function fetchProgress() {
|
||||
fetch(progressEndpoint, { credentials: 'same-origin' })
|
||||
.then(response => {
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
return response.json();
|
||||
})
|
||||
.then(data => {
|
||||
if (data.error) {
|
||||
console.error('Progress API error:', data.error, data.traceback);
|
||||
setOrchestratorState('error', 'Backend not responding');
|
||||
idleMessage.textContent = 'API Error: ' + data.error;
|
||||
idleMessage.style.color = '#f85149';
|
||||
return;
|
||||
}
|
||||
monitor.classList.toggle('is-guest', data.is_admin === false);
|
||||
updateProgress(data);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Progress fetch error:', error);
|
||||
setOrchestratorState('error', 'Backend not responding');
|
||||
idleMessage.textContent = 'Fetch Error: ' + error.message;
|
||||
idleMessage.style.color = '#f85149';
|
||||
});
|
||||
}
|
||||
|
||||
function startPolling() {
|
||||
if (pollInterval) return;
|
||||
fetchProgress();
|
||||
pollInterval = setInterval(fetchProgress, pollDelayMs);
|
||||
}
|
||||
|
||||
function stopPolling() {
|
||||
if (pollInterval) {
|
||||
clearInterval(pollInterval);
|
||||
pollInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
function setPollingDelay(ms) {
|
||||
pollDelayMs = ms;
|
||||
if (pollInterval) {
|
||||
clearInterval(pollInterval);
|
||||
pollInterval = setInterval(fetchProgress, pollDelayMs);
|
||||
}
|
||||
}
|
||||
|
||||
function setCollapsedState(collapsed) {
|
||||
isCollapsed = collapsed;
|
||||
if (isCollapsed) {
|
||||
monitor.classList.add('collapsed');
|
||||
collapseBtn.textContent = 'Details';
|
||||
collapseBtn.setAttribute('aria-expanded', 'false');
|
||||
} else {
|
||||
monitor.classList.remove('collapsed');
|
||||
collapseBtn.textContent = 'Hide';
|
||||
collapseBtn.setAttribute('aria-expanded', 'true');
|
||||
}
|
||||
}
|
||||
|
||||
function setActionButtonState(btn, busy) {
|
||||
if (!btn) return;
|
||||
const label = btn.dataset.label || '✕';
|
||||
btn.disabled = !!busy;
|
||||
btn.classList.toggle('is-busy', !!busy);
|
||||
btn.textContent = busy ? '…' : label;
|
||||
}
|
||||
|
||||
function patchProgressItem(url, action, btn, errorLabel) {
|
||||
if (!url || !action) return;
|
||||
setActionButtonState(btn, true);
|
||||
|
||||
fetch(ArchiveBoxUI.apiUrl(url), {
|
||||
method: 'PATCH',
|
||||
headers: ArchiveBoxUI.apiHeaders(),
|
||||
credentials: 'same-origin',
|
||||
body: JSON.stringify({ action: action }),
|
||||
})
|
||||
.then(response => response.json().then(data => ({response, data})))
|
||||
.then(({response, data}) => {
|
||||
if (!response.ok) throw new Error(data.detail || data.error || `HTTP ${response.status}`);
|
||||
if (data.error) console.error(`${errorLabel} error:`, data.error);
|
||||
fetchProgress();
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(`${errorLabel} failed:`, error);
|
||||
setActionButtonState(btn, false);
|
||||
});
|
||||
}
|
||||
|
||||
// Collapse toggle
|
||||
collapseBtn.addEventListener('click', function(event) {
|
||||
event.stopPropagation();
|
||||
setCollapsedState(!isCollapsed);
|
||||
});
|
||||
monitor.querySelector('.header-bar').addEventListener('click', function(event) {
|
||||
if (event.target.closest('button, a, input, select, textarea, [role="button"]')) return;
|
||||
if (isCollapsed) setCollapsedState(false);
|
||||
});
|
||||
|
||||
crawlTree.addEventListener('click', function(event) {
|
||||
const actionBtn = event.target.closest('.crawl-action-btn');
|
||||
if (actionBtn) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
patchProgressItem(
|
||||
actionBtn.dataset.crawlId ? `/api/v1/crawls/crawl/${actionBtn.dataset.crawlId}` : '',
|
||||
actionBtn.dataset.crawlAction,
|
||||
actionBtn,
|
||||
'Crawl action',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const btn = event.target.closest('.cancel-item-btn');
|
||||
if (!btn) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
|
||||
const cancelType = btn.dataset.cancelType;
|
||||
if (cancelType === 'crawl') {
|
||||
patchProgressItem(btn.dataset.crawlId ? `/api/v1/crawls/crawl/${btn.dataset.crawlId}` : '', 'cancel', btn, 'Cancel crawl');
|
||||
} else if (cancelType === 'snapshot') {
|
||||
patchProgressItem(btn.dataset.snapshotId ? `/api/v1/core/snapshot/${btn.dataset.snapshotId}` : '', 'cancel', btn, 'Cancel snapshot');
|
||||
}
|
||||
});
|
||||
|
||||
// Apply initial state
|
||||
setCollapsedState(isCollapsed);
|
||||
|
||||
// Start polling when page loads
|
||||
startPolling();
|
||||
setInterval(updateDurationBadges, 1000);
|
||||
|
||||
// Pause polling when tab is hidden
|
||||
document.addEventListener('visibilitychange', function() {
|
||||
if (document.hidden) {
|
||||
stopPolling();
|
||||
} else {
|
||||
startPolling();
|
||||
}
|
||||
});
|
||||
})();
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,23 +1,13 @@
|
||||
__package__ = "archivebox.progressmonitor"
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from abx_dl.events import PROCESS_EXIT_SKIPPED
|
||||
from django.conf import settings
|
||||
from django.db import DatabaseError
|
||||
from django.db.models import CharField, Count, Q, Sum
|
||||
from django.db.models.functions import Cast
|
||||
from django.http import HttpResponse, JsonResponse
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.permissions import can_view_snapshot, is_admin_user
|
||||
from archivebox.core.routes_util import build_snapshot_url, build_web_url, get_api_base_url
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.plugins.discovery import discover_plugin_configs
|
||||
|
||||
|
||||
def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_id: object | None = None) -> str:
|
||||
@@ -27,28 +17,10 @@ def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_
|
||||
return f"/progress.json?{scope}_id={str(object_id).replace('-', '')}"
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
|
||||
plugin_configs = discover_plugin_configs()
|
||||
indexing_plugin_names = frozenset(
|
||||
plugin_name
|
||||
for plugin_name, plugin_config in plugin_configs.items()
|
||||
if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
|
||||
)
|
||||
download_plugin_names = frozenset(
|
||||
plugin_name
|
||||
for plugin_name, plugin_config in plugin_configs.items()
|
||||
if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
|
||||
)
|
||||
return download_plugin_names, indexing_plugin_names
|
||||
|
||||
|
||||
def live_progress_view(request):
|
||||
"""Simple JSON endpoint for live progress status - used by admin progress monitor."""
|
||||
try:
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Machine, Process
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
snapshot_id_filter = (request.GET.get("snapshot_id") or "").strip().replace("-", "")
|
||||
crawl_id_filter = (request.GET.get("crawl_id") or "").strip().replace("-", "")
|
||||
@@ -65,883 +37,18 @@ def live_progress_view(request):
|
||||
scoped_snapshot = Snapshot.objects.filter(id=snapshot_id_filter).select_related("crawl").first()
|
||||
if scoped_snapshot is None or not can_view_snapshot(request, scoped_snapshot):
|
||||
return JsonResponse({"error": "Permission denied"}, status=403)
|
||||
elif crawl_id_filter:
|
||||
# Crawl-only scope still requires staff: there's no per-crawl ACL helper,
|
||||
# and a crawl can mix snapshot permissions levels.
|
||||
if not is_admin:
|
||||
return JsonResponse({"error": "Permission denied"}, status=403)
|
||||
else:
|
||||
if not is_admin:
|
||||
return JsonResponse({"error": "Permission denied"}, status=403)
|
||||
|
||||
request_config = request.archivebox_config
|
||||
now = timezone.now()
|
||||
crawl_scope = Crawl.objects.all()
|
||||
snapshot_scope = Snapshot.objects.all()
|
||||
archiveresult_scope = ArchiveResult.objects.all()
|
||||
if is_admin and not request.user.is_superuser:
|
||||
crawl_scope = crawl_scope.filter(created_by=request.user)
|
||||
snapshot_scope = snapshot_scope.filter(crawl__created_by=request.user)
|
||||
archiveresult_scope = archiveresult_scope.filter(snapshot__crawl__created_by=request.user)
|
||||
if scoped_snapshot is not None:
|
||||
snapshot_scope = Snapshot.objects.filter(id=scoped_snapshot.id)
|
||||
crawl_scope = Crawl.objects.filter(id=scoped_snapshot.crawl_id)
|
||||
archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=scoped_snapshot.id)
|
||||
elif crawl_id_filter:
|
||||
snapshot_scope = snapshot_scope.filter(crawl_id=crawl_id_filter)
|
||||
crawl_scope = crawl_scope.filter(id=crawl_id_filter)
|
||||
archiveresult_scope = archiveresult_scope.filter(snapshot__crawl_id=crawl_id_filter)
|
||||
|
||||
def is_current_run_timestamp(event_ts, run_started_at) -> bool:
|
||||
if run_started_at is None:
|
||||
return True
|
||||
if event_ts is None:
|
||||
return False
|
||||
return event_ts >= run_started_at
|
||||
|
||||
def archiveresult_matches_current_run(ar, run_started_at) -> bool:
|
||||
if run_started_at is None:
|
||||
return True
|
||||
if ar.status in (
|
||||
ArchiveResult.StatusChoices.QUEUED,
|
||||
ArchiveResult.StatusChoices.STARTED,
|
||||
ArchiveResult.StatusChoices.BACKOFF,
|
||||
):
|
||||
return True
|
||||
event_ts = ar.end_ts or ar.start_ts or ar.modified_at or ar.created_at
|
||||
return is_current_run_timestamp(event_ts, run_started_at)
|
||||
|
||||
def hook_details(hook_name: str, plugin: str = "setup") -> tuple[str, str, str, str]:
|
||||
normalized_hook_name = Path(hook_name).name if hook_name else ""
|
||||
if not normalized_hook_name:
|
||||
return (plugin, plugin, "unknown", "")
|
||||
|
||||
phase = "unknown"
|
||||
if normalized_hook_name == "InstallEvent":
|
||||
phase = "install"
|
||||
elif normalized_hook_name.startswith("on_CrawlSetup__"):
|
||||
phase = "crawl"
|
||||
elif normalized_hook_name.startswith("on_Snapshot__"):
|
||||
phase = "snapshot"
|
||||
|
||||
label = normalized_hook_name
|
||||
if "__" in normalized_hook_name:
|
||||
label = normalized_hook_name.split("__", 1)[1]
|
||||
label = label.rsplit(".", 1)[0]
|
||||
if len(label) > 3 and label[:2].isdigit() and label[2] == "_":
|
||||
label = label[3:]
|
||||
label = label.replace("_", " ").strip() or plugin
|
||||
|
||||
return (plugin, label, phase, normalized_hook_name)
|
||||
|
||||
def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]:
|
||||
hook_path = ""
|
||||
if isinstance(cmd, list) and cmd:
|
||||
hook_path = next(
|
||||
(arg for arg in cmd if isinstance(arg, str) and Path(arg).name.startswith("on_") and "__" in Path(arg).name),
|
||||
"",
|
||||
)
|
||||
if not hook_path and isinstance(cmd[0], str):
|
||||
hook_path = cmd[0]
|
||||
|
||||
if not hook_path:
|
||||
return ("", "setup", "unknown", "")
|
||||
|
||||
return hook_details(Path(hook_path).name, plugin=Path(hook_path).parent.name or "setup")
|
||||
|
||||
def archiveresult_output_path(ar) -> str | None:
|
||||
output_file_map = ar.output_files if isinstance(ar.output_files, dict) else {}
|
||||
|
||||
def is_root_relative(path: str) -> bool:
|
||||
metadata = output_file_map.get(path) or {}
|
||||
return bool(isinstance(metadata, dict) and metadata.get("root_relative"))
|
||||
|
||||
if ar.output_str:
|
||||
raw_output = str(ar.output_str).strip()
|
||||
if ar._looks_like_output_path(raw_output, ar.plugin):
|
||||
output_path = Path(raw_output)
|
||||
if output_path.is_absolute():
|
||||
return None
|
||||
|
||||
if raw_output.startswith(f"{ar.plugin}/"):
|
||||
candidates = [raw_output]
|
||||
elif len(output_path.parts) == 1:
|
||||
candidates = [f"{ar.plugin}/{raw_output}", raw_output]
|
||||
else:
|
||||
candidates = [raw_output]
|
||||
|
||||
if raw_output in output_file_map and is_root_relative(raw_output):
|
||||
return raw_output
|
||||
|
||||
for relative_path in candidates:
|
||||
plugin_relative = relative_path.removeprefix(f"{ar.plugin}/")
|
||||
if relative_path in output_file_map:
|
||||
return f"{ar.plugin}/{relative_path}" if not relative_path.startswith(f"{ar.plugin}/") else relative_path
|
||||
if plugin_relative in output_file_map:
|
||||
return f"{ar.plugin}/{plugin_relative}"
|
||||
|
||||
output_file_paths = list(output_file_map.keys())
|
||||
if output_file_paths:
|
||||
fallback_path = ArchiveResult._fallback_output_file_path(output_file_paths, ar.plugin, output_file_map)
|
||||
if fallback_path:
|
||||
if is_root_relative(fallback_path):
|
||||
return fallback_path
|
||||
return f"{ar.plugin}/{fallback_path}"
|
||||
|
||||
return None
|
||||
|
||||
def snapshot_output_url(snapshot, output_path: str) -> str:
|
||||
return build_snapshot_url(str(snapshot["id"]), output_path, request=request, config=request_config)
|
||||
|
||||
def snapshot_archive_path(snapshot) -> str:
|
||||
if snapshot["fs_version"] in ("0.7.0", "0.8.0"):
|
||||
return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{snapshot['timestamp']}"
|
||||
crawl = crawls_by_id.get(str(snapshot["crawl_id"]))
|
||||
username = "web"
|
||||
if crawl is not None and crawl["created_by_id"]:
|
||||
username = crawl["created_by__username"]
|
||||
if username == "system":
|
||||
username = "web"
|
||||
date_base = snapshot["bookmarked_at"] or snapshot["created_at"]
|
||||
date_str = date_base.strftime("%Y%m%d") if date_base else "unknown"
|
||||
domain = Snapshot.extract_domain_from_url(snapshot["url"])
|
||||
return f"{username}/{date_str}/{domain}/{snapshot['id']}"
|
||||
|
||||
def snapshot_view_url(snapshot, output_path: str = "") -> str:
|
||||
anchor = f"#{output_path}" if output_path else ""
|
||||
return build_web_url(
|
||||
f"/{snapshot_archive_path(snapshot)}/index.html{anchor}",
|
||||
request=request,
|
||||
config=request_config,
|
||||
)
|
||||
|
||||
def snapshot_display_url(url: str) -> str:
|
||||
url = str(url or "")
|
||||
return url if len(url) <= 96 else f"{url[:93]}..."
|
||||
|
||||
api_base = get_api_base_url(request=request, config=request_config) if scoped_snapshot is not None else ""
|
||||
|
||||
def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
|
||||
frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
|
||||
try:
|
||||
frame_stat = frame_path.stat()
|
||||
except OSError:
|
||||
return ""
|
||||
if frame_stat.st_size <= 0:
|
||||
return ""
|
||||
if now.timestamp() - frame_stat.st_mtime > 15:
|
||||
return ""
|
||||
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
|
||||
return f"{api_base}{rel}" if api_base else rel
|
||||
|
||||
machine_id = Machine.current().id
|
||||
orchestrator_proc = (
|
||||
Process.objects.filter(
|
||||
machine_id=machine_id,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
.only("id", "pid", "started_at", "machine_id", "process_type", "status")
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
if machine_id is not None
|
||||
else None
|
||||
)
|
||||
runner_worker = None
|
||||
orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
|
||||
if not orchestrator_proc_running:
|
||||
try:
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
|
||||
|
||||
supervisor = get_existing_supervisord_process(quiet=True)
|
||||
runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
|
||||
except (OSError, RuntimeError, TimeoutError):
|
||||
runner_worker = None
|
||||
|
||||
runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
|
||||
runner_worker_pid = runner_worker.get("pid") if runner_worker else None
|
||||
orchestrator_running = orchestrator_proc_running or runner_worker_running
|
||||
orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
|
||||
|
||||
# Get model counts by status
|
||||
crawl_status_counts = Crawl.status_counts(
|
||||
crawl_scope,
|
||||
(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
|
||||
)
|
||||
crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
|
||||
crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
|
||||
|
||||
# Get recent crawls (last 24 hours)
|
||||
from datetime import timedelta
|
||||
|
||||
one_day_ago = now - timedelta(days=1)
|
||||
paused_crawl_cutoff = now - timedelta(hours=12)
|
||||
crawls_recent = crawl_scope.filter(created_at__gte=one_day_ago).count()
|
||||
|
||||
snapshot_status_counts = Snapshot.status_counts(
|
||||
snapshot_scope,
|
||||
Snapshot.OPEN_STATES,
|
||||
)
|
||||
snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
|
||||
snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
|
||||
|
||||
download_plugin_names, indexing_plugin_names = _live_progress_plugin_names()
|
||||
result_statuses = (
|
||||
ArchiveResult.StatusChoices.QUEUED,
|
||||
ArchiveResult.StatusChoices.STARTED,
|
||||
)
|
||||
archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
|
||||
download_scope = archiveresult_scope.filter(
|
||||
plugin__in=download_plugin_names,
|
||||
snapshot__status__in=Snapshot.RUNNABLE_STATES,
|
||||
snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
|
||||
)
|
||||
indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
|
||||
download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
|
||||
indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
|
||||
archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
|
||||
archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
|
||||
|
||||
downloads_queued = download_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
|
||||
downloads_active = download_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
|
||||
indexing_queued = indexing_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
|
||||
indexing_active = indexing_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
|
||||
|
||||
# Build hierarchical active crawls with nested snapshots and archive results
|
||||
max_active_crawls = 10
|
||||
max_queued_crawls = 10
|
||||
max_started_snapshots_per_crawl = 50
|
||||
max_queued_snapshots_per_crawl = 50
|
||||
|
||||
active_crawl_fields = (
|
||||
"id",
|
||||
"created_at",
|
||||
"created_by_id",
|
||||
"modified_at",
|
||||
"urls",
|
||||
"config",
|
||||
"max_depth",
|
||||
"tags_str",
|
||||
"persona_id",
|
||||
"status",
|
||||
"retry_at",
|
||||
"label",
|
||||
"created_by__id",
|
||||
"created_by__username",
|
||||
)
|
||||
started_crawls = list(
|
||||
crawl_scope.filter(status=Crawl.StatusChoices.STARTED)
|
||||
.values(*active_crawl_fields)
|
||||
.order_by("-modified_at")[:max_active_crawls],
|
||||
)
|
||||
paused_crawls = list(
|
||||
crawl_scope.filter(
|
||||
Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
|
||||
| Q(
|
||||
status=Crawl.StatusChoices.PAUSED,
|
||||
snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
|
||||
snapshot_set__retry_at__lte=now,
|
||||
)
|
||||
| Q(
|
||||
status=Crawl.StatusChoices.PAUSED,
|
||||
snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
|
||||
),
|
||||
)
|
||||
.values(*active_crawl_fields)
|
||||
.distinct()
|
||||
.order_by("-modified_at")[:max_active_crawls],
|
||||
)
|
||||
queued_crawls = list(
|
||||
crawl_scope.filter(status=Crawl.StatusChoices.QUEUED).values(*active_crawl_fields).order_by("-modified_at")[:max_queued_crawls],
|
||||
)
|
||||
queued_crawls_hidden = max(crawls_queued - len(queued_crawls), 0)
|
||||
active_crawls_list = started_crawls + paused_crawls + queued_crawls
|
||||
for crawl in active_crawls_list:
|
||||
crawl["id"] = str(crawl["id"])
|
||||
if crawl["persona_id"]:
|
||||
crawl["persona_id"] = str(crawl["persona_id"])
|
||||
persona_details_by_id: dict[str, dict[str, str]] = {}
|
||||
persona_details_by_name: dict[str, dict[str, str]] = {}
|
||||
persona_objects_by_id = {}
|
||||
persona_objects_by_name = {}
|
||||
persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
|
||||
persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
|
||||
if persona_ids or persona_names:
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
|
||||
persona_details = {
|
||||
"name": persona.name,
|
||||
"admin_url": f"/admin/personas/persona/{persona.pk}/change/",
|
||||
}
|
||||
persona_details_by_id[str(persona.id)] = persona_details
|
||||
persona_details_by_name[persona.name] = persona_details
|
||||
persona_objects_by_id[str(persona.id)] = persona
|
||||
persona_objects_by_name[persona.name] = persona
|
||||
active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
|
||||
active_crawl_objects = {}
|
||||
if active_crawl_ids:
|
||||
for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
|
||||
crawl_obj._runtime_config = request_config
|
||||
active_crawl_objects[str(crawl_obj.id)] = crawl_obj
|
||||
snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
|
||||
cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
|
||||
if active_crawl_ids:
|
||||
for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
|
||||
snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
|
||||
|
||||
for row in (
|
||||
active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
|
||||
.values("crawl_id")
|
||||
.annotate(count=Count("id"))
|
||||
):
|
||||
cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
|
||||
|
||||
for row in (
|
||||
active_snapshot_scope.filter(
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
)
|
||||
.values("crawl_id")
|
||||
.annotate(size=Sum("output_size"))
|
||||
):
|
||||
crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
|
||||
|
||||
crawl_process_pids: dict[str, int] = {}
|
||||
snapshot_process_pids: dict[str, int] = {}
|
||||
process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
|
||||
process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
|
||||
seen_process_records: set[str] = set()
|
||||
crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list}
|
||||
started_snapshot_fields = (
|
||||
"id_str",
|
||||
"created_at",
|
||||
"modified_at",
|
||||
"url",
|
||||
"timestamp",
|
||||
"bookmarked_at",
|
||||
"crawl_id_str",
|
||||
"title",
|
||||
"downloaded_at",
|
||||
"fs_version",
|
||||
"status",
|
||||
)
|
||||
queued_snapshot_fields = (
|
||||
"id_str",
|
||||
"url",
|
||||
"crawl_id_str",
|
||||
"title",
|
||||
"status",
|
||||
)
|
||||
snapshots = []
|
||||
for crawl_id in active_crawl_ids:
|
||||
crawl_snapshot_scope = active_snapshot_scope.filter(crawl_id=crawl_id)
|
||||
snapshots.extend(
|
||||
crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED)
|
||||
.annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
|
||||
.values(*started_snapshot_fields)
|
||||
.order_by("-modified_at")[:max_started_snapshots_per_crawl],
|
||||
)
|
||||
queued_snapshots = list(
|
||||
crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
|
||||
.annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
|
||||
.values(
|
||||
*queued_snapshot_fields,
|
||||
)
|
||||
.order_by("modified_at")[:max_queued_snapshots_per_crawl],
|
||||
)
|
||||
queued_snapshot_overflow_by_crawl[str(crawl_id)] = max(
|
||||
snapshot_counts_by_crawl.get(str(crawl_id), {}).get(Snapshot.StatusChoices.QUEUED, 0) - len(queued_snapshots),
|
||||
0,
|
||||
)
|
||||
snapshots.extend(queued_snapshots)
|
||||
|
||||
for snapshot in snapshots:
|
||||
# Process.pwd points at Snapshot.output_dir, which uses CompactUUID
|
||||
# hex path components. Keep progress IDs compact too so process rows
|
||||
# can be matched without carrying dashed/undashed variants.
|
||||
snapshot["id"] = str(snapshot.pop("id_str")).replace("-", "")
|
||||
snapshot["crawl_id"] = str(snapshot.pop("crawl_id_str")).replace("-", "")
|
||||
snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
|
||||
displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids}
|
||||
for snapshot in snapshots:
|
||||
crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
|
||||
crawl_snapshots.append(snapshot)
|
||||
displayed_snapshot_ids = [
|
||||
snapshot["id"] for crawl_snapshots in displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots
|
||||
]
|
||||
detailed_snapshot_ids = [snapshot["id"] for snapshot in snapshots if snapshot["status"] != Snapshot.StatusChoices.QUEUED]
|
||||
process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "ended_at", "modified_at")
|
||||
if active_crawl_ids or displayed_snapshot_ids:
|
||||
process_scope = Process.objects.filter(
|
||||
machine_id=machine_id,
|
||||
process_type__in=[
|
||||
Process.TypeChoices.HOOK,
|
||||
Process.TypeChoices.BINARY,
|
||||
],
|
||||
)
|
||||
running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields)
|
||||
recent_processes = (
|
||||
process_scope.filter(modified_at__gte=now - timedelta(minutes=10)).values(*process_value_fields).order_by("-modified_at")
|
||||
)
|
||||
else:
|
||||
running_processes = Process.objects.none()
|
||||
recent_processes = Process.objects.none()
|
||||
|
||||
archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in detailed_snapshot_ids}
|
||||
if detailed_snapshot_ids:
|
||||
displayed_archiveresults = (
|
||||
archiveresult_scope.filter(snapshot_id__in=detailed_snapshot_ids)
|
||||
.select_related("process")
|
||||
.only(
|
||||
"id",
|
||||
"snapshot_id",
|
||||
"plugin",
|
||||
"hook_name",
|
||||
"status",
|
||||
"output_str",
|
||||
"output_files",
|
||||
"output_size",
|
||||
"start_ts",
|
||||
"end_ts",
|
||||
"created_at",
|
||||
"modified_at",
|
||||
"process_id",
|
||||
"process__id",
|
||||
"process__pid",
|
||||
"process__started_at",
|
||||
"process__timeout",
|
||||
)
|
||||
.order_by("snapshot_id", "start_ts", "created_at")
|
||||
)
|
||||
for archiveresult in displayed_archiveresults:
|
||||
archiveresults_by_snapshot.setdefault(str(archiveresult.snapshot_id), []).append(archiveresult)
|
||||
|
||||
def find_snapshot_for_process(proc_pwd: Path) -> Snapshot | None:
|
||||
for path_part in reversed(proc_pwd.parts):
|
||||
snapshot = snapshots_by_id.get(path_part.replace("-", ""))
|
||||
if snapshot:
|
||||
return snapshot
|
||||
return None
|
||||
|
||||
def find_crawl_for_process(proc_pwd: Path) -> Crawl | None:
|
||||
for path_part in reversed(proc_pwd.parts):
|
||||
crawl = crawls_by_id.get(path_part.replace("-", ""))
|
||||
if crawl:
|
||||
return crawl
|
||||
return None
|
||||
|
||||
running_worker_ids: set[str] = set()
|
||||
for proc in running_processes:
|
||||
if not proc["pwd"]:
|
||||
continue
|
||||
proc_pwd = Path(proc["pwd"])
|
||||
matched_snapshot = find_snapshot_for_process(proc_pwd)
|
||||
matched_crawl = (
|
||||
crawls_by_id.get(str(matched_snapshot["crawl_id"])) if matched_snapshot is not None else find_crawl_for_process(proc_pwd)
|
||||
)
|
||||
if matched_snapshot is None:
|
||||
if matched_crawl is None:
|
||||
continue
|
||||
crawl_id = str(matched_crawl["id"])
|
||||
snapshot_id = ""
|
||||
else:
|
||||
crawl_id = str(matched_snapshot["crawl_id"])
|
||||
snapshot_id = str(matched_snapshot["id"])
|
||||
running_worker_ids.add(str(proc["id"]))
|
||||
_plugin, _label, phase, _hook_name = process_label(proc["cmd"])
|
||||
if crawl_id and proc["pid"]:
|
||||
crawl_process_pids.setdefault(crawl_id, proc["pid"])
|
||||
if phase == "snapshot" and snapshot_id and proc["pid"]:
|
||||
snapshot_process_pids.setdefault(snapshot_id, proc["pid"])
|
||||
|
||||
for proc in recent_processes:
|
||||
if not proc["pwd"]:
|
||||
continue
|
||||
proc_pwd = Path(proc["pwd"])
|
||||
matched_snapshot = find_snapshot_for_process(proc_pwd)
|
||||
matched_crawl = (
|
||||
crawls_by_id.get(str(matched_snapshot["crawl_id"])) if matched_snapshot is not None else find_crawl_for_process(proc_pwd)
|
||||
)
|
||||
if matched_snapshot is None and matched_crawl is None:
|
||||
continue
|
||||
crawl_id = str(matched_snapshot["crawl_id"] if matched_snapshot is not None else matched_crawl["id"])
|
||||
snapshot_id = str(matched_snapshot["id"]) if matched_snapshot is not None else ""
|
||||
|
||||
plugin, label, phase, hook_name = process_label(proc["cmd"])
|
||||
|
||||
record_scope = str(snapshot_id) if phase == "snapshot" and snapshot_id else str(crawl_id)
|
||||
proc_key = f"{record_scope}:{plugin}:{label}:{proc['status']}:{proc['exit_code']}"
|
||||
if proc_key in seen_process_records:
|
||||
continue
|
||||
seen_process_records.add(proc_key)
|
||||
|
||||
status = (
|
||||
"started"
|
||||
if proc["status"] == Process.StatusChoices.RUNNING
|
||||
else (
|
||||
"skipped"
|
||||
if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0))
|
||||
else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded")
|
||||
)
|
||||
)
|
||||
payload: dict[str, object] = {
|
||||
"id": str(proc["id"]),
|
||||
"plugin": plugin,
|
||||
"label": label,
|
||||
"hook_name": hook_name,
|
||||
"status": status,
|
||||
"phase": phase,
|
||||
"source": "process",
|
||||
"process_id": str(proc["id"]),
|
||||
}
|
||||
if status == "started" and proc["pid"]:
|
||||
payload["pid"] = proc["pid"]
|
||||
proc_started_at = proc["started_at"] or proc["modified_at"]
|
||||
proc_run_at = (
|
||||
proc_started_at
|
||||
if proc["status"] == Process.StatusChoices.RUNNING
|
||||
else (proc["ended_at"] or proc["modified_at"] or proc_started_at)
|
||||
)
|
||||
if phase == "snapshot" and snapshot_id:
|
||||
process_records_by_snapshot.setdefault(snapshot_id, []).append((payload, proc_run_at))
|
||||
elif crawl_id:
|
||||
process_records_by_crawl.setdefault(crawl_id, []).append((payload, proc_run_at))
|
||||
|
||||
active_crawls = []
|
||||
total_workers = len(running_worker_ids)
|
||||
for crawl in active_crawls_list:
|
||||
crawl_id = str(crawl["id"])
|
||||
crawl_snapshot_counts = snapshot_counts_by_crawl.get(crawl_id, {})
|
||||
total_snapshots = sum(crawl_snapshot_counts.values())
|
||||
completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
|
||||
started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
|
||||
pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
|
||||
cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
|
||||
|
||||
# Count URLs in the crawl (for when snapshots haven't been created yet)
|
||||
urls_count = 0
|
||||
if crawl["urls"]:
|
||||
urls_count = len([u for u in crawl["urls"].split("\n") if u.strip() and not u.startswith("#")])
|
||||
|
||||
# Calculate crawl progress
|
||||
crawl_progress = int((completed_snapshots / total_snapshots) * 100) if total_snapshots > 0 else 0
|
||||
crawl_run_started_at = crawl["created_at"]
|
||||
crawl_setup_plugins = [
|
||||
payload
|
||||
for payload, proc_started_at in process_records_by_crawl.get(crawl_id, [])
|
||||
if is_current_run_timestamp(proc_started_at, crawl_run_started_at)
|
||||
]
|
||||
crawl_setup_total = len(crawl_setup_plugins)
|
||||
crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
|
||||
crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
|
||||
crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
|
||||
crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
|
||||
crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
|
||||
|
||||
# Get active snapshots for this crawl (already prefetched)
|
||||
active_snapshots_for_crawl = []
|
||||
for snapshot in displayed_snapshots_by_crawl.get(crawl_id, []):
|
||||
snapshot_run_started_at = snapshot.get("downloaded_at") or snapshot.get("created_at")
|
||||
snapshot_process_started_at = snapshot.get("created_at")
|
||||
# Get archive results only for displayed active snapshots. Large crawls can
|
||||
# contain thousands of sealed snapshots, and prefetching all their results
|
||||
# makes the progress endpoint compete with the runner.
|
||||
snapshot_results = [
|
||||
ar
|
||||
for ar in archiveresults_by_snapshot.get(str(snapshot["id"]), [])
|
||||
if archiveresult_matches_current_run(ar, snapshot_run_started_at)
|
||||
]
|
||||
if snapshot["status"] == Snapshot.StatusChoices.QUEUED:
|
||||
snapshot_results = []
|
||||
|
||||
plugin_progress_values: list[int] = []
|
||||
all_plugins: list[dict[str, object]] = []
|
||||
seen_plugin_keys: set[str] = set()
|
||||
snapshot_title = (
|
||||
str(snapshot["title"] or "")
|
||||
if snapshot["status"] == Snapshot.StatusChoices.QUEUED
|
||||
else Snapshot._normalize_title_candidate(snapshot["title"], snapshot_url=snapshot["url"])
|
||||
)
|
||||
snapshot_favicon_url = ""
|
||||
snapshot_preview_url = ""
|
||||
snapshot_preview_link = ""
|
||||
snapshot_screencast_url = ""
|
||||
snapshot_screencast_link = ""
|
||||
snapshot_fallback_urls: list[str] = []
|
||||
result_by_plugin = {result.plugin: result for result in snapshot_results}
|
||||
title_result = result_by_plugin.get("title")
|
||||
if not snapshot_title and title_result is not None and title_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
snapshot_title = Snapshot._normalize_title_candidate(title_result.output_str, snapshot_url=snapshot["url"])
|
||||
favicon_result = result_by_plugin.get("favicon")
|
||||
if favicon_result is not None and favicon_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
favicon_path = archiveresult_output_path(favicon_result) or "favicon/favicon.ico"
|
||||
snapshot_favicon_url = snapshot_output_url(snapshot, favicon_path)
|
||||
screenshot_result = result_by_plugin.get("screenshot")
|
||||
if screenshot_result is not None and screenshot_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
|
||||
snapshot_preview_link = snapshot_view_url(snapshot)
|
||||
screenshot_path = archiveresult_output_path(screenshot_result) or "screenshot/screenshot.png"
|
||||
snapshot_preview_url = snapshot_output_url(snapshot, screenshot_path)
|
||||
snapshot_preview_link = snapshot_view_url(snapshot, screenshot_path)
|
||||
if snapshot_favicon_url:
|
||||
snapshot_fallback_urls.append(snapshot_favicon_url)
|
||||
elif snapshot_favicon_url:
|
||||
snapshot_preview_url = snapshot_favicon_url
|
||||
|
||||
if snapshot["status"] == Snapshot.StatusChoices.STARTED:
|
||||
snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
|
||||
snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
|
||||
|
||||
def plugin_sort_key(ar):
|
||||
status_order = {
|
||||
ArchiveResult.StatusChoices.STARTED: 0,
|
||||
ArchiveResult.StatusChoices.QUEUED: 1,
|
||||
ArchiveResult.StatusChoices.SUCCEEDED: 2,
|
||||
ArchiveResult.StatusChoices.NORESULTS: 3,
|
||||
ArchiveResult.StatusChoices.FAILED: 4,
|
||||
}
|
||||
return (status_order.get(ar.status, 5), ar.plugin, ar.hook_name or "")
|
||||
|
||||
for ar in sorted(snapshot_results, key=plugin_sort_key):
|
||||
status = ar.status
|
||||
process = ar.process_record
|
||||
progress_value = 0
|
||||
if status in (
|
||||
ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
ArchiveResult.StatusChoices.FAILED,
|
||||
ArchiveResult.StatusChoices.SKIPPED,
|
||||
ArchiveResult.StatusChoices.NORESULTS,
|
||||
):
|
||||
progress_value = 100
|
||||
elif status == ArchiveResult.StatusChoices.STARTED:
|
||||
started_at = ar.start_ts or (process.started_at if process else None)
|
||||
timeout = process.timeout if process else 120
|
||||
if started_at and timeout:
|
||||
elapsed = max(0.0, (now - started_at).total_seconds())
|
||||
progress_value = int(min(99, max(1, (elapsed / float(timeout)) * 100)))
|
||||
else:
|
||||
progress_value = 1
|
||||
else:
|
||||
progress_value = 0
|
||||
|
||||
plugin_progress_values.append(progress_value)
|
||||
plugin, label, phase, hook_name = hook_details(ar.hook_name or ar.plugin, plugin=ar.plugin)
|
||||
|
||||
plugin_payload = {
|
||||
"id": str(ar.id),
|
||||
"plugin": ar.plugin,
|
||||
"label": label,
|
||||
"hook_name": hook_name,
|
||||
"phase": phase,
|
||||
"status": status,
|
||||
"process_id": str(process.id) if process else None,
|
||||
"admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
|
||||
}
|
||||
output_path = archiveresult_output_path(ar)
|
||||
if output_path:
|
||||
plugin_payload["output_path"] = output_path
|
||||
plugin_payload["output_url"] = snapshot_view_url(snapshot, output_path)
|
||||
if status == ArchiveResult.StatusChoices.STARTED and process:
|
||||
plugin_payload["pid"] = process.pid
|
||||
if status == ArchiveResult.StatusChoices.STARTED:
|
||||
plugin_payload["progress"] = progress_value
|
||||
plugin_payload["timeout"] = process.timeout if process else 120
|
||||
plugin_payload["source"] = "archiveresult"
|
||||
all_plugins.append(plugin_payload)
|
||||
seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
|
||||
|
||||
for proc_payload, proc_started_at in process_records_by_snapshot.get(str(snapshot["id"]), []):
|
||||
if not is_current_run_timestamp(proc_started_at, snapshot_process_started_at):
|
||||
continue
|
||||
proc_key = str(proc_payload.get("process_id") or f"{proc_payload.get('plugin')}:{proc_payload.get('hook_name')}")
|
||||
if proc_key in seen_plugin_keys:
|
||||
continue
|
||||
seen_plugin_keys.add(proc_key)
|
||||
all_plugins.append(proc_payload)
|
||||
|
||||
proc_status = proc_payload.get("status")
|
||||
if proc_status in ("succeeded", "failed", "skipped"):
|
||||
plugin_progress_values.append(100)
|
||||
elif proc_status == "started":
|
||||
plugin_progress_values.append(1)
|
||||
else:
|
||||
plugin_progress_values.append(0)
|
||||
|
||||
total_plugins = len(all_plugins)
|
||||
completed_plugins = sum(1 for item in all_plugins if item.get("status") == "succeeded")
|
||||
failed_plugins = sum(1 for item in all_plugins if item.get("status") == "failed")
|
||||
pending_plugins = sum(1 for item in all_plugins if item.get("status") == "queued")
|
||||
|
||||
snapshot_progress = int(sum(plugin_progress_values) / len(plugin_progress_values)) if plugin_progress_values else 0
|
||||
worker_state = "running" if snapshot_process_pids.get(str(snapshot["id"])) else "waiting"
|
||||
if (
|
||||
snapshot["status"] == Snapshot.StatusChoices.STARTED
|
||||
and worker_state == "waiting"
|
||||
and not all_plugins
|
||||
and snapshot["modified_at"]
|
||||
and (now - snapshot["modified_at"]).total_seconds() > 30
|
||||
):
|
||||
worker_state = "waiting" if orchestrator_running else "crashed"
|
||||
|
||||
if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not snapshot_process_pids.get(str(snapshot["id"])):
|
||||
compact_snapshot = [
|
||||
str(snapshot["id"]),
|
||||
snapshot_display_url(snapshot["url"]),
|
||||
]
|
||||
if snapshot_title:
|
||||
compact_snapshot.append(snapshot_title)
|
||||
active_snapshots_for_crawl.append(compact_snapshot)
|
||||
continue
|
||||
|
||||
snapshot_payload = {
|
||||
"id": str(snapshot["id"]),
|
||||
"url": snapshot_display_url(snapshot["url"]),
|
||||
"title": snapshot_title,
|
||||
"status": snapshot["status"],
|
||||
"worker_state": worker_state,
|
||||
}
|
||||
if snapshot["status"] != Snapshot.StatusChoices.QUEUED or all_plugins or snapshot_process_pids.get(str(snapshot["id"])):
|
||||
snapshot_payload.update(
|
||||
{
|
||||
"view_url": snapshot_view_url(snapshot),
|
||||
"started": (snapshot["downloaded_at"] or snapshot["created_at"]).isoformat()
|
||||
if (snapshot["downloaded_at"] or snapshot["created_at"])
|
||||
else None,
|
||||
"progress": snapshot_progress,
|
||||
"total_plugins": total_plugins,
|
||||
"completed_plugins": completed_plugins,
|
||||
"failed_plugins": failed_plugins,
|
||||
"pending_plugins": pending_plugins,
|
||||
"all_plugins": all_plugins,
|
||||
},
|
||||
)
|
||||
if snapshot_favicon_url:
|
||||
snapshot_payload["favicon_url"] = snapshot_favicon_url
|
||||
if snapshot_preview_url:
|
||||
snapshot_payload["preview_url"] = snapshot_preview_url
|
||||
snapshot_payload["preview_link"] = snapshot_preview_link
|
||||
if snapshot_screencast_url:
|
||||
snapshot_payload["screencast_url"] = snapshot_screencast_url
|
||||
snapshot_payload["screencast_link"] = snapshot_screencast_link
|
||||
if snapshot_fallback_urls:
|
||||
snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls
|
||||
if snapshot_process_pids.get(str(snapshot["id"])):
|
||||
snapshot_payload["worker_pid"] = snapshot_process_pids[str(snapshot["id"])]
|
||||
|
||||
active_snapshots_for_crawl.append(snapshot_payload)
|
||||
|
||||
# Check if crawl can start (for debugging stuck crawls)
|
||||
can_start = bool(crawl["urls"])
|
||||
urls_preview = crawl["urls"][:60] if crawl["urls"] else None
|
||||
crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
|
||||
persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
|
||||
persona_name = persona_details["name"] if persona_details else "Default"
|
||||
persona_details = persona_details or persona_details_by_name.get(persona_name)
|
||||
crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
|
||||
avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
|
||||
crawl_obj = active_crawl_objects[crawl_id]
|
||||
effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
|
||||
max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
|
||||
crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
|
||||
crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
|
||||
snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
|
||||
|
||||
# Check if retry_at is in the future (would prevent worker from claiming)
|
||||
retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
|
||||
is_paused = crawl_obj.is_paused
|
||||
seconds_until_retry = (
|
||||
0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
|
||||
)
|
||||
crawl_worker_state = (
|
||||
"running"
|
||||
if crawl_process_pids.get(crawl_id)
|
||||
or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl)
|
||||
else "waiting"
|
||||
)
|
||||
if is_paused:
|
||||
crawl_worker_state = "paused"
|
||||
elif (
|
||||
crawl["status"] == Crawl.StatusChoices.STARTED
|
||||
and crawl_worker_state == "waiting"
|
||||
and (started_snapshots or pending_snapshots)
|
||||
):
|
||||
crawl_worker_state = "waiting" if orchestrator_running else "crashed"
|
||||
|
||||
active_crawls.append(
|
||||
{
|
||||
"id": crawl_id,
|
||||
"label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60],
|
||||
"status": crawl["status"],
|
||||
"is_paused": is_paused,
|
||||
"started": crawl["created_at"].isoformat() if crawl["created_at"] else None,
|
||||
"progress": crawl_progress,
|
||||
"created_by": crawl["created_by__username"],
|
||||
"persona": persona_name,
|
||||
"persona_admin_url": persona_details["admin_url"] if persona_details else None,
|
||||
"max_depth": crawl["max_depth"],
|
||||
"max_urls": max_urls,
|
||||
"max_crawl_size": crawl_max_size,
|
||||
"crawl_timeout": crawl_timeout,
|
||||
"max_snapshot_size": snapshot_max_size,
|
||||
"max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited",
|
||||
"crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited",
|
||||
"max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
|
||||
"crawl_output_size": crawl_output_size,
|
||||
"avg_snapshot_size": avg_snapshot_size,
|
||||
"crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
|
||||
"avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
|
||||
"tags": crawl_tags,
|
||||
"urls_count": urls_count,
|
||||
"total_snapshots": total_snapshots,
|
||||
"completed_snapshots": completed_snapshots,
|
||||
"started_snapshots": started_snapshots,
|
||||
"failed_snapshots": 0,
|
||||
"pending_snapshots": pending_snapshots,
|
||||
"cancelled_snapshots": cancelled_snapshots,
|
||||
"setup_plugins": crawl_setup_plugins,
|
||||
"setup_total_plugins": crawl_setup_total,
|
||||
"setup_completed_plugins": crawl_setup_completed,
|
||||
"setup_failed_plugins": crawl_setup_failed,
|
||||
"setup_pending_plugins": crawl_setup_pending,
|
||||
"screencast_url": crawl_screencast_url,
|
||||
"screencast_link": crawl_screencast_link,
|
||||
"active_snapshots": active_snapshots_for_crawl,
|
||||
"queued_snapshots_hidden": queued_snapshot_overflow_by_crawl.get(crawl_id, 0),
|
||||
"can_start": can_start,
|
||||
"urls_preview": urls_preview,
|
||||
"retry_at_future": retry_at_future,
|
||||
"seconds_until_retry": seconds_until_retry,
|
||||
"worker_pid": crawl_process_pids.get(crawl_id),
|
||||
"worker_state": crawl_worker_state,
|
||||
},
|
||||
)
|
||||
|
||||
payload = {
|
||||
"is_admin": is_admin,
|
||||
"scope": {
|
||||
"snapshot_id": str(scoped_snapshot.id) if scoped_snapshot is not None else "",
|
||||
"crawl_id": crawl_id_filter,
|
||||
},
|
||||
"orchestrator_running": orchestrator_running,
|
||||
"orchestrator_pid": orchestrator_pid,
|
||||
"total_workers": total_workers,
|
||||
"crawls_active": crawls_active,
|
||||
"crawls_queued": crawls_queued,
|
||||
"crawls_recent": crawls_recent,
|
||||
"snapshots_active": snapshots_active,
|
||||
"snapshots_queued": snapshots_queued,
|
||||
"archiveresults_active": archiveresults_active,
|
||||
"archiveresults_queued": archiveresults_queued,
|
||||
"downloads_active": downloads_active,
|
||||
"downloads_queued": downloads_queued,
|
||||
"indexing_active": indexing_active,
|
||||
"indexing_queued": indexing_queued,
|
||||
"active_crawls": active_crawls,
|
||||
"queued_crawls_hidden": queued_crawls_hidden,
|
||||
"server_time": timezone.now().isoformat(),
|
||||
}
|
||||
elif not is_admin:
|
||||
# Crawl and global scopes require staff: a crawl can mix snapshot ACLs.
|
||||
return JsonResponse({"error": "Permission denied"}, status=403)
|
||||
|
||||
from .report import ProgressReport
|
||||
|
||||
payload = ProgressReport(
|
||||
request,
|
||||
scoped_snapshot=scoped_snapshot,
|
||||
crawl_id_filter=crawl_id_filter,
|
||||
is_admin=is_admin,
|
||||
).payload()
|
||||
if is_admin and not snapshot_id_filter and not crawl_id_filter and request.GET.get("collection") == "1":
|
||||
from archivebox.progressmonitor.collection import collection_summary
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ class SearchResultsChangeList(ChangeList):
|
||||
self.search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
|
||||
self.search_mode_backend = get_search_mode_backend(self.search_mode, config=request.archivebox_config)
|
||||
super().__init__(request, *args, **kwargs)
|
||||
self.embedded_changelist = request.GET.get("_embedded") == "crawl"
|
||||
self.embedded_changelist = request.GET.get("_embedded") in {"crawl", "snapshot"}
|
||||
|
||||
def get_results(self, request):
|
||||
"""Populate normal admin results plus search-index hint state."""
|
||||
@@ -69,7 +69,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
|
||||
if not search_term:
|
||||
return super().get_search_results(request, queryset, search_term)
|
||||
search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
|
||||
if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
|
||||
if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") not in {"crawl", "snapshot"}:
|
||||
cached_ids = get_cached_admin_search_ids(request)
|
||||
if cached_ids is not None:
|
||||
if not cached_ids:
|
||||
@@ -84,7 +84,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
|
||||
if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
|
||||
qs, use_distinct = super().get_search_results(request, queryset, search_term)
|
||||
return qs, use_distinct
|
||||
if request.GET.get("_embedded") == "crawl":
|
||||
if request.GET.get("_embedded") in {"crawl", "snapshot"}:
|
||||
try:
|
||||
return queryset.filter(
|
||||
pk__in=query_search_index(
|
||||
|
||||
@@ -309,6 +309,8 @@ def iter_query_search_ids(
|
||||
ids.close()
|
||||
successful_backends += 1
|
||||
except Exception as err:
|
||||
if isinstance(err, CalledProcessError) and err.stderr:
|
||||
err.add_note(f"{backend_name} search command stderr:\n{err.stderr.strip()}")
|
||||
errors.append(err)
|
||||
if not fallback_enabled:
|
||||
raise
|
||||
@@ -320,8 +322,6 @@ def iter_query_search_ids(
|
||||
f"[X] The search backend threw an exception={err}:",
|
||||
color="red",
|
||||
)
|
||||
if isinstance(err, CalledProcessError) and err.stderr:
|
||||
stderr(err.stderr.rstrip(), color="red")
|
||||
raise
|
||||
|
||||
|
||||
|
||||
@@ -16,45 +16,40 @@ def register_sonic_daemon_event_handler(bus) -> None:
|
||||
if not isinstance(record, dict) or record.get("type") != "SonicDaemonStartEvent":
|
||||
return
|
||||
|
||||
from abx_plugins.plugins.search_backend_sonic.daemon import (
|
||||
SonicDaemonStartEvent,
|
||||
is_port_listening,
|
||||
)
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
|
||||
from abx_plugins.plugins.search_backend_sonic.daemon import SonicDaemonStartEvent
|
||||
|
||||
daemon_event = SonicDaemonStartEvent.from_record(record)
|
||||
deadline = time.monotonic() + 30.0
|
||||
last_worker = None
|
||||
await asyncio.to_thread(wait_for_sonic_daemon, SonicDaemonStartEvent.from_record(record))
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
if is_port_listening(daemon_event.host, daemon_event.port):
|
||||
return
|
||||
bus.on(ProcessStdoutEvent, on_ProcessStdoutEvent__require_sonic_daemon)
|
||||
|
||||
supervisor = get_existing_supervisord_process(quiet=True)
|
||||
worker = get_worker(supervisor, daemon_event.worker_name) if supervisor is not None else None
|
||||
if isinstance(worker, dict):
|
||||
last_worker = worker
|
||||
if worker.get("statename") not in {"STARTING", "RUNNING", "BACKOFF", "STOPPING"}:
|
||||
break
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
def wait_for_sonic_daemon(daemon_event) -> None:
|
||||
"""Wait for the Sonic handshake used by both search commands and crawl hooks.
|
||||
|
||||
Supervisord RUNNING only means the process was spawned; its socket can still
|
||||
be opening. Fail immediately for a stopped worker and retain the existing
|
||||
30-second startup deadline for a live worker.
|
||||
"""
|
||||
from abx_plugins.plugins.search_backend_sonic.daemon import is_port_listening
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
|
||||
|
||||
deadline = time.monotonic() + 30.0
|
||||
while time.monotonic() < deadline:
|
||||
if is_port_listening(daemon_event.host, daemon_event.port):
|
||||
return
|
||||
if time.monotonic() < deadline:
|
||||
return
|
||||
break
|
||||
|
||||
supervisor = get_existing_supervisord_process(quiet=True)
|
||||
if supervisor is None:
|
||||
raise RuntimeError("Sonic search backend is required, but ArchiveBox supervisord is not running")
|
||||
|
||||
worker = get_worker(supervisor, daemon_event.worker_name)
|
||||
if worker is None and last_worker is not None:
|
||||
worker = last_worker
|
||||
if not worker:
|
||||
raise RuntimeError(f"Sonic search backend worker is not configured: {daemon_event.worker_name}")
|
||||
if worker.get("statename") != "RUNNING":
|
||||
if worker.get("statename") not in {"STARTING", "RUNNING"}:
|
||||
raise RuntimeError(
|
||||
f"Sonic search backend worker is {worker.get('statename')}: {worker.get('description')}",
|
||||
)
|
||||
raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}")
|
||||
time.sleep(min(0.1, max(0.0, deadline - time.monotonic())))
|
||||
|
||||
bus.on(ProcessStdoutEvent, on_ProcessStdoutEvent__require_sonic_daemon)
|
||||
raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}")
|
||||
@@ -56,14 +56,6 @@ def get_cached_admin_search_ids(request) -> list[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def get_cached_public_search_ids(request) -> list[str] | None:
|
||||
"""Return streamed public search IDs from Django cache."""
|
||||
cached = get_cached_public_search_state(request)
|
||||
if isinstance(cached, dict):
|
||||
return cached.get("ids") or []
|
||||
return None
|
||||
|
||||
|
||||
def get_cached_public_search_state(request) -> dict | None:
|
||||
"""Return streamed public search state from Django cache."""
|
||||
cached = cache.get(get_public_search_cache_key(request))
|
||||
|
||||
@@ -10,96 +10,38 @@ from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
|
||||
|
||||
|
||||
class CrawlService(BaseService):
|
||||
"""Project crawl phase changes without resurrecting paused or sealed work."""
|
||||
|
||||
LISTENS_TO = [CrawlSetupEvent, CrawlStartEvent, CrawlCleanupEvent, CrawlCompletedEvent]
|
||||
EMITS = []
|
||||
|
||||
def __init__(self, bus, *, crawl_id: str):
|
||||
self.crawl_id = crawl_id
|
||||
super().__init__(bus)
|
||||
self.bus.on(CrawlSetupEvent, self.on_CrawlSetupEvent__save_to_db)
|
||||
self.bus.on(CrawlStartEvent, self.on_CrawlStartEvent__save_to_db)
|
||||
self.bus.on(CrawlCleanupEvent, self.on_CrawlCleanupEvent__save_to_db)
|
||||
self.bus.on(CrawlCompletedEvent, self.on_CrawlCompletedEvent__save_to_db)
|
||||
for event_type in self.LISTENS_TO:
|
||||
self.bus.on(event_type, self.on_CrawlEvent__save_to_db)
|
||||
|
||||
async def on_CrawlSetupEvent__save_to_db(self, event: CrawlSetupEvent) -> None:
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
await (
|
||||
Crawl.objects.filter(id=self.crawl_id)
|
||||
.exclude(
|
||||
status__in=Crawl.INACTIVE_STATES,
|
||||
)
|
||||
.aupdate(
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
)
|
||||
|
||||
async def on_CrawlStartEvent__save_to_db(self, event: CrawlStartEvent) -> None:
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
await (
|
||||
Crawl.objects.filter(id=self.crawl_id)
|
||||
.exclude(
|
||||
status__in=Crawl.INACTIVE_STATES,
|
||||
)
|
||||
.aupdate(
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
)
|
||||
|
||||
async def on_CrawlCleanupEvent__save_to_db(self, event: CrawlCleanupEvent) -> None:
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
# Cleanup is still inside the active crawl lifecycle. Snapshot hooks may
|
||||
# have just written discovery output that the runner consumes before the
|
||||
# completion phase, so only CrawlCompleted/finalize_run_state makes the
|
||||
# final sealed-vs-requeue decision.
|
||||
await (
|
||||
Crawl.objects.filter(id=self.crawl_id)
|
||||
.exclude(
|
||||
status__in=Crawl.INACTIVE_STATES,
|
||||
)
|
||||
.aupdate(
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now(),
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
)
|
||||
|
||||
async def on_CrawlCompletedEvent__save_to_db(self, event: CrawlCompletedEvent) -> None:
|
||||
async def on_CrawlEvent__save_to_db(
|
||||
self,
|
||||
event: CrawlSetupEvent | CrawlStartEvent | CrawlCleanupEvent | CrawlCompletedEvent,
|
||||
) -> None:
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
crawl = await Crawl.objects.aget(id=self.crawl_id)
|
||||
if crawl.is_paused or crawl.status == Crawl.StatusChoices.SEALED:
|
||||
return
|
||||
is_finished = not await crawl.snapshot_set.filter(status__in=Snapshot.OPEN_STATES).aexists()
|
||||
if not is_finished:
|
||||
await (
|
||||
Crawl.objects.filter(id=self.crawl_id)
|
||||
.exclude(
|
||||
status__in=Crawl.INACTIVE_STATES,
|
||||
)
|
||||
.aupdate(
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now(),
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
status = Crawl.StatusChoices.STARTED
|
||||
retry_at = timezone.now()
|
||||
if isinstance(event, (CrawlSetupEvent, CrawlStartEvent)):
|
||||
retry_at += timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS)
|
||||
elif isinstance(event, CrawlCompletedEvent):
|
||||
crawl = await Crawl.objects.aget(id=self.crawl_id)
|
||||
if crawl.status in Crawl.INACTIVE_STATES:
|
||||
return
|
||||
if not await crawl.snapshot_set.filter(status__in=Snapshot.OPEN_STATES).aexists():
|
||||
status, retry_at = Crawl.StatusChoices.SEALED, None
|
||||
# Cleanup remains active: parser outputs may still be projected before
|
||||
# completion. Only completion can seal; every write rechecks cancellation.
|
||||
await (
|
||||
Crawl.objects.filter(id=self.crawl_id)
|
||||
.exclude(
|
||||
status__in=Crawl.INACTIVE_STATES,
|
||||
)
|
||||
.aupdate(
|
||||
status=Crawl.StatusChoices.SEALED,
|
||||
retry_at=None,
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
.exclude(status__in=Crawl.INACTIVE_STATES)
|
||||
.aupdate(status=status, retry_at=retry_at, modified_at=timezone.now())
|
||||
)
|
||||
@@ -63,8 +63,7 @@ class ProcessService(BaseService):
|
||||
|
||||
def __init__(self, bus):
|
||||
self._iface = None
|
||||
self._completed_queue: asyncio.Queue[ProcessCompletedEvent | None] = asyncio.Queue()
|
||||
self._completed_worker: asyncio.Task | None = None
|
||||
self._completed_lock = asyncio.Lock()
|
||||
super().__init__(bus)
|
||||
self.bus.on(ProcessStartedEvent, self.on_ProcessStartedEvent__save_to_db)
|
||||
self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
|
||||
@@ -76,7 +75,7 @@ class ProcessService(BaseService):
|
||||
self._iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
|
||||
return self._iface
|
||||
|
||||
async def on_ProcessStartedEvent__save_to_db(self, event: ProcessStartedEvent) -> None:
|
||||
async def _get_or_create_process(self, event: ProcessStartedEvent | ProcessCompletedEvent):
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
iface = await self.current_iface()
|
||||
@@ -84,7 +83,7 @@ class ProcessService(BaseService):
|
||||
worker_type = event.worker_type or ""
|
||||
started_at = parse_event_datetime(event.start_ts)
|
||||
if started_at is None:
|
||||
raise ValueError("ProcessStartedEvent.start_ts is required")
|
||||
raise ValueError(f"{type(event).__name__}.start_ts is required")
|
||||
if event.pid:
|
||||
process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
|
||||
else:
|
||||
@@ -113,7 +112,15 @@ class ProcessService(BaseService):
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
retry_at=None,
|
||||
)
|
||||
elif process.iface_id != iface.id or process.machine_id != iface.machine_id:
|
||||
return process, iface, started_at, process_env
|
||||
|
||||
async def on_ProcessStartedEvent__save_to_db(self, event: ProcessStartedEvent) -> None:
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
process, iface, started_at, process_env = await self._get_or_create_process(event)
|
||||
process_type = event.process_type or Process.TypeChoices.HOOK
|
||||
worker_type = event.worker_type or ""
|
||||
if process.iface_id != iface.id or process.machine_id != iface.machine_id:
|
||||
process.iface = iface
|
||||
process.machine = iface.machine
|
||||
await process.asave(update_fields=["iface", "machine", "modified_at"])
|
||||
@@ -165,31 +172,16 @@ class ProcessService(BaseService):
|
||||
process_id=str(process.id),
|
||||
)
|
||||
|
||||
async def _completed_worker_loop(self) -> None:
|
||||
while True:
|
||||
event = await self._completed_queue.get()
|
||||
try:
|
||||
if event is None:
|
||||
return
|
||||
await self._save_completed_process_to_db(event)
|
||||
finally:
|
||||
self._completed_queue.task_done()
|
||||
|
||||
def _ensure_completed_worker(self) -> None:
|
||||
if self._completed_worker is None or self._completed_worker.done():
|
||||
self._completed_worker = asyncio.create_task(self._completed_worker_loop())
|
||||
|
||||
async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
|
||||
self._ensure_completed_worker()
|
||||
completed_worker = self._completed_worker
|
||||
assert completed_worker is not None
|
||||
await self._completed_queue.put(event)
|
||||
await self.flush_completed()
|
||||
if completed_worker.done():
|
||||
await completed_worker
|
||||
# Serialize completion writes without a detached queue worker. Awaiting
|
||||
# the write here also delivers database failures to the originating event.
|
||||
async with self._completed_lock:
|
||||
await self._save_completed_process_to_db(event)
|
||||
|
||||
async def flush_completed(self) -> None:
|
||||
await self._completed_queue.join()
|
||||
# asyncio locks are FIFO: cleanup waits for earlier completion writers.
|
||||
async with self._completed_lock:
|
||||
pass
|
||||
|
||||
async def on_CrawlCleanupEvent__flush_completed(self, event: CrawlCleanupEvent) -> None:
|
||||
await self.flush_completed()
|
||||
@@ -200,43 +192,9 @@ class ProcessService(BaseService):
|
||||
async def _save_completed_process_to_db(self, event: ProcessCompletedEvent) -> None:
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
iface = await self.current_iface()
|
||||
process, iface, started_at, process_env = await self._get_or_create_process(event)
|
||||
process_type = event.process_type or Process.TypeChoices.HOOK
|
||||
worker_type = event.worker_type or ""
|
||||
started_at = parse_event_datetime(event.start_ts)
|
||||
if started_at is None:
|
||||
raise ValueError("ProcessCompletedEvent.start_ts is required")
|
||||
if event.pid:
|
||||
process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
|
||||
else:
|
||||
process_query = Process.objects.filter(
|
||||
process_type=process_type,
|
||||
worker_type=worker_type,
|
||||
pwd=event.output_dir,
|
||||
started_at=started_at,
|
||||
)
|
||||
process = await process_query.order_by("-modified_at").afirst()
|
||||
process_env = normalize_process_env(event.env)
|
||||
if process is None:
|
||||
await Process.objects.acreate(
|
||||
machine=iface.machine,
|
||||
iface=iface,
|
||||
parent_id=None,
|
||||
process_type=process_type,
|
||||
worker_type=worker_type,
|
||||
pwd=event.output_dir,
|
||||
cmd=[event.hook_path, *event.hook_args],
|
||||
env=process_env,
|
||||
timeout=event.timeout,
|
||||
pid=event.pid or None,
|
||||
url=event.url or None,
|
||||
started_at=started_at,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
retry_at=None,
|
||||
)
|
||||
process = await process_query.order_by("-modified_at").afirst()
|
||||
if process is None:
|
||||
return
|
||||
|
||||
missing_cmd = not process.cmd
|
||||
updates = {
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
"""ArchiveBox scheduling and execution entry points.
|
||||
|
||||
scheduler selects due work; dispatch claims it; crawl runs hook events and
|
||||
projects their results; install runs dependency-only event buses.
|
||||
"""
|
||||
|
||||
from .crawl import CrawlRunner as CrawlRunner
|
||||
from .scheduler import ensure_background_runner as ensure_background_runner
|
||||
from .crawl import run_crawl as run_crawl
|
||||
from .install import run_binary as run_binary
|
||||
from .install import run_install as run_install
|
||||
from .maintenance import run_snapshot_maintenance as run_snapshot_maintenance
|
||||
from .dispatch import run_due_crawl as run_due_crawl
|
||||
from .dispatch import run_due_snapshot as run_due_snapshot
|
||||
from .dispatch import run_due_binary as run_due_binary
|
||||
from .scheduler import run_pending_crawls as run_pending_crawls
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
from rich.console import Console
|
||||
from rich.text import Text
|
||||
from abx_dl.catalog import PluginCatalog
|
||||
|
||||
|
||||
def _runner_short_id(identifier) -> str:
|
||||
return str(identifier).replace("-", "")[-8:]
|
||||
|
||||
|
||||
def _runner_label(value: str, *, reserve: int) -> str:
|
||||
width = max(24, shutil.get_terminal_size(fallback=(120, 40)).columns - reserve)
|
||||
value = " ".join(str(value or "").split())
|
||||
if len(value) <= width:
|
||||
return value
|
||||
return f"{value[: max(0, width - 3)]}..."
|
||||
|
||||
|
||||
def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: str = "STARTED") -> None:
|
||||
crawl_id = crawl.id if crawl is not None else crawl_id
|
||||
line = Text()
|
||||
line.append(f"[Crawl#{_runner_short_id(crawl_id)}]", style="cyan bold")
|
||||
line.append(" ")
|
||||
if snapshot is not None:
|
||||
line.append(f"[Snapshot#{_runner_short_id(snapshot.id)}]", style="magenta bold")
|
||||
line.append(" ")
|
||||
status_styles = {
|
||||
"STARTED": "green bold",
|
||||
"SEALED": "blue bold",
|
||||
"PAUSED": "yellow bold",
|
||||
}
|
||||
line.append(f"[{status}]", style=status_styles.get(status, "white bold"))
|
||||
line.append(" ")
|
||||
prefix_width = len(line.plain)
|
||||
if snapshot is not None:
|
||||
label = snapshot.url
|
||||
else:
|
||||
label = (crawl.label or "").strip()
|
||||
if not label:
|
||||
label = (crawl.urls or "").partition("\n")[0].strip() or str(crawl_id)
|
||||
line.append(_runner_label(label, reserve=prefix_width))
|
||||
Console(highlight=False).print(line)
|
||||
|
||||
|
||||
def _count_selected_hooks(catalog: PluginCatalog, selected_plugins: list[str] | None) -> int:
|
||||
selected = catalog.select(selected_plugins) if selected_plugins else catalog
|
||||
return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
|
||||
|
||||
|
||||
def create_console():
|
||||
"""Build a correctly sized progress console and return its owned tty stream.
|
||||
|
||||
Callers close only the returned stream; stdout/stderr belong to the process.
|
||||
A crawl may use this console noninteractively, while installation displays
|
||||
the live UI only when a terminal is attached.
|
||||
"""
|
||||
stdout_is_tty = sys.stdout.isatty()
|
||||
stderr_is_tty = sys.stderr.isatty()
|
||||
interactive = stdout_is_tty or stderr_is_tty
|
||||
stream = sys.stderr if stderr_is_tty or not stdout_is_tty else sys.stdout
|
||||
owned_stream = None
|
||||
if interactive and os.path.exists("/dev/tty"):
|
||||
try:
|
||||
owned_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
|
||||
stream = owned_stream
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
size = os.get_terminal_size(stream.fileno())
|
||||
except (AttributeError, OSError, ValueError):
|
||||
size = shutil.get_terminal_size(fallback=(160, 40))
|
||||
console = Console(
|
||||
file=stream,
|
||||
force_terminal=interactive,
|
||||
width=size.columns,
|
||||
height=size.lines,
|
||||
_environ={"COLUMNS": str(size.columns), "LINES": str(size.lines)},
|
||||
)
|
||||
return console, owned_stream, interactive
|
||||
@@ -1,26 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.utils import timezone
|
||||
from rich.console import Console
|
||||
from rich.text import Text
|
||||
|
||||
from abxpkg.binary_service import BinaryRequestEvent, BinaryService
|
||||
from abxpkg.binary_service import BinaryService
|
||||
from abx_dl.events import (
|
||||
CrawlAbortEvent,
|
||||
CrawlCleanupEvent,
|
||||
@@ -57,28 +50,34 @@ from abx_dl.cli import LiveBusUI
|
||||
from abxbus import BaseEvent
|
||||
from abxbus.event_bus import EventBus, get_current_event, in_handler_context
|
||||
from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
|
||||
|
||||
from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, resource_admission
|
||||
|
||||
from archivebox.config.common import (
|
||||
ArchiveBoxBaseConfig,
|
||||
normalize_runtime_config,
|
||||
_plugin_enabled_config_keys,
|
||||
)
|
||||
from archivebox.misc.db import run_db_analyze_batch
|
||||
from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
|
||||
from archivebox.core.shutdown_util import foreground_shutdown_signals
|
||||
from archivebox.plugins.discovery import get_plugin_catalog
|
||||
from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
|
||||
from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
|
||||
from archivebox.crawls.locks import crawl_lifecycle_lock
|
||||
from ..archive_result_service import ArchiveResultService
|
||||
from ..binary_service import ArchiveBoxBinaryService, project_abxpkg_derived_cache_to_db
|
||||
from ..crawl_service import CrawlService
|
||||
from ..machine_service import MachineService
|
||||
from ..process_service import ProcessService as PersistedProcessService
|
||||
from ..snapshot_service import SnapshotService, project_discovered_snapshots
|
||||
from ..tag_service import TagService
|
||||
|
||||
from .archive_result_service import ArchiveResultService
|
||||
from .binary_service import ArchiveBoxBinaryService, project_abxpkg_derived_cache_to_db
|
||||
from .crawl_service import CrawlService
|
||||
from .machine_service import MachineService
|
||||
from .process_service import ProcessService as PersistedProcessService
|
||||
from .snapshot_service import SnapshotService, project_discovered_snapshots
|
||||
from .tag_service import TagService
|
||||
from .console import _count_selected_hooks, create_console
|
||||
from .maintenance import run_snapshot_maintenance
|
||||
|
||||
|
||||
async def _run_event_now(event, timeout: float | None = None):
|
||||
await event.now(timeout=timeout)
|
||||
await event.wait(timeout=timeout)
|
||||
await event.event_results_list()
|
||||
return event
|
||||
|
||||
|
||||
def _bus_name(prefix: str, identifier: str) -> str:
|
||||
@@ -86,49 +85,6 @@ def _bus_name(prefix: str, identifier: str) -> str:
|
||||
return f"{prefix}_{normalized}"
|
||||
|
||||
|
||||
def _runner_short_id(identifier) -> str:
|
||||
return str(identifier).replace("-", "")[-8:]
|
||||
|
||||
|
||||
def _runner_label(value: str, *, reserve: int) -> str:
|
||||
width = max(24, shutil.get_terminal_size(fallback=(120, 40)).columns - reserve)
|
||||
value = " ".join(str(value or "").split())
|
||||
if len(value) <= width:
|
||||
return value
|
||||
return f"{value[: max(0, width - 3)]}..."
|
||||
|
||||
|
||||
def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: str = "STARTED") -> None:
|
||||
crawl_id = crawl.id if crawl is not None else crawl_id
|
||||
line = Text()
|
||||
line.append(f"[Crawl#{_runner_short_id(crawl_id)}]", style="cyan bold")
|
||||
line.append(" ")
|
||||
if snapshot is not None:
|
||||
line.append(f"[Snapshot#{_runner_short_id(snapshot.id)}]", style="magenta bold")
|
||||
line.append(" ")
|
||||
status_styles = {
|
||||
"STARTED": "green bold",
|
||||
"SEALED": "blue bold",
|
||||
"PAUSED": "yellow bold",
|
||||
}
|
||||
line.append(f"[{status}]", style=status_styles.get(status, "white bold"))
|
||||
line.append(" ")
|
||||
prefix_width = len(line.plain)
|
||||
if snapshot is not None:
|
||||
label = snapshot.url
|
||||
else:
|
||||
label = (crawl.label or "").strip()
|
||||
if not label:
|
||||
label = (crawl.urls or "").partition("\n")[0].strip() or str(crawl_id)
|
||||
line.append(_runner_label(label, reserve=prefix_width))
|
||||
Console(highlight=False).print(line)
|
||||
|
||||
|
||||
def _count_selected_hooks(catalog: PluginCatalog, selected_plugins: list[str] | None) -> int:
|
||||
selected = catalog.select(selected_plugins) if selected_plugins else catalog
|
||||
return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
|
||||
|
||||
|
||||
def _enable_requested_plugins(config: dict[str, Any], plugins: PluginCatalog) -> None:
|
||||
for plugin in plugins.values():
|
||||
if plugin.enabled_key in plugin.config.properties:
|
||||
@@ -151,36 +107,27 @@ def _is_external_task_cancelled(error: asyncio.CancelledError) -> bool:
|
||||
return not isinstance(error, (EventHandlerAbortedError, EventHandlerCancelledError))
|
||||
|
||||
|
||||
async def _run_event_now(event, timeout: float | None = None):
|
||||
await event.now(timeout=timeout)
|
||||
await event.wait(timeout=timeout)
|
||||
await event.event_results_list()
|
||||
return event
|
||||
def defer_crawl_for_resources(crawl, *, excluded_snapshot_ids=()) -> bool:
|
||||
"""Leave extraction durable; never delay cancellation or final cleanup."""
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
|
||||
def ensure_background_runner() -> bool:
|
||||
from archivebox.machine.models import Machine, Process
|
||||
from archivebox.workers.supervisord_util import RUNNER_WORKER, get_existing_supervisord_process, get_worker, start_worker
|
||||
|
||||
supervisor = get_existing_supervisord_process()
|
||||
runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
|
||||
if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
|
||||
if crawl.status not in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
|
||||
return False
|
||||
if supervisor is not None:
|
||||
start_worker(supervisor, RUNNER_WORKER())
|
||||
return True
|
||||
|
||||
machine = Machine.current()
|
||||
Process.cleanup_stale_running(machine=machine)
|
||||
running_orchestrators = Process.objects.filter(
|
||||
machine=machine,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
reason = resource_admission.memory_pressure()
|
||||
if reason is None:
|
||||
return False
|
||||
pending = crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES).exclude(id__in=excluded_snapshot_ids)
|
||||
if crawl.snapshot_set.exists() and not pending.exists():
|
||||
return False
|
||||
now = timezone.now()
|
||||
retry_at = now + timedelta(seconds=RESOURCE_RECHECK_SECONDS)
|
||||
# Conditional short updates cannot resurrect a concurrent pause/cancel.
|
||||
type(crawl).objects.filter(id=crawl.id, status__in=[crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED]).update(
|
||||
retry_at=retry_at,
|
||||
modified_at=now,
|
||||
)
|
||||
if any(proc.is_running for proc in running_orchestrators):
|
||||
return False
|
||||
|
||||
return False
|
||||
pending.filter(retry_at__lte=now).update(retry_at=retry_at, modified_at=now)
|
||||
return True
|
||||
|
||||
|
||||
class CrawlRunner:
|
||||
@@ -693,34 +640,7 @@ class CrawlRunner:
|
||||
def _create_live_ui(self) -> LiveBusUI | None:
|
||||
if not self.show_progress:
|
||||
return None
|
||||
stdout_is_tty = sys.stdout.isatty()
|
||||
stderr_is_tty = sys.stderr.isatty()
|
||||
interactive_tty = stdout_is_tty or stderr_is_tty
|
||||
stream = sys.stderr if stderr_is_tty or not stdout_is_tty else sys.stdout
|
||||
if interactive_tty and os.path.exists("/dev/tty"):
|
||||
try:
|
||||
self._live_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
|
||||
stream = self._live_stream
|
||||
except OSError:
|
||||
self._live_stream = None
|
||||
try:
|
||||
terminal_size = os.get_terminal_size(stream.fileno())
|
||||
terminal_width = terminal_size.columns
|
||||
terminal_height = terminal_size.lines
|
||||
except (AttributeError, OSError, ValueError):
|
||||
terminal_size = shutil.get_terminal_size(fallback=(160, 40))
|
||||
terminal_width = terminal_size.columns
|
||||
terminal_height = terminal_size.lines
|
||||
ui_console = Console(
|
||||
file=stream,
|
||||
force_terminal=interactive_tty,
|
||||
width=terminal_width,
|
||||
height=terminal_height,
|
||||
_environ={
|
||||
"COLUMNS": str(terminal_width),
|
||||
"LINES": str(terminal_height),
|
||||
},
|
||||
)
|
||||
ui_console, self._live_stream, interactive_tty = create_console()
|
||||
plugins_label = ", ".join(self.selected_plugins) if self.selected_plugins else f"all ({len(self.catalog)} available)"
|
||||
live_ui = LiveBusUI(
|
||||
self.bus,
|
||||
@@ -1205,742 +1125,3 @@ def _run_crawl_locked(
|
||||
worker.join()
|
||||
if errors:
|
||||
raise errors[0]
|
||||
|
||||
|
||||
async def _run_binary(binary_id: str) -> None:
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
|
||||
binary = await Binary.objects.aget(id=binary_id)
|
||||
config = get_config(include_machine=False)
|
||||
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
|
||||
derived_config = normalize_runtime_config(machine.config)
|
||||
config = config.for_crawl()
|
||||
config = normalize_runtime_config(config)
|
||||
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
|
||||
process_service = PersistedProcessService(bus)
|
||||
binary_process_service = ArchiveBoxBinaryService(bus)
|
||||
BinaryService(bus, lib_dir=Path(config["ABXPKG_LIB_DIR"]))
|
||||
TagService(bus)
|
||||
MachineService(bus)
|
||||
catalog = get_plugin_catalog()
|
||||
config["ABX_RUNTIME"] = "archivebox"
|
||||
PluginBinaryEnvService(bus, catalog=catalog)
|
||||
HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
|
||||
await bus.emit(MachineEvent(config=config, config_type="user")).now()
|
||||
if derived_config:
|
||||
await bus.emit(MachineEvent(config=derived_config, config_type="derived")).now()
|
||||
|
||||
try:
|
||||
await bus.emit(
|
||||
BinaryRequestEvent(
|
||||
name=binary.name,
|
||||
binproviders=binary.binproviders,
|
||||
overrides=binary.overrides or None,
|
||||
extra_context={
|
||||
"plugin_name": "archivebox",
|
||||
"hook_name": "archivebox_binary_run",
|
||||
"output_dir": str(binary.output_dir),
|
||||
"binary_id": str(binary.id),
|
||||
"machine_id": str(binary.machine_id),
|
||||
},
|
||||
),
|
||||
).now(first_result=True)
|
||||
finally:
|
||||
await bus.wait_until_idle()
|
||||
await binary_process_service.flush_missing_finalizers()
|
||||
await process_service.flush_completed()
|
||||
|
||||
|
||||
def run_binary(binary_id: str) -> None:
|
||||
asyncio.run(_run_binary(binary_id))
|
||||
|
||||
|
||||
def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
|
||||
if snapshot is None:
|
||||
return False
|
||||
|
||||
# ArchiveBox owns filesystem and metadata maintenance at Snapshot
|
||||
# granularity. ArchiveResult rows are projections and never influence this
|
||||
# scheduler decision.
|
||||
current_retry_at = snapshot.retry_at
|
||||
has_pending_plugin_run = bool((snapshot.config or {}).get("RETRY_PLUGINS"))
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED and has_pending_plugin_run:
|
||||
next_retry_at = current_retry_at
|
||||
elif snapshot.status in Snapshot.OPEN_STATES:
|
||||
next_retry_at = timezone.now()
|
||||
else:
|
||||
next_retry_at = None
|
||||
if snapshot.fs_migration_needed:
|
||||
snapshot.migrate_filesystem_to_current_version()
|
||||
updated = snapshot.safe_update(
|
||||
{"retry_at": next_retry_at},
|
||||
refresh=False,
|
||||
extra_filter={
|
||||
"status": snapshot.status,
|
||||
"retry_at": current_retry_at,
|
||||
},
|
||||
)
|
||||
if not updated:
|
||||
return False
|
||||
snapshot.write_index_jsonl(output_dir=output_dir)
|
||||
return True
|
||||
|
||||
|
||||
def defer_crawl_for_resources(crawl, *, excluded_snapshot_ids=()) -> bool:
|
||||
"""Leave extraction durable; never delay cancellation or final cleanup."""
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
if crawl.status not in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
|
||||
return False
|
||||
reason = resource_admission.memory_pressure()
|
||||
if reason is None:
|
||||
return False
|
||||
pending = crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES).exclude(id__in=excluded_snapshot_ids)
|
||||
if crawl.snapshot_set.exists() and not pending.exists():
|
||||
return False
|
||||
now = timezone.now()
|
||||
retry_at = now + timedelta(seconds=RESOURCE_RECHECK_SECONDS)
|
||||
# Conditional short updates cannot resurrect a concurrent pause/cancel.
|
||||
type(crawl).objects.filter(id=crawl.id, status__in=[crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED]).update(
|
||||
retry_at=retry_at,
|
||||
modified_at=now,
|
||||
)
|
||||
pending.filter(retry_at__lte=now).update(retry_at=retry_at, modified_at=now)
|
||||
return True
|
||||
|
||||
|
||||
def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
|
||||
with crawl_lifecycle_lock(str(crawl.id)):
|
||||
return _run_due_crawl_locked(
|
||||
crawl,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
|
||||
|
||||
def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
|
||||
try:
|
||||
crawl.refresh_from_db(fields=["status", "retry_at", "modified_at"])
|
||||
except type(crawl).DoesNotExist:
|
||||
return False
|
||||
|
||||
if crawl.is_paused:
|
||||
_runner_console_line(crawl=crawl, status="PAUSED")
|
||||
return True
|
||||
if crawl.status in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
now = timezone.now()
|
||||
snapshot_count = crawl.snapshot_set.count()
|
||||
due_active_snapshots = crawl.snapshot_set.filter(
|
||||
status__in=Snapshot.RUNNABLE_STATES,
|
||||
retry_at__lte=now,
|
||||
).exists()
|
||||
if snapshot_count and due_active_snapshots:
|
||||
# Child Snapshot rows own active work. Do not rewrite the parent
|
||||
# row unless it is still the same STARTED row we selected; this
|
||||
# avoids hot-looping on the parent while child work is ready without
|
||||
# resurrecting a user cancellation that sealed the crawl after
|
||||
# selection.
|
||||
crawl.safe_update(
|
||||
{
|
||||
"status": crawl.StatusChoices.STARTED,
|
||||
"retry_at": now + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
|
||||
"modified_at": now,
|
||||
},
|
||||
refresh=False,
|
||||
extra_filter={"status": crawl.StatusChoices.STARTED},
|
||||
)
|
||||
return True
|
||||
if snapshot_count and not due_active_snapshots:
|
||||
if crawl.is_finished():
|
||||
if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
crawl.refresh_from_db()
|
||||
crawl.advance_lifecycle()
|
||||
return True
|
||||
|
||||
# retry_at is the only queue/ownership signal the runner sees.
|
||||
# Clearing it on an unfinished crawl hides the row forever, so keep
|
||||
# future snapshots scheduled and repair NULL queued child locks here.
|
||||
unlocked_children = crawl.snapshot_set.filter(
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
retry_at__isnull=True,
|
||||
).update(
|
||||
retry_at=now,
|
||||
modified_at=now,
|
||||
)
|
||||
if unlocked_children:
|
||||
crawl.update_and_requeue(status=crawl.StatusChoices.STARTED, retry_at=now)
|
||||
return True
|
||||
|
||||
next_snapshot_retry = (
|
||||
crawl.snapshot_set.filter(
|
||||
status__in=Snapshot.OPEN_STATES,
|
||||
retry_at__gt=now,
|
||||
)
|
||||
.order_by("retry_at", "created_at")
|
||||
.values_list("retry_at", flat=True)
|
||||
.first()
|
||||
)
|
||||
crawl.update_and_requeue(
|
||||
status=crawl.StatusChoices.STARTED,
|
||||
retry_at=next_snapshot_retry or now + timedelta(seconds=10),
|
||||
)
|
||||
return True
|
||||
if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
crawl.refresh_from_db()
|
||||
if crawl.status == crawl.StatusChoices.STARTED and crawl.is_finished():
|
||||
crawl.advance_lifecycle()
|
||||
return True
|
||||
_runner_console_line(crawl=crawl)
|
||||
run_crawl(str(crawl.id), process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts)
|
||||
return True
|
||||
|
||||
if crawl.status == crawl.StatusChoices.SEALED:
|
||||
if not type(crawl).claim_for_worker(crawl, lock_seconds=lock_seconds):
|
||||
return False
|
||||
_runner_console_line(crawl=crawl, status="SEALED")
|
||||
crawl.cleanup_runtime()
|
||||
crawl.update_and_requeue(retry_at=None)
|
||||
return True
|
||||
|
||||
crawl.update_and_requeue(retry_at=None)
|
||||
return True
|
||||
|
||||
|
||||
def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool:
|
||||
with crawl_lifecycle_lock(str(snapshot.crawl_id)):
|
||||
return _run_due_snapshot_locked(
|
||||
snapshot,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
|
||||
|
||||
def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
try:
|
||||
snapshot = Snapshot.objects.get(pk=snapshot.pk)
|
||||
except Snapshot.DoesNotExist:
|
||||
return False
|
||||
parent_reconciled = snapshot.reconcile_parent_lifecycle(lock_seconds=lock_seconds)
|
||||
if parent_reconciled is not None:
|
||||
if parent_reconciled:
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.fs_migration_needed:
|
||||
return run_snapshot_maintenance(str(snapshot.id))
|
||||
return parent_reconciled
|
||||
|
||||
if snapshot.is_paused:
|
||||
# Paused work never executes out of band. ArchiveResult rows are
|
||||
# historical projections and are not rewritten as scheduler state.
|
||||
snapshot.restore_paused_scheduler_marker()
|
||||
return True
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED:
|
||||
if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
|
||||
return False
|
||||
snapshot.refresh_from_db()
|
||||
owned_retry_at = snapshot.retry_at
|
||||
snapshot.finalize_completed_upload_results()
|
||||
if snapshot.fs_migration_needed:
|
||||
snapshot.migrate_filesystem_to_current_version()
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status != Snapshot.StatusChoices.SEALED or snapshot.retry_at != owned_retry_at:
|
||||
return True
|
||||
retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
|
||||
if retry_plugins:
|
||||
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
|
||||
run_crawl(
|
||||
str(snapshot.crawl_id),
|
||||
snapshot_ids=[str(snapshot.id)],
|
||||
selected_plugins=retry_plugins,
|
||||
process_discovered_snapshots_inline=False,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
return True
|
||||
return run_snapshot_maintenance(str(snapshot.id))
|
||||
|
||||
if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
snapshot.refresh_from_db()
|
||||
if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
|
||||
# The Snapshot lease may have expired while an abx-dl hook process is
|
||||
# still alive. Preserve the snapshot-level ownership boundary and do
|
||||
# not launch a second sequence; ArchiveResult status is irrelevant.
|
||||
snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
|
||||
return True
|
||||
if snapshot.fs_migration_needed:
|
||||
# Migrate before abx-dl writes new hook outputs. The claimed Snapshot
|
||||
# lease remains in place and the idempotent migration persists its
|
||||
# indexed fs_version marker only after copy/verification/cleanup.
|
||||
snapshot.migrate_filesystem_to_current_version()
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status == Snapshot.StatusChoices.QUEUED:
|
||||
snapshot.start_processing()
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status != Snapshot.StatusChoices.STARTED:
|
||||
return True
|
||||
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
|
||||
run_crawl(
|
||||
str(snapshot.crawl_id),
|
||||
snapshot_ids=[str(snapshot.id)],
|
||||
selected_plugins=None,
|
||||
process_discovered_snapshots_inline=True,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def run_due_binary(binary, *, lock_seconds: int) -> bool:
|
||||
from archivebox.crawls.locks import binary_lifecycle_lock
|
||||
|
||||
with binary_lifecycle_lock(str(binary.id)):
|
||||
binary.refresh_from_db()
|
||||
if binary.status == binary.StatusChoices.INSTALLED:
|
||||
return True
|
||||
if not binary.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
run_binary(str(binary.id))
|
||||
return True
|
||||
|
||||
|
||||
async def _run_install(plugin_names: list[str] | None = None) -> None:
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.plugins.discovery import get_enabled_plugins
|
||||
|
||||
catalog = get_plugin_catalog()
|
||||
config = get_config(include_machine=False)
|
||||
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
|
||||
derived_config = normalize_runtime_config(machine.config)
|
||||
config = config.for_crawl()
|
||||
config = normalize_runtime_config(config)
|
||||
bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
|
||||
PersistedProcessService(bus)
|
||||
ArchiveBoxBinaryService(bus)
|
||||
BinaryService(bus)
|
||||
TagService(bus)
|
||||
MachineService(bus)
|
||||
live_stream = None
|
||||
bus_destroyed = False
|
||||
|
||||
try:
|
||||
if plugin_names:
|
||||
selected_plugins = catalog.select(plugin_names)
|
||||
else:
|
||||
selected_plugins = catalog.select(get_enabled_plugins(config=config))
|
||||
if not selected_plugins:
|
||||
return
|
||||
plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(catalog)} available)"
|
||||
install_config = dict(config)
|
||||
for plugin in selected_plugins.values():
|
||||
if plugin.enabled_key in plugin.config.properties:
|
||||
install_config[plugin.enabled_key] = True
|
||||
install_config["ABX_RUNTIME"] = "archivebox"
|
||||
install_timeout = compute_install_phase_timeout(get_install_plugins(selected_plugins), install_config)
|
||||
timeout_seconds = config["TIMEOUT"]
|
||||
stdout_is_tty = sys.stdout.isatty()
|
||||
stderr_is_tty = sys.stderr.isatty()
|
||||
interactive_tty = stdout_is_tty or stderr_is_tty
|
||||
ui_console = None
|
||||
live_ui = None
|
||||
|
||||
if interactive_tty:
|
||||
stream = sys.stderr if stderr_is_tty else sys.stdout
|
||||
if os.path.exists("/dev/tty"):
|
||||
try:
|
||||
live_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
|
||||
stream = live_stream
|
||||
except OSError:
|
||||
live_stream = None
|
||||
try:
|
||||
terminal_size = os.get_terminal_size(stream.fileno())
|
||||
terminal_width = terminal_size.columns
|
||||
terminal_height = terminal_size.lines
|
||||
except (AttributeError, OSError, ValueError):
|
||||
terminal_size = shutil.get_terminal_size(fallback=(160, 40))
|
||||
terminal_width = terminal_size.columns
|
||||
terminal_height = terminal_size.lines
|
||||
ui_console = Console(
|
||||
file=stream,
|
||||
force_terminal=True,
|
||||
width=terminal_width,
|
||||
height=terminal_height,
|
||||
_environ={
|
||||
"COLUMNS": str(terminal_width),
|
||||
"LINES": str(terminal_height),
|
||||
},
|
||||
)
|
||||
|
||||
with TemporaryDirectory(prefix="archivebox-install-") as temp_dir:
|
||||
output_dir = Path(temp_dir)
|
||||
if ui_console is not None:
|
||||
live_ui = LiveBusUI(
|
||||
bus,
|
||||
total_hooks=_count_selected_hooks(selected_plugins, None),
|
||||
timeout_seconds=timeout_seconds,
|
||||
ui_console=ui_console,
|
||||
interactive_tty=interactive_tty,
|
||||
)
|
||||
live_ui.print_intro(
|
||||
url="install",
|
||||
output_dir=output_dir,
|
||||
plugins_label=plugins_label,
|
||||
)
|
||||
with live_ui if live_ui is not None else nullcontext():
|
||||
try:
|
||||
HookProcessService(bus, emit_jsonl=False, interactive_tty=interactive_tty)
|
||||
PluginBinaryEnvService(bus, catalog=selected_plugins)
|
||||
install_snapshot = AbxSnapshot(url="")
|
||||
PluginBinariesService(
|
||||
bus,
|
||||
catalog=selected_plugins,
|
||||
auto_install=True,
|
||||
install_plugins=get_install_plugins(selected_plugins),
|
||||
output_dir=output_dir,
|
||||
snapshot=install_snapshot,
|
||||
)
|
||||
await bus.emit(MachineEvent(config=install_config, config_type="user")).now()
|
||||
if derived_config:
|
||||
await bus.emit(MachineEvent(config=derived_config, config_type="derived")).now()
|
||||
install_event = bus.emit(
|
||||
InstallEvent(
|
||||
url="",
|
||||
snapshot_id=install_snapshot.id,
|
||||
output_dir=str(output_dir),
|
||||
event_timeout=install_timeout,
|
||||
event_handler_slow_timeout=slow_warning_timeout(install_timeout),
|
||||
),
|
||||
)
|
||||
await install_event.now(timeout=install_timeout)
|
||||
await install_event.wait(timeout=install_timeout)
|
||||
await install_event.event_results_list()
|
||||
finally:
|
||||
try:
|
||||
await bus.wait_until_idle()
|
||||
finally:
|
||||
await bus.destroy(clear=False)
|
||||
bus_destroyed = True
|
||||
finally:
|
||||
if not bus_destroyed:
|
||||
await bus.destroy(clear=False)
|
||||
try:
|
||||
if live_stream is not None:
|
||||
live_stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_install(*, plugin_names: list[str] | None = None) -> None:
|
||||
asyncio.run(_run_install(plugin_names=plugin_names))
|
||||
|
||||
|
||||
def _first_due_id(queryset):
|
||||
return queryset.order_by("retry_at", "created_at").values_list("id", flat=True).first()
|
||||
|
||||
|
||||
def _run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: int, interactive_interrupts: bool) -> bool:
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
due_crawls = Crawl.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
status=status,
|
||||
)
|
||||
if crawl_id:
|
||||
due_crawls = due_crawls.filter(id=crawl_id)
|
||||
due_crawl_id = _first_due_id(due_crawls)
|
||||
if due_crawl_id is None:
|
||||
return False
|
||||
due_crawl = Crawl.objects.filter(id=due_crawl_id).first()
|
||||
if due_crawl is None:
|
||||
return True
|
||||
run_due_crawl(
|
||||
due_crawl,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _run_due_snapshot_query(queryset, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool:
|
||||
due_snapshot_id = _first_due_id(queryset)
|
||||
return _run_due_snapshot_id(
|
||||
due_snapshot_id,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
|
||||
|
||||
def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
due_snapshot_id = snapshot_id
|
||||
if due_snapshot_id is None:
|
||||
return False
|
||||
due_snapshot = Snapshot.objects.filter(id=due_snapshot_id).first()
|
||||
if due_snapshot is None:
|
||||
return True
|
||||
run_due_snapshot(
|
||||
due_snapshot,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _run_due_binary() -> bool:
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
due_binary_id = (
|
||||
Binary.objects.filter(retry_at__lte=timezone.now())
|
||||
.exclude(status=Binary.StatusChoices.INSTALLED)
|
||||
.order_by("retry_at", "created_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
)
|
||||
if due_binary_id is None:
|
||||
return False
|
||||
due_binary = Binary.objects.filter(id=due_binary_id).first()
|
||||
if due_binary is None:
|
||||
return True
|
||||
run_due_binary(due_binary, lock_seconds=60)
|
||||
return True
|
||||
|
||||
|
||||
def run_pending_crawls(
|
||||
*,
|
||||
daemon: bool = False,
|
||||
crawl_id: str | None = None,
|
||||
maintenance_only: bool = False,
|
||||
interactive_interrupts: bool = False,
|
||||
) -> int:
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.crawls.models import Crawl, CrawlSchedule
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
crawl_claim_lock_seconds = 10
|
||||
runtime_config = get_config()
|
||||
last_recovery_at = 0.0
|
||||
last_retention_at = 0.0
|
||||
last_retention_repair_at = 0.0
|
||||
last_analyze_at = 0.0
|
||||
analyze_queue: list[str] | None = None
|
||||
analyze_sweep_started_at = 0.0
|
||||
orchestrator_started_at = time.monotonic()
|
||||
while True:
|
||||
raise_if_shutdown_requested()
|
||||
now_monotonic = time.monotonic()
|
||||
if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
|
||||
for model in (ArchiveResult, Snapshot, Crawl, Process):
|
||||
# Keep the tight scheduler loop anchored on indexed delete_at
|
||||
# columns only. Backfilling missing delete_at values has to read
|
||||
# config JSON for models whose retention policy is scoped to a
|
||||
# Crawl/Snapshot/Process. That repair is still required for
|
||||
# correctness, but it belongs in the idle maintenance block
|
||||
# below, not ahead of every claim attempt.
|
||||
model.delete_expired(batch_size=100, backfill_missing=False)
|
||||
last_retention_at = now_monotonic
|
||||
|
||||
if daemon and crawl_id is None:
|
||||
now = timezone.now()
|
||||
for schedule in CrawlSchedule.objects.filter(is_enabled=True).select_related("template", "template__created_by"):
|
||||
if schedule.is_due(now):
|
||||
schedule.dispatch(queued_at=now)
|
||||
|
||||
if maintenance_only:
|
||||
# Filesystem migration is independent of lifecycle status; do not
|
||||
# tick queued snapshots or start their extraction work here.
|
||||
filesystem_snapshot = (
|
||||
Snapshot.objects.filter(retry_at__lte=timezone.now())
|
||||
.exclude(fs_version=Snapshot._fs_current_version())
|
||||
.order_by("retry_at", "created_at")
|
||||
.first()
|
||||
)
|
||||
if filesystem_snapshot and Snapshot.claim_for_worker(filesystem_snapshot, lock_seconds=60):
|
||||
if run_snapshot_maintenance(str(filesystem_snapshot.id)):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
active_snapshots = Snapshot.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
crawl__status__in=Crawl.RUNNABLE_STATES,
|
||||
status__in=Snapshot.RUNNABLE_STATES,
|
||||
)
|
||||
if crawl_id:
|
||||
active_snapshots = active_snapshots.filter(crawl_id=crawl_id)
|
||||
if _run_due_snapshot_query(
|
||||
active_snapshots,
|
||||
lock_seconds=60,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
if _run_due_crawl_status(
|
||||
Crawl.StatusChoices.QUEUED,
|
||||
crawl_id=crawl_id,
|
||||
lock_seconds=crawl_claim_lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
if _run_due_crawl_status(
|
||||
Crawl.StatusChoices.STARTED,
|
||||
crawl_id=crawl_id,
|
||||
lock_seconds=crawl_claim_lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
# Canceled-crawl child sealing is important cleanup, but it must
|
||||
# not starve live crawl work when a large bulk cancel leaves many
|
||||
# children due at once.
|
||||
cancelling_snapshots = Snapshot.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
crawl__status=Crawl.StatusChoices.SEALED,
|
||||
status=Snapshot.StatusChoices.STARTED,
|
||||
)
|
||||
if crawl_id:
|
||||
cancelling_snapshots = cancelling_snapshots.filter(crawl_id=crawl_id)
|
||||
if _run_due_snapshot_query(
|
||||
cancelling_snapshots,
|
||||
lock_seconds=60,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
pausing_snapshots = Snapshot.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
crawl__status=Crawl.StatusChoices.PAUSED,
|
||||
status__in=Snapshot.RUNNABLE_STATES,
|
||||
)
|
||||
if crawl_id:
|
||||
pausing_snapshots = pausing_snapshots.filter(crawl_id=crawl_id)
|
||||
if _run_due_snapshot_query(
|
||||
pausing_snapshots,
|
||||
lock_seconds=60,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
):
|
||||
continue
|
||||
|
||||
# Final active-state fallback uses only the retry_at scheduler index and
|
||||
# selects an id first. Keep final SEALED rows out of this broad path so
|
||||
# large filesystem/index backfills cannot starve newly queued crawls.
|
||||
if not maintenance_only:
|
||||
due_snapshots = Snapshot.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
status__in=Snapshot.OPEN_STATES,
|
||||
)
|
||||
if crawl_id:
|
||||
due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
|
||||
if _run_due_snapshot_query(
|
||||
due_snapshots,
|
||||
lock_seconds=60,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
# Final snapshots can still have an explicit filesystem/index-json
|
||||
# maintenance tick. Search extraction is invoked directly by the
|
||||
# update path and never enters this scheduler through ArchiveResult.
|
||||
sealed_snapshots = Snapshot.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
)
|
||||
if crawl_id:
|
||||
sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)
|
||||
if _run_due_snapshot_query(
|
||||
sealed_snapshots,
|
||||
lock_seconds=60,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
):
|
||||
continue
|
||||
|
||||
if not maintenance_only:
|
||||
if _run_due_crawl_status(
|
||||
Crawl.StatusChoices.SEALED,
|
||||
crawl_id=crawl_id,
|
||||
lock_seconds=crawl_claim_lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
):
|
||||
continue
|
||||
|
||||
if crawl_id is None and not maintenance_only:
|
||||
if _run_due_binary():
|
||||
continue
|
||||
|
||||
now_monotonic = time.monotonic()
|
||||
if crawl_id is None and now_monotonic - last_retention_repair_at >= (60.0 if daemon else 0.0):
|
||||
for model in (ArchiveResult, Snapshot, Crawl, Process):
|
||||
# No runnable work was found on this scheduler pass. This is
|
||||
# the bounded repair point for missing retention deadlines,
|
||||
# including ArchiveResult rows intentionally saved without
|
||||
# delete_at in the plugin-result hot path. Running it here keeps
|
||||
# DELETE_AFTER resolution fresh without making every hook event
|
||||
# load parent Snapshot/Crawl config.
|
||||
model.delete_expired(batch_size=100, backfill_missing=True)
|
||||
last_retention_repair_at = now_monotonic
|
||||
|
||||
if daemon:
|
||||
now_monotonic = time.monotonic()
|
||||
if now_monotonic - last_recovery_at >= 30.0:
|
||||
from archivebox.core.recovery_util import recover_orchestrator_state
|
||||
|
||||
recover_orchestrator_state()
|
||||
last_recovery_at = now_monotonic
|
||||
# SQLite query plans degrade as the snapshot/archiveresult tables grow
|
||||
# past their last ANALYZE — stale stats make the optimizer start large
|
||||
# joins from auth_user/crawl instead of using the url index, blowing the
|
||||
# snapshot detail page out to ~500ms. Refresh stats at most once per
|
||||
# 24hr while the queue is idle, and only after the orchestrator has
|
||||
# been alive for at least an hour so short server boots / one-off work
|
||||
# never pay the cost. The sweep is batched one table per idle tick;
|
||||
# individual table ANALYZE statements abort after 2min (progress
|
||||
# handler) and the whole sweep is hard-capped at 5min so a
|
||||
# pathological table cannot wedge maintenance forever. Any failure
|
||||
# inside the maintenance hook is swallowed — orchestrator must never
|
||||
# be taken down by stats refresh.
|
||||
try:
|
||||
if (
|
||||
analyze_queue is None
|
||||
and now_monotonic - orchestrator_started_at >= 3600.0
|
||||
and now_monotonic - last_analyze_at >= 86400.0
|
||||
):
|
||||
analyze_sweep_started_at = now_monotonic
|
||||
analyze_queue = run_db_analyze_batch(None)
|
||||
elif analyze_queue and now_monotonic - analyze_sweep_started_at >= 300.0:
|
||||
# Sweep blew past the 5min hard cap — abandon what's left
|
||||
# and don't retry until the next 24hr window.
|
||||
analyze_queue = None
|
||||
last_analyze_at = now_monotonic
|
||||
elif analyze_queue:
|
||||
analyze_queue = run_db_analyze_batch(analyze_queue)
|
||||
if analyze_queue is not None and not analyze_queue:
|
||||
analyze_queue = None
|
||||
last_analyze_at = now_monotonic
|
||||
except Exception:
|
||||
analyze_queue = None
|
||||
last_analyze_at = now_monotonic
|
||||
time.sleep(2.0)
|
||||
continue
|
||||
return 0
|
||||
@@ -0,0 +1,209 @@
|
||||
from __future__ import annotations
|
||||
from datetime import timedelta
|
||||
from django.utils import timezone
|
||||
from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
|
||||
from archivebox.crawls.locks import crawl_lifecycle_lock
|
||||
|
||||
from .maintenance import run_snapshot_maintenance
|
||||
from .console import _runner_console_line
|
||||
from .install import run_binary
|
||||
from .crawl import run_crawl
|
||||
|
||||
|
||||
def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
|
||||
with crawl_lifecycle_lock(str(crawl.id)):
|
||||
return _run_due_crawl_locked(
|
||||
crawl,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
|
||||
|
||||
def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool:
|
||||
try:
|
||||
crawl.refresh_from_db(fields=["status", "retry_at", "modified_at"])
|
||||
except type(crawl).DoesNotExist:
|
||||
return False
|
||||
|
||||
if crawl.is_paused:
|
||||
_runner_console_line(crawl=crawl, status="PAUSED")
|
||||
return True
|
||||
if crawl.status in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
now = timezone.now()
|
||||
snapshot_count = crawl.snapshot_set.count()
|
||||
due_active_snapshots = crawl.snapshot_set.filter(
|
||||
status__in=Snapshot.RUNNABLE_STATES,
|
||||
retry_at__lte=now,
|
||||
).exists()
|
||||
if snapshot_count and due_active_snapshots:
|
||||
# Child Snapshot rows own active work. Do not rewrite the parent
|
||||
# row unless it is still the same STARTED row we selected; this
|
||||
# avoids hot-looping on the parent while child work is ready without
|
||||
# resurrecting a user cancellation that sealed the crawl after
|
||||
# selection.
|
||||
crawl.safe_update(
|
||||
{
|
||||
"status": crawl.StatusChoices.STARTED,
|
||||
"retry_at": now + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
|
||||
"modified_at": now,
|
||||
},
|
||||
refresh=False,
|
||||
extra_filter={"status": crawl.StatusChoices.STARTED},
|
||||
)
|
||||
return True
|
||||
if snapshot_count and not due_active_snapshots:
|
||||
if crawl.is_finished():
|
||||
if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
crawl.refresh_from_db()
|
||||
crawl.advance_lifecycle()
|
||||
return True
|
||||
|
||||
# retry_at is the only queue/ownership signal the runner sees.
|
||||
# Clearing it on an unfinished crawl hides the row forever, so keep
|
||||
# future snapshots scheduled and repair NULL queued child locks here.
|
||||
unlocked_children = crawl.snapshot_set.filter(
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
retry_at__isnull=True,
|
||||
).update(
|
||||
retry_at=now,
|
||||
modified_at=now,
|
||||
)
|
||||
if unlocked_children:
|
||||
crawl.update_and_requeue(status=crawl.StatusChoices.STARTED, retry_at=now)
|
||||
return True
|
||||
|
||||
next_snapshot_retry = (
|
||||
crawl.snapshot_set.filter(
|
||||
status__in=Snapshot.OPEN_STATES,
|
||||
retry_at__gt=now,
|
||||
)
|
||||
.order_by("retry_at", "created_at")
|
||||
.values_list("retry_at", flat=True)
|
||||
.first()
|
||||
)
|
||||
crawl.update_and_requeue(
|
||||
status=crawl.StatusChoices.STARTED,
|
||||
retry_at=next_snapshot_retry or now + timedelta(seconds=10),
|
||||
)
|
||||
return True
|
||||
if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
crawl.refresh_from_db()
|
||||
if crawl.status == crawl.StatusChoices.STARTED and crawl.is_finished():
|
||||
crawl.advance_lifecycle()
|
||||
return True
|
||||
_runner_console_line(crawl=crawl)
|
||||
run_crawl(str(crawl.id), process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts)
|
||||
return True
|
||||
|
||||
if crawl.status == crawl.StatusChoices.SEALED:
|
||||
if not type(crawl).claim_for_worker(crawl, lock_seconds=lock_seconds):
|
||||
return False
|
||||
_runner_console_line(crawl=crawl, status="SEALED")
|
||||
crawl.cleanup_runtime()
|
||||
crawl.update_and_requeue(retry_at=None)
|
||||
return True
|
||||
|
||||
crawl.update_and_requeue(retry_at=None)
|
||||
return True
|
||||
|
||||
|
||||
def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool:
|
||||
with crawl_lifecycle_lock(str(snapshot.crawl_id)):
|
||||
return _run_due_snapshot_locked(
|
||||
snapshot,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
|
||||
|
||||
def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
try:
|
||||
snapshot = Snapshot.objects.get(pk=snapshot.pk)
|
||||
except Snapshot.DoesNotExist:
|
||||
return False
|
||||
parent_reconciled = snapshot.reconcile_parent_lifecycle(lock_seconds=lock_seconds)
|
||||
if parent_reconciled is not None:
|
||||
if parent_reconciled:
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.fs_migration_needed:
|
||||
return run_snapshot_maintenance(str(snapshot.id))
|
||||
return parent_reconciled
|
||||
|
||||
if snapshot.is_paused:
|
||||
# Paused work never executes out of band. ArchiveResult rows are
|
||||
# historical projections and are not rewritten as scheduler state.
|
||||
snapshot.restore_paused_scheduler_marker()
|
||||
return True
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED:
|
||||
if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
|
||||
return False
|
||||
snapshot.refresh_from_db()
|
||||
owned_retry_at = snapshot.retry_at
|
||||
snapshot.finalize_completed_upload_results()
|
||||
if snapshot.fs_migration_needed:
|
||||
snapshot.migrate_filesystem_to_current_version()
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status != Snapshot.StatusChoices.SEALED or snapshot.retry_at != owned_retry_at:
|
||||
return True
|
||||
retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
|
||||
if retry_plugins:
|
||||
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
|
||||
run_crawl(
|
||||
str(snapshot.crawl_id),
|
||||
snapshot_ids=[str(snapshot.id)],
|
||||
selected_plugins=retry_plugins,
|
||||
process_discovered_snapshots_inline=False,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
return True
|
||||
return run_snapshot_maintenance(str(snapshot.id))
|
||||
|
||||
if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
snapshot.refresh_from_db()
|
||||
if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
|
||||
# The Snapshot lease may have expired while an abx-dl hook process is
|
||||
# still alive. Preserve the snapshot-level ownership boundary and do
|
||||
# not launch a second sequence; ArchiveResult status is irrelevant.
|
||||
snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
|
||||
return True
|
||||
if snapshot.fs_migration_needed:
|
||||
# Migrate before abx-dl writes new hook outputs. The claimed Snapshot
|
||||
# lease remains in place and the idempotent migration persists its
|
||||
# indexed fs_version marker only after copy/verification/cleanup.
|
||||
snapshot.migrate_filesystem_to_current_version()
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status == Snapshot.StatusChoices.QUEUED:
|
||||
snapshot.start_processing()
|
||||
snapshot.refresh_from_db()
|
||||
if snapshot.status != Snapshot.StatusChoices.STARTED:
|
||||
return True
|
||||
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
|
||||
run_crawl(
|
||||
str(snapshot.crawl_id),
|
||||
snapshot_ids=[str(snapshot.id)],
|
||||
selected_plugins=None,
|
||||
process_discovered_snapshots_inline=True,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def run_due_binary(binary, *, lock_seconds: int) -> bool:
|
||||
from archivebox.crawls.locks import binary_lifecycle_lock
|
||||
|
||||
with binary_lifecycle_lock(str(binary.id)):
|
||||
binary.refresh_from_db()
|
||||
if binary.status == binary.StatusChoices.INSTALLED:
|
||||
return True
|
||||
if not binary.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
run_binary(str(binary.id))
|
||||
return True
|
||||
@@ -0,0 +1,190 @@
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from asgiref.sync import sync_to_async
|
||||
from abxpkg.binary_service import BinaryRequestEvent, BinaryService
|
||||
from abx_dl.events import (
|
||||
InstallEvent,
|
||||
MachineEvent,
|
||||
slow_warning_timeout,
|
||||
)
|
||||
from abx_dl.models import Snapshot as AbxSnapshot
|
||||
from abx_dl.orchestrator import (
|
||||
compute_install_phase_timeout,
|
||||
create_bus,
|
||||
get_install_plugins,
|
||||
)
|
||||
from abx_dl.services.binary_service import PluginBinaryEnvService
|
||||
from abx_dl.services import PluginBinariesService
|
||||
from abx_dl.services.process_service import ProcessService as HookProcessService
|
||||
from abx_dl.cli import LiveBusUI
|
||||
from archivebox.config.common import (
|
||||
normalize_runtime_config,
|
||||
)
|
||||
from archivebox.plugins.discovery import get_plugin_catalog
|
||||
from ..binary_service import ArchiveBoxBinaryService
|
||||
from ..machine_service import MachineService
|
||||
from ..process_service import ProcessService as PersistedProcessService
|
||||
from ..tag_service import TagService
|
||||
|
||||
from .crawl import _bus_name
|
||||
from .console import _count_selected_hooks, create_console
|
||||
|
||||
|
||||
async def _run_binary(binary_id: str) -> None:
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
|
||||
binary = await Binary.objects.aget(id=binary_id)
|
||||
config = get_config(include_machine=False)
|
||||
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
|
||||
derived_config = normalize_runtime_config(machine.config)
|
||||
config = config.for_crawl()
|
||||
config = normalize_runtime_config(config)
|
||||
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
|
||||
process_service = PersistedProcessService(bus)
|
||||
binary_process_service = ArchiveBoxBinaryService(bus)
|
||||
BinaryService(bus, lib_dir=Path(config["ABXPKG_LIB_DIR"]))
|
||||
TagService(bus)
|
||||
MachineService(bus)
|
||||
catalog = get_plugin_catalog()
|
||||
config["ABX_RUNTIME"] = "archivebox"
|
||||
PluginBinaryEnvService(bus, catalog=catalog)
|
||||
HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
|
||||
await bus.emit(MachineEvent(config=config, config_type="user")).now()
|
||||
if derived_config:
|
||||
await bus.emit(MachineEvent(config=derived_config, config_type="derived")).now()
|
||||
|
||||
try:
|
||||
await bus.emit(
|
||||
BinaryRequestEvent(
|
||||
name=binary.name,
|
||||
binproviders=binary.binproviders,
|
||||
overrides=binary.overrides or None,
|
||||
extra_context={
|
||||
"plugin_name": "archivebox",
|
||||
"hook_name": "archivebox_binary_run",
|
||||
"output_dir": str(binary.output_dir),
|
||||
"binary_id": str(binary.id),
|
||||
"machine_id": str(binary.machine_id),
|
||||
},
|
||||
),
|
||||
).now(first_result=True)
|
||||
finally:
|
||||
await bus.wait_until_idle()
|
||||
await binary_process_service.flush_missing_finalizers()
|
||||
await process_service.flush_completed()
|
||||
|
||||
|
||||
def run_binary(binary_id: str) -> None:
|
||||
asyncio.run(_run_binary(binary_id))
|
||||
|
||||
|
||||
async def _run_install(plugin_names: list[str] | None = None) -> None:
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.plugins.discovery import get_enabled_plugins
|
||||
|
||||
catalog = get_plugin_catalog()
|
||||
config = get_config(include_machine=False)
|
||||
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
|
||||
derived_config = normalize_runtime_config(machine.config)
|
||||
config = config.for_crawl()
|
||||
config = normalize_runtime_config(config)
|
||||
bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
|
||||
PersistedProcessService(bus)
|
||||
ArchiveBoxBinaryService(bus)
|
||||
BinaryService(bus)
|
||||
TagService(bus)
|
||||
MachineService(bus)
|
||||
live_stream = None
|
||||
bus_destroyed = False
|
||||
|
||||
try:
|
||||
if plugin_names:
|
||||
selected_plugins = catalog.select(plugin_names)
|
||||
else:
|
||||
selected_plugins = catalog.select(get_enabled_plugins(config=config))
|
||||
if not selected_plugins:
|
||||
return
|
||||
plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(catalog)} available)"
|
||||
install_config = dict(config)
|
||||
for plugin in selected_plugins.values():
|
||||
if plugin.enabled_key in plugin.config.properties:
|
||||
install_config[plugin.enabled_key] = True
|
||||
install_config["ABX_RUNTIME"] = "archivebox"
|
||||
install_timeout = compute_install_phase_timeout(get_install_plugins(selected_plugins), install_config)
|
||||
timeout_seconds = config["TIMEOUT"]
|
||||
stdout_is_tty = sys.stdout.isatty()
|
||||
stderr_is_tty = sys.stderr.isatty()
|
||||
interactive_tty = stdout_is_tty or stderr_is_tty
|
||||
ui_console = None
|
||||
live_ui = None
|
||||
|
||||
if interactive_tty:
|
||||
ui_console, live_stream, interactive_tty = create_console()
|
||||
|
||||
with TemporaryDirectory(prefix="archivebox-install-") as temp_dir:
|
||||
output_dir = Path(temp_dir)
|
||||
if ui_console is not None:
|
||||
live_ui = LiveBusUI(
|
||||
bus,
|
||||
total_hooks=_count_selected_hooks(selected_plugins, None),
|
||||
timeout_seconds=timeout_seconds,
|
||||
ui_console=ui_console,
|
||||
interactive_tty=interactive_tty,
|
||||
)
|
||||
live_ui.print_intro(
|
||||
url="install",
|
||||
output_dir=output_dir,
|
||||
plugins_label=plugins_label,
|
||||
)
|
||||
with live_ui if live_ui is not None else nullcontext():
|
||||
try:
|
||||
HookProcessService(bus, emit_jsonl=False, interactive_tty=interactive_tty)
|
||||
PluginBinaryEnvService(bus, catalog=selected_plugins)
|
||||
install_snapshot = AbxSnapshot(url="")
|
||||
PluginBinariesService(
|
||||
bus,
|
||||
catalog=selected_plugins,
|
||||
auto_install=True,
|
||||
install_plugins=get_install_plugins(selected_plugins),
|
||||
output_dir=output_dir,
|
||||
snapshot=install_snapshot,
|
||||
)
|
||||
await bus.emit(MachineEvent(config=install_config, config_type="user")).now()
|
||||
if derived_config:
|
||||
await bus.emit(MachineEvent(config=derived_config, config_type="derived")).now()
|
||||
install_event = bus.emit(
|
||||
InstallEvent(
|
||||
url="",
|
||||
snapshot_id=install_snapshot.id,
|
||||
output_dir=str(output_dir),
|
||||
event_timeout=install_timeout,
|
||||
event_handler_slow_timeout=slow_warning_timeout(install_timeout),
|
||||
),
|
||||
)
|
||||
await install_event.now(timeout=install_timeout)
|
||||
await install_event.wait(timeout=install_timeout)
|
||||
await install_event.event_results_list()
|
||||
finally:
|
||||
try:
|
||||
await bus.wait_until_idle()
|
||||
finally:
|
||||
await bus.destroy(clear=False)
|
||||
bus_destroyed = True
|
||||
finally:
|
||||
if not bus_destroyed:
|
||||
await bus.destroy(clear=False)
|
||||
try:
|
||||
if live_stream is not None:
|
||||
live_stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_install(*, plugin_names: list[str] | None = None) -> None:
|
||||
asyncio.run(_run_install(plugin_names=plugin_names))
|
||||
@@ -0,0 +1,36 @@
|
||||
from pathlib import Path
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
|
||||
if snapshot is None:
|
||||
return False
|
||||
|
||||
# ArchiveBox owns filesystem and metadata maintenance at Snapshot
|
||||
# granularity. ArchiveResult rows are projections and never influence this
|
||||
# scheduler decision.
|
||||
current_retry_at = snapshot.retry_at
|
||||
has_pending_plugin_run = bool((snapshot.config or {}).get("RETRY_PLUGINS"))
|
||||
if snapshot.status == Snapshot.StatusChoices.SEALED and has_pending_plugin_run:
|
||||
next_retry_at = current_retry_at
|
||||
elif snapshot.status in Snapshot.OPEN_STATES:
|
||||
next_retry_at = timezone.now()
|
||||
else:
|
||||
next_retry_at = None
|
||||
if snapshot.fs_migration_needed:
|
||||
snapshot.migrate_filesystem_to_current_version()
|
||||
updated = snapshot.safe_update(
|
||||
{"retry_at": next_retry_at},
|
||||
refresh=False,
|
||||
extra_filter={
|
||||
"status": snapshot.status,
|
||||
"retry_at": current_retry_at,
|
||||
},
|
||||
)
|
||||
if not updated:
|
||||
return False
|
||||
snapshot.write_index_jsonl(output_dir=output_dir)
|
||||
return True
|
||||
@@ -0,0 +1,269 @@
|
||||
from __future__ import annotations
|
||||
import time
|
||||
from django.utils import timezone
|
||||
from archivebox.misc.db import run_db_analyze_batch
|
||||
from archivebox.core.shutdown_util import raise_if_shutdown_requested
|
||||
|
||||
from .dispatch import run_due_binary
|
||||
from .dispatch import run_due_crawl
|
||||
from .dispatch import run_due_snapshot
|
||||
from .maintenance import run_snapshot_maintenance
|
||||
|
||||
|
||||
def ensure_background_runner() -> bool:
|
||||
from archivebox.machine.models import Machine, Process
|
||||
from archivebox.workers.supervisord_util import RUNNER_WORKER, get_existing_supervisord_process, get_worker, start_worker
|
||||
|
||||
supervisor = get_existing_supervisord_process()
|
||||
runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
|
||||
if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
|
||||
return False
|
||||
if supervisor is not None:
|
||||
start_worker(supervisor, RUNNER_WORKER())
|
||||
return True
|
||||
|
||||
machine = Machine.current()
|
||||
Process.cleanup_stale_running(machine=machine)
|
||||
running_orchestrators = Process.objects.filter(
|
||||
machine=machine,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
)
|
||||
if any(proc.is_running for proc in running_orchestrators):
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _first_due_id(queryset):
|
||||
return queryset.order_by("retry_at", "created_at").values_list("id", flat=True).first()
|
||||
|
||||
|
||||
def _run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: int, interactive_interrupts: bool) -> bool:
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
due_crawls = Crawl.objects.filter(
|
||||
retry_at__lte=timezone.now(),
|
||||
status=status,
|
||||
)
|
||||
if crawl_id:
|
||||
due_crawls = due_crawls.filter(id=crawl_id)
|
||||
due_crawl_id = _first_due_id(due_crawls)
|
||||
if due_crawl_id is None:
|
||||
return False
|
||||
due_crawl = Crawl.objects.filter(id=due_crawl_id).first()
|
||||
if due_crawl is None:
|
||||
return True
|
||||
run_due_crawl(
|
||||
due_crawl,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _run_due_snapshot_query(queryset, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool:
|
||||
due_snapshot_id = _first_due_id(queryset)
|
||||
return _run_due_snapshot_id(
|
||||
due_snapshot_id,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
|
||||
|
||||
def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
due_snapshot_id = snapshot_id
|
||||
if due_snapshot_id is None:
|
||||
return False
|
||||
due_snapshot = Snapshot.objects.filter(id=due_snapshot_id).first()
|
||||
if due_snapshot is None:
|
||||
return True
|
||||
run_due_snapshot(
|
||||
due_snapshot,
|
||||
lock_seconds=lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _run_due_binary() -> bool:
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
due_binary_id = (
|
||||
Binary.objects.filter(retry_at__lte=timezone.now())
|
||||
.exclude(status=Binary.StatusChoices.INSTALLED)
|
||||
.order_by("retry_at", "created_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
)
|
||||
if due_binary_id is None:
|
||||
return False
|
||||
due_binary = Binary.objects.filter(id=due_binary_id).first()
|
||||
if due_binary is None:
|
||||
return True
|
||||
run_due_binary(due_binary, lock_seconds=60)
|
||||
return True
|
||||
|
||||
|
||||
def _run_scheduled_work(*, crawl_id, interactive_interrupts, runtime_config, crawl_lock_seconds) -> bool:
|
||||
"""Try one work item in priority order; query each tier only when reached."""
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
# Active children run before parent bookkeeping. Cancellation and pause
|
||||
# cleanup come next, then the broad active fallback, and finally sealed
|
||||
# maintenance. Keep this ordering so a backfill cannot starve new work.
|
||||
priorities = (
|
||||
(Snapshot, {"crawl__status__in": Crawl.RUNNABLE_STATES, "status__in": Snapshot.RUNNABLE_STATES}),
|
||||
(Crawl, {"status": Crawl.StatusChoices.QUEUED}),
|
||||
(Crawl, {"status": Crawl.StatusChoices.STARTED}),
|
||||
(Snapshot, {"crawl__status": Crawl.StatusChoices.SEALED, "status": Snapshot.StatusChoices.STARTED}),
|
||||
(Snapshot, {"crawl__status": Crawl.StatusChoices.PAUSED, "status__in": Snapshot.RUNNABLE_STATES}),
|
||||
(Snapshot, {"status__in": Snapshot.OPEN_STATES}),
|
||||
(Snapshot, {"status": Snapshot.StatusChoices.SEALED}),
|
||||
(Crawl, {"status": Crawl.StatusChoices.SEALED}),
|
||||
)
|
||||
for model, filters in priorities:
|
||||
if model is Crawl:
|
||||
ran = _run_due_crawl_status(
|
||||
filters["status"],
|
||||
crawl_id=crawl_id,
|
||||
lock_seconds=crawl_lock_seconds,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
)
|
||||
else:
|
||||
queryset = Snapshot.objects.filter(retry_at__lte=timezone.now(), **filters)
|
||||
if crawl_id:
|
||||
queryset = queryset.filter(crawl_id=crawl_id)
|
||||
ran = _run_due_snapshot_query(
|
||||
queryset,
|
||||
lock_seconds=60,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
)
|
||||
if ran:
|
||||
return True
|
||||
return crawl_id is None and _run_due_binary()
|
||||
|
||||
|
||||
def run_pending_crawls(
|
||||
*,
|
||||
daemon: bool = False,
|
||||
crawl_id: str | None = None,
|
||||
maintenance_only: bool = False,
|
||||
interactive_interrupts: bool = False,
|
||||
) -> int:
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.crawls.models import Crawl, CrawlSchedule
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
crawl_claim_lock_seconds = 10
|
||||
runtime_config = get_config()
|
||||
last_recovery_at = 0.0
|
||||
last_retention_at = 0.0
|
||||
last_retention_repair_at = 0.0
|
||||
last_analyze_at = 0.0
|
||||
analyze_queue: list[str] | None = None
|
||||
analyze_sweep_started_at = 0.0
|
||||
orchestrator_started_at = time.monotonic()
|
||||
while True:
|
||||
raise_if_shutdown_requested()
|
||||
now_monotonic = time.monotonic()
|
||||
if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
|
||||
for model in (ArchiveResult, Snapshot, Crawl, Process):
|
||||
# Keep the tight scheduler loop anchored on indexed delete_at
|
||||
# columns only. Backfilling missing delete_at values has to read
|
||||
# config JSON for models whose retention policy is scoped to a
|
||||
# Crawl/Snapshot/Process. That repair is still required for
|
||||
# correctness, but it belongs in the idle maintenance block
|
||||
# below, not ahead of every claim attempt.
|
||||
model.delete_expired(batch_size=100, backfill_missing=False)
|
||||
last_retention_at = now_monotonic
|
||||
|
||||
if daemon and crawl_id is None:
|
||||
now = timezone.now()
|
||||
for schedule in CrawlSchedule.objects.filter(is_enabled=True).select_related("template", "template__created_by"):
|
||||
if schedule.is_due(now):
|
||||
schedule.dispatch(queued_at=now)
|
||||
|
||||
if maintenance_only:
|
||||
# Filesystem migration is independent of lifecycle status; do not
|
||||
# tick queued snapshots or start their extraction work here.
|
||||
filesystem_snapshot = (
|
||||
Snapshot.objects.filter(retry_at__lte=timezone.now())
|
||||
.exclude(fs_version=Snapshot._fs_current_version())
|
||||
.order_by("retry_at", "created_at")
|
||||
.first()
|
||||
)
|
||||
if filesystem_snapshot and Snapshot.claim_for_worker(filesystem_snapshot, lock_seconds=60):
|
||||
if run_snapshot_maintenance(str(filesystem_snapshot.id)):
|
||||
continue
|
||||
|
||||
if not maintenance_only and _run_scheduled_work(
|
||||
crawl_id=crawl_id,
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
runtime_config=runtime_config,
|
||||
crawl_lock_seconds=crawl_claim_lock_seconds,
|
||||
):
|
||||
continue
|
||||
|
||||
now_monotonic = time.monotonic()
|
||||
if crawl_id is None and now_monotonic - last_retention_repair_at >= (60.0 if daemon else 0.0):
|
||||
for model in (ArchiveResult, Snapshot, Crawl, Process):
|
||||
# No runnable work was found on this scheduler pass. This is
|
||||
# the bounded repair point for missing retention deadlines,
|
||||
# including ArchiveResult rows intentionally saved without
|
||||
# delete_at in the plugin-result hot path. Running it here keeps
|
||||
# DELETE_AFTER resolution fresh without making every hook event
|
||||
# load parent Snapshot/Crawl config.
|
||||
model.delete_expired(batch_size=100, backfill_missing=True)
|
||||
last_retention_repair_at = now_monotonic
|
||||
|
||||
if daemon:
|
||||
now_monotonic = time.monotonic()
|
||||
if now_monotonic - last_recovery_at >= 30.0:
|
||||
from archivebox.core.recovery_util import recover_orchestrator_state
|
||||
|
||||
recover_orchestrator_state()
|
||||
last_recovery_at = now_monotonic
|
||||
# SQLite query plans degrade as the snapshot/archiveresult tables grow
|
||||
# past their last ANALYZE — stale stats make the optimizer start large
|
||||
# joins from auth_user/crawl instead of using the url index, blowing the
|
||||
# snapshot detail page out to ~500ms. Refresh stats at most once per
|
||||
# 24hr while the queue is idle, and only after the orchestrator has
|
||||
# been alive for at least an hour so short server boots / one-off work
|
||||
# never pay the cost. The sweep is batched one table per idle tick;
|
||||
# individual table ANALYZE statements abort after 2min (progress
|
||||
# handler) and the whole sweep is hard-capped at 5min so a
|
||||
# pathological table cannot wedge maintenance forever. Any failure
|
||||
# inside the maintenance hook is swallowed — orchestrator must never
|
||||
# be taken down by stats refresh.
|
||||
try:
|
||||
if (
|
||||
analyze_queue is None
|
||||
and now_monotonic - orchestrator_started_at >= 3600.0
|
||||
and now_monotonic - last_analyze_at >= 86400.0
|
||||
):
|
||||
analyze_sweep_started_at = now_monotonic
|
||||
analyze_queue = run_db_analyze_batch(None)
|
||||
elif analyze_queue and now_monotonic - analyze_sweep_started_at >= 300.0:
|
||||
# Sweep blew past the 5min hard cap — abandon what's left
|
||||
# and don't retry until the next 24hr window.
|
||||
analyze_queue = None
|
||||
last_analyze_at = now_monotonic
|
||||
elif analyze_queue:
|
||||
analyze_queue = run_db_analyze_batch(analyze_queue)
|
||||
if analyze_queue is not None and not analyze_queue:
|
||||
analyze_queue = None
|
||||
last_analyze_at = now_monotonic
|
||||
except Exception:
|
||||
analyze_queue = None
|
||||
last_analyze_at = now_monotonic
|
||||
time.sleep(2.0)
|
||||
continue
|
||||
return 0
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,142 +0,0 @@
|
||||
{% extends "admin/base_site.html" %}
|
||||
{% load i18n admin_urls static admin_list %}
|
||||
|
||||
{% block title %}{% if cl.formset and cl.formset.errors %}{% translate "Error:" %} {% endif %}{{ block.super }}{% endblock %}
|
||||
{% block extrastyle %}
|
||||
{{ block.super }}
|
||||
<link rel="stylesheet" href="{% static "admin/css/changelists.css" %}">
|
||||
{% if cl.formset %}
|
||||
<link rel="stylesheet" href="{% static "admin/css/forms.css" %}">
|
||||
{% endif %}
|
||||
{% if cl.formset or action_form %}
|
||||
<script src="{% url 'admin:jsi18n' %}"></script>
|
||||
{% endif %}
|
||||
{{ media.css }}
|
||||
{% if not actions_on_top and not actions_on_bottom %}
|
||||
<style>
|
||||
#changelist table thead th:first-child {width: inherit}
|
||||
#changelist-toolbar-filter-toggle {
|
||||
margin-left: 6px;
|
||||
}
|
||||
</style>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
{% block extrahead %}
|
||||
{{ block.super }}
|
||||
{{ media.js }}
|
||||
{% endblock %}
|
||||
|
||||
{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %}
|
||||
|
||||
{% if not is_popup %}
|
||||
{% block breadcrumbs %}
|
||||
<div class="breadcrumbs">
|
||||
<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
|
||||
› <a href="{% url 'admin:app_list' app_label=cl.opts.app_label %}">{{ cl.opts.app_config.verbose_name }}</a>
|
||||
› {{ cl.opts.verbose_name_plural|capfirst }}
|
||||
</div>
|
||||
{% endblock %}
|
||||
{% endif %}
|
||||
|
||||
{% block coltype %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div id="content-main">
|
||||
{% block object-tools %}
|
||||
<ul class="object-tools">
|
||||
{% block object-tools-items %}
|
||||
{% change_list_object_tools %}
|
||||
{% endblock %}
|
||||
</ul>
|
||||
{% endblock %}
|
||||
{% if cl.formset and cl.formset.errors %}
|
||||
<p class="errornote">
|
||||
{% if cl.formset.total_error_count == 1 %}{% translate "Please correct the error below." %}{% else %}{% translate "Please correct the errors below." %}{% endif %}
|
||||
</p>
|
||||
{{ cl.formset.non_form_errors }}
|
||||
{% endif %}
|
||||
<div class="module{% if cl.has_filters %} filtered{% endif %}" id="changelist">
|
||||
<div class="changelist-form-container">
|
||||
<div>
|
||||
{% block search %}{% search_form cl %}{% endblock %}
|
||||
{% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %}
|
||||
|
||||
<form id="changelist-form" method="post"{% if cl.formset and cl.formset.is_multipart %} enctype="multipart/form-data"{% endif %} novalidate>{% csrf_token %}
|
||||
{% if cl.formset %}
|
||||
<div>{{ cl.formset.management_form }}</div>
|
||||
{% endif %}
|
||||
|
||||
{% block result_list %}
|
||||
{% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %}
|
||||
{% result_list cl %}
|
||||
{% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %}
|
||||
{% endblock %}
|
||||
{% block pagination %}
|
||||
<div class="changelist-footer">
|
||||
{% pagination cl %}
|
||||
{% if cl.formset and cl.result_count %}<input type="submit" name="_save" class="default" value="{% translate 'Save' %}">{% endif %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% if cl.has_filters %}
|
||||
<div id="changelist-filter">
|
||||
<h2>
|
||||
{% translate 'Filter' %}
|
||||
<button
|
||||
type="button"
|
||||
id="changelist-filter-toggle"
|
||||
class="filter-toggle"
|
||||
aria-expanded="true"
|
||||
data-show-label="{% translate 'Filters' %}"
|
||||
data-hide-label="{% translate 'Hide' %}"
|
||||
>
|
||||
{% translate 'Hide' %}
|
||||
</button>
|
||||
</h2>
|
||||
{% if cl.has_active_filters %}<h3 id="changelist-filter-clear">
|
||||
<a href="{{ cl.clear_all_filters_qs }}">✖ {% translate "Clear all filters" %}</a>
|
||||
</h3>{% endif %}
|
||||
{% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% if cl.has_filters %}
|
||||
<script>
|
||||
(function() {
|
||||
var storageKey = 'admin-filters-collapsed';
|
||||
var toggle = document.getElementById('changelist-filter-toggle');
|
||||
var toolbarToggle = document.getElementById('changelist-toolbar-filter-toggle');
|
||||
if (!toggle) return;
|
||||
|
||||
function applyState() {
|
||||
var collapsed = localStorage.getItem(storageKey) === 'true';
|
||||
document.body.classList.toggle('filters-collapsed', collapsed);
|
||||
toggle.textContent = collapsed ? toggle.dataset.showLabel : toggle.dataset.hideLabel;
|
||||
toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true');
|
||||
if (toolbarToggle) {
|
||||
toolbarToggle.textContent = collapsed ? 'Filters ▸' : 'Filters ◂';
|
||||
toolbarToggle.style.display = 'inline-flex';
|
||||
}
|
||||
}
|
||||
|
||||
toggle.addEventListener('click', function() {
|
||||
var collapsed = !document.body.classList.contains('filters-collapsed');
|
||||
localStorage.setItem(storageKey, collapsed ? 'true' : 'false');
|
||||
applyState();
|
||||
});
|
||||
if (toolbarToggle) {
|
||||
toolbarToggle.addEventListener('click', function() {
|
||||
localStorage.setItem(storageKey, 'false');
|
||||
applyState();
|
||||
});
|
||||
}
|
||||
|
||||
applyState();
|
||||
})();
|
||||
</script>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
Loaded 100 of 360 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user