Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nick Sweeting
|
5750f9be76 |
No files matched your search
@@ -2,6 +2,7 @@ name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [dev, main]
|
||||
push:
|
||||
branches: [dev, main]
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -1,50 +1,78 @@
|
||||
name: Deploy Publicsite to GitHub Pages
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev]
|
||||
pull_request:
|
||||
branches:
|
||||
- dev
|
||||
paths:
|
||||
- publicsite/**
|
||||
- archivebox/**
|
||||
- bin/collect_ui_screenshots.*
|
||||
- bin/generate_ui_screenshot_gallery.py
|
||||
- bin/take_screenshot.js
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
- .github/workflows/deploy-publicsite.yml
|
||||
workflow_dispatch:
|
||||
workflow_run:
|
||||
workflows: ['Capture publicsite screenshots']
|
||||
types: [completed]
|
||||
branches: [dev]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# Failed captures must not displace a pending push deployment.
|
||||
group: fast-pages-${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' && github.run_id || github.ref }}
|
||||
cancel-in-progress: false
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.ref || 'dev' }}
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- name: Restore latest complete screenshots
|
||||
run: uv run --no-project python bin/restore_site_screenshots.py archivebox publicsite/screenshots
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
- name: Render current gallery layout without recapturing images
|
||||
run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: publicsite
|
||||
deploy:
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/configure-pages@v5
|
||||
- uses: actions/deploy-pages@v4
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Python and uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: false
|
||||
|
||||
- name: Install ArchiveBox and screenshot dependencies
|
||||
run: uv sync --no-cache --frozen
|
||||
|
||||
- name: Build UI screenshot gallery
|
||||
run: bin/collect_ui_screenshots.sh
|
||||
env:
|
||||
BASE_URL: http://archivebox.localhost:8000
|
||||
|
||||
- name: Verify gallery was generated from this exact revision
|
||||
run: |
|
||||
uv run --no-cache --no-sync python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
|
||||
version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
|
||||
assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
|
||||
assert provenance["version"] == version, provenance
|
||||
assert provenance["capture_count"] >= 3 * 35, provenance
|
||||
assert len(provenance["files"]) == provenance["capture_count"], provenance
|
||||
PY
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: ./publicsite
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
@@ -1,33 +0,0 @@
|
||||
name: Documentation version policy
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [Release]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '17 * * * *'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: documentation-version-policy
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
reconcile:
|
||||
if: github.repository == 'ArchiveBox/ArchiveBox' && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success')
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
ref: dev
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: '0.11.3'
|
||||
- name: Reconcile published versions without removing historical URLs
|
||||
env:
|
||||
READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }}
|
||||
run: uv run --no-project --with 'requests==2.34.2' --with 'packaging==26.2' python bin/sync_docs_versions.py --apply
|
||||
@@ -9,38 +9,6 @@ env:
|
||||
SHOW_PROGRESS: "False"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build Sphinx site and generated API reference
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
- name: Install documentation build dependencies
|
||||
run: uv sync --locked --dev
|
||||
- name: Build documentation from source, including untracked API pages
|
||||
env:
|
||||
READTHEDOCS: "True"
|
||||
READTHEDOCS_VERSION: ${{ github.head_ref || github.ref_name }}
|
||||
READTHEDOCS_GIT_COMMIT_HASH: ${{ github.sha }}
|
||||
run: uv run --no-sync sphinx-build -W --keep-going -b html docs "$RUNNER_TEMP/docs-html"
|
||||
- name: Verify API navigation and Read the Docs version-selector assets
|
||||
run: |
|
||||
test -s "$RUNNER_TEMP/docs-html/apidocs/index.html"
|
||||
test -s "$RUNNER_TEMP/docs-html/apidocs/archivebox/archivebox.core.models.snapshots.html"
|
||||
test -s "$RUNNER_TEMP/docs-html/apidocs/archivebox/archivebox.api.v1_core.snapshots.html"
|
||||
grep -q 'apidocs/index.html' "$RUNNER_TEMP/docs-html/Contents.html"
|
||||
grep -q 'class="version-switch"' "$RUNNER_TEMP/docs-html/index.html"
|
||||
grep -q 'readthedocs-addons-data-ready' "$RUNNER_TEMP/docs-html/_static/js/versions.js"
|
||||
grep -q 'const themeVersionSelector = true' "$RUNNER_TEMP/docs-html/_static/js/versions.js"
|
||||
! grep -q 'Edit on GitHub' "$RUNNER_TEMP/docs-html/apidocs/archivebox/archivebox.core.models.snapshots.html"
|
||||
grep -q "github.com/ArchiveBox/ArchiveBox/blob/$GITHUB_SHA/README.md" "$RUNNER_TEMP/docs-html/README.html"
|
||||
|
||||
discover:
|
||||
name: Discover documentation examples
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
name: Capture publicsite screenshots
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
paths:
|
||||
- publicsite/**
|
||||
- archivebox/**
|
||||
- bin/collect_ui_screenshots.*
|
||||
- bin/generate_ui_screenshot_gallery.py
|
||||
- bin/take_screenshot.js
|
||||
- docs/Screenshots.md
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
- .github/workflows/screenshots.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Python and uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: false
|
||||
|
||||
- name: Install ArchiveBox and screenshot dependencies
|
||||
run: uv sync --no-cache --frozen
|
||||
|
||||
- name: Build UI screenshot gallery
|
||||
run: bin/collect_ui_screenshots.sh
|
||||
env:
|
||||
BASE_URL: http://archivebox.localhost:8000
|
||||
|
||||
- name: Verify gallery was generated from this exact revision
|
||||
run: |
|
||||
uv run --no-cache --no-sync python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
|
||||
version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
|
||||
assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
|
||||
assert provenance["version"] == version, provenance
|
||||
assert provenance["capture_count"] >= 3 * 35, provenance
|
||||
assert len(provenance["files"]) == provenance["capture_count"], provenance
|
||||
PY
|
||||
|
||||
- name: Save validated screenshots
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: site-screenshots
|
||||
path: publicsite/screenshots
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
+2
-2
@@ -10,10 +10,10 @@
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.289 \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.282 \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.289
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.282
|
||||
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
<br/>
|
||||
|
||||
▶️ <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> | <a href="https://demo.archivebox.io">Demo</a> | <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a> | <a href="https://github.com/ArchiveBox/ArchiveBox#background--motivation">Info & Motivation</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a>
|
||||
▶️ <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> | <a href="https://demo.archivebox.io">Demo</a> | <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a> | <a href="#background--motivation">Info & Motivation</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a>
|
||||
|
||||
<br/>
|
||||
|
||||
@@ -152,18 +152,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
|
||||
#### ✳️ Easy Setup
|
||||
|
||||
<details>
|
||||
<summary><b><img style="height: 28px" src="https://archivebox.github.io/ios-archivebox/docs/icons/apple.svg" alt="Apple" align="top"/> <code>ArchiveBox Server.app</code></b> (macOS)</summary>
|
||||
<br/>
|
||||
<ol>
|
||||
<li><a href="https://github.com/ArchiveBox/ios-archivebox/releases">Download <code>ArchiveBox Server.app</code></a>, move it to Applications, and open it.</li>
|
||||
<li>Set up an admin account and generate an API Key.</li>
|
||||
<li>Use the Web UI or <a href="https://archivebox.github.io/ios-archivebox/"><code>ArchiveBox.app</code></a>, and optionally connect REST API clients, the <a href="https://archivebox.github.io/ios-archivebox/">mobile app</a>, the <a href="https://archivebox.github.io/archivebox-browser-extension/">browser extension</a>, and other integrations to your server using your API Key.</li>
|
||||
</ol>
|
||||
<br/>
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker-compose</code></b> (macOS/Linux) <b>👈 recommended</b> <i>(click to expand)</i></summary>
|
||||
<summary><b><img style="height: 28px" src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" align="top"/> <code>docker-compose</code></b> (macOS/Linux) <b>👈 recommended</b> <i>(click to expand)</i></summary>
|
||||
<br/>
|
||||
<i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all <a href="#dependencies">extras</a> out-of-the-box.</i>
|
||||
<br/><br/>
|
||||
@@ -187,7 +176,7 @@ docker compose exec archivebox archivebox help
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">filesystem/SQL/Python</a> to manage your archive.
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">filesystem/SQL/Python</a> to manage your archive.
|
||||
<br/><br/>
|
||||
</details>
|
||||
|
||||
@@ -211,7 +200,7 @@ docker exec archivebox archivebox help
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
|
||||
<br/><br/>
|
||||
</details>
|
||||
|
||||
@@ -225,7 +214,7 @@ See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
See <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/bin/setup.sh"><code>setup.sh</code></a> for the source code of the <code>uv</code> install shortcut.<br/>
|
||||
See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as an install method"</a> blog post for my thoughts on the shortcomings of this install method.
|
||||
<br/><br/>
|
||||
@@ -265,7 +254,7 @@ archivebox help
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
<br/>
|
||||
<sub>See the <a href="https://docs.astral.sh/uv/guides/tools/"><code>uv tool</code> documentation</a> for more details about this installation method.</sub>
|
||||
<br/><br/>
|
||||
@@ -300,7 +289,7 @@ archivebox help
|
||||
</code></pre>
|
||||
</li>
|
||||
</ol>
|
||||
See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. <code>sudo archivebox install</code> uses apt for missing system dependencies while preserving ownership of the user-owned collection. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details.</sub>
|
||||
<br/><br/>
|
||||
</details>
|
||||
@@ -334,7 +323,7 @@ archivebox help
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
|
||||
<sub>See the <a href="https://github.com/ArchiveBox/homebrew-archivebox"><code>homebrew-archivebox</code></a> repo for more details about this distribution.</sub>
|
||||
<br/><br/>
|
||||
</details>
|
||||
@@ -352,7 +341,7 @@ See <a href="#usage">below</a> for more usage examples using the CLI, Web UI, or
|
||||
<li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
|
||||
<li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
|
||||
</ul>
|
||||
See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
|
||||
<br/><br/>
|
||||
</details>
|
||||
|
||||
@@ -399,7 +388,7 @@ See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or file
|
||||
|
||||
<li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
|
||||
</ul>
|
||||
See <a href="#usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
|
||||
See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.
|
||||
<br/><br/>
|
||||
</details>
|
||||
|
||||
@@ -437,7 +426,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
|
||||
#### ➡️ Next Steps
|
||||
|
||||
- Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
|
||||
- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile).
|
||||
- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
|
||||
- Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
|
||||
- Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
|
||||
- Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
|
||||
@@ -572,20 +561,6 @@ docker compose run --rm archivebox config --set PERMISSIONS=public
|
||||
</code></pre>
|
||||
</details>
|
||||
|
||||
<br/>
|
||||
|
||||
<details>
|
||||
<summary><b><img style="height: 22px" src="https://archivebox.github.io/ios-archivebox/docs/icons/apple.svg" alt="Apple" align="top"/> <img style="height: 22px" src="https://cdn.jsdelivr.net/gh/devicons/devicon@v2.17.0/icons/windows11/windows11-original.svg" alt="Windows" align="top"/> 🐧 Desktop / Mobile / Extension Usage</b></summary>
|
||||
<br/>
|
||||
|
||||
- 📱 [<code>ArchiveBox</code> iOS Mobile App](https://archivebox.github.io/ios-archivebox/)
|
||||
- 🖥️ [<code>ArchiveBox.app</code> macOS Desktop App](https://archivebox.github.io/ios-archivebox/)
|
||||
- 🧩 [ArchiveBox Browser Extension](https://archivebox.github.io/archivebox-browser-extension/)
|
||||
- 💻 [Linux/Windows Desktop App (Requires Docker Desktop)](https://github.com/ArchiveBox/electron-archivebox)
|
||||
|
||||
<br/>
|
||||
</details>
|
||||
|
||||
<br/>
|
||||
<br/>
|
||||
|
||||
@@ -617,7 +592,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
|
||||
<sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
|
||||
<br/><br/>
|
||||
<a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
|
||||
<br/>
|
||||
</div>
|
||||
|
||||
@@ -638,11 +613,8 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
|
||||
## Input Formats: How to pass URLs into ArchiveBox for saving
|
||||
|
||||
|
||||
- <img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" /> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b>
|
||||
<i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox/Safari browsers.</i>
|
||||
|
||||
- <img style="height: 28px" src="https://archivebox.github.io/ios-archivebox/App/AppIcon.icon/Assets/ArchiveBox.png" alt="ArchiveBox app icon" /> <b>From the official <a href="https://archivebox.github.io/ios-archivebox/">ArchiveBox macOS Desktop App or iOS Mobile App</a></b><br/>
|
||||
<i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
|
||||
- <img style="height: 28px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e"/> <b>From the official <a href="https://github.com/ArchiveBox/archivebox-extension">ArchiveBox Browser Extension</a></b><br>
|
||||
<i>Provides realtime archiving of browsing history or selected pages from Chrome/Chromium/Firefox browsers.</i>
|
||||
|
||||
- <img style="height: 22px" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files<br>
|
||||
<i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from urllib.parse import quote
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
|
||||
@@ -7,7 +7,6 @@ from django.core import signing
|
||||
from django.http import Http404, HttpRequest, HttpResponseForbidden
|
||||
from django.shortcuts import redirect
|
||||
from django.views import View
|
||||
from django.utils.http import url_has_allowed_host_and_scheme
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import (
|
||||
@@ -42,7 +41,8 @@ def _replay_cookie_name(snapshot: Snapshot) -> str:
|
||||
def _clean_replay_next(path: str | None) -> str:
|
||||
"""Only allow same-snap relative replay paths; grants must never redirect off-host."""
|
||||
path = f"/{(path or 'index.html').lstrip('/')}"
|
||||
if not url_has_allowed_host_and_scheme(path, allowed_hosts=set()):
|
||||
parsed = urlparse(path)
|
||||
if parsed.scheme or parsed.netloc or path.startswith("//"):
|
||||
return "/index.html"
|
||||
return path
|
||||
|
||||
|
||||
@@ -130,13 +130,10 @@ class _ZipBuffer(io.RawIOBase):
|
||||
def _iter_visible_files(root: Path):
|
||||
"""Yield non-hidden files in a stable order so ZIP output is deterministic."""
|
||||
|
||||
resolved_root = root.resolve()
|
||||
for current_root, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = sorted(dirname for dirname in dirnames if not dirname.startswith("."))
|
||||
for filename in sorted(name for name in filenames if not name.startswith(".")):
|
||||
entry = Path(current_root) / filename
|
||||
if entry.resolve().is_relative_to(resolved_root):
|
||||
yield entry
|
||||
yield Path(current_root) / filename
|
||||
|
||||
|
||||
def _iter_directory_zip(fullpath: Path, root_name: str):
|
||||
@@ -372,8 +369,6 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
if config is None:
|
||||
config = get_config(resolve_plugins=False)
|
||||
fullpath, path = _resolve_archive_path(document_root, path)
|
||||
if not fullpath.resolve().is_relative_to(Path(document_root).resolve()):
|
||||
raise Http404(_("Path is outside the archive directory."))
|
||||
replay_response = partial(_apply_archive_replay_headers, fullpath=fullpath, is_archive_replay=is_archive_replay, config=config)
|
||||
if os.access(fullpath, os.R_OK) and fullpath.is_dir():
|
||||
if request.GET.get("download") == "zip" and show_indexes:
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
__package__ = "archivebox.progressmonitor"
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import DatabaseError
|
||||
from django.http import HttpResponse, JsonResponse
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.core.permissions import can_view_snapshot, is_admin_user
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_id: object | None = None) -> str:
|
||||
"""Return the canonical same-origin progress endpoint for monitor embeds."""
|
||||
@@ -61,10 +59,9 @@ def live_progress_view(request):
|
||||
return HttpResponse(ujson.dumps(payload), content_type="application/json")
|
||||
except ImportError:
|
||||
return JsonResponse(payload)
|
||||
except (DatabaseError, OSError, RuntimeError, TypeError, ValueError):
|
||||
logger.exception("Unable to load progress")
|
||||
except (DatabaseError, OSError, RuntimeError, TypeError, ValueError) as e:
|
||||
error_payload = {
|
||||
"error": "Unable to load progress",
|
||||
"error": str(e),
|
||||
"orchestrator_running": False,
|
||||
"total_workers": 0,
|
||||
"crawls_active": 0,
|
||||
@@ -81,4 +78,8 @@ def live_progress_view(request):
|
||||
"active_crawls": [],
|
||||
"server_time": timezone.now().isoformat(),
|
||||
}
|
||||
if settings.DEBUG:
|
||||
import traceback
|
||||
|
||||
error_payload["traceback"] = traceback.format_exc()
|
||||
return JsonResponse(error_payload, status=500)
|
||||
@@ -1246,36 +1246,20 @@ body.change-list #changelist .actions .actions-tags-with-buttons[hidden] {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
/* The native select receives the actual click, including in WKWebView. */
|
||||
body.change-list #changelist .actions .action-rearchive-select,
|
||||
body.change-list #changelist .actions .action-permissions-select {
|
||||
.action-rearchive-select,
|
||||
.action-permissions-select {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
left: 0;
|
||||
bottom: 0;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
opacity: 0;
|
||||
cursor: pointer;
|
||||
z-index: 1;
|
||||
pointer-events: none;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
body.change-list #changelist .action-rearchive-wrapper > .button,
|
||||
body.change-list #changelist .action-permissions-wrapper > .button {
|
||||
padding: 8px 16px !important;
|
||||
width: max-content;
|
||||
min-width: max-content;
|
||||
flex: 0 0 auto;
|
||||
gap: 6px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.action-rearchive-wrapper:focus-within,
|
||||
.action-permissions-wrapper:focus-within {
|
||||
outline: 2px solid #2563eb;
|
||||
outline-offset: 2px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
/* Object count */
|
||||
.actions .action-counter {
|
||||
color: #64748b;
|
||||
|
||||
@@ -90,10 +90,22 @@ function fix_actions() {
|
||||
.addClass('action-rearchive-select')
|
||||
.attr('aria-label', 'Re-Archive')
|
||||
.appendTo(wrapper)
|
||||
$('<span>')
|
||||
.attr('aria-hidden', 'true')
|
||||
$('<button>')
|
||||
.attr('type', 'button')
|
||||
.addClass('button')
|
||||
.html('Re-Archive <span aria-hidden="true">▾</span>')
|
||||
.click(function (e) {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
const el = select[0]
|
||||
if (typeof el.showPicker === 'function') {
|
||||
try { el.showPicker() } catch (err) { el.focus(); el.click() }
|
||||
} else {
|
||||
el.focus()
|
||||
el.click()
|
||||
}
|
||||
return false
|
||||
})
|
||||
.appendTo(wrapper)
|
||||
$('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
|
||||
rearchiveActions.forEach(function (action_value) {
|
||||
@@ -139,11 +151,23 @@ function fix_actions() {
|
||||
.addClass('action-permissions-select')
|
||||
.attr('aria-label', 'Permissions')
|
||||
.appendTo(wrapper)
|
||||
$('<span>')
|
||||
.attr('aria-hidden', 'true')
|
||||
$('<button>')
|
||||
.attr('type', 'button')
|
||||
.attr('name', action_type)
|
||||
.addClass('button')
|
||||
.text(this.text)
|
||||
.click(function (e) {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
const el = select[0]
|
||||
if (typeof el.showPicker === 'function') {
|
||||
try { el.showPicker() } catch (err) { el.focus(); el.click() }
|
||||
} else {
|
||||
el.focus()
|
||||
el.click()
|
||||
}
|
||||
return false
|
||||
})
|
||||
.appendTo(wrapper)
|
||||
$('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
|
||||
;[
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import json
|
||||
from functools import partial
|
||||
import re
|
||||
import secrets
|
||||
import signal
|
||||
@@ -449,6 +450,12 @@ def initialized_archive(tmp_path):
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def archivebox_cli(initialized_archive):
|
||||
"""Public CLI bound to this test's collection with extraction opt-in."""
|
||||
return partial(run_archivebox_cmd, cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recursive_test_site():
|
||||
pages = {
|
||||
@@ -1920,94 +1927,3 @@ def install_real_chrome(data_dir, env, *, isolation):
|
||||
timeout=600,
|
||||
)
|
||||
assert install_process.returncode == 0, install_process.stderr or install_process.stdout
|
||||
|
||||
|
||||
def write_import_format_files(base_dir: Path, urls: dict[str, str] | None = None) -> dict[str, Path]:
|
||||
urls = {
|
||||
"rss": "https://example.com/",
|
||||
"netscape": "https://www.iana.org/domains/reserved",
|
||||
"dom": "https://www.iana.org/help/example-domains",
|
||||
"json": "https://example.com/?archivebox-json-import=1",
|
||||
"jsonl": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"txt": "https://example.org/",
|
||||
**(urls or {}),
|
||||
}
|
||||
files = {
|
||||
"rss": base_dir / "test_rss.xml",
|
||||
"netscape": base_dir / "test_netscape.html",
|
||||
"dom": base_dir / "test_dom.html",
|
||||
"json": base_dir / "test_bookmarks.json",
|
||||
"jsonl": base_dir / "test_bookmarks.jsonl",
|
||||
"txt": base_dir / "test_urls.txt",
|
||||
}
|
||||
files["rss"].write_text(
|
||||
f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>ArchiveBox RSS import fixture</title>
|
||||
<link>{urls["rss"]}</link>
|
||||
<description>ArchiveBox RSS import fixture</description>
|
||||
<item>
|
||||
<title>RSS Example Import</title>
|
||||
<link>{urls["rss"]}</link>
|
||||
<guid>{urls["rss"]}</guid>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<category>rss-tag</category>
|
||||
<category>metadata</category>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["netscape"].write_text(
|
||||
f"""<!DOCTYPE NETSCAPE-Bookmark-file-1>
|
||||
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
|
||||
<TITLE>Bookmarks</TITLE>
|
||||
<H1>Bookmarks</H1>
|
||||
<DL><p>
|
||||
<DT><A HREF="{urls["netscape"]}" ADD_DATE="1704153600" TAGS="netscape-tag,metadata">IANA Reserved Domains</A>
|
||||
</DL><p>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["dom"].write_text(
|
||||
f"""<!doctype html>
|
||||
<html>
|
||||
<head><title>DOM import fixture</title></head>
|
||||
<body>
|
||||
<a href="{urls["dom"]}">IANA Example Domains</a>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["json"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"url": urls["json"],
|
||||
"title": "JSON Import Example",
|
||||
"tags": ["json-tag", "metadata"],
|
||||
"bookmarked_at": "2024-01-03T00:00:00+00:00",
|
||||
},
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["jsonl"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"url": urls["jsonl"],
|
||||
"title": "JSONL Import Example",
|
||||
"tags": "jsonl-tag,metadata",
|
||||
"bookmarked_at": "2024-01-04T00:00:00+00:00",
|
||||
},
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["txt"].write_text(
|
||||
f"Plain text import fixture containing {urls['txt']} as a real live URL.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return files
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Shared real import files and assertions for CLI, REST, and web add flows."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from archivebox.core.models import Snapshot, SnapshotTag
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
IMPORT_FORMAT_EXPECTATIONS = {
|
||||
"rss": {
|
||||
"url": "https://example.com/",
|
||||
"title": "RSS Example Import",
|
||||
"date": "2024-01-01",
|
||||
"tags": {"rss-tag", "metadata"},
|
||||
},
|
||||
"netscape": {
|
||||
"url": "https://www.iana.org/domains/reserved",
|
||||
"title": "IANA Reserved Domains",
|
||||
"date": "2024-01-02",
|
||||
"tags": {"netscape-tag", "metadata"},
|
||||
},
|
||||
"dom": {
|
||||
"url": "https://www.iana.org/help/example-domains",
|
||||
},
|
||||
"json": {
|
||||
"url": "https://example.com/?archivebox-json-import=1",
|
||||
"title": "JSON Import Example",
|
||||
"date": "2024-01-03",
|
||||
"tags": {"json-tag", "metadata"},
|
||||
},
|
||||
"jsonl": {
|
||||
"url": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"title": "JSONL Import Example",
|
||||
"date": "2024-01-04",
|
||||
"tags": {"jsonl-tag", "metadata"},
|
||||
},
|
||||
"txt": {
|
||||
"url": "https://example.org/",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def assert_expected_import_snapshots(
|
||||
cwd: Path,
|
||||
expected_urls: set[str],
|
||||
*,
|
||||
expected_tags: set[str] | None = None,
|
||||
) -> None:
|
||||
allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
with use_archivebox_db(cwd):
|
||||
snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("id", "url", "status"))
|
||||
tag_names_by_snapshot_id = {}
|
||||
if expected_tags and snapshots:
|
||||
for snapshot_id, tag_name in SnapshotTag.objects.filter(
|
||||
snapshot_id__in=[snapshot["id"] for snapshot in snapshots],
|
||||
).values_list("snapshot_id", "tag__name"):
|
||||
tag_names_by_snapshot_id.setdefault(snapshot_id, set()).add(tag_name)
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
bad_statuses = []
|
||||
missing_tags = {}
|
||||
for snapshot in snapshots:
|
||||
counts[snapshot["url"]] += 1
|
||||
if snapshot["status"] not in allowed_statuses:
|
||||
bad_statuses.append((snapshot["url"], snapshot["status"]))
|
||||
if expected_tags:
|
||||
tag_names = tag_names_by_snapshot_id.get(snapshot["id"], set())
|
||||
missing = expected_tags - tag_names
|
||||
if missing:
|
||||
missing_tags[snapshot["url"]] = missing
|
||||
assert all(count == 1 for count in counts.values()), counts
|
||||
assert not bad_statuses, bad_statuses
|
||||
assert not missing_tags, missing_tags
|
||||
|
||||
|
||||
def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> None:
|
||||
with use_archivebox_db(cwd):
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
assert not canary.exists()
|
||||
assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
|
||||
for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
|
||||
assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
|
||||
|
||||
|
||||
def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
|
||||
other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
|
||||
other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
|
||||
other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
|
||||
canary = tmp_path / "archivebox_shell_injection_canary"
|
||||
return (
|
||||
[
|
||||
safe_url,
|
||||
"file:///etc/hosts",
|
||||
"/etc/hosts",
|
||||
"../../../../etc/passwd",
|
||||
f"file://{other_crawl_source}",
|
||||
str(other_crawl_source),
|
||||
f"'; touch {canary}; #",
|
||||
f'" && touch {canary} && echo "',
|
||||
f"$(touch {canary})",
|
||||
f"`touch {canary}`",
|
||||
"""<?xml version="1.0"?>
|
||||
<!DOCTYPE rss [
|
||||
<!ENTITY localfile SYSTEM "file:///etc/hosts">
|
||||
]>
|
||||
<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<channel>
|
||||
<item><title>&localfile;</title><link>file:///etc/passwd</link></item>
|
||||
<xi:include href="file:///etc/hosts" parse="text"/>
|
||||
</channel>
|
||||
</rss>""",
|
||||
],
|
||||
canary,
|
||||
)
|
||||
|
||||
|
||||
def write_import_format_files(base_dir: Path, urls: dict[str, str] | None = None) -> dict[str, Path]:
|
||||
urls = {
|
||||
"rss": "https://example.com/",
|
||||
"netscape": "https://www.iana.org/domains/reserved",
|
||||
"dom": "https://www.iana.org/help/example-domains",
|
||||
"json": "https://example.com/?archivebox-json-import=1",
|
||||
"jsonl": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"txt": "https://example.org/",
|
||||
**(urls or {}),
|
||||
}
|
||||
files = {
|
||||
"rss": base_dir / "test_rss.xml",
|
||||
"netscape": base_dir / "test_netscape.html",
|
||||
"dom": base_dir / "test_dom.html",
|
||||
"json": base_dir / "test_bookmarks.json",
|
||||
"jsonl": base_dir / "test_bookmarks.jsonl",
|
||||
"txt": base_dir / "test_urls.txt",
|
||||
}
|
||||
files["rss"].write_text(
|
||||
f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>ArchiveBox RSS import fixture</title>
|
||||
<link>{urls["rss"]}</link>
|
||||
<description>ArchiveBox RSS import fixture</description>
|
||||
<item>
|
||||
<title>RSS Example Import</title>
|
||||
<link>{urls["rss"]}</link>
|
||||
<guid>{urls["rss"]}</guid>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<category>rss-tag</category>
|
||||
<category>metadata</category>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["netscape"].write_text(
|
||||
f"""<!DOCTYPE NETSCAPE-Bookmark-file-1>
|
||||
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
|
||||
<TITLE>Bookmarks</TITLE>
|
||||
<H1>Bookmarks</H1>
|
||||
<DL><p>
|
||||
<DT><A HREF="{urls["netscape"]}" ADD_DATE="1704153600" TAGS="netscape-tag,metadata">IANA Reserved Domains</A>
|
||||
</DL><p>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["dom"].write_text(
|
||||
f"""<!doctype html>
|
||||
<html>
|
||||
<head><title>DOM import fixture</title></head>
|
||||
<body>
|
||||
<a href="{urls["dom"]}">IANA Example Domains</a>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["json"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"url": urls["json"],
|
||||
"title": "JSON Import Example",
|
||||
"tags": ["json-tag", "metadata"],
|
||||
"bookmarked_at": "2024-01-03T00:00:00+00:00",
|
||||
},
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["jsonl"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"url": urls["jsonl"],
|
||||
"title": "JSONL Import Example",
|
||||
"tags": "jsonl-tag,metadata",
|
||||
"bookmarked_at": "2024-01-04T00:00:00+00:00",
|
||||
},
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["txt"].write_text(
|
||||
f"Plain text import fixture containing {urls['txt']} as a real live URL.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return files
|
||||
@@ -23,32 +23,8 @@ from archivebox.uuid_compat import uuid7
|
||||
# Schema Definitions for Each Version
|
||||
# =============================================================================
|
||||
|
||||
SCHEMA_0_4 = """
|
||||
-- Django system tables (minimal)
|
||||
CREATE TABLE IF NOT EXISTS django_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
applied DATETIME NOT NULL
|
||||
);
|
||||
|
||||
-- Core tables for 0.4.x
|
||||
CREATE TABLE IF NOT EXISTS core_snapshot (
|
||||
id CHAR(32) PRIMARY KEY,
|
||||
url VARCHAR(200) NOT NULL UNIQUE,
|
||||
timestamp VARCHAR(32) UNIQUE,
|
||||
title VARCHAR(128),
|
||||
tags VARCHAR(256),
|
||||
added DATETIME NOT NULL,
|
||||
updated DATETIME
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS core_snapshot_url ON core_snapshot(url);
|
||||
CREATE INDEX IF NOT EXISTS core_snapshot_timestamp ON core_snapshot(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS core_snapshot_added ON core_snapshot(added);
|
||||
"""
|
||||
|
||||
SCHEMA_0_7 = """
|
||||
-- Django system tables (complete for 0.7.x)
|
||||
# Django auth and admin tables were unchanged between the 0.7 and 0.8 fixtures.
|
||||
DJANGO_AUTH_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS django_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app VARCHAR(255) NOT NULL,
|
||||
@@ -128,7 +104,35 @@ CREATE TABLE IF NOT EXISTS django_session (
|
||||
expire_date DATETIME NOT NULL
|
||||
);
|
||||
|
||||
-- Core tables for 0.7.x
|
||||
"""
|
||||
|
||||
SCHEMA_0_4 = """
|
||||
-- Django system tables (minimal)
|
||||
CREATE TABLE IF NOT EXISTS django_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
applied DATETIME NOT NULL
|
||||
);
|
||||
|
||||
-- Core tables for 0.4.x
|
||||
CREATE TABLE IF NOT EXISTS core_snapshot (
|
||||
id CHAR(32) PRIMARY KEY,
|
||||
url VARCHAR(200) NOT NULL UNIQUE,
|
||||
timestamp VARCHAR(32) UNIQUE,
|
||||
title VARCHAR(128),
|
||||
tags VARCHAR(256),
|
||||
added DATETIME NOT NULL,
|
||||
updated DATETIME
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS core_snapshot_url ON core_snapshot(url);
|
||||
CREATE INDEX IF NOT EXISTS core_snapshot_timestamp ON core_snapshot(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS core_snapshot_added ON core_snapshot(added);
|
||||
"""
|
||||
|
||||
SCHEMA_0_7 = (
|
||||
DJANGO_AUTH_SCHEMA
|
||||
+ """-- Core tables for 0.7.x
|
||||
CREATE TABLE IF NOT EXISTS core_tag (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(100) NOT NULL UNIQUE,
|
||||
@@ -183,89 +187,11 @@ INSERT INTO django_content_type (app_label, model) VALUES
|
||||
('core', 'archiveresult'),
|
||||
('core', 'tag');
|
||||
"""
|
||||
)
|
||||
|
||||
SCHEMA_0_8 = """
|
||||
-- Django system tables (complete for 0.8.x)
|
||||
CREATE TABLE IF NOT EXISTS django_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
applied DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS django_content_type (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app_label VARCHAR(100) NOT NULL,
|
||||
model VARCHAR(100) NOT NULL,
|
||||
UNIQUE(app_label, model)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_permission (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
content_type_id INTEGER NOT NULL REFERENCES django_content_type(id),
|
||||
codename VARCHAR(100) NOT NULL,
|
||||
UNIQUE(content_type_id, codename)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_group (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name VARCHAR(150) NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_group_permissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
group_id INTEGER NOT NULL REFERENCES auth_group(id),
|
||||
permission_id INTEGER NOT NULL REFERENCES auth_permission(id),
|
||||
UNIQUE(group_id, permission_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_user (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
password VARCHAR(128) NOT NULL,
|
||||
last_login DATETIME,
|
||||
is_superuser BOOL NOT NULL,
|
||||
username VARCHAR(150) NOT NULL UNIQUE,
|
||||
first_name VARCHAR(150) NOT NULL,
|
||||
last_name VARCHAR(150) NOT NULL,
|
||||
email VARCHAR(254) NOT NULL,
|
||||
is_staff BOOL NOT NULL,
|
||||
is_active BOOL NOT NULL,
|
||||
date_joined DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_user_groups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES auth_user(id),
|
||||
group_id INTEGER NOT NULL REFERENCES auth_group(id),
|
||||
UNIQUE(user_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auth_user_user_permissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES auth_user(id),
|
||||
permission_id INTEGER NOT NULL REFERENCES auth_permission(id),
|
||||
UNIQUE(user_id, permission_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS django_admin_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
action_time DATETIME NOT NULL,
|
||||
object_id TEXT,
|
||||
object_repr VARCHAR(200) NOT NULL,
|
||||
action_flag SMALLINT UNSIGNED NOT NULL,
|
||||
change_message TEXT NOT NULL,
|
||||
content_type_id INTEGER REFERENCES django_content_type(id),
|
||||
user_id INTEGER NOT NULL REFERENCES auth_user(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS django_session (
|
||||
session_key VARCHAR(40) NOT NULL PRIMARY KEY,
|
||||
session_data TEXT NOT NULL,
|
||||
expire_date DATETIME NOT NULL
|
||||
);
|
||||
|
||||
-- Machine app tables (added in 0.8.x)
|
||||
SCHEMA_0_8 = (
|
||||
DJANGO_AUTH_SCHEMA
|
||||
+ """-- Machine app tables (added in 0.8.x)
|
||||
CREATE TABLE IF NOT EXISTS machine_machine (
|
||||
id CHAR(36) PRIMARY KEY,
|
||||
created_at DATETIME NOT NULL,
|
||||
@@ -501,6 +427,7 @@ INSERT INTO django_content_type (app_label, model) VALUES
|
||||
('api', 'apitoken'),
|
||||
('api', 'outboundwebhook');
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -748,482 +675,302 @@ def seed_0_4_data(db_path: Path) -> dict[str, list[dict]]:
|
||||
return created_data
|
||||
|
||||
|
||||
def seed_0_7_data(db_path: Path) -> dict[str, list[dict]]:
|
||||
"""Seed a 0.7.x database with realistic test data."""
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
MIGRATIONS_0_7 = [
|
||||
("contenttypes", "0001_initial"),
|
||||
("contenttypes", "0002_remove_content_type_name"),
|
||||
("auth", "0001_initial"),
|
||||
("auth", "0002_alter_permission_name_max_length"),
|
||||
("auth", "0003_alter_user_email_max_length"),
|
||||
("auth", "0004_alter_user_username_opts"),
|
||||
("auth", "0005_alter_user_last_login_null"),
|
||||
("auth", "0006_require_contenttypes_0002"),
|
||||
("auth", "0007_alter_validators_add_error_messages"),
|
||||
("auth", "0008_alter_user_username_max_length"),
|
||||
("auth", "0009_alter_user_last_name_max_length"),
|
||||
("auth", "0010_alter_group_name_max_length"),
|
||||
("auth", "0011_update_proxy_permissions"),
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
("admin", "0001_initial"),
|
||||
("admin", "0002_logentry_remove_auto_add"),
|
||||
("admin", "0003_logentry_add_action_flag_choices"),
|
||||
("sessions", "0001_initial"),
|
||||
("core", "0001_initial"),
|
||||
("core", "0002_auto_20200625_1521"),
|
||||
("core", "0003_auto_20200630_1034"),
|
||||
("core", "0004_auto_20200713_1552"),
|
||||
("core", "0005_auto_20200728_0326"),
|
||||
("core", "0006_auto_20201012_1520"),
|
||||
("core", "0007_archiveresult"),
|
||||
("core", "0008_auto_20210105_1421"),
|
||||
("core", "0009_auto_20210216_1038"),
|
||||
("core", "0010_auto_20210216_1055"),
|
||||
("core", "0011_auto_20210216_1331"),
|
||||
("core", "0012_auto_20210216_1425"),
|
||||
("core", "0013_auto_20210218_0729"),
|
||||
("core", "0014_auto_20210218_0729"),
|
||||
("core", "0015_auto_20210218_0730"),
|
||||
("core", "0016_auto_20210218_1204"),
|
||||
("core", "0017_auto_20210219_0211"),
|
||||
("core", "0018_auto_20210327_0952"),
|
||||
("core", "0019_auto_20210401_0654"),
|
||||
("core", "0020_auto_20210410_1031"),
|
||||
("core", "0021_auto_20220914_0934"),
|
||||
("core", "0022_auto_20231023_2008"),
|
||||
]
|
||||
|
||||
created_data = {
|
||||
"users": [],
|
||||
"snapshots": [],
|
||||
"tags": [],
|
||||
"archiveresults": [],
|
||||
}
|
||||
MIGRATIONS_0_8 = MIGRATIONS_0_7 + [
|
||||
# For 0.8.x (dev branch), record the migrations that 0023_new_schema replaces
|
||||
("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
|
||||
("core", "0024_auto_20240513_1143"),
|
||||
("core", "0025_alter_archiveresult_uuid"),
|
||||
("core", "0026_archiveresult_created_archiveresult_created_by_and_more"),
|
||||
("core", "0027_update_snapshot_ids"),
|
||||
("core", "0028_alter_archiveresult_uuid"),
|
||||
("core", "0029_alter_archiveresult_id"),
|
||||
("core", "0030_alter_archiveresult_uuid"),
|
||||
("core", "0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more"),
|
||||
("core", "0032_alter_archiveresult_id"),
|
||||
("core", "0033_rename_id_archiveresult_old_id"),
|
||||
("core", "0034_alter_archiveresult_old_id_alter_archiveresult_uuid"),
|
||||
("core", "0035_remove_archiveresult_uuid_archiveresult_id"),
|
||||
("core", "0036_alter_archiveresult_id_alter_archiveresult_old_id"),
|
||||
("core", "0037_rename_id_snapshot_old_id"),
|
||||
("core", "0038_rename_uuid_snapshot_id"),
|
||||
("core", "0039_rename_snapshot_archiveresult_snapshot_old"),
|
||||
("core", "0040_archiveresult_snapshot"),
|
||||
("core", "0041_alter_archiveresult_snapshot_and_more"),
|
||||
("core", "0042_remove_archiveresult_snapshot_old"),
|
||||
("core", "0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more"),
|
||||
("core", "0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more"),
|
||||
("core", "0045_alter_snapshot_old_id"),
|
||||
("core", "0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more"),
|
||||
("core", "0047_alter_snapshottag_unique_together_and_more"),
|
||||
("core", "0048_alter_archiveresult_snapshot_and_more"),
|
||||
("core", "0049_rename_snapshot_snapshottag_snapshot_old_and_more"),
|
||||
("core", "0050_alter_snapshottag_snapshot_old"),
|
||||
("core", "0051_snapshottag_snapshot_alter_snapshottag_snapshot_old"),
|
||||
("core", "0052_alter_snapshottag_unique_together_and_more"),
|
||||
("core", "0053_remove_snapshottag_snapshot_old"),
|
||||
("core", "0054_alter_snapshot_timestamp"),
|
||||
("core", "0055_alter_tag_slug"),
|
||||
("core", "0056_remove_tag_uuid"),
|
||||
("core", "0057_rename_id_tag_old_id"),
|
||||
("core", "0058_alter_tag_old_id"),
|
||||
("core", "0059_tag_id"),
|
||||
("core", "0060_alter_tag_id"),
|
||||
("core", "0061_rename_tag_snapshottag_old_tag_and_more"),
|
||||
("core", "0062_alter_snapshottag_old_tag"),
|
||||
("core", "0063_snapshottag_tag_alter_snapshottag_old_tag"),
|
||||
("core", "0064_alter_snapshottag_unique_together_and_more"),
|
||||
("core", "0065_remove_snapshottag_old_tag"),
|
||||
("core", "0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id"),
|
||||
("core", "0067_alter_snapshottag_tag"),
|
||||
("core", "0068_alter_archiveresult_options"),
|
||||
("core", "0069_alter_archiveresult_created_alter_snapshot_added_and_more"),
|
||||
("core", "0070_alter_archiveresult_created_by_alter_snapshot_added_and_more"),
|
||||
("core", "0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more"),
|
||||
("core", "0072_rename_added_snapshot_bookmarked_at_and_more"),
|
||||
("core", "0073_rename_created_archiveresult_created_at_and_more"),
|
||||
("core", "0074_alter_snapshot_downloaded_at"),
|
||||
# For 0.8.x: DO NOT record 0023_new_schema - it replaces 0023-0074 for fresh installs
|
||||
# We already recorded 0023-0074 above, so Django will know the state
|
||||
# For 0.8.x: Record original machine migrations (before squashing)
|
||||
# DO NOT record 0001_squashed here - it replaces 0001-0004 for fresh installs
|
||||
("machine", "0001_initial"),
|
||||
("machine", "0002_alter_machine_stats_installedbinary"),
|
||||
("machine", "0003_alter_installedbinary_options_and_more"),
|
||||
("machine", "0004_alter_installedbinary_abspath_and_more"),
|
||||
# Then the new migrations after squashing
|
||||
("machine", "0003_alter_dependency_id_alter_installedbinary_dependency_and_more"),
|
||||
("machine", "0004_drop_dependency_table"),
|
||||
# Crawls must come before core.0024 because 0024_b depends on it
|
||||
("crawls", "0001_initial"),
|
||||
# Core 0024 migrations chain (in dependency order)
|
||||
("core", "0024_b_clear_config_fields"),
|
||||
("core", "0024_c_disable_fk_checks"),
|
||||
("core", "0024_d_fix_crawls_config"),
|
||||
("core", "0024_snapshot_crawl"),
|
||||
("core", "0024_f_add_snapshot_config"),
|
||||
("core", "0025_allow_duplicate_urls_per_crawl"),
|
||||
# For 0.8.x: Record original api migration (before squashing)
|
||||
# DO NOT record 0001_squashed here - it replaces 0001 for fresh installs
|
||||
("api", "0001_initial"),
|
||||
("api", "0002_alter_apitoken_options"),
|
||||
("api", "0003_rename_user_apitoken_created_by_apitoken_abid_and_more"),
|
||||
("api", "0004_alter_apitoken_id_alter_apitoken_uuid"),
|
||||
("api", "0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more"),
|
||||
("api", "0006_remove_outboundwebhook_uuid_apitoken_id_and_more"),
|
||||
("api", "0007_alter_apitoken_created_by"),
|
||||
("api", "0008_alter_apitoken_created_alter_apitoken_created_by_and_more"),
|
||||
("api", "0009_rename_created_apitoken_created_at_and_more"),
|
||||
# Note: crawls.0001_initial moved earlier (before core.0024) due to dependencies
|
||||
# Stop here - 0.8.x ends at core.0025, crawls.0001, and we want to TEST the later migrations
|
||||
# Do NOT record 0026+ as they need to be tested during migration
|
||||
]
|
||||
|
||||
# Create a user
|
||||
cursor.execute("""
|
||||
INSERT INTO auth_user (password, is_superuser, username, first_name, last_name,
|
||||
email, is_staff, is_active, date_joined)
|
||||
VALUES ('pbkdf2_sha256$test', 1, 'admin', 'Admin', 'User',
|
||||
'admin@example.com', 1, 1, datetime('now'))
|
||||
""")
|
||||
user_id = cursor.lastrowid
|
||||
created_data["users"].append({"id": user_id, "username": "admin"})
|
||||
|
||||
# Create 5 tags
|
||||
tag_names = ["news", "tech", "blog", "reference", "code"]
|
||||
for name in tag_names:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_tag (name, slug) VALUES (?, ?)
|
||||
""",
|
||||
(name, name.lower()),
|
||||
)
|
||||
tag_id = cursor.lastrowid
|
||||
created_data["tags"].append({"id": tag_id, "name": name, "slug": name.lower()})
|
||||
def insert_legacy_row(conn: sqlite3.Connection, table: str, values: dict):
|
||||
"""Insert fixture data with column names beside their values, using real SQLite."""
|
||||
columns = ", ".join(values)
|
||||
placeholders = ", ".join("?" for _ in values)
|
||||
return conn.execute(f"INSERT INTO {table} ({columns}) VALUES ({placeholders})", tuple(values.values())).lastrowid
|
||||
|
||||
# Create 5 snapshots
|
||||
test_urls = [
|
||||
("https://example.com/page1", "Example Page 1"),
|
||||
("https://example.org/article", "Article Title"),
|
||||
("https://github.com/user/repo", "GitHub Repository"),
|
||||
("https://news.ycombinator.com/item?id=12345", "HN Discussion"),
|
||||
("https://en.wikipedia.org/wiki/Test", "Wikipedia Test"),
|
||||
]
|
||||
|
||||
for i, (url, title) in enumerate(test_urls):
|
||||
snapshot_id = generate_uuid()
|
||||
timestamp = f"2024010{i + 1}120000.000000"
|
||||
added = f"2024-01-0{i + 1} 12:00:00"
|
||||
def seed_legacy_data(db_path: Path, *, with_crawls: bool) -> dict:
|
||||
"""Seed the shared 0.7/0.8 scenarios, with explicit 0.8-only column additions."""
|
||||
from contextlib import closing
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_snapshot (id, url, timestamp, title, added, updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(snapshot_id, url, timestamp, title, added, added),
|
||||
)
|
||||
created_data = {name: [] for name in ("users", "snapshots", "tags", "archiveresults")}
|
||||
if with_crawls:
|
||||
created_data.update(seeds=[], crawls=[])
|
||||
|
||||
created_data["snapshots"].append(
|
||||
with closing(sqlite3.connect(db_path)) as conn:
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
user_id = insert_legacy_row(
|
||||
conn,
|
||||
"auth_user",
|
||||
{
|
||||
"id": snapshot_id,
|
||||
"url": url,
|
||||
"timestamp": timestamp,
|
||||
"title": title,
|
||||
"password": "pbkdf2_sha256$test",
|
||||
"is_superuser": 1,
|
||||
"username": "admin",
|
||||
"first_name": "Admin",
|
||||
"last_name": "User",
|
||||
"email": "admin@example.com",
|
||||
"is_staff": 1,
|
||||
"is_active": 1,
|
||||
"date_joined": now,
|
||||
},
|
||||
)
|
||||
created_data["users"].append({"id": user_id, "username": "admin"})
|
||||
ownership = {"created_at": now, "modified_at": now, "created_by_id": user_id} if with_crawls else {}
|
||||
for name in ("news", "tech", "blog", "reference", "code"):
|
||||
tag = {"name": name, "slug": name.lower()}
|
||||
tag_id = insert_legacy_row(conn, "core_tag", {**tag, **ownership})
|
||||
created_data["tags"].append({"id": tag_id, **tag})
|
||||
|
||||
# Assign 2 tags to each snapshot
|
||||
tag_ids = [created_data["tags"][i % 5]["id"], created_data["tags"][(i + 1) % 5]["id"]]
|
||||
for tag_id in tag_ids:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_snapshot_tags (snapshot_id, tag_id) VALUES (?, ?)
|
||||
""",
|
||||
(snapshot_id, tag_id),
|
||||
)
|
||||
if with_crawls:
|
||||
# Real 0.8 Seed/Crawl pairs: Crawl.urls replaced Seed.uri in 0.9.
|
||||
for uri, max_depth in (("https://example.com", 0), ("https://github.com/ArchiveBox", 1)):
|
||||
seed_id, crawl_id = generate_uuid(), generate_uuid()
|
||||
insert_legacy_row(
|
||||
conn,
|
||||
"seeds_seed",
|
||||
{
|
||||
"id": seed_id,
|
||||
**ownership,
|
||||
"uri": uri,
|
||||
"extractor": "auto",
|
||||
"tags_str": "",
|
||||
"config": "{}",
|
||||
"num_uses_failed": 0,
|
||||
"num_uses_succeeded": 0,
|
||||
},
|
||||
)
|
||||
insert_legacy_row(
|
||||
conn,
|
||||
"crawls_crawl",
|
||||
{
|
||||
"id": crawl_id,
|
||||
**ownership,
|
||||
"seed_id": seed_id,
|
||||
"config": "{}",
|
||||
"max_depth": max_depth,
|
||||
"tags_str": "",
|
||||
"persona": "auto",
|
||||
"status": "queued",
|
||||
"retry_at": now,
|
||||
"num_uses_failed": 0,
|
||||
"num_uses_succeeded": 0,
|
||||
},
|
||||
)
|
||||
created_data["seeds"].append({"id": seed_id, "uri": uri})
|
||||
created_data["crawls"].append({"id": crawl_id, "urls": uri, "max_depth": max_depth, "label": "", "status": "queued"})
|
||||
|
||||
# Create 5 archive results for each snapshot
|
||||
extractors = ["title", "favicon", "screenshot", "singlefile", "wget"]
|
||||
statuses = ["succeeded", "succeeded", "failed", "succeeded", "skipped"]
|
||||
test_urls = [
|
||||
("https://example.com/page1", "Example Page 1"),
|
||||
("https://example.org/article", "Article Title"),
|
||||
("https://github.com/user/repo", "GitHub Repository"),
|
||||
("https://news.ycombinator.com/item?id=12345", "HN Discussion"),
|
||||
("https://en.wikipedia.org/wiki/Test", "Wikipedia Test"),
|
||||
]
|
||||
result_states = [
|
||||
("title", "succeeded"),
|
||||
("favicon", "succeeded"),
|
||||
("screenshot", "failed"),
|
||||
("singlefile", "succeeded"),
|
||||
("wget", "skipped"),
|
||||
]
|
||||
for i, (url, title) in enumerate(test_urls):
|
||||
snapshot_id = generate_uuid()
|
||||
timestamp = f"2024010{i + 1}120000.000000"
|
||||
added = f"2024-01-0{i + 1} 12:00:00"
|
||||
snapshot = {"id": snapshot_id, "url": url, "timestamp": timestamp, "title": title}
|
||||
if with_crawls:
|
||||
snapshot["crawl_id"] = created_data["crawls"][i // 2]["id"] if i < 3 else None
|
||||
dates = {
|
||||
"created_by_id": user_id,
|
||||
"created_at": added,
|
||||
"modified_at": added,
|
||||
"bookmarked_at": added,
|
||||
"depth": 0,
|
||||
"status": "queued",
|
||||
"config": "{}",
|
||||
"notes": "",
|
||||
}
|
||||
else:
|
||||
dates = {"added": added, "updated": added}
|
||||
insert_legacy_row(conn, "core_snapshot", {**snapshot, **dates})
|
||||
created_data["snapshots"].append(snapshot)
|
||||
for tag in (created_data["tags"][i % 5], created_data["tags"][(i + 1) % 5]):
|
||||
insert_legacy_row(conn, "core_snapshot_tags", {"snapshot_id": snapshot_id, "tag_id": tag["id"]})
|
||||
|
||||
for j, (extractor, status) in enumerate(zip(extractors, statuses)):
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_archiveresult
|
||||
(uuid, snapshot_id, extractor, cmd, pwd, cmd_version, output, start_ts, end_ts, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
generate_uuid(),
|
||||
snapshot_id,
|
||||
extractor,
|
||||
json.dumps([extractor, "--version"]),
|
||||
f"/data/archive/{timestamp}",
|
||||
"1.0.0",
|
||||
f"{extractor}/index.html" if status == "succeeded" else "",
|
||||
f"2024-01-0{i + 1} 12:00:0{j}",
|
||||
f"2024-01-0{i + 1} 12:00:1{j}",
|
||||
status,
|
||||
),
|
||||
)
|
||||
for j, (extractor, status) in enumerate(result_states):
|
||||
result_uuid = generate_uuid()
|
||||
start_ts, end_ts = f"2024-01-0{i + 1} 12:00:0{j}", f"2024-01-0{i + 1} 12:00:1{j}"
|
||||
result = {"snapshot_id": snapshot_id, "extractor": extractor, "status": status}
|
||||
extra = {}
|
||||
if with_crawls:
|
||||
result["uuid"] = result_uuid
|
||||
extra = {
|
||||
"created_by_id": user_id,
|
||||
"created_at": start_ts,
|
||||
"modified_at": end_ts,
|
||||
"retry_at": now,
|
||||
"notes": "",
|
||||
"output_dir": extractor,
|
||||
}
|
||||
insert_legacy_row(
|
||||
conn,
|
||||
"core_archiveresult",
|
||||
{
|
||||
**result,
|
||||
**extra,
|
||||
"uuid": result_uuid,
|
||||
"cmd": json.dumps([extractor, "--version"]),
|
||||
"pwd": f"/data/archive/{timestamp}",
|
||||
"cmd_version": "1.0.0",
|
||||
"output": f"{extractor}/index.html" if status == "succeeded" else "",
|
||||
"start_ts": start_ts,
|
||||
"end_ts": end_ts,
|
||||
},
|
||||
)
|
||||
created_data["archiveresults"].append(result)
|
||||
|
||||
created_data["archiveresults"].append(
|
||||
{
|
||||
"snapshot_id": snapshot_id,
|
||||
"extractor": extractor,
|
||||
"status": status,
|
||||
},
|
||||
)
|
||||
|
||||
# Record migrations as applied (0.7.x migrations up to 0022)
|
||||
migrations = [
|
||||
("contenttypes", "0001_initial"),
|
||||
("contenttypes", "0002_remove_content_type_name"),
|
||||
("auth", "0001_initial"),
|
||||
("auth", "0002_alter_permission_name_max_length"),
|
||||
("auth", "0003_alter_user_email_max_length"),
|
||||
("auth", "0004_alter_user_username_opts"),
|
||||
("auth", "0005_alter_user_last_login_null"),
|
||||
("auth", "0006_require_contenttypes_0002"),
|
||||
("auth", "0007_alter_validators_add_error_messages"),
|
||||
("auth", "0008_alter_user_username_max_length"),
|
||||
("auth", "0009_alter_user_last_name_max_length"),
|
||||
("auth", "0010_alter_group_name_max_length"),
|
||||
("auth", "0011_update_proxy_permissions"),
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
("admin", "0001_initial"),
|
||||
("admin", "0002_logentry_remove_auto_add"),
|
||||
("admin", "0003_logentry_add_action_flag_choices"),
|
||||
("sessions", "0001_initial"),
|
||||
("core", "0001_initial"),
|
||||
("core", "0002_auto_20200625_1521"),
|
||||
("core", "0003_auto_20200630_1034"),
|
||||
("core", "0004_auto_20200713_1552"),
|
||||
("core", "0005_auto_20200728_0326"),
|
||||
("core", "0006_auto_20201012_1520"),
|
||||
("core", "0007_archiveresult"),
|
||||
("core", "0008_auto_20210105_1421"),
|
||||
("core", "0009_auto_20210216_1038"),
|
||||
("core", "0010_auto_20210216_1055"),
|
||||
("core", "0011_auto_20210216_1331"),
|
||||
("core", "0012_auto_20210216_1425"),
|
||||
("core", "0013_auto_20210218_0729"),
|
||||
("core", "0014_auto_20210218_0729"),
|
||||
("core", "0015_auto_20210218_0730"),
|
||||
("core", "0016_auto_20210218_1204"),
|
||||
("core", "0017_auto_20210219_0211"),
|
||||
("core", "0018_auto_20210327_0952"),
|
||||
("core", "0019_auto_20210401_0654"),
|
||||
("core", "0020_auto_20210410_1031"),
|
||||
("core", "0021_auto_20220914_0934"),
|
||||
("core", "0022_auto_20231023_2008"),
|
||||
]
|
||||
|
||||
for app, name in migrations:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO django_migrations (app, name, applied)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
""",
|
||||
(app, name),
|
||||
conn.executemany(
|
||||
"INSERT INTO django_migrations (app, name, applied) VALUES (?, ?, datetime('now'))",
|
||||
MIGRATIONS_0_8 if with_crawls else MIGRATIONS_0_7,
|
||||
)
|
||||
|
||||
created_data["preserved_rows"] = seed_security_metadata(conn, include_api=False)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
created_data["preserved_rows"] = seed_security_metadata(conn, include_api=with_crawls)
|
||||
conn.commit()
|
||||
return created_data
|
||||
|
||||
|
||||
def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]:
|
||||
"""Seed a 0.8.x database with realistic test data including Crawls."""
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
def seed_0_7_data(db_path: Path) -> dict:
|
||||
return seed_legacy_data(db_path, with_crawls=False)
|
||||
|
||||
created_data = {
|
||||
"users": [],
|
||||
"seeds": [],
|
||||
"crawls": [],
|
||||
"snapshots": [],
|
||||
"tags": [],
|
||||
"archiveresults": [],
|
||||
}
|
||||
|
||||
# Create a user
|
||||
cursor.execute("""
|
||||
INSERT INTO auth_user (password, is_superuser, username, first_name, last_name,
|
||||
email, is_staff, is_active, date_joined)
|
||||
VALUES ('pbkdf2_sha256$test', 1, 'admin', 'Admin', 'User',
|
||||
'admin@example.com', 1, 1, datetime('now'))
|
||||
""")
|
||||
user_id = cursor.lastrowid
|
||||
created_data["users"].append({"id": user_id, "username": "admin"})
|
||||
|
||||
# Create 5 tags
|
||||
tag_names = ["news", "tech", "blog", "reference", "code"]
|
||||
for name in tag_names:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_tag (name, slug, created_at, modified_at, created_by_id)
|
||||
VALUES (?, ?, datetime('now'), datetime('now'), ?)
|
||||
""",
|
||||
(name, name.lower(), user_id),
|
||||
)
|
||||
tag_id = cursor.lastrowid
|
||||
created_data["tags"].append({"id": tag_id, "name": name, "slug": name.lower()})
|
||||
|
||||
# Create two real 0.8.x Seed/Crawl pairs. Crawl.urls replaced Seed.uri in 0.9.
|
||||
test_crawls = [
|
||||
("https://example.com", 0),
|
||||
("https://github.com/ArchiveBox", 1),
|
||||
]
|
||||
|
||||
for uri, max_depth in test_crawls:
|
||||
seed_id = generate_uuid()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO seeds_seed (
|
||||
id, created_at, created_by_id, modified_at, uri, extractor,
|
||||
tags_str, config, num_uses_failed, num_uses_succeeded
|
||||
)
|
||||
VALUES (?, datetime('now'), ?, datetime('now'), ?, 'auto', '', '{}', 0, 0)
|
||||
""",
|
||||
(seed_id, user_id, uri),
|
||||
)
|
||||
created_data["seeds"].append({"id": seed_id, "uri": uri})
|
||||
|
||||
crawl_id = generate_uuid()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO crawls_crawl (id, created_at, created_by_id, modified_at, seed_id,
|
||||
config, max_depth, tags_str, persona, status, retry_at,
|
||||
num_uses_failed, num_uses_succeeded)
|
||||
VALUES (?, datetime('now'), ?, datetime('now'), ?, '{}', ?, '', 'auto', 'queued', datetime('now'), 0, 0)
|
||||
""",
|
||||
(crawl_id, user_id, seed_id, max_depth),
|
||||
)
|
||||
|
||||
created_data["crawls"].append(
|
||||
{
|
||||
"id": crawl_id,
|
||||
"urls": uri,
|
||||
"max_depth": max_depth,
|
||||
"label": "",
|
||||
"status": "queued",
|
||||
},
|
||||
)
|
||||
|
||||
# Create 5 snapshots linked to crawls
|
||||
test_urls = [
|
||||
("https://example.com/page1", "Example Page 1", created_data["crawls"][0]["id"]),
|
||||
("https://example.org/article", "Article Title", created_data["crawls"][0]["id"]),
|
||||
("https://github.com/user/repo", "GitHub Repository", created_data["crawls"][1]["id"]),
|
||||
("https://news.ycombinator.com/item?id=12345", "HN Discussion", None),
|
||||
("https://en.wikipedia.org/wiki/Test", "Wikipedia Test", None),
|
||||
]
|
||||
|
||||
for i, (url, title, crawl_id) in enumerate(test_urls):
|
||||
snapshot_id = generate_uuid()
|
||||
timestamp = f"2024010{i + 1}120000.000000"
|
||||
created_at = f"2024-01-0{i + 1} 12:00:00"
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_snapshot (id, created_by_id, created_at, modified_at, url, timestamp,
|
||||
bookmarked_at, crawl_id, title, depth, status, config, notes)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 'queued', '{}', '')
|
||||
""",
|
||||
(snapshot_id, user_id, created_at, created_at, url, timestamp, created_at, crawl_id, title),
|
||||
)
|
||||
|
||||
created_data["snapshots"].append(
|
||||
{
|
||||
"id": snapshot_id,
|
||||
"url": url,
|
||||
"timestamp": timestamp,
|
||||
"title": title,
|
||||
"crawl_id": crawl_id,
|
||||
},
|
||||
)
|
||||
|
||||
# Assign 2 tags to each snapshot
|
||||
tag_ids = [created_data["tags"][i % 5]["id"], created_data["tags"][(i + 1) % 5]["id"]]
|
||||
for tag_id in tag_ids:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_snapshot_tags (snapshot_id, tag_id) VALUES (?, ?)
|
||||
""",
|
||||
(snapshot_id, tag_id),
|
||||
)
|
||||
|
||||
# Create 5 archive results for each snapshot
|
||||
extractors = ["title", "favicon", "screenshot", "singlefile", "wget"]
|
||||
statuses = ["succeeded", "succeeded", "failed", "succeeded", "skipped"]
|
||||
|
||||
for j, (extractor, status) in enumerate(zip(extractors, statuses)):
|
||||
result_uuid = generate_uuid()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO core_archiveresult
|
||||
(uuid, created_by_id, created_at, modified_at, snapshot_id, extractor, pwd,
|
||||
cmd, cmd_version, output, start_ts, end_ts, status, retry_at, notes, output_dir)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), '', ?)
|
||||
""",
|
||||
(
|
||||
result_uuid,
|
||||
user_id,
|
||||
f"2024-01-0{i + 1} 12:00:0{j}",
|
||||
f"2024-01-0{i + 1} 12:00:1{j}",
|
||||
snapshot_id,
|
||||
extractor,
|
||||
f"/data/archive/{timestamp}",
|
||||
json.dumps([extractor, "--version"]),
|
||||
"1.0.0",
|
||||
f"{extractor}/index.html" if status == "succeeded" else "",
|
||||
f"2024-01-0{i + 1} 12:00:0{j}",
|
||||
f"2024-01-0{i + 1} 12:00:1{j}",
|
||||
status,
|
||||
f"{extractor}",
|
||||
),
|
||||
)
|
||||
|
||||
created_data["archiveresults"].append(
|
||||
{
|
||||
"uuid": result_uuid,
|
||||
"snapshot_id": snapshot_id,
|
||||
"extractor": extractor,
|
||||
"status": status,
|
||||
},
|
||||
)
|
||||
|
||||
# Record migrations as applied (0.8.x migrations)
|
||||
migrations = [
|
||||
("contenttypes", "0001_initial"),
|
||||
("contenttypes", "0002_remove_content_type_name"),
|
||||
("auth", "0001_initial"),
|
||||
("auth", "0002_alter_permission_name_max_length"),
|
||||
("auth", "0003_alter_user_email_max_length"),
|
||||
("auth", "0004_alter_user_username_opts"),
|
||||
("auth", "0005_alter_user_last_login_null"),
|
||||
("auth", "0006_require_contenttypes_0002"),
|
||||
("auth", "0007_alter_validators_add_error_messages"),
|
||||
("auth", "0008_alter_user_username_max_length"),
|
||||
("auth", "0009_alter_user_last_name_max_length"),
|
||||
("auth", "0010_alter_group_name_max_length"),
|
||||
("auth", "0011_update_proxy_permissions"),
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
("admin", "0001_initial"),
|
||||
("admin", "0002_logentry_remove_auto_add"),
|
||||
("admin", "0003_logentry_add_action_flag_choices"),
|
||||
("sessions", "0001_initial"),
|
||||
("core", "0001_initial"),
|
||||
("core", "0002_auto_20200625_1521"),
|
||||
("core", "0003_auto_20200630_1034"),
|
||||
("core", "0004_auto_20200713_1552"),
|
||||
("core", "0005_auto_20200728_0326"),
|
||||
("core", "0006_auto_20201012_1520"),
|
||||
("core", "0007_archiveresult"),
|
||||
("core", "0008_auto_20210105_1421"),
|
||||
("core", "0009_auto_20210216_1038"),
|
||||
("core", "0010_auto_20210216_1055"),
|
||||
("core", "0011_auto_20210216_1331"),
|
||||
("core", "0012_auto_20210216_1425"),
|
||||
("core", "0013_auto_20210218_0729"),
|
||||
("core", "0014_auto_20210218_0729"),
|
||||
("core", "0015_auto_20210218_0730"),
|
||||
("core", "0016_auto_20210218_1204"),
|
||||
("core", "0017_auto_20210219_0211"),
|
||||
("core", "0018_auto_20210327_0952"),
|
||||
("core", "0019_auto_20210401_0654"),
|
||||
("core", "0020_auto_20210410_1031"),
|
||||
("core", "0021_auto_20220914_0934"),
|
||||
("core", "0022_auto_20231023_2008"),
|
||||
# For 0.8.x (dev branch), record the migrations that 0023_new_schema replaces
|
||||
("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
|
||||
("core", "0024_auto_20240513_1143"),
|
||||
("core", "0025_alter_archiveresult_uuid"),
|
||||
("core", "0026_archiveresult_created_archiveresult_created_by_and_more"),
|
||||
("core", "0027_update_snapshot_ids"),
|
||||
("core", "0028_alter_archiveresult_uuid"),
|
||||
("core", "0029_alter_archiveresult_id"),
|
||||
("core", "0030_alter_archiveresult_uuid"),
|
||||
("core", "0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more"),
|
||||
("core", "0032_alter_archiveresult_id"),
|
||||
("core", "0033_rename_id_archiveresult_old_id"),
|
||||
("core", "0034_alter_archiveresult_old_id_alter_archiveresult_uuid"),
|
||||
("core", "0035_remove_archiveresult_uuid_archiveresult_id"),
|
||||
("core", "0036_alter_archiveresult_id_alter_archiveresult_old_id"),
|
||||
("core", "0037_rename_id_snapshot_old_id"),
|
||||
("core", "0038_rename_uuid_snapshot_id"),
|
||||
("core", "0039_rename_snapshot_archiveresult_snapshot_old"),
|
||||
("core", "0040_archiveresult_snapshot"),
|
||||
("core", "0041_alter_archiveresult_snapshot_and_more"),
|
||||
("core", "0042_remove_archiveresult_snapshot_old"),
|
||||
("core", "0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more"),
|
||||
("core", "0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more"),
|
||||
("core", "0045_alter_snapshot_old_id"),
|
||||
("core", "0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more"),
|
||||
("core", "0047_alter_snapshottag_unique_together_and_more"),
|
||||
("core", "0048_alter_archiveresult_snapshot_and_more"),
|
||||
("core", "0049_rename_snapshot_snapshottag_snapshot_old_and_more"),
|
||||
("core", "0050_alter_snapshottag_snapshot_old"),
|
||||
("core", "0051_snapshottag_snapshot_alter_snapshottag_snapshot_old"),
|
||||
("core", "0052_alter_snapshottag_unique_together_and_more"),
|
||||
("core", "0053_remove_snapshottag_snapshot_old"),
|
||||
("core", "0054_alter_snapshot_timestamp"),
|
||||
("core", "0055_alter_tag_slug"),
|
||||
("core", "0056_remove_tag_uuid"),
|
||||
("core", "0057_rename_id_tag_old_id"),
|
||||
("core", "0058_alter_tag_old_id"),
|
||||
("core", "0059_tag_id"),
|
||||
("core", "0060_alter_tag_id"),
|
||||
("core", "0061_rename_tag_snapshottag_old_tag_and_more"),
|
||||
("core", "0062_alter_snapshottag_old_tag"),
|
||||
("core", "0063_snapshottag_tag_alter_snapshottag_old_tag"),
|
||||
("core", "0064_alter_snapshottag_unique_together_and_more"),
|
||||
("core", "0065_remove_snapshottag_old_tag"),
|
||||
("core", "0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id"),
|
||||
("core", "0067_alter_snapshottag_tag"),
|
||||
("core", "0068_alter_archiveresult_options"),
|
||||
("core", "0069_alter_archiveresult_created_alter_snapshot_added_and_more"),
|
||||
("core", "0070_alter_archiveresult_created_by_alter_snapshot_added_and_more"),
|
||||
("core", "0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more"),
|
||||
("core", "0072_rename_added_snapshot_bookmarked_at_and_more"),
|
||||
("core", "0073_rename_created_archiveresult_created_at_and_more"),
|
||||
("core", "0074_alter_snapshot_downloaded_at"),
|
||||
# For 0.8.x: DO NOT record 0023_new_schema - it replaces 0023-0074 for fresh installs
|
||||
# We already recorded 0023-0074 above, so Django will know the state
|
||||
# For 0.8.x: Record original machine migrations (before squashing)
|
||||
# DO NOT record 0001_squashed here - it replaces 0001-0004 for fresh installs
|
||||
("machine", "0001_initial"),
|
||||
("machine", "0002_alter_machine_stats_installedbinary"),
|
||||
("machine", "0003_alter_installedbinary_options_and_more"),
|
||||
("machine", "0004_alter_installedbinary_abspath_and_more"),
|
||||
# Then the new migrations after squashing
|
||||
("machine", "0003_alter_dependency_id_alter_installedbinary_dependency_and_more"),
|
||||
("machine", "0004_drop_dependency_table"),
|
||||
# Crawls must come before core.0024 because 0024_b depends on it
|
||||
("crawls", "0001_initial"),
|
||||
# Core 0024 migrations chain (in dependency order)
|
||||
("core", "0024_b_clear_config_fields"),
|
||||
("core", "0024_c_disable_fk_checks"),
|
||||
("core", "0024_d_fix_crawls_config"),
|
||||
("core", "0024_snapshot_crawl"),
|
||||
("core", "0024_f_add_snapshot_config"),
|
||||
("core", "0025_allow_duplicate_urls_per_crawl"),
|
||||
# For 0.8.x: Record original api migration (before squashing)
|
||||
# DO NOT record 0001_squashed here - it replaces 0001 for fresh installs
|
||||
("api", "0001_initial"),
|
||||
("api", "0002_alter_apitoken_options"),
|
||||
("api", "0003_rename_user_apitoken_created_by_apitoken_abid_and_more"),
|
||||
("api", "0004_alter_apitoken_id_alter_apitoken_uuid"),
|
||||
("api", "0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more"),
|
||||
("api", "0006_remove_outboundwebhook_uuid_apitoken_id_and_more"),
|
||||
("api", "0007_alter_apitoken_created_by"),
|
||||
("api", "0008_alter_apitoken_created_alter_apitoken_created_by_and_more"),
|
||||
("api", "0009_rename_created_apitoken_created_at_and_more"),
|
||||
# Note: crawls.0001_initial moved earlier (before core.0024) due to dependencies
|
||||
# Stop here - 0.8.x ends at core.0025, crawls.0001, and we want to TEST the later migrations
|
||||
# Do NOT record 0026+ as they need to be tested during migration
|
||||
]
|
||||
|
||||
for app, name in migrations:
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO django_migrations (app, name, applied)
|
||||
VALUES (?, ?, datetime('now'))
|
||||
""",
|
||||
(app, name),
|
||||
)
|
||||
|
||||
created_data["preserved_rows"] = seed_security_metadata(conn, include_api=True)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return created_data
|
||||
def seed_0_8_data(db_path: Path) -> dict:
|
||||
return seed_legacy_data(db_path, with_crawls=True)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -1469,3 +1216,13 @@ def verify_process_migration(db_path: Path, expected_archiveresult_count: int) -
|
||||
|
||||
conn.close()
|
||||
return True, f"Process migration verified: {process_count} Processes created"
|
||||
|
||||
|
||||
def create_legacy_archive(work_dir, version):
|
||||
"""Build a real legacy SQLite collection for upgrade integration tests."""
|
||||
schema, seed = {"0.7": (SCHEMA_0_7, seed_0_7_data), "0.8": (SCHEMA_0_8, seed_0_8_data)}[version]
|
||||
create_data_dir_structure(work_dir)
|
||||
db_path = work_dir / "index.sqlite3"
|
||||
with sqlite3.connect(db_path) as conn:
|
||||
conn.executescript(schema)
|
||||
return work_dir, db_path, seed(db_path)
|
||||
@@ -5,7 +5,6 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Event
|
||||
|
||||
from .conftest import (
|
||||
write_import_format_files,
|
||||
api_client_request,
|
||||
cli_env,
|
||||
create_admin_and_token,
|
||||
@@ -24,38 +23,13 @@ from archivebox.core.models import Snapshot, SnapshotTag
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
IMPORT_FORMAT_EXPECTATIONS = {
|
||||
"rss": {
|
||||
"url": "https://example.com/",
|
||||
"title": "RSS Example Import",
|
||||
"date": "2024-01-01",
|
||||
"tags": {"rss-tag", "metadata"},
|
||||
},
|
||||
"netscape": {
|
||||
"url": "https://www.iana.org/domains/reserved",
|
||||
"title": "IANA Reserved Domains",
|
||||
"date": "2024-01-02",
|
||||
"tags": {"netscape-tag", "metadata"},
|
||||
},
|
||||
"dom": {
|
||||
"url": "https://www.iana.org/help/example-domains",
|
||||
},
|
||||
"json": {
|
||||
"url": "https://example.com/?archivebox-json-import=1",
|
||||
"title": "JSON Import Example",
|
||||
"date": "2024-01-03",
|
||||
"tags": {"json-tag", "metadata"},
|
||||
},
|
||||
"jsonl": {
|
||||
"url": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"title": "JSONL Import Example",
|
||||
"date": "2024-01-04",
|
||||
"tags": {"jsonl-tag", "metadata"},
|
||||
},
|
||||
"txt": {
|
||||
"url": "https://example.org/",
|
||||
},
|
||||
}
|
||||
from archivebox.tests.import_helpers import (
|
||||
IMPORT_FORMAT_EXPECTATIONS,
|
||||
assert_expected_import_snapshots,
|
||||
assert_no_file_or_shell_payload_snapshots,
|
||||
malicious_add_inputs,
|
||||
write_import_format_files,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
@@ -234,13 +208,7 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
|
||||
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
|
||||
assert root_input == crawl.urls
|
||||
api_server = start_api_server_without_runner(tmp_path, env, port)
|
||||
with use_archivebox_db(tmp_path):
|
||||
imported_snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("url", "status"))
|
||||
assert sorted(row["url"] for row in imported_snapshots) == sorted(expected_urls), imported_snapshots
|
||||
assert all(
|
||||
row["status"] in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
for row in imported_snapshots
|
||||
), imported_snapshots
|
||||
assert_expected_import_snapshots(tmp_path, expected_urls)
|
||||
|
||||
for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
|
||||
with use_archivebox_db(tmp_path):
|
||||
@@ -288,32 +256,7 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
|
||||
"""REST add must not let path, file://, traversal, or shell strings become archiveable URLs."""
|
||||
init_archive(tmp_path)
|
||||
safe_url = "https://example.com/?archivebox-api-security=1"
|
||||
other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
|
||||
other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
|
||||
other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
|
||||
canary = tmp_path / "archivebox_shell_injection_canary"
|
||||
inputs = [
|
||||
safe_url,
|
||||
"file:///etc/hosts",
|
||||
"/etc/hosts",
|
||||
"../../../../etc/passwd",
|
||||
f"file://{other_crawl_source}",
|
||||
str(other_crawl_source),
|
||||
f"'; touch {canary}; #",
|
||||
f'" && touch {canary} && echo "',
|
||||
f"$(touch {canary})",
|
||||
f"`touch {canary}`",
|
||||
"""<?xml version="1.0"?>
|
||||
<!DOCTYPE rss [
|
||||
<!ENTITY localfile SYSTEM "file:///etc/hosts">
|
||||
]>
|
||||
<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<channel>
|
||||
<item><title>&localfile;</title><link>file:///etc/passwd</link></item>
|
||||
<xi:include href="file:///etc/hosts" parse="text"/>
|
||||
</channel>
|
||||
</rss>""",
|
||||
]
|
||||
inputs, canary = malicious_add_inputs(tmp_path, safe_url=safe_url)
|
||||
port = get_free_port()
|
||||
env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
|
||||
api_token = create_admin_and_token(tmp_path)
|
||||
@@ -343,19 +286,8 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
|
||||
if api_server is not None:
|
||||
stop_archivebox_process(api_server)
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
imported_snapshots = list(Snapshot.objects.filter(url__in={safe_url}).values("url", "status"))
|
||||
assert sorted(row["url"] for row in imported_snapshots) == sorted({safe_url}), imported_snapshots
|
||||
assert all(
|
||||
row["status"] in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
for row in imported_snapshots
|
||||
), imported_snapshots
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
assert not canary.exists()
|
||||
assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
|
||||
for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
|
||||
assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
|
||||
assert_expected_import_snapshots(tmp_path, {safe_url}, expected_tags={"api-security"})
|
||||
assert_no_file_or_shell_payload_snapshots(tmp_path, canary=canary)
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshot = Snapshot.objects.get(url=safe_url)
|
||||
crawl = Crawl.objects.get()
|
||||
|
||||
@@ -97,17 +97,21 @@ class TestLDAPIntegration:
|
||||
ldap_backends = [b for b in settings.AUTHENTICATION_BACKENDS if "ldap" in b.lower()]
|
||||
assert len(ldap_backends) == 0, "LDAP backend should not be present when LDAP_ENABLED=False"
|
||||
|
||||
def test_django_settings_with_ldap_enabled(self, initialized_archive):
|
||||
def test_django_settings_with_ldap_enabled(self, archivebox_cli, initialized_archive):
|
||||
"""Test the real enabled LDAP settings path with installed libraries."""
|
||||
assert find_spec("django_auth_ldap") is not None
|
||||
assert find_spec("ldap") is not None
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
[
|
||||
"manage",
|
||||
"shell",
|
||||
"-c",
|
||||
"from django.conf import settings; print('LDAP_BACKENDS=' + ','.join(settings.AUTHENTICATION_BACKENDS)); print('LDAP_SERVER_URI=' + settings.AUTH_LDAP_SERVER_URI)",
|
||||
(
|
||||
"from django.conf import settings; "
|
||||
"print('LDAP_BACKENDS=' + ','.join(settings.AUTHENTICATION_BACKENDS)); "
|
||||
"print('LDAP_SERVER_URI=' + settings.AUTH_LDAP_SERVER_URI)"
|
||||
),
|
||||
],
|
||||
timeout=45,
|
||||
env={
|
||||
@@ -117,9 +121,6 @@ class TestLDAPIntegration:
|
||||
"LDAP_BIND_PASSWORD": "password",
|
||||
"LDAP_USER_BASE": "ou=users,dc=example,dc=com",
|
||||
},
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
@@ -92,7 +92,7 @@ def test_binary_request_preserves_native_overrides_in_db():
|
||||
assert binary_events[-1].overrides == overrides
|
||||
|
||||
|
||||
def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized_archive, tmp_path):
|
||||
def test_binary_request_installs_env_binary_and_recovers_stale_cache(archivebox_cli, initialized_archive, tmp_path):
|
||||
name = "archivebox"
|
||||
provider_bin_dir = initialized_archive / "lib" / "env" / "bin"
|
||||
host_binary = shutil.which(name)
|
||||
@@ -128,13 +128,10 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
assert binary_processes[-1].started_at < binary_processes[-1].ended_at
|
||||
assert any(f"--name={name}" in arg for arg in binary_processes[-1].cmd)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["version"],
|
||||
timeout=60,
|
||||
env=runtime_env,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
version_stdout, version_stderr, version_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert version_code == 0, version_stderr
|
||||
@@ -143,13 +140,10 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
|
||||
first_abspath.unlink()
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run", f"--binary-id={first_binary_id}"],
|
||||
timeout=120,
|
||||
env=runtime_env,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
rerun_stdout, rerun_stderr, rerun_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -177,13 +171,10 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
retry_at=None,
|
||||
)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run", f"--binary-id={first_binary_id}"],
|
||||
timeout=120,
|
||||
env=changed_runtime_env,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
relib_stdout, relib_stderr, relib_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -198,7 +189,7 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
assert Path(relibbed.abspath).resolve() == Path(host_binary).resolve()
|
||||
|
||||
|
||||
def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
|
||||
def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(archivebox_cli, initialized_archive, tmp_path):
|
||||
name = "http"
|
||||
provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
|
||||
runtime_env = _runtime_env(initialized_archive)
|
||||
@@ -230,13 +221,10 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
|
||||
queued.save(update_fields=["binproviders", "overrides", "modified_at"])
|
||||
recovered_runtime_env = _runtime_env(initialized_archive)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run", f"--binary-id={queued_id}"],
|
||||
timeout=120,
|
||||
env=recovered_runtime_env,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@ from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
write_import_format_files,
|
||||
cli_env,
|
||||
find_snapshot_dir,
|
||||
run_archivebox_cmd,
|
||||
@@ -25,38 +24,13 @@ from archivebox.tests.conftest import (
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
IMPORT_FORMAT_EXPECTATIONS = {
|
||||
"rss": {
|
||||
"url": "https://example.com/",
|
||||
"title": "RSS Example Import",
|
||||
"date": "2024-01-01",
|
||||
"tags": {"rss-tag", "metadata"},
|
||||
},
|
||||
"netscape": {
|
||||
"url": "https://www.iana.org/domains/reserved",
|
||||
"title": "IANA Reserved Domains",
|
||||
"date": "2024-01-02",
|
||||
"tags": {"netscape-tag", "metadata"},
|
||||
},
|
||||
"dom": {
|
||||
"url": "https://www.iana.org/help/example-domains",
|
||||
},
|
||||
"json": {
|
||||
"url": "https://example.com/?archivebox-json-import=1",
|
||||
"title": "JSON Import Example",
|
||||
"date": "2024-01-03",
|
||||
"tags": {"json-tag", "metadata"},
|
||||
},
|
||||
"jsonl": {
|
||||
"url": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"title": "JSONL Import Example",
|
||||
"date": "2024-01-04",
|
||||
"tags": {"jsonl-tag", "metadata"},
|
||||
},
|
||||
"txt": {
|
||||
"url": "https://example.org/",
|
||||
},
|
||||
}
|
||||
from archivebox.tests.import_helpers import (
|
||||
IMPORT_FORMAT_EXPECTATIONS,
|
||||
assert_expected_import_snapshots,
|
||||
assert_no_file_or_shell_payload_snapshots,
|
||||
malicious_add_inputs,
|
||||
write_import_format_files,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
@@ -159,13 +133,7 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
|
||||
assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
|
||||
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
|
||||
assert root_input == crawl.urls
|
||||
with use_archivebox_db(initialized_archive):
|
||||
imported_snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("url", "status"))
|
||||
assert sorted(row["url"] for row in imported_snapshots) == sorted(expected_urls), imported_snapshots
|
||||
assert all(
|
||||
row["status"] in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
for row in imported_snapshots
|
||||
), imported_snapshots
|
||||
assert_expected_import_snapshots(initialized_archive, expected_urls)
|
||||
|
||||
list_result = run_archivebox_cmd(
|
||||
["list", "--json"],
|
||||
@@ -204,32 +172,7 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
|
||||
def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive):
|
||||
"""CLI add must not turn user-supplied local paths or shell payloads into snapshots."""
|
||||
safe_url = "https://example.com/?archivebox-cli-security=1"
|
||||
other_crawl_source = initialized_archive / "sources" / "other_crawl_source.txt"
|
||||
other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
|
||||
other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
|
||||
canary = initialized_archive / "archivebox_shell_injection_canary"
|
||||
inputs = [
|
||||
safe_url,
|
||||
"file:///etc/hosts",
|
||||
"/etc/hosts",
|
||||
"../../../../etc/passwd",
|
||||
f"file://{other_crawl_source}",
|
||||
str(other_crawl_source),
|
||||
f"'; touch {canary}; #",
|
||||
f'" && touch {canary} && echo "',
|
||||
f"$(touch {canary})",
|
||||
f"`touch {canary}`",
|
||||
"""<?xml version="1.0"?>
|
||||
<!DOCTYPE rss [
|
||||
<!ENTITY localfile SYSTEM "file:///etc/hosts">
|
||||
]>
|
||||
<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<channel>
|
||||
<item><title>&localfile;</title><link>file:///etc/passwd</link></item>
|
||||
<xi:include href="file:///etc/hosts" parse="text"/>
|
||||
</channel>
|
||||
</rss>""",
|
||||
]
|
||||
inputs, canary = malicious_add_inputs(initialized_archive, safe_url=safe_url)
|
||||
env = cli_env(**IMPORT_FORMAT_ENV)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
@@ -242,20 +185,9 @@ def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
run_queued_crawls(initialized_archive, env=env, timeout=120)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
imported_snapshots = list(Snapshot.objects.filter(url__in={safe_url}).values("url", "status"))
|
||||
assert sorted(row["url"] for row in imported_snapshots) == sorted({safe_url}), imported_snapshots
|
||||
assert all(
|
||||
row["status"] in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
for row in imported_snapshots
|
||||
), imported_snapshots
|
||||
assert_expected_import_snapshots(initialized_archive, {safe_url})
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
assert not canary.exists()
|
||||
assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
|
||||
for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
|
||||
assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
|
||||
assert_no_file_or_shell_payload_snapshots(initialized_archive, canary=canary)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshots = list(Snapshot.objects.filter(url=safe_url).values_list("url", "status", "tags__name"))
|
||||
crawl = Crawl.objects.get()
|
||||
|
||||
@@ -24,30 +24,37 @@ PROJECTOR_TEST_ENV = {
|
||||
}
|
||||
|
||||
|
||||
def create_projected_favicon(archive):
|
||||
"""Run the public create/request/run pipeline and return its persisted result."""
|
||||
common = {"cwd": archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
return records[0]
|
||||
|
||||
|
||||
class TestArchiveResultCreate:
|
||||
"""Tests for `archivebox archiveresult create`."""
|
||||
|
||||
def test_create_from_snapshot_jsonl(self, initialized_archive):
|
||||
def test_create_from_snapshot_jsonl(self, archivebox_cli, initialized_archive):
|
||||
"""Create archive results from Snapshot JSONL input."""
|
||||
url = create_test_url()
|
||||
|
||||
# Create a snapshot first
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
# Pipe snapshot to archiveresult create
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "create", "--plugin=title"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -67,24 +74,18 @@ class TestArchiveResultCreate:
|
||||
assert ar["hook_name"] == "on_Snapshot__54_title"
|
||||
assert "id" not in ar
|
||||
|
||||
def test_create_with_specific_plugin(self, initialized_archive):
|
||||
def test_create_with_specific_plugin(self, archivebox_cli, initialized_archive):
|
||||
"""Create archive result for specific plugin."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "create", "--plugin=screenshot"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -95,7 +96,7 @@ class TestArchiveResultCreate:
|
||||
assert all(record["plugin"] == "screenshot" for record in ar_records)
|
||||
assert [record["hook_name"] for record in ar_records] == ["on_Snapshot__51_screenshot"]
|
||||
|
||||
def test_create_pass_through_crawl(self, initialized_archive):
|
||||
def test_create_pass_through_crawl(self, archivebox_cli, initialized_archive):
|
||||
"""Pass-through Crawl records unchanged."""
|
||||
url = create_test_url()
|
||||
|
||||
@@ -104,22 +105,16 @@ class TestArchiveResultCreate:
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
# Now pipe all to archiveresult create
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "create", "--plugin=title"],
|
||||
stdin=stdout2,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -131,23 +126,17 @@ class TestArchiveResultCreate:
|
||||
assert "Snapshot" in types
|
||||
assert "ArchiveResult" in types
|
||||
|
||||
def test_create_passes_through_cli_crawl_when_no_snapshots(self, initialized_archive):
|
||||
def test_create_passes_through_cli_crawl_when_no_snapshots(self, archivebox_cli, initialized_archive):
|
||||
"""A real Crawl with no Snapshot input passes through successfully."""
|
||||
crawl_result = run_archivebox_cmd(
|
||||
crawl_result = archivebox_cli(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert crawl_result.returncode == 0, crawl_result.stderr
|
||||
crawl_record = parse_jsonl_output(crawl_result.stdout)[0]
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "create"],
|
||||
stdin=crawl_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -160,38 +149,26 @@ class TestArchiveResultCreate:
|
||||
class TestArchiveResultList:
|
||||
"""Tests for `archivebox archiveresult list`."""
|
||||
|
||||
def test_list_empty(self, initialized_archive):
|
||||
def test_list_empty(self, archivebox_cli, initialized_archive):
|
||||
"""List with no archive results returns empty."""
|
||||
result = run_archivebox_cmd(["archiveresult", "list"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "list"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "Listed 0 archive results" in result.stderr
|
||||
|
||||
def test_list_filter_by_status(self, initialized_archive):
|
||||
def test_list_filter_by_status(self, archivebox_cli, initialized_archive):
|
||||
"""Filter archive results by status."""
|
||||
# Create snapshot and materialize an archive result via the runner
|
||||
common = {"cwd": initialized_archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
created = records[0]
|
||||
run_archivebox_cmd(
|
||||
created = create_projected_favicon(initialized_archive)
|
||||
archivebox_cli(
|
||||
["archiveresult", "update", "--status=queued"],
|
||||
stdin=json.dumps(created),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "list", "--status=queued"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -199,22 +176,12 @@ class TestArchiveResultList:
|
||||
for r in records:
|
||||
assert r["status"] == "queued"
|
||||
|
||||
def test_list_filter_by_plugin(self, initialized_archive):
|
||||
def test_list_filter_by_plugin(self, archivebox_cli, initialized_archive):
|
||||
"""Filter archive results by plugin."""
|
||||
common = {"cwd": initialized_archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
create_projected_favicon(initialized_archive)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -222,24 +189,14 @@ class TestArchiveResultList:
|
||||
for r in records:
|
||||
assert r["plugin"] == "favicon"
|
||||
|
||||
def test_list_with_limit(self, initialized_archive):
|
||||
def test_list_with_limit(self, archivebox_cli, initialized_archive):
|
||||
"""Limit number of results."""
|
||||
# Create multiple archive results
|
||||
for _ in range(3):
|
||||
common = {"cwd": initialized_archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
create_projected_favicon(initialized_archive)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "list", "--limit=2"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -250,24 +207,13 @@ class TestArchiveResultList:
|
||||
class TestArchiveResultUpdate:
|
||||
"""Tests for `archivebox archiveresult update`."""
|
||||
|
||||
def test_update_status(self, initialized_archive):
|
||||
def test_update_status(self, archivebox_cli, initialized_archive):
|
||||
"""Update archive result status."""
|
||||
common = {"cwd": initialized_archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
ar = records[0]
|
||||
ar = create_projected_favicon(initialized_archive)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "update", "--status=failed"],
|
||||
stdin=json.dumps(ar),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -280,47 +226,25 @@ class TestArchiveResultUpdate:
|
||||
class TestArchiveResultDelete:
|
||||
"""Tests for `archivebox archiveresult delete`."""
|
||||
|
||||
def test_delete_requires_yes(self, initialized_archive):
|
||||
def test_delete_requires_yes(self, archivebox_cli, initialized_archive):
|
||||
"""Delete requires --yes flag."""
|
||||
common = {"cwd": initialized_archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
ar = records[0]
|
||||
ar = create_projected_favicon(initialized_archive)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "delete"],
|
||||
stdin=json.dumps(ar),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "--yes" in result.stderr
|
||||
|
||||
def test_delete_with_yes(self, initialized_archive):
|
||||
def test_delete_with_yes(self, archivebox_cli, initialized_archive):
|
||||
"""Delete with --yes flag works."""
|
||||
common = {"cwd": initialized_archive, "default_cli_env": True, "disable_extractors": True, "check": True}
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], **common)
|
||||
snapshot = parse_jsonl_output(created.stdout)[0]
|
||||
requested = run_archivebox_cmd(["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), **common)
|
||||
run_archivebox_cmd(["run"], stdin=requested.stdout, timeout=120, env=PROJECTOR_TEST_ENV, **common)
|
||||
listed = run_archivebox_cmd(["archiveresult", "list", "--plugin=favicon", f"--snapshot-id={snapshot['id']}"], **common)
|
||||
records = parse_jsonl_output(listed.stdout)
|
||||
assert len(records) == 1
|
||||
ar = records[0]
|
||||
ar = create_projected_favicon(initialized_archive)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["archiveresult", "delete", "--yes"],
|
||||
stdin=json.dumps(ar),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
@@ -29,11 +29,13 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
class TestCrawlCreate:
|
||||
"""Tests for `archivebox crawl create`."""
|
||||
|
||||
def test_create_from_url_args(self, initialized_archive):
|
||||
def test_create_from_url_args(self, archivebox_cli, initialized_archive):
|
||||
"""Create crawl from URL arguments."""
|
||||
url = create_test_url()
|
||||
|
||||
result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["crawl", "create", url],
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
||||
assert "Created crawl" in result.stderr
|
||||
@@ -44,17 +46,14 @@ class TestCrawlCreate:
|
||||
assert records[0]["type"] == "Crawl"
|
||||
assert url in records[0]["urls"]
|
||||
|
||||
def test_create_from_stdin_urls(self, initialized_archive):
|
||||
def test_create_from_stdin_urls(self, archivebox_cli, initialized_archive):
|
||||
"""Create crawl from stdin URLs (one per line)."""
|
||||
urls = [create_test_url() for _ in range(3)]
|
||||
stdin = "\n".join(urls)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["crawl", "create"],
|
||||
stdin=stdin,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
||||
@@ -67,55 +66,43 @@ class TestCrawlCreate:
|
||||
for url in urls:
|
||||
assert url in crawl["urls"]
|
||||
|
||||
def test_create_with_depth(self, initialized_archive):
|
||||
def test_create_with_depth(self, archivebox_cli, initialized_archive):
|
||||
"""Create crawl with --depth flag."""
|
||||
url = create_test_url()
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["crawl", "create", "--depth=2", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert records[0]["max_depth"] == 2
|
||||
|
||||
def test_create_with_tag(self, initialized_archive):
|
||||
def test_create_with_tag(self, archivebox_cli, initialized_archive):
|
||||
"""Create crawl with --tag flag."""
|
||||
url = create_test_url()
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["crawl", "create", "--tag=test-tag", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert "test-tag" in records[0].get("tags_str", "")
|
||||
|
||||
def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
|
||||
def test_create_passes_through_tag_emitted_by_cli(self, archivebox_cli, initialized_archive):
|
||||
"""A real Tag emitted by the CLI remains available to the next stage."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
tag_result = archivebox_cli(
|
||||
["tag", "create", "crawl-input-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
tag_record = parse_jsonl_output(tag_result.stdout)[0]
|
||||
url = create_test_url()
|
||||
stdin = tag_result.stdout + json.dumps({"url": url}) + "\n"
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["crawl", "create"],
|
||||
stdin=stdin,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -124,7 +111,7 @@ class TestCrawlCreate:
|
||||
assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
|
||||
assert any(record.get("type") == "Crawl" and url in record["urls"] for record in records)
|
||||
|
||||
def test_create_pass_through_existing_crawl(self, initialized_archive):
|
||||
def test_create_pass_through_existing_crawl(self, archivebox_cli, initialized_archive):
|
||||
"""Existing Crawl records (with id) are passed through."""
|
||||
# First create a crawl
|
||||
url = create_test_url()
|
||||
@@ -133,12 +120,9 @@ class TestCrawlCreate:
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
# Now pipe it back - should pass through
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "create"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -151,35 +135,36 @@ class TestCrawlCreate:
|
||||
class TestCrawlList:
|
||||
"""Tests for `archivebox crawl list`."""
|
||||
|
||||
def test_list_empty(self, initialized_archive):
|
||||
def test_list_empty(self, archivebox_cli, initialized_archive):
|
||||
"""List with no crawls returns empty."""
|
||||
result = run_archivebox_cmd(["crawl", "list"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["crawl", "list"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "Listed 0 crawls" in result.stderr
|
||||
|
||||
def test_list_returns_created(self, initialized_archive):
|
||||
def test_list_returns_created(self, archivebox_cli, initialized_archive):
|
||||
"""List returns previously created crawls."""
|
||||
url = create_test_url()
|
||||
run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(["crawl", "list"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["crawl", "list"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert len(records) >= 1
|
||||
assert any(url in r.get("urls", "") for r in records)
|
||||
|
||||
def test_list_filter_by_status(self, initialized_archive):
|
||||
def test_list_filter_by_status(self, archivebox_cli, initialized_archive):
|
||||
"""Filter crawls by status."""
|
||||
url = create_test_url()
|
||||
run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["crawl", "list", "--status=queued"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -187,18 +172,17 @@ class TestCrawlList:
|
||||
for r in records:
|
||||
assert r["status"] == "queued"
|
||||
|
||||
def test_list_with_limit(self, initialized_archive):
|
||||
def test_list_with_limit(self, archivebox_cli, initialized_archive):
|
||||
"""Limit number of results."""
|
||||
# Create multiple crawls
|
||||
for _ in range(3):
|
||||
run_archivebox_cmd(
|
||||
archivebox_cli(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(["crawl", "list", "--limit=2"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["crawl", "list", "--limit=2"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
@@ -208,7 +192,7 @@ class TestCrawlList:
|
||||
class TestCrawlUpdate:
|
||||
"""Tests for `archivebox crawl update`."""
|
||||
|
||||
def test_update_status(self, initialized_archive):
|
||||
def test_update_status(self, archivebox_cli, initialized_archive):
|
||||
"""Update crawl status."""
|
||||
# Create a crawl
|
||||
url = create_test_url()
|
||||
@@ -217,12 +201,9 @@ class TestCrawlUpdate:
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
# Update it
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "update", "--status=started"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -236,57 +217,48 @@ class TestCrawlUpdate:
|
||||
class TestCrawlDelete:
|
||||
"""Tests for `archivebox crawl delete`."""
|
||||
|
||||
def test_delete_requires_yes(self, initialized_archive):
|
||||
def test_delete_requires_yes(self, archivebox_cli, initialized_archive):
|
||||
"""Delete requires --yes flag."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "delete"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert code == 1
|
||||
assert "--yes" in stderr
|
||||
|
||||
def test_delete_with_yes(self, initialized_archive):
|
||||
def test_delete_with_yes(self, archivebox_cli, initialized_archive):
|
||||
"""Delete with --yes flag works."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "delete", "--yes"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert code == 0
|
||||
assert "Deleted 1 crawls" in stderr
|
||||
|
||||
def test_delete_dry_run(self, initialized_archive):
|
||||
def test_delete_dry_run(self, archivebox_cli, initialized_archive):
|
||||
"""Dry run shows what would be deleted."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "delete", "--dry-run"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
|
||||
@@ -167,30 +167,15 @@ def test_extract_no_wait_keeps_sealed_snapshot_sealed(initialized_archive):
|
||||
assert result.output_str == "Example Domain"
|
||||
|
||||
|
||||
def test_extract_runs_on_snapshot_id(initialized_archive):
|
||||
"""Test that extract command accepts a snapshot ID."""
|
||||
env = cli_env(PLUGINS="wget,title")
|
||||
create_extract_snapshot(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
|
||||
|
||||
# Run extract on the snapshot
|
||||
result = run_archivebox_cmd(
|
||||
["extract", "--plugins=wget,title", str(snapshot_id)],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=90,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
def assert_example_outputs(initialized_archive, result, snapshot_id):
|
||||
"""Verify emitted JSONL, persisted results, and real wget/title output bytes."""
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
result_records = {
|
||||
record["plugin"]: record
|
||||
for record in records
|
||||
if record.get("type") == "ArchiveResult"
|
||||
and record.get("snapshot_id") == str(snapshot_id)
|
||||
and (record.get("plugin") in {"wget", "title"})
|
||||
and record.get("plugin") in {"wget", "title"}
|
||||
}
|
||||
assert set(result_records) == {"wget", "title"}, records
|
||||
assert result_records["title"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
@@ -215,6 +200,26 @@ def test_extract_runs_on_snapshot_id(initialized_archive):
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
|
||||
|
||||
def test_extract_runs_on_snapshot_id(initialized_archive):
|
||||
"""Test that extract command accepts a snapshot ID."""
|
||||
env = cli_env(PLUGINS="wget,title")
|
||||
create_extract_snapshot(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
|
||||
|
||||
# Run extract on the snapshot
|
||||
result = run_archivebox_cmd(
|
||||
["extract", "--plugins=wget,title", str(snapshot_id)],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=90,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert_example_outputs(initialized_archive, result, snapshot_id)
|
||||
|
||||
|
||||
def test_extract_with_enabled_extractor_creates_archiveresult(initialized_archive):
|
||||
"""Test that extract creates ArchiveResult when extractor is enabled."""
|
||||
env = cli_env(PLUGINS="wget,title")
|
||||
@@ -233,35 +238,7 @@ def test_extract_with_enabled_extractor_creates_archiveresult(initialized_archiv
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
result_records = {
|
||||
record["plugin"]: record
|
||||
for record in records
|
||||
if record.get("type") == "ArchiveResult"
|
||||
and record.get("snapshot_id") == str(snapshot_id)
|
||||
and (record.get("plugin") in {"wget", "title"})
|
||||
}
|
||||
assert set(result_records) == {"wget", "title"}, records
|
||||
assert result_records["title"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["title"]["output_str"] == "Example Domain"
|
||||
assert result_records["wget"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["wget"]["output_str"] == "wget/example.com/index.html"
|
||||
with use_archivebox_db(initialized_archive):
|
||||
archiveresults = {row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin__in=("wget", "title"))}
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
|
||||
assert snapshot_dir is not None
|
||||
title_path = snapshot_dir / "title" / "title.txt"
|
||||
wget_path = snapshot_dir / "wget" / "example.com" / "index.html"
|
||||
assert title_path.is_file()
|
||||
assert wget_path.is_file()
|
||||
assert title_path.read_text(encoding="utf-8").strip() == "Example Domain"
|
||||
assert "Example Domain" in wget_path.read_text(encoding="utf-8")
|
||||
assert archiveresults["title"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["title"].output_str == "Example Domain"
|
||||
assert archiveresults["title"].output_files["title.txt"]["size"] == title_path.stat().st_size
|
||||
assert archiveresults["wget"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["wget"].output_str == "wget/example.com/index.html"
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
assert_example_outputs(initialized_archive, result, snapshot_id)
|
||||
|
||||
|
||||
def test_extract_plugin_option_accepted(initialized_archive):
|
||||
@@ -280,35 +257,7 @@ def test_extract_plugin_option_accepted(initialized_archive):
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
result_records = {
|
||||
record["plugin"]: record
|
||||
for record in records
|
||||
if record.get("type") == "ArchiveResult"
|
||||
and record.get("snapshot_id") == str(snapshot_id)
|
||||
and (record.get("plugin") in {"wget", "title"})
|
||||
}
|
||||
assert set(result_records) == {"wget", "title"}, records
|
||||
assert result_records["title"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["title"]["output_str"] == "Example Domain"
|
||||
assert result_records["wget"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["wget"]["output_str"] == "wget/example.com/index.html"
|
||||
with use_archivebox_db(initialized_archive):
|
||||
archiveresults = {row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin__in=("wget", "title"))}
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
|
||||
assert snapshot_dir is not None
|
||||
title_path = snapshot_dir / "title" / "title.txt"
|
||||
wget_path = snapshot_dir / "wget" / "example.com" / "index.html"
|
||||
assert title_path.is_file()
|
||||
assert wget_path.is_file()
|
||||
assert title_path.read_text(encoding="utf-8").strip() == "Example Domain"
|
||||
assert "Example Domain" in wget_path.read_text(encoding="utf-8")
|
||||
assert archiveresults["title"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["title"].output_str == "Example Domain"
|
||||
assert archiveresults["title"].output_files["title.txt"]["size"] == title_path.stat().st_size
|
||||
assert archiveresults["wget"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["wget"].output_str == "wget/example.com/index.html"
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
assert_example_outputs(initialized_archive, result, snapshot_id)
|
||||
|
||||
|
||||
def test_extract_stdin_snapshot_id(initialized_archive):
|
||||
@@ -328,35 +277,7 @@ def test_extract_stdin_snapshot_id(initialized_archive):
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
result_records = {
|
||||
record["plugin"]: record
|
||||
for record in records
|
||||
if record.get("type") == "ArchiveResult"
|
||||
and record.get("snapshot_id") == str(snapshot_id)
|
||||
and (record.get("plugin") in {"wget", "title"})
|
||||
}
|
||||
assert set(result_records) == {"wget", "title"}, records
|
||||
assert result_records["title"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["title"]["output_str"] == "Example Domain"
|
||||
assert result_records["wget"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["wget"]["output_str"] == "wget/example.com/index.html"
|
||||
with use_archivebox_db(initialized_archive):
|
||||
archiveresults = {row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin__in=("wget", "title"))}
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
|
||||
assert snapshot_dir is not None
|
||||
title_path = snapshot_dir / "title" / "title.txt"
|
||||
wget_path = snapshot_dir / "wget" / "example.com" / "index.html"
|
||||
assert title_path.is_file()
|
||||
assert wget_path.is_file()
|
||||
assert title_path.read_text(encoding="utf-8").strip() == "Example Domain"
|
||||
assert "Example Domain" in wget_path.read_text(encoding="utf-8")
|
||||
assert archiveresults["title"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["title"].output_str == "Example Domain"
|
||||
assert archiveresults["title"].output_files["title.txt"]["size"] == title_path.stat().st_size
|
||||
assert archiveresults["wget"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["wget"].output_str == "wget/example.com/index.html"
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
assert_example_outputs(initialized_archive, result, snapshot_id)
|
||||
|
||||
|
||||
def test_extract_stdin_jsonl_input(initialized_archive):
|
||||
@@ -382,35 +303,7 @@ def test_extract_stdin_jsonl_input(initialized_archive):
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
result_records = {
|
||||
record["plugin"]: record
|
||||
for record in records
|
||||
if record.get("type") == "ArchiveResult"
|
||||
and record.get("snapshot_id") == str(snapshot_id)
|
||||
and (record.get("plugin") in {"wget", "title"})
|
||||
}
|
||||
assert set(result_records) == {"wget", "title"}, records
|
||||
assert result_records["title"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["title"]["output_str"] == "Example Domain"
|
||||
assert result_records["wget"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["wget"]["output_str"] == "wget/example.com/index.html"
|
||||
with use_archivebox_db(initialized_archive):
|
||||
archiveresults = {row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin__in=("wget", "title"))}
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
|
||||
assert snapshot_dir is not None
|
||||
title_path = snapshot_dir / "title" / "title.txt"
|
||||
wget_path = snapshot_dir / "wget" / "example.com" / "index.html"
|
||||
assert title_path.is_file()
|
||||
assert wget_path.is_file()
|
||||
assert title_path.read_text(encoding="utf-8").strip() == "Example Domain"
|
||||
assert "Example Domain" in wget_path.read_text(encoding="utf-8")
|
||||
assert archiveresults["title"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["title"].output_str == "Example Domain"
|
||||
assert archiveresults["title"].output_files["title.txt"]["size"] == title_path.stat().st_size
|
||||
assert archiveresults["wget"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["wget"].output_str == "wget/example.com/index.html"
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
assert_example_outputs(initialized_archive, result, snapshot_id)
|
||||
|
||||
|
||||
def test_extract_pipeline_from_snapshot(initialized_archive):
|
||||
@@ -431,35 +324,7 @@ def test_extract_pipeline_from_snapshot(initialized_archive):
|
||||
snapshot = Snapshot.objects.filter(url="https://example.com").first()
|
||||
|
||||
assert snapshot is not None, "Snapshot should be created by pipeline"
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
result_records = {
|
||||
record["plugin"]: record
|
||||
for record in records
|
||||
if record.get("type") == "ArchiveResult"
|
||||
and record.get("snapshot_id") == str(snapshot.id)
|
||||
and (record.get("plugin") in {"wget", "title"})
|
||||
}
|
||||
assert set(result_records) == {"wget", "title"}, records
|
||||
assert result_records["title"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["title"]["output_str"] == "Example Domain"
|
||||
assert result_records["wget"]["status"] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result_records["wget"]["output_str"] == "wget/example.com/index.html"
|
||||
with use_archivebox_db(initialized_archive):
|
||||
archiveresults = {row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin__in=("wget", "title"))}
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot.id))
|
||||
assert snapshot_dir is not None
|
||||
title_path = snapshot_dir / "title" / "title.txt"
|
||||
wget_path = snapshot_dir / "wget" / "example.com" / "index.html"
|
||||
assert title_path.is_file()
|
||||
assert wget_path.is_file()
|
||||
assert title_path.read_text(encoding="utf-8").strip() == "Example Domain"
|
||||
assert "Example Domain" in wget_path.read_text(encoding="utf-8")
|
||||
assert archiveresults["title"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["title"].output_str == "Example Domain"
|
||||
assert archiveresults["title"].output_files["title.txt"]["size"] == title_path.stat().st_size
|
||||
assert archiveresults["wget"].status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert archiveresults["wget"].output_str == "wget/example.com/index.html"
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
assert_example_outputs(initialized_archive, result, snapshot.id)
|
||||
|
||||
|
||||
def test_extract_multiple_snapshots(initialized_archive):
|
||||
|
||||
@@ -3,8 +3,6 @@ Tests for archivebox list command.
|
||||
Verify list emits snapshot JSONL and applies the documented filters.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
@@ -201,7 +199,7 @@ def test_streaming_json_matches_snapshot_serializer(initialized_archive):
|
||||
assert [record["url"] for record in actual] == ["https://example.com/a", "https://example.com/b"]
|
||||
|
||||
|
||||
def test_list_limit_zero_streams_one_million_snapshots_without_materializing(initialized_archive, tmp_path):
|
||||
def test_list_limit_zero_streams_one_million_snapshots_without_materializing(archivebox_cli, initialized_archive, tmp_path):
|
||||
"""Regression: archivebox list --limit=0 must stream unbounded result sets."""
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
@@ -272,12 +270,9 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini
|
||||
|
||||
output_path = tmp_path / "million-snapshots.jsonl"
|
||||
with output_path.open("w") as stdout:
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["list", "--limit=0"],
|
||||
stdout=stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
@@ -424,48 +419,39 @@ def test_list_allows_sort_with_limit(initialized_archive):
|
||||
assert len(rows) == 2
|
||||
|
||||
|
||||
def test_snapshot_list_search_meta(initialized_archive):
|
||||
def test_snapshot_list_search_meta(archivebox_cli, initialized_archive):
|
||||
"""snapshot list should support metadata search mode."""
|
||||
url = create_test_url(domain="meta-search-example.com")
|
||||
run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list", "--search=meta", "meta-search-example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert len(records) == 1
|
||||
assert records[0]["url"] == url
|
||||
assert "meta-search-example.com" in records[0]["url"]
|
||||
|
||||
|
||||
def test_list_search_meta_matches_metadata(initialized_archive):
|
||||
def test_list_search_meta_matches_metadata(archivebox_cli, initialized_archive):
|
||||
"""top-level list --search=meta should apply metadata search to the queryset."""
|
||||
url = create_test_url(domain="top-level-meta-search-example.com")
|
||||
run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["list", "--search=meta", "top-level-meta-search-example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert len(records) == 1
|
||||
assert records[0]["url"] == url
|
||||
assert "top-level-meta-search-example.com" in records[0]["url"]
|
||||
|
||||
|
||||
def test_search_command_finds_snapshots(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_finds_snapshots(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
@@ -474,15 +460,14 @@ def test_search_command_finds_snapshots(initialized_archive):
|
||||
assert "example" in result.stdout
|
||||
|
||||
|
||||
def test_search_command_returns_no_results_for_missing_term(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_returns_no_results_for_missing_term(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(["search", "nonexistentterm12345"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["search", "nonexistentterm12345"],
|
||||
)
|
||||
|
||||
assert result.returncode in [0, 1]
|
||||
|
||||
@@ -493,89 +478,69 @@ def test_search_command_on_empty_archive(initialized_archive):
|
||||
assert result.returncode in [0, 1]
|
||||
|
||||
|
||||
def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_outputs_matching_snapshots_as_jsonl(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert [row["url"] for row in records] == ["https://example.com"]
|
||||
assert any("example.com" in row.get("url", "") for row in records)
|
||||
|
||||
|
||||
def test_search_command_json_outputs_matching_snapshots(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_json_outputs_matching_snapshots(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(["search", "--json"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert [row["url"] for row in payload] == ["https://example.com"]
|
||||
assert any("example.com" in row.get("url", "") for row in payload)
|
||||
|
||||
|
||||
def test_search_command_json_with_headers_wraps_links_payload(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_json_with_headers_wraps_links_payload(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["search", "--json", "--with-headers"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert "links" in payload
|
||||
assert [row["url"] for row in payload["links"]] == ["https://example.com"]
|
||||
assert any("example.com" in row.get("url", "") for row in payload["links"])
|
||||
|
||||
|
||||
def test_search_command_html_outputs_markup(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_html_outputs_markup(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(["search", "--html"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "<" in result.stdout
|
||||
assert 'href="https://example.com"' in result.stdout
|
||||
assert "example.com" in result.stdout
|
||||
|
||||
|
||||
def test_search_command_csv_outputs_requested_column(initialized_archive):
|
||||
run_archivebox_cmd(
|
||||
def test_search_command_csv_outputs_requested_column(archivebox_cli, initialized_archive):
|
||||
archivebox_cli(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["search", "--csv", "url", "--with-headers"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert list(csv.reader(io.StringIO(result.stdout))) == [["url"], ["https://example.com"]]
|
||||
assert "url" in result.stdout
|
||||
assert "example.com" in result.stdout
|
||||
|
||||
|
||||
def test_search_command_with_headers_requires_structured_output_format(initialized_archive):
|
||||
@@ -586,19 +551,16 @@ def test_search_command_with_headers_requires_structured_output_format(initializ
|
||||
assert "json" in result.stderr.lower()
|
||||
|
||||
|
||||
def test_search_command_sort_option_runs_successfully(initialized_archive):
|
||||
def test_search_command_sort_option_runs_successfully(archivebox_cli, initialized_archive):
|
||||
for url in ["https://iana.org", "https://example.com"]:
|
||||
run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["search", "--csv", "url", "--sort=url"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert list(csv.reader(io.StringIO(result.stdout))) == [["https://example.com"], ["https://iana.org"]]
|
||||
assert "example.com" in result.stdout or "iana.org" in result.stdout
|
||||
|
||||
|
||||
def test_search_command_help_lists_supported_filters(initialized_archive):
|
||||
|
||||
@@ -18,10 +18,10 @@ def test_persona_help_runs_successfully(tmp_path):
|
||||
assert "list" in result.stdout
|
||||
|
||||
|
||||
def test_persona_commands_emit_consistent_model_records(initialized_archive):
|
||||
def test_persona_commands_emit_consistent_model_records(archivebox_cli, initialized_archive):
|
||||
import json
|
||||
|
||||
created = run_archivebox_cmd(["persona", "create", "portable"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
created = archivebox_cli(["persona", "create", "portable"])
|
||||
assert created.returncode == 0, created.stderr
|
||||
record = json.loads(created.stdout)
|
||||
assert record == {
|
||||
@@ -32,7 +32,7 @@ def test_persona_commands_emit_consistent_model_records(initialized_archive):
|
||||
"COOKIES_FILE": "",
|
||||
}
|
||||
for command in (["persona", "list", "--name=portable"], ["persona", "update"]):
|
||||
result = run_archivebox_cmd(command, input=created.stdout, cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(command, input=created.stdout)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert json.loads(result.stdout) == record
|
||||
assert Path(record["CHROME_USER_DATA_DIR"]).is_dir()
|
||||
|
||||
@@ -14,7 +14,6 @@ from importlib.resources import files
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.machine.models import Binary
|
||||
@@ -76,7 +75,7 @@ def test_parse_line_accepts_supported_piping_inputs():
|
||||
assert parse_line("file:///tmp/example.txt") is None
|
||||
|
||||
|
||||
def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path, initialized_archive):
|
||||
def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(archivebox_cli, tmp_path, initialized_archive):
|
||||
"""Piping helpers should consume args, structured JSONL, and pass-through records."""
|
||||
from archivebox.misc.jsonl import TYPE_CRAWL, read_args_or_stdin
|
||||
|
||||
@@ -87,19 +86,13 @@ def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path, initial
|
||||
local_file.write_text("https://from-file-arg.example\n")
|
||||
assert list(read_args_or_stdin((str(local_file),))) == []
|
||||
|
||||
snapshot_result = run_archivebox_cmd(
|
||||
snapshot_result = archivebox_cli(
|
||||
["snapshot", "create", "--tag=test", "https://jsonl-url.com"],
|
||||
check=True,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
crawl_result = run_archivebox_cmd(
|
||||
crawl_result = archivebox_cli(
|
||||
["crawl", "create", "https://crawl-url.com"],
|
||||
check=True,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
snapshot_record = next(record for record in parse_jsonl_output(snapshot_result.stdout) if record.get("type") == "Snapshot")
|
||||
|
||||
@@ -164,25 +157,24 @@ def test_collect_urls_from_plugins_trims_trailing_punctuation(tmp_path):
|
||||
]
|
||||
|
||||
|
||||
def test_crawl_create_stdout_pipes_into_run(initialized_archive):
|
||||
def test_crawl_create_stdout_pipes_into_run(archivebox_cli, initialized_archive):
|
||||
"""`archivebox crawl create | archivebox run` should queue and materialize snapshots."""
|
||||
url = create_test_url()
|
||||
|
||||
_cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "create", url],
|
||||
)
|
||||
create_stdout, create_stderr, create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert create_code == 0, create_stderr
|
||||
assert_jsonl_only(create_stdout)
|
||||
|
||||
crawl = next(record for record in parse_jsonl_output(create_stdout) if record.get("type") == "Crawl")
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=create_stdout,
|
||||
timeout=120,
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert run_code == 0, run_stderr
|
||||
@@ -197,41 +189,34 @@ def test_crawl_create_stdout_pipes_into_run(initialized_archive):
|
||||
assert snapshot_count >= 1
|
||||
|
||||
|
||||
def test_snapshot_list_stdout_pipes_into_run(initialized_archive):
|
||||
def test_snapshot_list_stdout_pipes_into_run(archivebox_cli, initialized_archive):
|
||||
"""`archivebox snapshot list | archivebox run` should requeue listed snapshots."""
|
||||
url = create_test_url()
|
||||
|
||||
_cmd_result = run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
)
|
||||
create_stdout, create_stderr, create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert create_code == 0, create_stderr
|
||||
snapshot = next(record for record in parse_jsonl_output(create_stdout) if record.get("type") == "Snapshot")
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "list", "--status=queued", f"--url__icontains={snapshot['id']}"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
list_stdout, list_stderr, list_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
if list_code != 0 or not parse_jsonl_output(list_stdout):
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "list", f"--url__icontains={url}"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
list_stdout, list_stderr, list_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert list_code == 0, list_stderr
|
||||
assert_jsonl_only(list_stdout)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=list_stdout,
|
||||
timeout=120,
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert run_code == 0, run_stderr
|
||||
@@ -245,47 +230,35 @@ def test_snapshot_list_stdout_pipes_into_run(initialized_archive):
|
||||
assert snapshot_status == "sealed"
|
||||
|
||||
|
||||
def test_archiveresult_list_stdout_pipes_into_run(initialized_archive):
|
||||
def test_archiveresult_list_stdout_pipes_into_run(archivebox_cli, initialized_archive):
|
||||
"""`archivebox archiveresult list | archivebox run` should preserve clean JSONL stdout."""
|
||||
url = create_test_url()
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
snapshot_stdout, snapshot_stderr, snapshot_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert snapshot_code == 0, snapshot_stderr
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "create", "--plugin=favicon"],
|
||||
stdin=snapshot_stdout,
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
ar_create_stdout, ar_create_stderr, ar_create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert ar_create_code == 0, ar_create_stderr
|
||||
|
||||
run_archivebox_cmd(
|
||||
archivebox_cli(
|
||||
["run"],
|
||||
stdin=ar_create_stdout,
|
||||
timeout=120,
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
list_stdout, list_stderr, list_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert list_code == 0, list_stderr
|
||||
@@ -293,14 +266,11 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive):
|
||||
listed_records = parse_jsonl_output(list_stdout)
|
||||
archiveresult = next(record for record in listed_records if record.get("type") == "ArchiveResult")
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=list_stdout,
|
||||
timeout=120,
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert run_code == 0, run_stderr
|
||||
@@ -310,13 +280,10 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive):
|
||||
assert any(record.get("type") == "ArchiveResult" and record.get("id") == archiveresult["id"] for record in run_records)
|
||||
|
||||
|
||||
def test_binary_create_stdout_pipes_into_run(initialized_archive):
|
||||
def test_binary_create_stdout_pipes_into_run(archivebox_cli, initialized_archive):
|
||||
"""`archivebox binary create | archivebox run` should queue the binary record for processing."""
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["binary", "create", "--name=python3", f"--abspath={sys.executable}", "--version=test"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
create_stdout, create_stderr, create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert create_code == 0, create_stderr
|
||||
@@ -324,13 +291,10 @@ def test_binary_create_stdout_pipes_into_run(initialized_archive):
|
||||
|
||||
binary = next(record for record in parse_jsonl_output(create_stdout) if record.get("type") in {"BinaryRequest", "Binary"})
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=create_stdout,
|
||||
timeout=120,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert run_code == 0, run_stderr
|
||||
@@ -344,45 +308,38 @@ def test_binary_create_stdout_pipes_into_run(initialized_archive):
|
||||
assert status in {"queued", "installed"}
|
||||
|
||||
|
||||
def test_multi_stage_pipeline_into_run(initialized_archive):
|
||||
def test_multi_stage_pipeline_into_run(archivebox_cli, initialized_archive):
|
||||
"""`crawl create | snapshot create | archiveresult create | run` should preserve JSONL and finish work."""
|
||||
url = create_test_url()
|
||||
|
||||
_cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
_cmd_result = archivebox_cli(
|
||||
["crawl", "create", url],
|
||||
)
|
||||
crawl_stdout, crawl_stderr, crawl_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert crawl_code == 0, crawl_stderr
|
||||
assert_jsonl_only(crawl_stdout)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create"],
|
||||
stdin=crawl_stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
snapshot_stdout, snapshot_stderr, snapshot_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert snapshot_code == 0, snapshot_stderr
|
||||
assert_jsonl_only(snapshot_stdout)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "create", "--plugin=favicon"],
|
||||
stdin=snapshot_stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
archiveresult_stdout, archiveresult_stderr, archiveresult_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert archiveresult_code == 0, archiveresult_stderr
|
||||
assert_jsonl_only(archiveresult_stdout)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=archiveresult_stdout,
|
||||
timeout=120,
|
||||
env=PIPE_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert run_code == 0, run_stderr
|
||||
|
||||
@@ -85,27 +85,13 @@ class TestRunInput:
|
||||
|
||||
@pytest.mark.parametrize("source", ["crawl", "snapshot"])
|
||||
@pytest.mark.parametrize("runs", [1, 2], ids=["new-record", "existing-record"])
|
||||
def test_run_created_record(self, initialized_archive, source, runs):
|
||||
created = run_archivebox_cmd(
|
||||
[source, "create", create_test_url()],
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
def test_run_created_record(self, archivebox_cli, initialized_archive, source, runs):
|
||||
created = archivebox_cli([source, "create", create_test_url()], env=RUN_TEST_ENV)
|
||||
assert created.returncode == 0, created.stderr
|
||||
created_record = next(record for record in parse_jsonl_output(created.stdout) if record.get("type") == source.title())
|
||||
stdin = created.stdout
|
||||
for _ in range(runs):
|
||||
result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=stdin,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
result = archivebox_cli(["run"], stdin=stdin, timeout=120, env=RUN_TEST_ENV)
|
||||
assert result.returncode == 0, result.stderr
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert len(records) >= 1
|
||||
@@ -115,17 +101,14 @@ class TestRunInput:
|
||||
assert matching[0]["id"] == created_record["id"]
|
||||
stdin = result.stdout
|
||||
|
||||
def test_run_with_plain_url(self, initialized_archive):
|
||||
def test_run_with_plain_url(self, archivebox_cli, initialized_archive):
|
||||
"""Run accepts plain URL records (no type field)."""
|
||||
url = create_test_url()
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=url + "\n",
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -137,18 +120,15 @@ class TestRunWithArchiveResult:
|
||||
"""Tests for `archivebox run` with ArchiveResult input."""
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, initialized_archive):
|
||||
def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, archivebox_cli, initialized_archive):
|
||||
import json
|
||||
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
create_result = run_archivebox_cmd(
|
||||
create_result = archivebox_cli(
|
||||
["snapshot", "create", create_test_url()],
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
snapshot_id = next(record["id"] for record in parse_jsonl_output(create_result.stdout) if record.get("type") == "Snapshot")
|
||||
missing_hook = "on_Snapshot__99_missing_favicon_hook"
|
||||
@@ -162,14 +142,11 @@ class TestRunWithArchiveResult:
|
||||
},
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=f"{request}\n",
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
@@ -186,66 +163,48 @@ class TestRunWithArchiveResult:
|
||||
assert rows[0][0].startswith("on_Snapshot__")
|
||||
assert rows[0][1] in ArchiveResult.FINAL_STATES
|
||||
|
||||
def test_run_requeues_failed_archiveresult(self, initialized_archive):
|
||||
def test_run_requeues_failed_archiveresult(self, archivebox_cli, initialized_archive):
|
||||
"""Run uses a failed ArchiveResult as a parent Snapshot/plugin reference."""
|
||||
url = create_test_url()
|
||||
|
||||
# Create snapshot and archive result
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["archiveresult", "create", "--plugin=favicon"],
|
||||
stdin=stdout1,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(stdout2))
|
||||
|
||||
initial_run = run_archivebox_cmd(
|
||||
initial_run = archivebox_cli(
|
||||
["run"],
|
||||
stdin=stdout2,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert initial_run.returncode == 0, initial_run.stderr
|
||||
persisted_result = run_archivebox_cmd(
|
||||
persisted_result = archivebox_cli(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert persisted_result.returncode == 0, persisted_result.stderr
|
||||
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(persisted_result.stdout))
|
||||
|
||||
# Update to failed
|
||||
update_result = run_archivebox_cmd(
|
||||
update_result = archivebox_cli(
|
||||
["archiveresult", "update", "--status=failed"],
|
||||
stdin=persisted_result.stdout,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert update_result.returncode == 0, update_result.stderr
|
||||
failed_result = run_archivebox_cmd(
|
||||
failed_result = archivebox_cli(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert failed_result.returncode == 0, failed_result.stderr
|
||||
failed_records = [record for record in parse_jsonl_output(failed_result.stdout) if record.get("type") == "ArchiveResult"]
|
||||
@@ -254,14 +213,11 @@ class TestRunWithArchiveResult:
|
||||
failed_jsonl = next(line for line in failed_result.stdout.splitlines() if failed_records[0]["id"] in line) + "\n"
|
||||
|
||||
# Now run should re-queue it
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=failed_jsonl,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -273,7 +229,7 @@ class TestRunWithArchiveResult:
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
class TestRunRecovery:
|
||||
def test_run_maintenance_logs_unfinished_crawl_repair(self, initialized_archive):
|
||||
def test_run_maintenance_logs_unfinished_crawl_repair(self, archivebox_cli, initialized_archive):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
@@ -302,13 +258,10 @@ class TestRunRecovery:
|
||||
crawl_id = crawl.id
|
||||
snapshot_id = snapshot.id
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run", "--maintenance-only"],
|
||||
timeout=90,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
@@ -329,23 +282,17 @@ class TestRunRecovery:
|
||||
class TestRunPassThrough:
|
||||
"""Tests for pass-through behavior in `archivebox run`."""
|
||||
|
||||
def test_run_passes_through_tag_emitted_by_cli(self, initialized_archive):
|
||||
def test_run_passes_through_tag_emitted_by_cli(self, archivebox_cli, initialized_archive):
|
||||
"""Run passes through a real non-runnable Tag record."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
tag_result = archivebox_cli(
|
||||
["tag", "create", "run-input-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
tag_record = parse_jsonl_output(tag_result.stdout)[0]
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=tag_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -354,26 +301,20 @@ class TestRunPassThrough:
|
||||
assert len(tag_records) == 1
|
||||
assert tag_records[0]["id"] == tag_record["id"]
|
||||
|
||||
def test_run_outputs_all_processed_records(self, initialized_archive):
|
||||
def test_run_outputs_all_processed_records(self, archivebox_cli, initialized_archive):
|
||||
"""Run outputs all processed records for chaining."""
|
||||
url = create_test_url()
|
||||
create_result = run_archivebox_cmd(
|
||||
create_result = archivebox_cli(
|
||||
["crawl", "create", url],
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert create_result.returncode == 0, create_result.stderr
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=create_result.stdout,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -385,39 +326,27 @@ class TestRunPassThrough:
|
||||
class TestRunMixedInput:
|
||||
"""Tests for `archivebox run` with mixed record types."""
|
||||
|
||||
def test_run_handles_mixed_records_emitted_by_cli(self, initialized_archive):
|
||||
def test_run_handles_mixed_records_emitted_by_cli(self, archivebox_cli, initialized_archive):
|
||||
"""Run handles real Crawl, Snapshot, and Tag records from CLI stages."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
tag_result = archivebox_cli(
|
||||
["tag", "create", "mixed-run-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
crawl_result = run_archivebox_cmd(
|
||||
crawl_result = archivebox_cli(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert crawl_result.returncode == 0, crawl_result.stderr
|
||||
snapshot_result = run_archivebox_cmd(
|
||||
snapshot_result = archivebox_cli(
|
||||
["snapshot", "create"],
|
||||
stdin=crawl_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert snapshot_result.returncode == 0, snapshot_result.stderr
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=tag_result.stdout + snapshot_result.stdout,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -430,28 +359,25 @@ class TestRunMixedInput:
|
||||
class TestRunEmpty:
|
||||
"""Tests for `archivebox run` edge cases."""
|
||||
|
||||
def test_run_empty_stdin(self, initialized_archive):
|
||||
def test_run_empty_stdin(self, archivebox_cli, initialized_archive):
|
||||
"""Run with empty stdin returns success."""
|
||||
result = run_archivebox_cmd(["run"], stdin="", cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin="",
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
def test_run_no_runnable_records_to_process(self, initialized_archive):
|
||||
def test_run_no_runnable_records_to_process(self, archivebox_cli, initialized_archive):
|
||||
"""Run with only a real non-runnable Tag reports no work."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
tag_result = archivebox_cli(
|
||||
["tag", "create", "non-runnable-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["run"],
|
||||
stdin=tag_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
@@ -29,11 +29,13 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
class TestSnapshotCreate:
|
||||
"""Tests for `archivebox snapshot create`."""
|
||||
|
||||
def test_create_from_url_args(self, initialized_archive):
|
||||
def test_create_from_url_args(self, archivebox_cli, initialized_archive):
|
||||
"""Create snapshot from URL arguments."""
|
||||
url = create_test_url()
|
||||
|
||||
result = run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
||||
assert "Created" in result.stderr
|
||||
@@ -43,7 +45,7 @@ class TestSnapshotCreate:
|
||||
assert records[0]["type"] == "Snapshot"
|
||||
assert records[0]["url"] == url
|
||||
|
||||
def test_create_from_crawl_jsonl(self, initialized_archive):
|
||||
def test_create_from_crawl_jsonl(self, archivebox_cli, initialized_archive):
|
||||
"""Create snapshots from Crawl JSONL input."""
|
||||
url = create_test_url()
|
||||
|
||||
@@ -53,12 +55,9 @@ class TestSnapshotCreate:
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
# Pipe crawl to snapshot create
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create"],
|
||||
stdin=json.dumps(crawl),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -73,44 +72,35 @@ class TestSnapshotCreate:
|
||||
snapshot = next(r for r in records if r["type"] == "Snapshot")
|
||||
assert snapshot["url"] == url
|
||||
assert snapshot["crawl_id"] == crawl["id"]
|
||||
listed = run_archivebox_cmd(["crawl", "list"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
listed = archivebox_cli(["crawl", "list"])
|
||||
assert listed.returncode == 0, listed.stderr
|
||||
assert [record["id"] for record in parse_jsonl_output(listed.stdout)] == [crawl["id"]]
|
||||
|
||||
def test_create_with_tag(self, initialized_archive):
|
||||
def test_create_with_tag(self, archivebox_cli, initialized_archive):
|
||||
"""Create snapshot with --tag flag."""
|
||||
url = create_test_url()
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "create", "--tag=test-tag", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert "test-tag" in records[0].get("tags", "")
|
||||
|
||||
def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
|
||||
def test_create_passes_through_tag_emitted_by_cli(self, archivebox_cli, initialized_archive):
|
||||
"""A real Tag emitted by the CLI remains available to the next stage."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
tag_result = archivebox_cli(
|
||||
["tag", "create", "snapshot-input-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
tag_record = parse_jsonl_output(tag_result.stdout)[0]
|
||||
url = create_test_url()
|
||||
stdin = tag_result.stdout + url + "\n"
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "create"],
|
||||
stdin=stdin,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -119,11 +109,13 @@ class TestSnapshotCreate:
|
||||
assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
|
||||
assert any(record.get("type") == "Snapshot" and record["url"] == url for record in records)
|
||||
|
||||
def test_create_multiple_urls(self, initialized_archive):
|
||||
def test_create_multiple_urls(self, archivebox_cli, initialized_archive):
|
||||
"""Create snapshots from multiple URLs."""
|
||||
urls = [create_test_url() for _ in range(3)]
|
||||
|
||||
result = run_archivebox_cmd(["snapshot", "create"] + urls, cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["snapshot", "create"] + urls,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
@@ -137,35 +129,36 @@ class TestSnapshotCreate:
|
||||
class TestSnapshotList:
|
||||
"""Tests for `archivebox snapshot list`."""
|
||||
|
||||
def test_list_empty(self, initialized_archive):
|
||||
def test_list_empty(self, archivebox_cli, initialized_archive):
|
||||
"""List with no snapshots returns empty."""
|
||||
result = run_archivebox_cmd(["snapshot", "list"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "Listed 0 snapshots" in result.stderr
|
||||
|
||||
def test_list_returns_created(self, initialized_archive):
|
||||
def test_list_returns_created(self, archivebox_cli, initialized_archive):
|
||||
"""List returns previously created snapshots."""
|
||||
url = create_test_url()
|
||||
run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(["snapshot", "list"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert len(records) >= 1
|
||||
assert any(r.get("url") == url for r in records)
|
||||
|
||||
def test_list_filter_by_status(self, initialized_archive):
|
||||
def test_list_filter_by_status(self, archivebox_cli, initialized_archive):
|
||||
"""Filter snapshots by status."""
|
||||
url = create_test_url()
|
||||
run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list", "--status=queued"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -173,16 +166,13 @@ class TestSnapshotList:
|
||||
for r in records:
|
||||
assert r["status"] == "queued"
|
||||
|
||||
def test_list_filter_by_url_contains(self, initialized_archive):
|
||||
def test_list_filter_by_url_contains(self, archivebox_cli, initialized_archive):
|
||||
"""Filter snapshots by URL contains."""
|
||||
url = create_test_url(domain="unique-domain-12345.com")
|
||||
run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list", "--url__icontains=unique-domain-12345"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
@@ -190,42 +180,30 @@ class TestSnapshotList:
|
||||
assert len(records) == 1
|
||||
assert "unique-domain-12345" in records[0]["url"]
|
||||
|
||||
def test_list_with_limit(self, initialized_archive):
|
||||
def test_list_with_limit(self, archivebox_cli, initialized_archive):
|
||||
"""Limit number of results."""
|
||||
for _ in range(3):
|
||||
run_archivebox_cmd(
|
||||
archivebox_cli(
|
||||
["snapshot", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list", "--limit=2"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
assert len(records) == 2
|
||||
|
||||
def test_list_with_sort_and_limit(self, initialized_archive):
|
||||
def test_list_with_sort_and_limit(self, archivebox_cli, initialized_archive):
|
||||
"""Sorting should be applied before limiting."""
|
||||
for _ in range(3):
|
||||
run_archivebox_cmd(
|
||||
archivebox_cli(
|
||||
["snapshot", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["snapshot", "list", "--limit=2", "--sort=-created_at"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Command failed: {result.stderr}"
|
||||
@@ -236,24 +214,18 @@ class TestSnapshotList:
|
||||
class TestSnapshotUpdate:
|
||||
"""Tests for `archivebox snapshot update`."""
|
||||
|
||||
def test_update_status(self, initialized_archive):
|
||||
def test_update_status(self, archivebox_cli, initialized_archive):
|
||||
"""Update snapshot status."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "update", "--status=started"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -263,24 +235,18 @@ class TestSnapshotUpdate:
|
||||
records = parse_jsonl_output(stdout2)
|
||||
assert records[0]["status"] == "started"
|
||||
|
||||
def test_update_add_tag(self, initialized_archive):
|
||||
def test_update_add_tag(self, archivebox_cli, initialized_archive):
|
||||
"""Update snapshot by adding tag."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "update", "--tag=new-tag"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
@@ -291,72 +257,54 @@ class TestSnapshotUpdate:
|
||||
class TestSnapshotDelete:
|
||||
"""Tests for `archivebox snapshot delete`."""
|
||||
|
||||
def test_delete_requires_yes(self, initialized_archive):
|
||||
def test_delete_requires_yes(self, archivebox_cli, initialized_archive):
|
||||
"""Delete requires --yes flag."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "delete"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert code == 1
|
||||
assert "--yes" in stderr
|
||||
|
||||
def test_delete_with_yes(self, initialized_archive):
|
||||
def test_delete_with_yes(self, archivebox_cli, initialized_archive):
|
||||
"""Delete with --yes flag works."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "delete", "--yes"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert code == 0
|
||||
assert "Deleted 1 snapshots" in stderr
|
||||
|
||||
def test_delete_dry_run(self, initialized_archive):
|
||||
def test_delete_dry_run(self, archivebox_cli, initialized_archive):
|
||||
"""Dry run shows what would be deleted."""
|
||||
url = create_test_url()
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
_cmd_result = archivebox_cli(
|
||||
["snapshot", "delete", "--dry-run"],
|
||||
stdin=json.dumps(snapshot),
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
"""Version selection uses numeric release ordering across actual ArchiveBox tags."""
|
||||
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_docs_picker_keeps_latest_rc_and_historical_patch_per_minor():
|
||||
select = runpy.run_path(str(Path(__file__).resolve().parents[2] / "bin/sync_docs_versions.py"))["selected_tags"]
|
||||
tags = [
|
||||
"v0.9.35rc99",
|
||||
"v0.9.35rc469",
|
||||
"v0.9.31-rc",
|
||||
"v0.8.5-rc",
|
||||
"v0.8.6rc1",
|
||||
"v0.7.3",
|
||||
"v0.7.4",
|
||||
"v0.6.0",
|
||||
"v0.6.2",
|
||||
"v0.5.4",
|
||||
"v0.5.6",
|
||||
"v0.4.9",
|
||||
"v0.4.24",
|
||||
"v0.2.4",
|
||||
"v0.1.0",
|
||||
"v0.0.3",
|
||||
]
|
||||
assert select(tags) == [
|
||||
"v0.9.35rc469",
|
||||
"v0.8.6rc1",
|
||||
"v0.7.4",
|
||||
"v0.6.2",
|
||||
"v0.5.6",
|
||||
"v0.4.24",
|
||||
"v0.2.4",
|
||||
"v0.1.0",
|
||||
"v0.0.3",
|
||||
]
|
||||
assert select(reversed(tags)) == select(tags)
|
||||
|
||||
|
||||
def test_readiness_requires_current_successful_build_from_expected_source():
|
||||
ready = runpy.run_path(str(Path(__file__).resolve().parents[2] / "bin/sync_docs_versions.py"))["build_ready"]
|
||||
version = {"slug": "v0.7.4", "verbose_name": "docs/restore-main-sphinx", "active": True, "built": True}
|
||||
build = {"version": "v0.7.4", "state": {"code": "finished"}, "success": True, "commit": "current"}
|
||||
assert ready(version, build, "docs/restore-main-sphinx", "current")
|
||||
assert not ready(version, {**build, "success": False}, "docs/restore-main-sphinx", "current")
|
||||
assert not ready(version, {**build, "commit": "previous"}, "docs/restore-main-sphinx", "current")
|
||||
assert not ready(version, {**build, "version": "dev"}, "docs/restore-main-sphinx", "current")
|
||||
assert not ready(version, {**build, "state": {"code": "building"}}, "docs/restore-main-sphinx", "current")
|
||||
assert not ready(version, build, "v0.7.4", "current")
|
||||
assert not ready(version, build, "docs/restore-main-sphinx", None)
|
||||
assert not ready(version, None, "docs/restore-main-sphinx", "current")
|
||||
assert not ready({**version, "active": False}, build, "docs/restore-main-sphinx", "current")
|
||||
|
||||
|
||||
def test_latest_readiness_checks_tracking_branch_identifier():
|
||||
ready = runpy.run_path(str(Path(__file__).resolve().parents[2] / "bin/sync_docs_versions.py"))["build_ready"]
|
||||
version = {"slug": "latest", "verbose_name": "latest", "identifier": "main", "active": True}
|
||||
build = {"version": "latest", "state": {"code": "finished"}, "success": True, "commit": "current"}
|
||||
assert ready(version, build, "main", "current")
|
||||
assert not ready({**version, "identifier": "dev"}, build, "main", "current")
|
||||
@@ -17,7 +17,6 @@ from importlib.resources import files
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
|
||||
|
||||
@@ -483,17 +482,14 @@ class TestDependencyRecordOutput:
|
||||
"""Test Binary JSONL emitted by the real CLI and persisted model."""
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_binary_cli_emits_resolved_dependency_record(self, initialized_archive, hermetic_lib_dir):
|
||||
def test_binary_cli_emits_resolved_dependency_record(self, archivebox_cli, initialized_archive, hermetic_lib_dir):
|
||||
install_real_binary("wget", binproviders="env,apt,brew")
|
||||
wget_path = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=WGET_CONFIG)["WGET_BINARY"]
|
||||
version = subprocess.run([wget_path, "--version"], capture_output=True, text=True, check=True).stdout.split()[2]
|
||||
from archivebox.tests.conftest import parse_jsonl_output
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
result = archivebox_cli(
|
||||
["binary", "create", "--name=wget", f"--abspath={wget_path}", f"--version={version}"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
@@ -503,11 +499,8 @@ class TestDependencyRecordOutput:
|
||||
assert data["abspath"] == wget_path
|
||||
assert data["version"] == version
|
||||
|
||||
list_result = run_archivebox_cmd(
|
||||
list_result = archivebox_cli(
|
||||
["binary", "list", "--name=wget"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert list_result.returncode == 0, list_result.stderr
|
||||
listed = parse_jsonl_output(list_result.stdout)
|
||||
|
||||
@@ -15,9 +15,7 @@ from uuid import UUID
|
||||
import pytest
|
||||
|
||||
from .migrations_helpers import (
|
||||
SCHEMA_0_7,
|
||||
seed_0_7_data,
|
||||
create_data_dir_structure,
|
||||
create_legacy_archive,
|
||||
run_archivebox_migration_cmd,
|
||||
verify_archiveresult_count,
|
||||
verify_tag_count,
|
||||
@@ -27,21 +25,8 @@ from .migrations_helpers import (
|
||||
|
||||
@pytest.fixture
|
||||
def archive_07(tmp_path):
|
||||
"""Create a temporary directory with 0.7.x schema and data."""
|
||||
db_path = tmp_path / "index.sqlite3"
|
||||
|
||||
# Create directory structure
|
||||
create_data_dir_structure(tmp_path)
|
||||
|
||||
# Create database with 0.7.x schema
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.executescript(SCHEMA_0_7)
|
||||
conn.close()
|
||||
|
||||
# Seed with test data
|
||||
original_data = seed_0_7_data(db_path)
|
||||
|
||||
return tmp_path, db_path, original_data
|
||||
"""Create a collection using the real 0.7.x schema and seed data."""
|
||||
return create_legacy_archive(tmp_path, "0.7")
|
||||
|
||||
|
||||
def test_migration_preserves_tags(archive_07):
|
||||
|
||||
@@ -19,8 +19,7 @@ from urllib.parse import urlparse
|
||||
import pytest
|
||||
|
||||
from .migrations_helpers import (
|
||||
SCHEMA_0_8,
|
||||
seed_0_8_data,
|
||||
create_legacy_archive,
|
||||
SCHEMA_0_7,
|
||||
current_snapshot_dir,
|
||||
filesystem_manifest,
|
||||
@@ -37,18 +36,8 @@ from .migrations_helpers import (
|
||||
|
||||
@pytest.fixture
|
||||
def migration_08_data(tmp_path):
|
||||
"""Create a temporary directory with 0.8.x schema and data."""
|
||||
work_dir = tmp_path
|
||||
db_path = work_dir / "index.sqlite3"
|
||||
|
||||
create_data_dir_structure(work_dir)
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.executescript(SCHEMA_0_8)
|
||||
conn.close()
|
||||
|
||||
original_data = seed_0_8_data(db_path)
|
||||
return work_dir, db_path, original_data
|
||||
"""Create a collection using the real 0.8.x schema and seed data."""
|
||||
return create_legacy_archive(tmp_path, "0.8")
|
||||
|
||||
|
||||
def convert_legacy_tags_to_uuid(conn: sqlite3.Connection) -> None:
|
||||
|
||||
@@ -5,11 +5,7 @@ import sqlite3
|
||||
import pytest
|
||||
|
||||
from .migrations_helpers import (
|
||||
SCHEMA_0_7,
|
||||
SCHEMA_0_8,
|
||||
seed_0_7_data,
|
||||
seed_0_8_data,
|
||||
create_data_dir_structure,
|
||||
create_legacy_archive,
|
||||
run_archivebox_migration_cmd,
|
||||
verify_all_snapshots_in_output,
|
||||
verify_foreign_keys,
|
||||
@@ -21,13 +17,7 @@ from .migrations_helpers import (
|
||||
|
||||
@pytest.fixture(params=("0.7", "0.8"), ids=("from-0.7", "from-0.8"))
|
||||
def legacy_archive(tmp_path, request):
|
||||
schema, seed = {"0.7": (SCHEMA_0_7, seed_0_7_data), "0.8": (SCHEMA_0_8, seed_0_8_data)}[request.param]
|
||||
create_data_dir_structure(tmp_path)
|
||||
db_path = tmp_path / "index.sqlite3"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.executescript(schema)
|
||||
conn.close()
|
||||
return tmp_path, db_path, seed(db_path)
|
||||
return create_legacy_archive(tmp_path, request.param)
|
||||
|
||||
|
||||
def test_migration_preserves_snapshot_count(legacy_archive):
|
||||
|
||||
@@ -635,22 +635,7 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
|
||||
for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
|
||||
if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
|
||||
]
|
||||
# This fixture serves HTTP. TLSNotary must reject every plaintext document;
|
||||
# a setup, timeout, or verifier error must not count as that expected refusal.
|
||||
tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
|
||||
assert {row[1] for row in tlsnotary_results} == expected_urls
|
||||
assert len(tlsnotary_results) == len(expected_urls)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
|
||||
assert status == ArchiveResult.StatusChoices.FAILED
|
||||
assert not files
|
||||
assert size == 0
|
||||
process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
|
||||
assert process is not None
|
||||
assert process.status == Process.StatusChoices.EXITED
|
||||
assert process.exit_code == 1
|
||||
assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
|
||||
assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
|
||||
assert not unexpected_results
|
||||
ytdlp_results = [
|
||||
(url, status, output_str)
|
||||
for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
|
||||
@@ -697,6 +682,19 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
|
||||
assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
|
||||
|
||||
assert processes
|
||||
failed_hook_results = [
|
||||
{
|
||||
"url": url,
|
||||
"depth": depth,
|
||||
"plugin": plugin,
|
||||
"hook_name": hook_name,
|
||||
"status": status,
|
||||
"output_str": output_str,
|
||||
}
|
||||
for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
|
||||
if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg"
|
||||
]
|
||||
assert not failed_hook_results
|
||||
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
|
||||
|
||||
intervals = []
|
||||
|
||||
@@ -8,7 +8,6 @@ from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl, CrawlSchedule
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
from .conftest import (
|
||||
write_import_format_files,
|
||||
cli_env,
|
||||
create_admin_and_token,
|
||||
get_depth_counts,
|
||||
@@ -20,38 +19,12 @@ from .conftest import (
|
||||
get_http_response,
|
||||
)
|
||||
|
||||
IMPORT_FORMAT_EXPECTATIONS = {
|
||||
"rss": {
|
||||
"url": "https://example.com/",
|
||||
"title": "RSS Example Import",
|
||||
"date": "2024-01-01",
|
||||
"tags": {"rss-tag", "metadata"},
|
||||
},
|
||||
"netscape": {
|
||||
"url": "https://www.iana.org/domains/reserved",
|
||||
"title": "IANA Reserved Domains",
|
||||
"date": "2024-01-02",
|
||||
"tags": {"netscape-tag", "metadata"},
|
||||
},
|
||||
"dom": {
|
||||
"url": "https://www.iana.org/help/example-domains",
|
||||
},
|
||||
"json": {
|
||||
"url": "https://example.com/?archivebox-json-import=1",
|
||||
"title": "JSON Import Example",
|
||||
"date": "2024-01-03",
|
||||
"tags": {"json-tag", "metadata"},
|
||||
},
|
||||
"jsonl": {
|
||||
"url": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"title": "JSONL Import Example",
|
||||
"date": "2024-01-04",
|
||||
"tags": {"jsonl-tag", "metadata"},
|
||||
},
|
||||
"txt": {
|
||||
"url": "https://example.org/",
|
||||
},
|
||||
}
|
||||
from archivebox.tests.import_helpers import (
|
||||
IMPORT_FORMAT_EXPECTATIONS,
|
||||
assert_no_file_or_shell_payload_snapshots,
|
||||
malicious_add_inputs,
|
||||
write_import_format_files,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
@@ -297,10 +270,6 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou
|
||||
URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
|
||||
)
|
||||
create_admin_and_token(tmp_path)
|
||||
# These tests deliberately stop the server while crawls are queued. Finish
|
||||
# dependency installation first so shutdown cannot leave a half-built venv.
|
||||
install_result = run_archivebox_cmd(["install", *env["PLUGINS"].split(",")], cwd=tmp_path, env=env, timeout=180)
|
||||
assert install_result.returncode == 0, install_result.stderr or install_result.stdout
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
@@ -393,32 +362,7 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path
|
||||
"""Public /add/ must not archive local paths or execute shell-like textarea content."""
|
||||
init_archive(tmp_path)
|
||||
safe_url = "https://example.com/?archivebox-public-ui-security=1"
|
||||
other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
|
||||
other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
|
||||
other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
|
||||
canary = tmp_path / "archivebox_shell_injection_canary"
|
||||
inputs = [
|
||||
safe_url,
|
||||
"file:///etc/hosts",
|
||||
"/etc/hosts",
|
||||
"../../../../etc/passwd",
|
||||
f"file://{other_crawl_source}",
|
||||
str(other_crawl_source),
|
||||
f"'; touch {canary}; #",
|
||||
f'" && touch {canary} && echo "',
|
||||
f"$(touch {canary})",
|
||||
f"`touch {canary}`",
|
||||
"""<?xml version="1.0"?>
|
||||
<!DOCTYPE rss [
|
||||
<!ENTITY localfile SYSTEM "file:///etc/hosts">
|
||||
]>
|
||||
<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<channel>
|
||||
<item><title>&localfile;</title><link>file:///etc/passwd</link></item>
|
||||
<xi:include href="file:///etc/hosts" parse="text"/>
|
||||
</channel>
|
||||
</rss>""",
|
||||
]
|
||||
inputs, canary = malicious_add_inputs(tmp_path, safe_url=safe_url)
|
||||
|
||||
port = get_free_port()
|
||||
env = cli_env(
|
||||
@@ -431,10 +375,6 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path
|
||||
URL_ALLOWLIST=r"example\.com|example\.org|iana\.org|www\.iana\.org",
|
||||
)
|
||||
create_admin_and_token(tmp_path)
|
||||
# These tests deliberately stop the server while crawls are queued. Finish
|
||||
# dependency installation first so shutdown cannot leave a half-built venv.
|
||||
install_result = run_archivebox_cmd(["install", *env["PLUGINS"].split(",")], cwd=tmp_path, env=env, timeout=180)
|
||||
assert install_result.returncode == 0, install_result.stderr or install_result.stdout
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
@@ -483,12 +423,7 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path
|
||||
finally:
|
||||
stop_server(tmp_path)
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
assert not canary.exists()
|
||||
assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
|
||||
for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
|
||||
assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
|
||||
assert_no_file_or_shell_payload_snapshots(tmp_path, canary=canary)
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshot = Snapshot.objects.get(url=safe_url)
|
||||
crawl = Crawl.objects.get()
|
||||
|
||||
@@ -18,6 +18,7 @@ from pathlib import Path
|
||||
from threading import Thread
|
||||
|
||||
import pytest
|
||||
from django.test import override_settings
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
|
||||
@@ -195,9 +196,8 @@ class TestLiveProgressView:
|
||||
assert b"active_crawls" not in response.content
|
||||
assert b"traceback" not in response.content
|
||||
|
||||
@pytest.mark.parametrize("debug", [False, True])
|
||||
def test_live_progress_error_response_hides_internal_details(self, client, admin_user, crawl, settings, debug):
|
||||
settings.DEBUG = debug
|
||||
@override_settings(DEBUG=False)
|
||||
def test_live_progress_error_response_hides_traceback_without_debug(self, client, admin_user, crawl):
|
||||
|
||||
Crawl.objects.filter(pk=crawl.pk).update(
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
@@ -211,8 +211,7 @@ class TestLiveProgressView:
|
||||
|
||||
assert response.status_code == 500
|
||||
payload = response.json()
|
||||
assert payload["error"] == "Unable to load progress"
|
||||
assert "not-an-integer" not in response.content.decode()
|
||||
assert "error" in payload
|
||||
assert "traceback" not in payload
|
||||
assert payload["active_crawls"] == []
|
||||
|
||||
|
||||
@@ -717,117 +717,6 @@ class TestUrlRouting:
|
||||
""",
|
||||
)
|
||||
|
||||
def test_replay_auth_rejects_backslash_network_paths(self) -> None:
|
||||
self._run(
|
||||
r"""
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
ensure_admin_user()
|
||||
snapshot = get_snapshot()
|
||||
admin_client = Client()
|
||||
assert admin_client.login(username="testadmin", password="testpassword")
|
||||
for next_path, expected in [
|
||||
(r"/\attacker.example/private", "/index.html"),
|
||||
(r"\attacker.example/private", "/index.html"),
|
||||
("/screenshot/output.png?download=1", "/screenshot/output.png?download=1"),
|
||||
]:
|
||||
handoff = admin_client.get(
|
||||
"/admin/core/snapshot/replay-auth/",
|
||||
{"snapshot": str(snapshot.id), "next": next_path},
|
||||
HTTP_HOST=get_admin_host(),
|
||||
)
|
||||
assert handoff.status_code == 302
|
||||
target = urlsplit(handoff["Location"])
|
||||
query = parse_qs(target.query)
|
||||
assert query["next"] == [expected], (next_path, query["next"])
|
||||
replay_client = Client()
|
||||
replay = replay_client.get(
|
||||
"/_auth", {"grant": query["grant"][0], "next": next_path},
|
||||
HTTP_HOST=get_snapshot_host(str(snapshot.id)),
|
||||
)
|
||||
assert replay.status_code == 302
|
||||
assert replay["Location"] == expected, (next_path, replay["Location"])
|
||||
print("OK")
|
||||
""",
|
||||
mode="safe-subdomains-fullreplay",
|
||||
)
|
||||
|
||||
def test_replay_rejects_parent_paths_for_files_and_zip_downloads(self) -> None:
|
||||
self._run(
|
||||
"""
|
||||
snapshot = get_snapshot()
|
||||
outside = Path(snapshot.output_dir).parent / "outside-replay-probe.txt"
|
||||
outside.write_text("private-outside-replay-probe")
|
||||
ensure_admin_user()
|
||||
client = Client()
|
||||
assert client.login(username="testadmin", password="testpassword")
|
||||
try:
|
||||
for parent in ("..", "%2e%2e"):
|
||||
for suffix in ("/outside-replay-probe.txt", "/?files=1&download=zip"):
|
||||
for host, prefix in (
|
||||
(get_snapshot_host(str(snapshot.id)), ""),
|
||||
(get_web_host(), f"/{snapshot.url_path}"),
|
||||
):
|
||||
response = client.get(f"{prefix}/{parent}{suffix}", HTTP_HOST=host, follow=True)
|
||||
assert response.status_code in (400, 404), (host, prefix, parent, suffix, response.status_code)
|
||||
assert b"private-outside-replay-probe" not in response_body(response)
|
||||
finally:
|
||||
outside.unlink()
|
||||
print("OK")
|
||||
""",
|
||||
mode="safe-subdomains-fullreplay",
|
||||
)
|
||||
|
||||
def test_replay_does_not_follow_symlinks_outside_snapshot(self) -> None:
|
||||
self._run(
|
||||
"""
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
snapshot = get_snapshot()
|
||||
root = Path(snapshot.output_dir)
|
||||
outside = root.parent / "outside-symlink-probe.txt"
|
||||
outside.write_text("private-outside-symlink-probe")
|
||||
linked = root / "linked-secret.txt"
|
||||
linked.symlink_to(outside)
|
||||
linked_dir = root / "linked-parent"
|
||||
linked_dir.symlink_to(root.parent, target_is_directory=True)
|
||||
inside = root / "inside-symlink-probe.txt"
|
||||
inside.write_text("public-inside-symlink-probe")
|
||||
linked_inside = root / "linked-inside.txt"
|
||||
linked_inside.symlink_to(inside)
|
||||
client = Client()
|
||||
try:
|
||||
for path in (
|
||||
"/linked-secret.txt",
|
||||
"/LINKED-SECRET.TXT",
|
||||
"/linked-parent/outside-symlink-probe.txt",
|
||||
"/linked-parent/?files=1&download=zip",
|
||||
):
|
||||
response = client.get(path, HTTP_HOST=get_snapshot_host(str(snapshot.id)))
|
||||
assert response.status_code == 404, (path, response.status_code)
|
||||
assert b"private-outside-symlink-probe" not in response_body(response)
|
||||
response = client.get("/linked-inside.txt", HTTP_HOST=get_snapshot_host(str(snapshot.id)))
|
||||
assert response.status_code == 200
|
||||
assert response_body(response) == b"public-inside-symlink-probe"
|
||||
response = client.get("/?files=1&download=zip", HTTP_HOST=get_snapshot_host(str(snapshot.id)))
|
||||
assert response.status_code == 200
|
||||
with zipfile.ZipFile(io.BytesIO(response_body(response))) as archive:
|
||||
assert not any(name.endswith("/linked-secret.txt") for name in archive.namelist())
|
||||
assert not any("/linked-parent/" in name for name in archive.namelist())
|
||||
linked_name = next(name for name in archive.namelist() if name.endswith("/linked-inside.txt"))
|
||||
assert archive.read(linked_name) == b"public-inside-symlink-probe"
|
||||
finally:
|
||||
linked.unlink()
|
||||
linked_dir.unlink()
|
||||
linked_inside.unlink()
|
||||
inside.unlink()
|
||||
outside.unlink()
|
||||
print("OK")
|
||||
""",
|
||||
mode="safe-subdomains-fullreplay",
|
||||
)
|
||||
|
||||
def test_snapshot_routing_and_hosts(self) -> None:
|
||||
self._run(
|
||||
"""
|
||||
|
||||
@@ -4,7 +4,6 @@ import hashlib
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import struct
|
||||
import sys
|
||||
@@ -124,53 +123,6 @@ def append_manifest(manifest_path: Path, screenshot_path: Path) -> None:
|
||||
manifest.write(json.dumps(item) + "\n")
|
||||
|
||||
|
||||
def render_gallery(gallery_header: str, gallery_sections: str) -> str:
|
||||
return (
|
||||
'<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
||||
"<title>ArchiveBox UI Screenshots</title><style>"
|
||||
":root{--ink:#28252a;--muted:#716c74;--accent:#9b2854;--paper:#fcfaf7;--line:#e9e2e3}"
|
||||
"*{box-sizing:border-box}body{margin:0;font:16px/1.75 system-ui,sans-serif;"
|
||||
"background:var(--paper);color:var(--ink)}.screenshot-content{max-width:1100px;margin:0 auto;padding:28px}"
|
||||
"a{color:var(--accent)}h1,h2{line-height:1.15;letter-spacing:-.045em}"
|
||||
"article{margin:40px 0 65px}article h2{font-size:29px;margin:0 0 10px}article p{font-size:13px;color:var(--muted);margin:0 0 20px}"
|
||||
".profile-switch{display:flex;flex-wrap:wrap;gap:4px;background:#f0e8eb;border:1px solid var(--line);padding:5px;"
|
||||
"border-radius:28px;width:max-content;max-width:100%;margin:30px 0}.profile-switch button{font:600 13px system-ui;"
|
||||
"cursor:pointer;border:0;border-radius:22px;background:transparent;color:var(--muted);padding:10px 20px}"
|
||||
".profile-switch button[aria-pressed=true]{background:var(--accent);color:white;box-shadow:0 3px 10px #9b285425}"
|
||||
"a:focus-visible,button:focus-visible{outline:3px solid var(--accent);outline-offset:4px}"
|
||||
"figure{min-width:0;margin:0 0 20px;padding:18px;border:1px solid var(--line);border-radius:20px;"
|
||||
"background:linear-gradient(140deg,#fff,#f8edf2)}figure[hidden]{display:none}"
|
||||
".shot-mobile{max-width:440px;margin-inline:auto}.shot-tablet{max-width:800px;margin-inline:auto}"
|
||||
"figcaption{font-size:12px;color:var(--muted);text-align:center;margin:0 0 12px}"
|
||||
"img{display:block;width:100%;height:auto;border-radius:8px}code{overflow-wrap:anywhere}"
|
||||
"@media(max-width:550px){.screenshot-content{padding:20px}figure{padding:8px;border-radius:12px}.profile-switch button{padding:9px 16px}}"
|
||||
'</style><link rel="stylesheet" href="../site-chrome.css"></head><body>'
|
||||
+ (REPO_DIR / "bin/templates/screenshots-header.html").read_text(encoding="utf-8")
|
||||
+ '<div class="screenshot-content">'
|
||||
+ gallery_header
|
||||
+ '<div class="profile-switch" role="group" aria-label="Screenshot viewport">'
|
||||
+ "".join(
|
||||
f'<button type="button" data-viewport="{profile}" aria-pressed="{str(profile == "desktop").lower()}">{profile.title()}</button>'
|
||||
for profile in CAPTURE_PROFILES
|
||||
)
|
||||
+ "</div><main>"
|
||||
+ gallery_sections
|
||||
+ """</main><script>
|
||||
const figures = document.querySelectorAll('.shot');
|
||||
const buttons = document.querySelectorAll('[data-viewport]');
|
||||
function selectProfile(profile) {
|
||||
for (const figure of figures) figure.hidden = figure.dataset.profile !== profile;
|
||||
for (const button of buttons) button.setAttribute('aria-pressed', String(button.dataset.viewport === profile));
|
||||
}
|
||||
for (const button of buttons) button.addEventListener('click', () => selectProfile(button.dataset.viewport));
|
||||
selectProfile('desktop');
|
||||
</script></div>"""
|
||||
+ (REPO_DIR / "bin/templates/screenshots-footer.html").read_text(encoding="utf-8")
|
||||
+ "</body></html>\n"
|
||||
)
|
||||
|
||||
|
||||
def build_gallery(manifest_path: Path, html_path: Path) -> None:
|
||||
provenance = build_provenance()
|
||||
source_base_url = f"https://github.com/ArchiveBox/ArchiveBox/blob/{provenance['revision']}/"
|
||||
@@ -246,7 +198,7 @@ def build_gallery(manifest_path: Path, html_path: Path) -> None:
|
||||
cache_version = hashlib.sha256(screenshot_path.read_bytes()).hexdigest()[:12]
|
||||
versioned_filename = f"{html.escape(filename)}?v={cache_version}"
|
||||
html_figures.append(
|
||||
f'<figure class="shot shot-{profile}" data-profile="{profile}"><figcaption>{label}</figcaption>'
|
||||
f'<figure class="shot shot-{profile}"><figcaption>{label}</figcaption>'
|
||||
f'<a href="./{versioned_filename}">'
|
||||
f'<img src="./{versioned_filename}" width="{width}" height="{height}" loading="lazy" '
|
||||
f'alt="{html.escape(str(capture["name"]))} — {profile}"></a></figure>',
|
||||
@@ -258,15 +210,25 @@ def build_gallery(manifest_path: Path, html_path: Path) -> None:
|
||||
f'<div class="shots">{"".join(html_figures)}</div></article>',
|
||||
)
|
||||
|
||||
gallery_header = (
|
||||
'<header><p><a href="../">← ArchiveBox</a></p><h1>ArchiveBox UI Screenshots</h1>'
|
||||
+ f"<p>Generated from ArchiveBox <code>{html.escape(provenance['version'])}</code> at "
|
||||
html_path.write_text(
|
||||
'<!doctype html><html lang="en"><head><meta charset="utf-8">'
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
||||
"<title>ArchiveBox UI Screenshots</title><style>"
|
||||
"body{max-width:1900px;margin:0 auto;padding:20px;font:16px system-ui,sans-serif;background:#f6f7f9;color:#18202a}"
|
||||
"a{color:#2563eb}article{margin:32px 0 64px}.shots{display:grid;grid-template-columns:2fr 1.35fr .8fr;gap:14px;align-items:start}"
|
||||
"figure{min-width:0;margin:0}figcaption{font-weight:650;margin:0 0 8px}img{display:block;width:100%;height:auto;"
|
||||
"border:1px solid #cbd5e1;border-radius:8px;background:white;box-shadow:0 8px 24px #0f172a18}code{overflow-wrap:anywhere}"
|
||||
"@media(max-width:900px){.shots{grid-template-columns:1fr}body{padding:12px}}"
|
||||
'</style></head><body><header><p><a href="../">← ArchiveBox</a></p><h1>ArchiveBox UI Screenshots</h1>'
|
||||
f"<p>Generated from ArchiveBox <code>{html.escape(provenance['version'])}</code> at "
|
||||
f'<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{html.escape(provenance["revision"])}">'
|
||||
f"<code>{html.escape(provenance['revision'][:12])}</code></a> on "
|
||||
f'<time datetime="{html.escape(provenance["generated_at"])}">{html.escape(provenance["generated_at"])}</time>. '
|
||||
"Desktop, tablet, and mobile viewports are captured from the same build.</p></header>"
|
||||
"Desktop, tablet, and mobile viewports are captured from the same build.</p></header><main>"
|
||||
+ "".join(html_sections)
|
||||
+ "</main></body></html>\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
html_path.write_text(render_gallery(gallery_header, "".join(html_sections)), encoding="utf-8")
|
||||
|
||||
file_hashes = {
|
||||
capture["filename"]: hashlib.sha256((html_path.parent / capture["filename"]).read_bytes()).hexdigest() for capture in captures
|
||||
@@ -292,27 +254,7 @@ def build_gallery(manifest_path: Path, html_path: Path) -> None:
|
||||
screenshot_path.unlink()
|
||||
|
||||
|
||||
def refresh_gallery(html_path: Path) -> None:
|
||||
"""Apply current presentation to published capture content without changing provenance."""
|
||||
document = html_path.read_text(encoding="utf-8")
|
||||
header = re.search(r"<header>.*?</header>", document, re.DOTALL)
|
||||
sections = re.search(r"<main>(.*?)</main>", document, re.DOTALL)
|
||||
if not header or not sections:
|
||||
raise SystemExit("Published screenshot gallery is missing its capture header or content")
|
||||
content = sections[1]
|
||||
for profile in CAPTURE_PROFILES:
|
||||
# Galleries published before the switcher used classes alone.
|
||||
content = content.replace(
|
||||
f'<figure class="shot shot-{profile}">',
|
||||
f'<figure class="shot shot-{profile}" data-profile="{profile}">',
|
||||
)
|
||||
html_path.write_text(render_gallery(header[0], content), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "refresh":
|
||||
refresh_gallery(Path(sys.argv[2]))
|
||||
return
|
||||
if len(sys.argv) == 4 and sys.argv[1] == "validate":
|
||||
validate_navigation(Path(sys.argv[2]), sys.argv[3])
|
||||
return
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restore validated capture output without running application CI."""
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import subprocess
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import urlopen
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("kind", choices=["archivebox", "ios", "extension"])
|
||||
parser.add_argument("destination", type=Path)
|
||||
args = parser.parse_args()
|
||||
args.destination.mkdir(parents=True, exist_ok=True)
|
||||
# Only successful default-branch capture runs are eligible, never PR artifacts.
|
||||
if os.environ.get("GH_TOKEN"):
|
||||
repo = os.environ["GITHUB_REPOSITORY"]
|
||||
branch = "dev" if args.kind == "archivebox" else "main"
|
||||
runs = json.loads(
|
||||
subprocess.check_output(
|
||||
["gh", "api", f"repos/{repo}/actions/workflows/screenshots.yml/runs?branch={branch}&status=success&per_page=100"],
|
||||
),
|
||||
)
|
||||
for run in runs["workflow_runs"]:
|
||||
if run["event"] not in ("push", "workflow_dispatch"):
|
||||
continue
|
||||
artifacts = json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/actions/runs/{run['id']}/artifacts"]))
|
||||
if any(a["name"] == "site-screenshots" and not a["expired"] for a in artifacts["artifacts"]):
|
||||
subprocess.run(
|
||||
["gh", "run", "download", str(run["id"]), "--repo", repo, "--name", "site-screenshots", "--dir", str(args.destination)],
|
||||
check=True,
|
||||
)
|
||||
print(f"Restored validated captures from run {run['id']} ({run['head_sha']})")
|
||||
raise SystemExit(0)
|
||||
# Published captures remain available even after Actions artifacts expire.
|
||||
base, manifest_name = {
|
||||
"archivebox": ("https://archivebox.io/screenshots/", "build.json"),
|
||||
"ios": ("https://archivebox.github.io/ios-archivebox/screenshots/", "manifest.json"),
|
||||
"extension": ("https://archivebox.github.io/archivebox-browser-extension/screenshots/", "manifest.json"),
|
||||
}[args.kind]
|
||||
|
||||
|
||||
def fetch(name):
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or ".." in path.parts or ":" in name or "\\" in name:
|
||||
raise ValueError(f"Unsafe capture path: {name}")
|
||||
with urlopen(base + name, timeout=60) as response:
|
||||
data = response.read()
|
||||
expected = hashes.get(name)
|
||||
if expected and hashlib.sha256(data).hexdigest() != expected:
|
||||
raise ValueError(f"Capture checksum mismatch: {name}")
|
||||
target = args.destination / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(data)
|
||||
|
||||
|
||||
try:
|
||||
with urlopen(base + manifest_name, timeout=60) as response:
|
||||
raw = response.read()
|
||||
except HTTPError as error:
|
||||
if error.code == 404 and args.kind == "ios":
|
||||
print("No complete native gallery published yet; use the curated README screenshots.")
|
||||
raise SystemExit(0)
|
||||
raise
|
||||
manifest = json.loads(raw)
|
||||
hashes = {}
|
||||
if args.kind == "archivebox":
|
||||
hashes = manifest["files"]
|
||||
names = [*hashes, "index.html"]
|
||||
elif args.kind == "extension":
|
||||
names = [image["file"] for capture in manifest["screenshots"] for image in capture["images"]]
|
||||
else:
|
||||
if manifest.get("complete") is not True:
|
||||
raise ValueError("Published native gallery must be complete")
|
||||
names = [capture["path"] for capture in manifest["captures"]]
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
list(pool.map(fetch, names))
|
||||
(args.destination / manifest_name).write_bytes(raw)
|
||||
print(f"Restored {len(names)} published capture files, retaining revision {manifest['revision']}")
|
||||
@@ -1,164 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep one published docs version per minor, plus latest/main and dev.
|
||||
|
||||
Run with uv run bin/sync_docs_versions.py for a read-only plan. Pass --apply
|
||||
with READTHEDOCS_TOKEN to reconcile. Retired versions are hidden, preserving
|
||||
existing URLs. Missing builds must succeed before older entries are hidden.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
API = "https://app.readthedocs.org/api/v3/projects/archivebox/"
|
||||
# Deployment maps these branch-backed versions to their release URL slugs manually.
|
||||
# The hourly reconciler must never rename versions or rewrite release tags.
|
||||
REPAIR_REFS = {
|
||||
"v0.7.4": "docs/restore-main-sphinx",
|
||||
"v0.8.6rc1": "docs/restore-08-sphinx",
|
||||
"v0.2.4": "docs/restore-02-sphinx",
|
||||
"v0.1.0": "docs/restore-01-sphinx",
|
||||
"v0.0.3": "docs/restore-00-sphinx",
|
||||
}
|
||||
|
||||
|
||||
def build_ready(version, build, expected_ref, expected_commit):
|
||||
"""Only a successful current build from the intended Git source can replace docs."""
|
||||
source = version["identifier"] if version["slug"] == "latest" else version["verbose_name"]
|
||||
return bool(
|
||||
expected_commit
|
||||
and source == expected_ref
|
||||
and version["active"]
|
||||
and build
|
||||
and build["version"] == version["slug"]
|
||||
and build["state"]["code"] == "finished"
|
||||
and build["success"] is True
|
||||
and build["commit"] == expected_commit,
|
||||
)
|
||||
|
||||
|
||||
def selected_tags(tags):
|
||||
releases = {}
|
||||
for tag in tags:
|
||||
if not tag.startswith("v"):
|
||||
continue
|
||||
try:
|
||||
version = Version(tag[1:])
|
||||
except InvalidVersion:
|
||||
continue
|
||||
releases.setdefault((version.major, version.minor), []).append((version, tag))
|
||||
minors = sorted(releases, reverse=True)
|
||||
selected = []
|
||||
for index, minor in enumerate(minors):
|
||||
if index < 2:
|
||||
candidates = [(version, tag) for version, tag in releases[minor] if version.pre and version.pre[0] == "rc"]
|
||||
else:
|
||||
candidates = [(version, tag) for version, tag in releases[minor] if not version.is_prerelease and not version.is_devrelease]
|
||||
if not candidates:
|
||||
raise ValueError(f"No {'RC' if index < 2 else 'stable'} release for minor {minor}")
|
||||
selected.append(max(candidates)[1])
|
||||
return selected
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args()
|
||||
token = os.environ.get("READTHEDOCS_TOKEN")
|
||||
if args.apply and not token:
|
||||
parser.error("--apply requires READTHEDOCS_TOKEN")
|
||||
session = requests.Session()
|
||||
session.headers["User-Agent"] = "ArchiveBox-docs-version-policy"
|
||||
if token:
|
||||
session.headers["Authorization"] = f"Token {token}"
|
||||
|
||||
def request(method, path, **kwargs):
|
||||
response = session.request(method, API + path, timeout=30, **kwargs)
|
||||
response.raise_for_status()
|
||||
return response.json() if response.content else None
|
||||
|
||||
refs = subprocess.check_output(["git", "ls-remote", "--heads", "--tags", "origin"], text=True)
|
||||
remote_refs = {ref: commit for commit, ref in (line.split() for line in refs.splitlines())}
|
||||
tags = [ref.removeprefix("refs/tags/") for ref in remote_refs if ref.startswith("refs/tags/") and not ref.endswith("^{}")]
|
||||
expected_refs = {"latest": "main", "dev": "dev", **REPAIR_REFS}
|
||||
wanted = ["latest", "dev", *selected_tags(tags)]
|
||||
project = request("GET", "")
|
||||
page = request("GET", "versions/?limit=1000")
|
||||
versions = {version["slug"]: version for version in page["results"]}
|
||||
while page["next"]:
|
||||
response = session.get(page["next"], timeout=30)
|
||||
response.raise_for_status()
|
||||
page = response.json()
|
||||
versions.update((version["slug"], version) for version in page["results"])
|
||||
missing = set(wanted) - versions.keys()
|
||||
if missing:
|
||||
raise SystemExit(f"Read the Docs has not synchronized these tags: {sorted(missing)}")
|
||||
print("Desired picker:", ", ".join(wanted))
|
||||
print(f"latest currently tracks {project['default_branch']}; required branch: main")
|
||||
for slug in wanted:
|
||||
expected = expected_refs.get(slug, slug)
|
||||
source = versions[slug]["identifier"] if slug == "latest" else versions[slug]["verbose_name"]
|
||||
if source != expected:
|
||||
print(f"{slug}: source is {source}; requires staged deployment from {expected}")
|
||||
print("Current-build readiness is verified only with --apply (authenticated per-version API reads).")
|
||||
if not args.apply:
|
||||
return
|
||||
|
||||
if project["default_branch"] != "main":
|
||||
subprocess.run(["git", "fetch", "--depth=1", "origin", "main"], check=True)
|
||||
configured = subprocess.run(["git", "cat-file", "-e", "FETCH_HEAD:.github/.readthedocs.yaml"], capture_output=True)
|
||||
if configured.returncode:
|
||||
raise SystemExit("main needs the docs build configuration PR before latest can safely track it")
|
||||
request("PATCH", "", json={"default_branch": "main"})
|
||||
request("POST", "versions/latest/builds/")
|
||||
|
||||
pending = []
|
||||
for slug in wanted:
|
||||
version = versions[slug]
|
||||
expected = expected_refs.get(slug, slug)
|
||||
source = version["identifier"] if slug == "latest" else version["verbose_name"]
|
||||
if source != expected:
|
||||
# Refresh latest after changing the project's default branch.
|
||||
if slug == "latest":
|
||||
version = request("GET", "versions/latest/")
|
||||
else:
|
||||
pending.append(f"{slug} (deploy repair branch {expected})")
|
||||
continue
|
||||
is_branch = slug in expected_refs
|
||||
ref = f"refs/heads/{expected}" if is_branch else f"refs/tags/{expected}"
|
||||
commit = remote_refs.get(ref + "^{}", remote_refs.get(ref))
|
||||
if not commit:
|
||||
pending.append(f"{slug} (missing remote ref {expected})")
|
||||
continue
|
||||
if not version["active"]:
|
||||
request("PATCH", f"versions/{slug}/", json={"active": True, "hidden": True})
|
||||
# Activation starts a build; keep it out of the picker until verified.
|
||||
pending.append(f"{slug} (activated, awaiting build)")
|
||||
continue
|
||||
builds = request("GET", f"versions/{slug}/builds/?limit=1")["results"]
|
||||
if not build_ready(version, builds[0] if builds else None, expected, commit):
|
||||
pending.append(f"{slug} (newest build must succeed at {commit})")
|
||||
if pending:
|
||||
raise SystemExit("Keeping existing picker; awaiting: " + "; ".join(pending))
|
||||
for slug in wanted:
|
||||
if versions[slug]["hidden"]:
|
||||
request("PATCH", f"versions/{slug}/", json={"hidden": False})
|
||||
for slug, version in versions.items():
|
||||
if slug not in wanted and version["active"] and not version["hidden"]:
|
||||
request("PATCH", f"versions/{slug}/", json={"hidden": True})
|
||||
print("Hidden (URL preserved):", slug)
|
||||
if project["default_version"] != "latest":
|
||||
request("PATCH", "", json={"default_version": "latest"})
|
||||
print("Version visibility reconciled successfully")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except requests.RequestException as error:
|
||||
print(f"Read the Docs API request failed: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -1,26 +0,0 @@
|
||||
<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps & tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/ios-archivebox/#get-started">ArchiveBox for macOS & iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS & iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/archivebox-browser-extension/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/abx-plugins/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/abx-dl/">One-shot abx-dl CLI</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-dl" aria-label="One-shot abx-dl CLI source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/ios-archivebox/#your-server-your-choice">ArchiveBox Server.app</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox Server.app source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 1Line truncated
|
||||
|
||||
<div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security & privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
|
||||
</footer>
|
||||
<script>
|
||||
(() => {
|
||||
const menu = document.querySelector('.abx-apps');
|
||||
document.addEventListener('keydown', event => {
|
||||
if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
|
||||
});
|
||||
document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
|
||||
function revealAnchor() {
|
||||
let target;
|
||||
try { target = document.getElementById(decodeURIComponent(location.hash.slice(1))); } catch { return; }
|
||||
if (!target) return;
|
||||
let details = target.closest('details');
|
||||
let opened = false;
|
||||
while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
|
||||
if (opened) requestAnimationFrame(() => target.scrollIntoView());
|
||||
}
|
||||
addEventListener('hashchange', revealAnchor);
|
||||
revealAnchor();
|
||||
})();
|
||||
</script>
|
||||
@@ -1,9 +0,0 @@
|
||||
<header class="abx-header">
|
||||
<a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">UI Screenshots</span></a>
|
||||
<nav class="abx-nav" aria-label="Main navigation">
|
||||
<details class="abx-apps"><summary>Apps</summary><div class="abx-app-links"><a href="https://archivebox.io/">ArchiveBox Server</a><a href="https://archivebox.github.io/ios-archivebox/">ArchiveBox for macOS & iOS</a><a href="https://archivebox.github.io/archivebox-browser-extension/">ArchiveBox Browser Extension</a><a href="https://archivebox.github.io/abx-plugins/">ArchiveBox Plugin Library</a><a href="https://archivebox.github.io/abx-dl/"><span>One-shot <code>abx-dl</code> CLI</span></a><a href="https://github.com/ArchiveBox">More on Github...</a></div></details>
|
||||
<a href="https://archivebox.io/screenshots/">Screenshots</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
|
||||
<a class="abx-cta" href="https://archivebox.io/#install">Get started</a>
|
||||
</nav>
|
||||
</header>
|
||||
@@ -11,7 +11,7 @@ TIMEOUT=120
|
||||
env TIMEOUT=120 archivebox add ~/Downloads/bookmarks_export.html
|
||||
```
|
||||
|
||||
Environment variables seed process-level defaults. Persisted Machine, Persona, Crawl, and Snapshot settings can override them depending on scope, and existing Crawl config is not silently overwritten by later environment changes. Runtime-derived values like crawl/snapshot output dirs are resolved fresh for each run instead of being stored in frozen crawl config. For more examples see [Usage: Configuration](Usage.md#run-archivebox-with-configuration-options)...
|
||||
Environment variables seed process-level defaults. Persisted Machine, Persona, Crawl, and Snapshot settings can override them depending on scope, and existing Crawl config is not silently overwritten by later environment changes. Runtime-derived values like crawl/snapshot output dirs are resolved fresh for each run instead of being stored in frozen crawl config. For more examples see [Usage: Configuration](Usage#run-archivebox-with-configuration-options)...
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ Overview
|
||||
|
||||
Home.md
|
||||
README.md
|
||||
Screenshots.md
|
||||
|
||||
|
||||
Getting Started
|
||||
@@ -74,7 +73,6 @@ Meta
|
||||
|
||||
Roadmap.md
|
||||
Changelog.md
|
||||
Release-Notes-v0.9.md
|
||||
Donations.md
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -8,7 +8,7 @@ Running ArchiveBox with Docker allows you to manage it in a container without ex
|
||||
|
||||
<img src="https://imgur.zervice.io/qFAPRwC.png" width="20%" align="right"/>
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Overview](#Overview)
|
||||
- [Docker Compose](#docker-compose) ⭐️ (recommended)
|
||||
- [Setup](#setup)
|
||||
- [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-docker-compose-%EF%B8%8F)
|
||||
@@ -154,7 +154,7 @@ Then open [`http://web.archivebox.localhost:8000`](http://web.archivebox.localho
|
||||
|
||||
### Configuration
|
||||
|
||||
ArchiveBox running with `docker compose` accepts all the same config options as other ArchiveBox distributions, see the full list of options available on the [Configuration](Configuration.md) page.
|
||||
ArchiveBox running with `docker compose` accepts all the same config options as other ArchiveBox distributions, see the full list of options available on the [[Configuration]] page.
|
||||
|
||||
The recommended way configure ArchiveBox in Docker Compose is using `archivebox config --set ...` or by editing `ArchiveBox.conf`.
|
||||
```bash
|
||||
@@ -287,7 +287,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev config --set TIMEOUT=120
|
||||
TIMEOUT=120
|
||||
```
|
||||
|
||||
ArchiveBox in Docker also accepts config as environment variables, see more on the [Configuration](Configuration.md) page (and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options).
|
||||
ArchiveBox in Docker also accepts config as environment variables, see more on the [[Configuration]] page (and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options).
|
||||
|
||||
For example, this disables the screenshot extractor for a single run (without persisting for other runs):
|
||||
```bash
|
||||
|
||||
+12
-12
@@ -8,11 +8,11 @@ ArchiveBox is primarily distributed as a Python package installed with `uv`, but
|
||||
|
||||
- *[Supported Systems](#supported-systems)*
|
||||
- Install Instructions
|
||||
- **[Option A. Docker / Docker Compose ⭐️](#option-a-docker--docker-compose-setup-)**
|
||||
- **[Option A. Docker / Docker Compose ⭐️](#option-a-docker--docker-compose-setup-%EF%B8%8F)**
|
||||
- [Option B. uv Install Shortcut](#option-b-uv-install-shortcut)
|
||||
- [Option C. System Package Manager Setup](#option-c-bare-metal-setup)
|
||||
- *[Upgrading ArchiveBox to a new version](#next-steps-upgrading-archivebox-to-a-new-version)*
|
||||
- *[Next Steps](#next-steps-add-some-urls-to-archive-and-try-out-cli--web-ui)*
|
||||
- *[Upgrading ArchiveBox to a new version](#upgrading-archivebox-to-a-new-version)*
|
||||
- *[Next Steps](#next-steps)*
|
||||
|
||||
|
||||
## Supported Systems
|
||||
@@ -50,7 +50,7 @@ It's also recommended to use a filesystem with compression and/or [deduplication
|
||||
1. If you don't already have Docker installed, follow the official instructions for Linux or macOS:
|
||||
https://docs.docker.com/install/#supported-platforms ➡️
|
||||
|
||||
2. Then follow the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#quickstart) guide and read the [Docker](Docker.md) wiki page for next steps. ➡️
|
||||
2. Then follow the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#quickstart) guide and read the [[Docker]] wiki page for next steps. ➡️
|
||||
|
||||
> You can also run Dockerized ArchiveBox using [UNRAID/TrueNAS/Proxmox/etc.](https://github.com/ArchiveBox/ArchiveBox#-other-options) or Kubernetes.
|
||||
|
||||
@@ -99,7 +99,7 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
|
||||
<img src="https://avatars0.githubusercontent.com/u/1503512?s=200&v=4" width="100px" align="right"/>
|
||||
|
||||
**More info:**
|
||||
- For help installing these, see the [Manual Setup](#option-c-bare-metal-setup), [Troubleshooting](Troubleshooting.md) and [Chromium Install](Chromium-Install.md) pages.
|
||||
- For help installing these, see the [Manual Setup](#manual-setup), [[Troubleshooting]] and [[Chromium Install]] pages.
|
||||
- For per-plugin binary and enable/disable options (CHROME_BINARY, RIPGREP_BINARY, `<plugin>_ENABLED`, etc.) see the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/).
|
||||
|
||||
|
||||
@@ -227,7 +227,7 @@ archivebox help
|
||||
|
||||
ArchiveBox can be launched as `root` by setup and package-manager flows; it creates or selects the `archivebox` service account and drops privileges before writing collection data. Run collection commands such as `init`, `install`, `add`, and `status` inside the data directory. Informational commands such as `version` and `help` can run anywhere.
|
||||
|
||||
If you have issues getting Chromium / Google Chrome or other dependencies working with ArchiveBox, see the [Chromium Install](Chromium-Install.md) and [Troubleshooting](Troubleshooting.md) pages for more detailed instructions.
|
||||
If you have issues getting Chromium / Google Chrome or other dependencies working with ArchiveBox, see the [[Chromium Install]] and [[Troubleshooting]] pages for more detailed instructions.
|
||||
|
||||
<br/>
|
||||
|
||||
@@ -254,7 +254,7 @@ archivebox status
|
||||
archivebox server 0.0.0.0:8000
|
||||
# Visit http://admin.archivebox.localhost:8000/admin/ in a browser to finish setup
|
||||
```
|
||||
See our [Usage](Usage.md) Wiki documentation page for more examples.
|
||||
See our [[Usage]] Wiki documentation page for more examples.
|
||||
|
||||
<br/>
|
||||
|
||||
@@ -290,8 +290,8 @@ Check our more detailed [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wik
|
||||
|
||||
### Further Reading
|
||||
|
||||
- Read [Usage](Usage.md) to learn how to use the ArchiveBox CLI and HTML output
|
||||
- Read [Configuration](Configuration.md) to learn about the various archive method options
|
||||
- Read [Scheduled Archiving](Scheduled-Archiving.md) to learn how to set up automatic daily archiving
|
||||
- Read [Publishing Your Archive](Publishing-Your-Archive.md) if you want to host your archive for others to access online
|
||||
- Read [Troubleshooting](Troubleshooting.md) if you encounter any problems
|
||||
- Read [[Usage]] to learn how to use the ArchiveBox CLI and HTML output
|
||||
- Read [[Configuration]] to learn about the various archive method options
|
||||
- Read [[Scheduled Archiving]] to learn how to set up automatic daily archiving
|
||||
- Read [[Publishing Your Archive]] if you want to host your archive for others to access online
|
||||
- Read [[Troubleshooting]] if you encounter any problems
|
||||
+6
-6
@@ -8,7 +8,7 @@
|
||||
|
||||
ArchiveBox [officially supports](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems) **macOS** and **Ubuntu** on `amd64` or `arm64`, plus **Docker** on Linux and macOS. Other operating systems are not tested for this release.
|
||||
|
||||
For more detailed Docker and Docker Compose-specific instructions, see the [Docker](Docker.md) page.
|
||||
For more detailed Docker and Docker Compose-specific instructions, see the [[Docker]] page.
|
||||
|
||||
---
|
||||
|
||||
@@ -84,8 +84,8 @@ archivebox server
|
||||
archivebox help # see info about all the available commands
|
||||
```
|
||||
|
||||
- Read [Usage](Usage.md) to learn about the various CLI and web UI functions
|
||||
- Read [Configuration](Configuration.md) to learn about the various archive method options
|
||||
- Read [Scheduled Archiving](Scheduled-Archiving.md) to learn how to set up automatic daily archiving
|
||||
- Read [Publishing Your Archive](Publishing-Your-Archive.md) if you want to host your archive for others to access online
|
||||
- Read [Troubleshooting](Troubleshooting.md) if you encounter any problems
|
||||
- Read [[Usage]] to learn about the various CLI and web UI functions
|
||||
- Read [[Configuration]] to learn about the various archive method options
|
||||
- Read [[Scheduled Archiving]] to learn how to set up automatic daily archiving
|
||||
- Read [[Publishing Your Archive]] if you want to host your archive for others to access online
|
||||
- Read [[Troubleshooting]] if you encounter any problems
|
||||
@@ -10,7 +10,7 @@ archivebox config --set PERMISSIONS=private # default new snapshots to login
|
||||
archivebox manage [createsuperuser|changepassword] # create/modify admin UI users
|
||||
```
|
||||
|
||||
See [Setting Up Authentication](Setting-up-Authentication.md) for more...
|
||||
See [[Setting Up Authentication]] for more...
|
||||
|
||||
<br/>
|
||||
|
||||
@@ -50,7 +50,7 @@ To get started, import a dedicated browser profile into a [persona](https://gith
|
||||
|
||||
➡️ For full instructions on setting up a Chromium user profile see here: https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile
|
||||
|
||||
If you're importing private links or authenticated content, you probably don't want to share your archive folder publicly on a webserver, so don't follow the [Publishing Your Archive](Publishing-Your-Archive.md) instructions unless you are only serving it on a trusted LAN or have some sort of authentication in front of it. Make sure to point ArchiveBox to an output folder with conservative permissions, as it may contain archived content with secret session tokens or pieces of your user data. You may also wish to encrypt the archive using an encrypted disk image or filesystem like ZFS as it will contain all requests and response data, including session keys, user data, usernames, etc.
|
||||
If you're importing private links or authenticated content, you probably don't want to share your archive folder publicly on a webserver, so don't follow the [[Publishing Your Archive]] instructions unless you are only serving it on a trusted LAN or have some sort of authentication in front of it. Make sure to point ArchiveBox to an output folder with conservative permissions, as it may contain archived content with secret session tokens or pieces of your user data. You may also wish to encrypt the archive using an encrypted disk image or filesystem like ZFS as it will contain all requests and response data, including session keys, user data, usernames, etc.
|
||||
|
||||
#### ⚠️ Things to watch out for: ⚠️
|
||||
|
||||
|
||||
@@ -4,18 +4,18 @@
|
||||
|
||||
What are you having an issue with?:
|
||||
|
||||
- [Installing ArchiveBox](#installing)
|
||||
- [Installing ArchiveBox](#Installing)
|
||||
- [Upgrading ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives)
|
||||
- [Configuring ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
|
||||
- [Archiving content with ArchiveBox](#archiving)
|
||||
- [Hosting your collection publicly](#hosting-the-archive)
|
||||
- [Archiving content with ArchiveBox](#Archiving)
|
||||
- [Hosting your collection publicly](#Hosting-the-Archive)
|
||||
- [Database and filesystem issues](#database)
|
||||
|
||||
---
|
||||
|
||||
## Installing
|
||||
|
||||
If using `archivebox` without Docker, make sure you've followed the full guide in the [Install](Install.md) instructions first. Then check here for help depending on what component you need help with.
|
||||
If using `archivebox` without Docker, make sure you've followed the full guide in the [[Install]] instructions first. Then check here for help depending on what component you need help with.
|
||||
|
||||
Then make sure `archivebox` is installed available in your `$PATH`.
|
||||
```bash
|
||||
@@ -57,11 +57,11 @@ uv tool list
|
||||
archivebox version
|
||||
```
|
||||
|
||||
If `archivebox` is missing, repeat the `uv tool install` command from the [Install](Install.md) guide.
|
||||
If `archivebox` is missing, repeat the `uv tool install` command from the [[Install]] guide.
|
||||
|
||||
### Chromium/Google Chrome
|
||||
|
||||
For more info, see the [Chromium Install](Chromium-Install.md) page.
|
||||
For more info, see the [[Chromium Install]] page.
|
||||
|
||||
ArchiveBox resolves Chrome through `abxpkg`, preferring a compatible browser already installed on the host and otherwise installing a managed build:
|
||||
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
|
||||
Moved to:
|
||||
|
||||
- [Upgrading](Upgrading.md)
|
||||
- [Merging Collections](Merging-Collections.md)
|
||||
- [Database Troubleshooting](Troubleshooting.md#database)
|
||||
- [[Upgrading]]
|
||||
- [[Merging Collections]]
|
||||
- [Database Troubleshooting](./Troubleshooting#database)
|
||||
+1
-1
@@ -156,7 +156,7 @@ More info:
|
||||
|
||||
## Merge two or more existing archives
|
||||
|
||||
See [Merging Collections](Merging-Collections.md)...
|
||||
See [[Merging Collections]]...
|
||||
|
||||
<br/>
|
||||
|
||||
|
||||
+17
-17
@@ -6,18 +6,18 @@
|
||||
|
||||
<img src="https://imgur.zervice.io/aQZZcku.png" width="20%" align="right"/>
|
||||
|
||||
- [CLI Usage](#cli-usage): Docs and examples for the ArchiveBox command line interface.
|
||||
- [Admin UI Usage](#ui-usage): Docs and screenshots for the outputted HTML archive interface.
|
||||
- [Browser Extension Usage](#browser-extension-usage): Docs and screenshots for the outputted HTML archive interface.
|
||||
- [Disk Layout](#disk-layout): Description of the archive folder structure and contents.
|
||||
- [CLI Usage](#CLI-Usage): Docs and examples for the ArchiveBox command line interface.
|
||||
- [Admin UI Usage](#UI-Usage): Docs and screenshots for the outputted HTML archive interface.
|
||||
- [Browser Extension Usage](#Browser-Extension-Usage): Docs and screenshots for the outputted HTML archive interface.
|
||||
- [Disk Layout](#Disk-Layout): Description of the archive folder structure and contents.
|
||||
|
||||
**Related:**
|
||||
|
||||
- [Docker](Docker.md): Learn about ArchiveBox usage with Docker and Docker Compose
|
||||
- [Configuration](Configuration.md): Learn about the various archive method options
|
||||
- [Scheduled Archiving](Scheduled-Archiving.md): Learn how to set up automatic daily archiving
|
||||
- [Publishing Your Archive](Publishing-Your-Archive.md): Learn how to host your archive for others to access
|
||||
- [Troubleshooting](Troubleshooting.md): Resources if you encounter any problems
|
||||
- [[Docker]]: Learn about ArchiveBox usage with Docker and Docker Compose
|
||||
- [[Configuration]]: Learn about the various archive method options
|
||||
- [[Scheduled Archiving]]: Learn how to set up automatic daily archiving
|
||||
- [[Publishing Your Archive]]: Learn how to host your archive for others to access
|
||||
- [[Troubleshooting]]: Resources if you encounter any problems
|
||||
|
||||
## CLI Usage
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
All three of these ways of running ArchiveBox are equivalent and interchangeable:
|
||||
|
||||
- `archivebox [subcommand] [...args]`
|
||||
*Using the Python package via the current `uv` instructions in [Install](Install.md)*
|
||||
*Using the Python package via the current `uv` instructions in [[Install]]*
|
||||
- `docker run ... archivebox/archivebox [subcommand] [...args]`
|
||||
*Using the official Docker image*
|
||||
- `docker compose run --rm archivebox [subcommand] [...args]`
|
||||
@@ -34,12 +34,12 @@ All three of these ways of running ArchiveBox are equivalent and interchangeable
|
||||
|
||||
You can share a single archivebox data directory between Docker and non-Docker instances as well, allowing you to run the server in a container but still execute CLI commands on the host for example.
|
||||
|
||||
For more examples see [README: Usage](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-cli-usage) and [Docker](Docker.md) pages.
|
||||
For more examples see [README: Usage](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-cli-usage) and [[Docker]] pages.
|
||||
|
||||
- [Run ArchiveBox with configuration options](#run-archivebox-with-configuration-options)
|
||||
- [Import a single URL](#import-a-single-url)
|
||||
- [Import a list of URLs from a text file](#import-a-list-of-urls-from-a-text-file)
|
||||
- [Import list of links from browser history](#import-list-of-links-from-browser-history)
|
||||
- [Run ArchiveBox with configuration options](#Run-ArchiveBox-with-configuration-options)
|
||||
- [Import a single URL](#Import-a-single-URL)
|
||||
- [Import a list of URLs from a text file](#Import-a-list-of-URLs-from-a-text-file)
|
||||
- [Import list of links from browser history](#Import-list-of-links-from-browser-history)
|
||||
|
||||
---
|
||||
|
||||
@@ -58,8 +58,8 @@ TIMEOUT=120
|
||||
env TIMEOUT=120 archivebox add 'https://example.com'
|
||||
```
|
||||
|
||||
See [Configuration](Configuration.md) page for core ArchiveBox config options and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options (e.g. `YTDLP_MAX_SIZE`, `CHROME_USER_DATA_DIR`, `WGET_ARGS`, etc.).
|
||||
If you're using Docker, also make sure to read the Configuration section on the [Docker](Docker.md) page.
|
||||
See [[Configuration]] page for core ArchiveBox config options and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options (e.g. `YTDLP_MAX_SIZE`, `CHROME_USER_DATA_DIR`, `WGET_ARGS`, etc.).
|
||||
If you're using Docker, also make sure to read the Configuration section on the [[Docker]] page.
|
||||
|
||||
> [!TIP]
|
||||
> You can run ArchiveBox commands from anywhere (without having to `cd` into a data directory first):
|
||||
|
||||
@@ -16,10 +16,10 @@ Whether you want to learn which organizations are the big players in the web arc
|
||||
|
||||
<img src="https://imgur.zervice.io/duS8Lm7.png" width="200px" align="right" style="float: right; margin: 5px"/>
|
||||
|
||||
- [The Master Lists](#the-master-lists)
|
||||
- [The Master Lists](#The-Master-Lists)
|
||||
*Community-maintained indexes of web archiving tools and groups by IIPC, COPTR, ArchiveTeam, Wikipedia, & the ASA.*
|
||||
|
||||
- [Web Archiving Software](#web-archiving-projects)
|
||||
- [Web Archiving Software](#Web-Archiving-Projects)
|
||||
*Open source tools and projects in the internet archiving space.*
|
||||
- [Bookmarking Services](#bookmarking-services)
|
||||
- [Well-Known Open Source Projects](#from-the-archiveorg--archive-it-teams)
|
||||
@@ -27,14 +27,14 @@ Whether you want to learn which organizations are the big players in the web arc
|
||||
- [ArchiveBox Alternatives](#other-archivebox-alternatives)
|
||||
- [Smaller Utilities](#smaller-utilities)
|
||||
|
||||
- [Reading List](#reading-list)
|
||||
- [Reading List](#Reading-List)
|
||||
*Articles, posts, and blogs relevant to ArchiveBox and web archiving in general.*
|
||||
- [Blogs](#blogs-friends-of-archivebox)
|
||||
- [Articles](#articles-we-like-about-internet-archiving)
|
||||
- [Blogs](#Blogs)
|
||||
- [Articles](#Articles)
|
||||
- [ArchiveBox-Specific Posts, Tutorials, and Guides](#archivebox-specific-posts-tutorials-and-guides)
|
||||
- [ArchiveBox Discussions in News & Social Media](#archivebox-discussions-in-news--social-media)
|
||||
|
||||
- [Communities](#communities)
|
||||
- [Communities](#Communities)
|
||||
*A collection of the most active internet archiving communities and initiatives.*
|
||||
- [Most Active Web-Archiving Communities](#most-active-communities)
|
||||
- [Other Web Archiving Communities](#web-archiving-communities)
|
||||
@@ -504,5 +504,5 @@ You can find more organizations and initiatives on these other lists:
|
||||
[](https://archive.org/donate/)
|
||||
|
||||
<br/><br/>
|
||||
<small><a href="#web-archiving-community">^ Back to Top ^</a></small>
|
||||
<small><a href="#contents">^ Back to Top ^</a></small>
|
||||
</div>
|
||||
Vendored
+2
-2
@@ -1,12 +1,12 @@
|
||||
{% extends "!layout.html" %}
|
||||
|
||||
{% block body %}
|
||||
{% if is_dev %}
|
||||
{% if READTHEDOCS and current_version != "stable" %}
|
||||
<div class="admonition warning">
|
||||
<p class="admonition-title">Development Docs</p>
|
||||
<p>You are reading docs for <strong>ArchiveBox {{ version }}</strong> (pre-release).
|
||||
These docs may include unreleased features and breaking changes.
|
||||
Use the version selector to choose another published version.</p>
|
||||
For the latest stable release, see the <a href="/en/stable/">stable docs</a>.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
{{ super() }}
|
||||
|
||||
+5
-24
@@ -26,6 +26,7 @@ project = "ArchiveBox"
|
||||
copyright = f"{datetime.date.today().year} ArchiveBox"
|
||||
author = "Nick Sweeting"
|
||||
github_url = "https://github.com/ArchiveBox/ArchiveBox"
|
||||
github_doc_root = "https://github.com/ArchiveBox/docs/tree/master/" # docs repo uses master branch
|
||||
github_view_style = "blob"
|
||||
language = "en"
|
||||
|
||||
@@ -37,10 +38,6 @@ tag = release
|
||||
if release[0].isdigit():
|
||||
tag = f"v{release}" # .split('rc')[0]
|
||||
|
||||
# Branch and PR builds must link to the code they actually document.
|
||||
tag = os.environ.get("READTHEDOCS_GIT_COMMIT_HASH") or os.environ.get("READTHEDOCS_GIT_IDENTIFIER") or tag
|
||||
github_doc_root = f"{github_url}/tree/{tag}/docs/"
|
||||
|
||||
# Detect if this is a dev/pre-release build using PEP 440 parsing.
|
||||
# A version like "0.9.10" with no suffix is stable.
|
||||
# A version like "0.9.10rc1", "0.9.10.dev1", "0.9.10a1" is pre-release.
|
||||
@@ -64,7 +61,6 @@ if rtd_version in ("latest", "dev", "main", "master"):
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = [
|
||||
"sphinx_rtd_theme",
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.linkcode",
|
||||
@@ -111,8 +107,6 @@ autodoc2_skip_module_regexes = [
|
||||
autodoc2_hidden_regexes = [
|
||||
r".*__package__",
|
||||
]
|
||||
# Preserve GitHub-compatible heading fragments used by the shared wiki pages.
|
||||
myst_heading_anchors = 6
|
||||
myst_enable_extensions = ["linkify"] # pip install linkify-it-py
|
||||
myst_fence_as_directive = ["mermaid"] # render ```mermaid blocks via sphinxcontrib-mermaid
|
||||
|
||||
@@ -174,14 +168,13 @@ html_theme_options = {
|
||||
html_context = {
|
||||
"display_github": True,
|
||||
"github_user": "ArchiveBox",
|
||||
"github_repo": "ArchiveBox",
|
||||
"github_version": tag,
|
||||
"conf_py_path": "/docs/",
|
||||
"github_repo": "docs",
|
||||
"github_version": "master", # docs repo uses master branch
|
||||
"conf_py_path": "/",
|
||||
# RTD injects these automatically when building on RTD:
|
||||
# current_version, versions, downloads, READTHEDOCS, etc.
|
||||
# For local/non-RTD builds, set version info explicitly:
|
||||
"current_version": rtd_version or release,
|
||||
"is_dev": is_dev,
|
||||
"current_version": f"{release} (dev)" if is_dev else release,
|
||||
}
|
||||
html_show_sphinx = False
|
||||
|
||||
@@ -247,15 +240,3 @@ def linkcode_resolve(domain, info):
|
||||
except ValueError:
|
||||
return None
|
||||
return f"{github_url}/{github_view_style}/{tag}/{relative.as_posix()}{anchor}"
|
||||
|
||||
|
||||
def configure_source_links(app, pagename, templatename, context, doctree):
|
||||
"""Generated API pages have Python source links, not editable Markdown files."""
|
||||
if pagename.startswith("apidocs/"):
|
||||
context["display_github"] = False
|
||||
elif pagename == "README":
|
||||
context["conf_py_path"] = "/"
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.connect("html-page-context", configure_source_links)
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.35rc477",
|
||||
"version": "0.9.35rc461",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
+25
-41
@@ -42,20 +42,25 @@
|
||||
}
|
||||
}
|
||||
</script>
|
||||
<link rel="stylesheet" href="site-chrome.css">
|
||||
</head>
|
||||
<body>
|
||||
<!-- BEGIN ARCHIVEBOX HEADER — generated by monorepo/site-chrome/sync.py -->
|
||||
<header class="abx-header">
|
||||
<a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">Web Archive</span></a>
|
||||
<nav class="abx-nav" aria-label="Main navigation">
|
||||
<details class="abx-apps"><summary>Apps</summary><div class="abx-app-links"><a href="https://archivebox.io/">ArchiveBox Server</a><a href="https://archivebox.github.io/ios-archivebox/">ArchiveBox for macOS & iOS</a><a href="https://archivebox.github.io/archivebox-browser-extension/">ArchiveBox Browser Extension</a><a href="https://archivebox.github.io/abx-plugins/">ArchiveBox Plugin Library</a><a href="https://archivebox.github.io/abx-dl/"><span>One-shot <code>abx-dl</code> CLI</span></a><a href="https://github.com/ArchiveBox">More on Github...</a></div></details>
|
||||
<a href="#use-cases">Use cases</a><a href="#configuration">Configuration</a><a href="#layout-security">Layout & security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
|
||||
<a class="abx-cta" href="#install">Get started</a>
|
||||
</nav>
|
||||
</header>
|
||||
<!-- END ARCHIVEBOX HEADER -->
|
||||
<header class="site-header">
|
||||
<a class="brand" href="#top" aria-label="ArchiveBox home">
|
||||
<img src="./icon.png" alt="" width="42" height="42">
|
||||
<span>ArchiveBox</span>
|
||||
</a>
|
||||
<nav aria-label="Primary navigation">
|
||||
<a href="#install"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 3v11"/><path d="m7 10 5 5 5-5"/><path d="M5 21h14"/></svg><span>Install</span></a>
|
||||
<a href="#use-cases"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><path d="M22 21v-2a4 4 0 0 0-3-3.87"/><path d="M16 3.13a4 4 0 0 1 0 7.75"/></svg><span>Use cases</span></a>
|
||||
<a href="#configuration"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 21v-7"/><path d="M4 10V3"/><path d="M12 21v-9"/><path d="M12 8V3"/><path d="M20 21v-5"/><path d="M20 12V3"/><path d="M2 14h4"/><path d="M10 8h4"/><path d="M18 16h4"/></svg><span>Configuration</span></a>
|
||||
<a href="#layout-security"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10Z"/><path d="m9 12 2 2 4-4"/></svg><span>Layout & security</span></a>
|
||||
<a href="#background"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 8v4l3 3"/><circle cx="12" cy="12" r="9"/></svg><span>Backstory</span></a>
|
||||
<a href="#ecosystem"><svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="14" width="7" height="7" rx="1.5"/><path d="M10 6.5h4"/><path d="M17.5 10v4"/><path d="M10 17.5h4"/></svg><span>Ecosystem</span></a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"/><path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2Z"/></svg><span>Docs</span></a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M15 22v-4a4.8 4.8 0 0 0-.09-.93"/><path d="M9 22v-4a4.8 4.8 0 0 1 .09-.93"/><path d="M17.8 20.2c1.4-.5 2.2-1.7 2.2-3.7V12.7c0-.7-.2-1.4-.7-2 .1-.3.3-1.7-.1-3.2 0 0-1.1-.4-3.4.9a11.5 11.5 0 0 0-6 0C7.5 7.1 6.4 7.5 6.4 7.5c-.4 1.5-.2 2.9-.1 3.2-.5.6-.7 1.3-.7 2v3.8c0 2 1 3.2 2.4 3.7"/><path d="M9 18c-3 1.3-5 0-6-2"/></svg><span>GitHub</span></a>
|
||||
<a href="https://pypi.org/project/archivebox/"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16.5 9.4 7.5 4.2"/><path d="M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16Z"/><path d="M3.3 7 12 12l8.7-5"/><path d="M12 22V12"/></svg><span>PyPI</span></a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main id="top">
|
||||
<section class="hero section">
|
||||
@@ -458,39 +463,18 @@
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<!-- BEGIN ARCHIVEBOX FOOTER — generated by monorepo/site-chrome/sync.py -->
|
||||
<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps & tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/ios-archivebox/#get-started">ArchiveBox for macOS & iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS & iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/archivebox-browser-extension/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/abx-plugins/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/abx-dl/">One-shot abx-dl CLI</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-dl" aria-label="One-shot abx-dl CLI source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://archivebox.github.io/ios-archivebox/#your-server-your-choice">ArchiveBox Server.app</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox Server.app source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 1Line truncated
|
||||
<div class="abx-footer-local"><p>Open-source self-hosted web archiving. MIT licensed, nonprofit-supported, and built for durable collections.</p>
|
||||
<footer class="site-footer">
|
||||
<div class="brand">
|
||||
<img src="./icon.png" alt="" width="34" height="34">
|
||||
<span>ArchiveBox</span>
|
||||
</div>
|
||||
<p>Open-source self-hosted web archiving. MIT licensed, nonprofit-supported, and built for durable collections.</p>
|
||||
<div>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE">License</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/releases">Changelog</a>
|
||||
<a href="https://zulip.archivebox.io">Community chat</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations">Donations</a>
|
||||
</div></div>
|
||||
<div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security & privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
|
||||
</footer>
|
||||
<script>
|
||||
(() => {
|
||||
const menu = document.querySelector('.abx-apps');
|
||||
document.addEventListener('keydown', event => {
|
||||
if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
|
||||
});
|
||||
document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
|
||||
function revealAnchor() {
|
||||
let target;
|
||||
try { target = document.getElementById(decodeURIComponent(location.hash.slice(1))); } catch { return; }
|
||||
if (!target) return;
|
||||
let details = target.closest('details');
|
||||
let opened = false;
|
||||
while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
|
||||
if (opened) requestAnimationFrame(() => target.scrollIntoView());
|
||||
}
|
||||
addEventListener('hashchange', revealAnchor);
|
||||
revealAnchor();
|
||||
})();
|
||||
</script>
|
||||
<!-- END ARCHIVEBOX FOOTER -->
|
||||
</div>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,75 +0,0 @@
|
||||
/* Shared ArchiveBox site chrome. Source: ArchiveBox/monorepo/site-chrome. */
|
||||
.abx-header, .abx-footer { box-sizing: border-box; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
|
||||
.abx-header *, .abx-footer * { box-sizing: border-box; }
|
||||
.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
|
||||
.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
|
||||
.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
|
||||
.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
|
||||
.abx-header .abx-subsite { margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
|
||||
.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
|
||||
.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
|
||||
.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
|
||||
.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
|
||||
.abx-header .abx-nav a:hover { color: #9b2854; }
|
||||
.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
|
||||
.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
|
||||
.abx-header .abx-apps { position: relative; }
|
||||
.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
|
||||
.abx-apps > summary::-webkit-details-marker { display: none; }
|
||||
.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
|
||||
.abx-apps[open] > summary::before { transform: rotate(90deg); }
|
||||
.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 305px; max-height: min(70vh, 580px); overflow-y: auto; padding: 10px; border: 1px solid #e9e2e3; border-radius: 14px; background: #fcfaf7; box-shadow: 0 16px 48px #2d152522; }
|
||||
.abx-header .abx-app-links a { display: flex; min-height: 42px; padding: 9px 12px; border-radius: 7px; white-space: normal; }
|
||||
.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0 .3em; padding: 0; border: 0; background: transparent; color: inherit; }
|
||||
.abx-header .abx-app-links a:hover { background: #f2e8ed; }
|
||||
.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
|
||||
.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
|
||||
.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
|
||||
.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
|
||||
.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
|
||||
.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
|
||||
.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
|
||||
.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
|
||||
.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; letter-spacing: .12em; text-transform: uppercase; }
|
||||
.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
|
||||
.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
|
||||
.abx-footer .abx-footer-column li::before { content: none; }
|
||||
.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
|
||||
.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
|
||||
.abx-footer a:hover { color: #fff; text-decoration: underline; }
|
||||
.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
|
||||
.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
|
||||
.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
|
||||
.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
|
||||
.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
|
||||
.abx-footer .abx-footer-bottom a { color: #cebbc9; }
|
||||
.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
|
||||
.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
|
||||
.abx-footer-local > div { padding-top: 18px; }
|
||||
.abx-footer-local a { color: #e5aabd; }
|
||||
.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
|
||||
.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
|
||||
.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
|
||||
.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
|
||||
.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
|
||||
@media (max-width: 1100px) {
|
||||
.abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
|
||||
.abx-header .abx-nav { justify-content: flex-start; }
|
||||
.abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: 20px; width: auto; margin-top: 10px; }
|
||||
}
|
||||
@media (max-width: 700px) {
|
||||
.abx-header { position: relative; padding: 14px 18px; }
|
||||
.abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
|
||||
.abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
|
||||
.abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
|
||||
.abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
|
||||
.abx-header .abx-app-links { left: 12px; right: 12px; max-height: 60vh; }
|
||||
.abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
|
||||
.abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
|
||||
.abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
|
||||
.abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
|
||||
.abx-footer .abx-footer-column li { min-height: 40px; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before { transition: none; } }
|
||||
|
||||
.abx-footer-local > div > a + a { margin-left: 12px; }
|
||||
+4
-20
@@ -125,8 +125,7 @@ code {
|
||||
gap: 8px 18px;
|
||||
}
|
||||
|
||||
.site-header nav a,
|
||||
.site-header nav .project-menu > summary {
|
||||
.site-header nav a {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
@@ -141,8 +140,7 @@ code {
|
||||
transition: border-color 160ms ease, color 160ms ease;
|
||||
}
|
||||
|
||||
.site-header nav a:hover,
|
||||
.site-header nav .project-menu > summary:hover {
|
||||
.site-header nav a:hover {
|
||||
color: var(--text);
|
||||
text-decoration: none;
|
||||
border-color: hsl(var(--primary) / 0.42);
|
||||
@@ -808,7 +806,7 @@ h3 {
|
||||
}
|
||||
|
||||
.site-header {
|
||||
position: relative;
|
||||
position: static;
|
||||
width: 100%;
|
||||
margin: 0;
|
||||
padding: 14px 16px;
|
||||
@@ -818,8 +816,7 @@ h3 {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.site-header nav a,
|
||||
.site-header nav .project-menu > summary {
|
||||
.site-header nav a {
|
||||
font-size: 0.74rem;
|
||||
letter-spacing: 0.1em;
|
||||
}
|
||||
@@ -884,16 +881,3 @@ h3 {
|
||||
font-size: 0.82rem;
|
||||
}
|
||||
}
|
||||
|
||||
/* Shared project navigation stays usable without JavaScript. */
|
||||
.project-menu { position: relative; flex: 0 0 auto; }
|
||||
.project-menu > summary { cursor: pointer; list-style: none; }
|
||||
.project-menu > summary::-webkit-details-marker { display: none; }
|
||||
.project-menu > summary::after { content: "▾"; }
|
||||
.project-menu > summary:focus-visible, .project-menu__links a:focus-visible { outline: 3px solid #9b2854; outline-offset: 3px; }
|
||||
.project-menu__links { position: absolute; right: 0; top: 100%; z-index: 30; width: 220px; max-width: calc(100vw - 32px); padding: 8px; border: 1px solid #e9e2e3; border-radius: 12px; background: #fff; box-shadow: 0 12px 32px #28252a20; }
|
||||
header nav .project-menu__links a { display: flex; align-items: center; min-height: 44px; padding: 8px 12px; color: #28252a; font-size: 13px; letter-spacing: normal; text-transform: none; text-decoration: none; white-space: nowrap; }
|
||||
header nav .project-menu__links a:hover, header nav .project-menu__links a[aria-current="page"] { background: #f8edf2; color: #9b2854; border-radius: 6px; }
|
||||
@media (max-width: 700px) { header.site-header { flex-wrap: wrap; } header.site-header nav { flex-wrap: wrap; gap: 8px 16px; } }
|
||||
|
||||
@media (max-width: 1100px) { .project-menu__links { position: fixed; left: 16px; right: 16px; top: auto; width: auto; } }
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.35rc477"
|
||||
version = "0.9.35rc461"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@@ -79,9 +79,9 @@ dependencies = [
|
||||
### Extractor dependencies (runtime binaries resolved through abxpkg)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.56", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abxpkg==1.12.118", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.12.252", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.12.289", # shared ArchiveBox downloader package
|
||||
"abxpkg==1.12.117", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.12.244", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.12.282", # shared ArchiveBox downloader package
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
"browser-cookie3>=0.20.1",
|
||||
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
|
||||
|
||||
|
||||
[tool.bumpver]
|
||||
current_version = "v0.9.35rc477"
|
||||
current_version = "v0.9.35rc461"
|
||||
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
|
||||
commit_message = "bump version {old_version} -> {new_version}"
|
||||
tag_message = "{new_version}"
|
||||
|
||||
@@ -13,18 +13,18 @@ supported-markers = [
|
||||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-09-13T06:06:59.555346705Z"
|
||||
exclude-newer = "2026-09-12T23:41:52.996600165Z"
|
||||
exclude-newer-span = "P5D"
|
||||
|
||||
[options.exclude-newer-package]
|
||||
abxbus = { timestamp = "2026-09-18T06:06:58.555362062Z", span = "PT1S" }
|
||||
abxbus = { timestamp = "2026-09-17T23:41:51.996613044Z", span = "PT1S" }
|
||||
abx-plugins = "2100-01-01T00:00:00Z"
|
||||
abx-dl = "2100-01-01T00:00:00Z"
|
||||
abxpkg = "2100-01-01T00:00:00Z"
|
||||
|
||||
[[package]]
|
||||
name = "abx-dl"
|
||||
version = "1.12.289"
|
||||
version = "1.12.282"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -38,14 +38,14 @@ dependencies = [
|
||||
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e3/18/99113fc3117ab0cb272582e2ed66b959fc7f0efcb763f5d1ccc04095b7cc/abx_dl-1.12.289.tar.gz", hash = "sha256:b91ed92fa4d431a01165880054e85e4da00e6ebbd6d24b2472eb4e7b341f2ef5", size = 88248 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e8/61/6cb403767815780042875c68137a924e1a20862a50c9e7eefe7c09671817/abx_dl-1.12.282.tar.gz", hash = "sha256:3c49774432a3777c4441d5306ad174019653f8a668f46b1ee57258c5f65d6361", size = 88227 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/af/56/daf1fc30cce421356d6cea1497fa7e3fa31d19b84f4f554c161b50ccf479/abx_dl-1.12.289-py3-none-any.whl", hash = "sha256:bd94d50010470393b5ad6438487d63e9bfb84eb6b26217c1585be81e862cf803", size = 93264 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/b6/a19d8a07aa4e375c880101d99d8a63749b5cc6cb1542ea4761a9d233f089/abx_dl-1.12.282-py3-none-any.whl", hash = "sha256:7a922ee8b35b711c26c489549845b49897c712664f47bc08b8a85f2bd8577a8d", size = 93263 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "abx-plugins"
|
||||
version = "1.12.252"
|
||||
version = "1.12.244"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -54,9 +54,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ee/8c/66d64ce850fbb41b8e941f2b95d79b7ddc9ebd4f7238f447a7bf98fd91d0/abx_plugins-1.12.252.tar.gz", hash = "sha256:dac5a4529fc439e37c2442d5f5a06e17dbafa33fb4a4662123e02147b2e18e41", size = 463016, upload-time = "2026-09-18T05:34:20.338Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/03/bf/dfa79260a5d019452a912371c2fedee9cf7cc85aeadf0f14c6110b1c2dca/abx_plugins-1.12.244.tar.gz", hash = "sha256:c9f0170fcd0e1a83aada3f425c0999dcb9355ae75dfd6410a1cab32f15c1789b", size = 293567, upload-time = "2026-09-17T23:27:22.652Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/2d/151511349ff8f2ca6e11c756c98c6797030511fbad95d66461feb07a895a/abx_plugins-1.12.252-py3-none-any.whl", hash = "sha256:1d2a42d055a26d448b3a27664ff6ca80aa01498636377052eda6664e16206afa", size = 629564, upload-time = "2026-09-18T05:34:18.845Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/1b/5056288814dc2c07dde0f220ca9d1871ae8c508d5602a5aa6bb7cc1bcf88/abx_plugins-1.12.244-py3-none-any.whl", hash = "sha256:92f3cd8fbab5ee248e6460deda715225a165386693acd5c783571a56bc4d7acd", size = 457740, upload-time = "2026-09-17T23:27:20.918Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -82,7 +82,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "abxpkg"
|
||||
version = "1.12.118"
|
||||
version = "1.12.117"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -90,9 +90,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c1/c6/d6d0c7c2481a0525e451230e34ab832c4abc71dcbadd734fdcacac18465f/abxpkg-1.12.118.tar.gz", hash = "sha256:a68b0d72b495142e87a014cd90cfe949524eb7187d981bf0775579454f7c29ef", size = 239381, upload-time = "2026-09-18T05:27:25.739Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/00/35/bbfaccae109328c716cea4e0cdbf2d2dc5975f448203bb3f4daa605591a6/abxpkg-1.12.117.tar.gz", hash = "sha256:4d88cc293f343eee2be7ecaab1203dfbc20571a4a39b2afa2f866c9ef8032dec", size = 239387, upload-time = "2026-09-02T19:57:26.142Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/67/860d7e3d81e7f15f836d1157f77955b047808237fa8ef8949b21be38adc9/abxpkg-1.12.118-py3-none-any.whl", hash = "sha256:b632be829ae692d35b88da605341bf4d4fd78af6fb7cca2ea1feddf4a39290e0", size = 253748, upload-time = "2026-09-18T05:27:24.028Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/46/372be1f8fa915eb254ad865b97fb41073acdb5573dbce3e23a6083417d87/abxpkg-1.12.117-py3-none-any.whl", hash = "sha256:76ec8afaac7b2d7aece7bdda2bbf606e6373db51eb4dcb906008052d09e59577", size = 253749, upload-time = "2026-09-02T19:57:27.437Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -127,7 +127,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "archivebox"
|
||||
version = "0.9.35rc477"
|
||||
version = "0.9.35rc461"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -227,10 +227,10 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "abx-dl", specifier = "==1.12.289" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.12.252" },
|
||||
{ name = "abx-dl", specifier = "==1.12.282" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.12.244" },
|
||||
{ name = "abxbus", specifier = "==2.5.56" },
|
||||
{ name = "abxpkg", specifier = "==1.12.118" },
|
||||
{ name = "abxpkg", specifier = "==1.12.117" },
|
||||
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
|
||||
{ name = "atomicwrites", specifier = "==1.4.1" },
|
||||
{ name = "base32-crockford", specifier = ">=0.3.0" },
|
||||
|
||||
Reference in new issue
Block a user