Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nick Sweeting
|
2ad7a5a1ac | ||
|
Nick Sweeting
|
ca1a8c908c |
No files matched your search
-104
@@ -1,104 +0,0 @@
|
||||
.DS_Store
|
||||
._*
|
||||
/.heartbeat.json
|
||||
/package.json
|
||||
/package-lock.json
|
||||
__pycache__
|
||||
**/__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.uv-cache/
|
||||
.cache/
|
||||
.cache*/
|
||||
.github/
|
||||
.pdm-build/
|
||||
.pdm-build/**
|
||||
.pdm-python
|
||||
.eggs/
|
||||
*.egg-info/
|
||||
.git/
|
||||
!.git/
|
||||
.git/*
|
||||
.vscode/
|
||||
!.git/HEAD
|
||||
!.git/packed-refs
|
||||
!.git/refs/
|
||||
!.git/refs/heads/
|
||||
!.git/refs/heads/**
|
||||
|
||||
venv/
|
||||
.venv/
|
||||
.venv-old/
|
||||
.docker_venv/
|
||||
.docker-venv/
|
||||
node_modules/
|
||||
abx-dl/
|
||||
abxpkg/
|
||||
abx-plugins/
|
||||
abxbus/
|
||||
chrome/
|
||||
chromeprofile/
|
||||
chrome_profile/
|
||||
lib/
|
||||
out/
|
||||
tests/out/
|
||||
users/
|
||||
archive/
|
||||
crawls/
|
||||
snapshots/
|
||||
logs/
|
||||
archivebox-docker-smoke*/
|
||||
archivebox-compose-smoke*/
|
||||
docker-test/
|
||||
docker-test*/
|
||||
core
|
||||
*.core
|
||||
|
||||
pdm.dev.lock
|
||||
pdm.lock
|
||||
|
||||
docs/
|
||||
build/
|
||||
dist/
|
||||
brew_dist/
|
||||
deb_dist/
|
||||
pip_dist/
|
||||
cache/
|
||||
caches/
|
||||
assets/
|
||||
docker/
|
||||
website/
|
||||
typings/
|
||||
|
||||
# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
|
||||
# development UI screenshots or test trees. PNGs are already compressed, so
|
||||
# including them creates a large image layer that later cleanup cannot remove.
|
||||
publicsite/
|
||||
old/
|
||||
test/
|
||||
tests/
|
||||
archivebox/tests/
|
||||
|
||||
tmp/
|
||||
.tmp/
|
||||
data/
|
||||
data*/
|
||||
-
|
||||
personas/
|
||||
sources/
|
||||
output/
|
||||
index.sqlite3
|
||||
index.sqlite3-wal
|
||||
queue.sqlite3
|
||||
*.sqlite*
|
||||
data.*
|
||||
.archivebox_id
|
||||
ArchiveBox.conf
|
||||
*.stdout
|
||||
*.stderr
|
||||
*.log
|
||||
+12
-21
@@ -1,26 +1,17 @@
|
||||
# Read the Docs config for https://docs.archivebox.io
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html
|
||||
|
||||
version: 2
|
||||
|
||||
submodules:
|
||||
include: all
|
||||
recursive: true
|
||||
|
||||
build:
|
||||
os: ubuntu-22.04
|
||||
os: ubuntu-24.04
|
||||
tools:
|
||||
python: "3.12"
|
||||
#nodejs: "20" # not needed unless we need the full archivebox to run while building docs for some reason
|
||||
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
|
||||
formats:
|
||||
- pdf
|
||||
- epub
|
||||
|
||||
# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
|
||||
python: "3.13"
|
||||
jobs:
|
||||
build:
|
||||
html:
|
||||
- uv run --no-project --with-requirements .github/docs/requirements.txt sphinx-build -W --keep-going -b html -c .github/docs . "$READTHEDOCS_OUTPUT/html"
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- method: uv
|
||||
command: pip
|
||||
requirements: .github/docs/requirements.txt
|
||||
sphinx:
|
||||
configuration: .github/docs/conf.py
|
||||
fail_on_warning: true
|
||||
@@ -1,76 +0,0 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as
|
||||
contributors and maintainers pledge to making participation in our project and
|
||||
our community a harassment-free experience for everyone, regardless of age, body
|
||||
size, disability, ethnicity, sex characteristics, gender identity and expression,
|
||||
level of experience, education, socio-economic status, nationality, personal
|
||||
appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment
|
||||
include:
|
||||
|
||||
* Using welcoming and inclusive language
|
||||
* Being respectful of differing viewpoints and experiences
|
||||
* Gracefully accepting constructive criticism
|
||||
* Focusing on what is best for the community
|
||||
* Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery and unwelcome sexual attention or
|
||||
advances
|
||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or electronic
|
||||
address, without explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable
|
||||
behavior and are expected to take appropriate and fair corrective action in
|
||||
response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or
|
||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
||||
permanently any contributor for other behaviors that they deem inappropriate,
|
||||
threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies both within project spaces and in public spaces
|
||||
when an individual is representing the project or its community. Examples of
|
||||
representing a project or community include using an official project e-mail
|
||||
address, posting via an official social media account, or acting as an appointed
|
||||
representative at an online or offline event. Representation of a project may be
|
||||
further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported by contacting the project team at archivebox-coc@sweeting.me. All
|
||||
complaints will be reviewed and investigated and will result in a response that
|
||||
is deemed necessary and appropriate to the circumstances. The project team is
|
||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
||||
Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
||||
faith may face temporary or permanent repercussions as determined by other
|
||||
members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see
|
||||
https://www.contributor-covenant.org/faq
|
||||
@@ -1,38 +0,0 @@
|
||||
# Contribution Process
|
||||
|
||||
1. Confirm your desired features fit into our bigger project goals [Roadmap](https://github.com/pirate/ArchiveBox/wiki/Roadmap).
|
||||
2. Open an issue with your planned implementation to discuss
|
||||
3. Check in with me before starting development to make sure your work wont conflict with or duplicate existing work
|
||||
4. Setup your dev environment, make some changes, and test using the test input files
|
||||
5. Commit, push, and submit a PR and wait for review feedback
|
||||
6. Have patience, don't abandon your PR! We love contributors but we all have day jobs and don't always have time to respond to notifications instantly. If you want a faster response, ping @theSquashSH on twitter or Patreon.
|
||||
|
||||
**Useful links:**
|
||||
|
||||
- https://github.com/ArchiveBox/ArchiveBox/issues
|
||||
- https://github.com/ArchiveBox/ArchiveBox/pulls
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
|
||||
|
||||
### Development Setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ArchiveBox/ArchiveBox
|
||||
cd ArchiveBox
|
||||
# Ideally do this in a virtualenv
|
||||
pip install -e '.[dev]' # or use: pipenv install --dev
|
||||
```
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
./bin/lint.sh
|
||||
./bin/test.sh
|
||||
./bin/build.sh
|
||||
```
|
||||
|
||||
For more common tasks see the `Development` section at the bottom of the README.
|
||||
|
||||
### Getting Help
|
||||
|
||||
Open issues on Github or message me https://sweeting.me/#contact.
|
||||
@@ -1,2 +0,0 @@
|
||||
github: ["ArchiveBox", "pirate"]
|
||||
custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
|
||||
@@ -1,197 +0,0 @@
|
||||
name: 🐞 Bug report
|
||||
description: Report a bug or error you encountered in ArchiveBox
|
||||
title: "Bug: ..."
|
||||
assignees:
|
||||
- pirate
|
||||
type: 'Bug'
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
*Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).
|
||||
Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Provide a screenshot and describe the bug
|
||||
description: |
|
||||
Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.
|
||||
placeholder: |
|
||||
Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
|
||||
I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: steps_to_reproduce
|
||||
attributes:
|
||||
label: Steps to reproduce
|
||||
description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
|
||||
render: markdown
|
||||
placeholder: |
|
||||
1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
|
||||
2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
|
||||
3. Typed 'https://example.com/xyz' into the 'Add URL' input field
|
||||
4. Clicked the 'Add+' button
|
||||
5. Got a 500 error and saw the errors below in terminal
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs or errors
|
||||
description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
|
||||
placeholder: |
|
||||
╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567 │
|
||||
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
||||
|
||||
[+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
|
||||
> Saved verbatim input to sources/1730577269-import.txt
|
||||
> Parsed 1 URLs from input (Generic TXT)
|
||||
...
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: ArchiveBox Version
|
||||
description: |
|
||||
**REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).
|
||||
For Docker Compose run: `docker compose run archivebox version`
|
||||
For plain Docker run: `docker run -v $PWD:/data archivebox/archivebox version`
|
||||
render: shell
|
||||
placeholder: |
|
||||
0.8.6
|
||||
ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
|
||||
IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
|
||||
EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
|
||||
DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
|
||||
|
||||
Binary Dependencies:
|
||||
√ python 3.14.0 venv_pip ~/.venv/bin/python
|
||||
√ django 6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/__init__.py
|
||||
√ sqlite 2.6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
|
||||
√ pip 24.3.1 venv_pip ~/.venv/bin/pip
|
||||
...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: install_method
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: How did you install the version of ArchiveBox you are using?
|
||||
multiple: false
|
||||
options:
|
||||
- pip
|
||||
- apt
|
||||
- brew
|
||||
- nix
|
||||
- Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
|
||||
- Other
|
||||
|
||||
- type: dropdown
|
||||
id: operating_system
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: What operating system are you running on?
|
||||
description: |
|
||||
Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
|
||||
multiple: false
|
||||
options:
|
||||
- Linux (Ubuntu/Debian/Arch/Alpine/etc.)
|
||||
- macOS (including Docker on macOS)
|
||||
- BSD (FreeBSD/OpenBSD/NetBSD/etc.)
|
||||
- Windows (including WSL, WSL2, Docker Desktop on Windows)
|
||||
- Other
|
||||
|
||||
- type: checkboxes
|
||||
id: filesystem
|
||||
attributes:
|
||||
label: What type of drive are you using to store your ArchiveBox data?
|
||||
description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
|
||||
options:
|
||||
- label: "some of `data/` is on a local SSD or NVMe drive"
|
||||
required: false
|
||||
- label: "some of `data/` is on a spinning hard drive or external USB drive"
|
||||
required: false
|
||||
- label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
|
||||
required: false
|
||||
- label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
|
||||
required: false
|
||||
|
||||
|
||||
- type: textarea
|
||||
id: docker_compose_yml
|
||||
attributes:
|
||||
label: Docker Compose Configuration
|
||||
description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
|
||||
placeholder: |
|
||||
services:
|
||||
archivebox:
|
||||
image: archivebox/archivebox:latest
|
||||
ports:
|
||||
- 5797:5797
|
||||
volumes:
|
||||
- ./data:/data
|
||||
environment:
|
||||
- ADMIN_USERNAME=admin
|
||||
- ADMIN_PASSWORD=****<redact any passwords>****
|
||||
- ALLOWED_HOSTS=*
|
||||
- CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
|
||||
- PUBLIC_INDEX=True
|
||||
- PUBLIC_SNAPSHOTS=True
|
||||
- PUBLIC_ADD_VIEW=False
|
||||
...
|
||||
|
||||
archivebox_scheduler:
|
||||
image: archivebox/archivebox:latest
|
||||
command: schedule --foreground --update --every=day
|
||||
environment:
|
||||
...
|
||||
|
||||
...
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: configuration
|
||||
attributes:
|
||||
label: ArchiveBox Configuration
|
||||
description: "Please share your full `data/ArchiveBox.conf` file here."
|
||||
render: shell
|
||||
placeholder: |
|
||||
[SERVER_CONFIG]
|
||||
SECRET_KEY = "*********<redact any secrets/passwords>************"
|
||||
|
||||
WGET_RESTRICT_FILE_NAMES=windows
|
||||
USE_SYSTEM_WGET=true
|
||||
CHECK_SSL_VALIDITY=false
|
||||
...
|
||||
validations:
|
||||
required: false
|
||||
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
|
||||
We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.
|
||||
Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.
|
||||
|
||||
In the meantime please consider:
|
||||
|
||||
- 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
|
||||
- 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
|
||||
- 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
|
||||
- ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
name: 💡 Feature or enhancement request
|
||||
description: Suggest an idea or improvement for this project
|
||||
title: "Feature Request: ..."
|
||||
assignees:
|
||||
- pirate
|
||||
type: 'Enhancement'
|
||||
labels: 'status: idea phase'
|
||||
body:
|
||||
- type: dropdown
|
||||
id: suggestion_type
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: "What type of suggestion are you making?"
|
||||
multiple: false
|
||||
options:
|
||||
- New extractor / type of content to save
|
||||
- Proposing a new feature
|
||||
- Modification of existing behavior
|
||||
- Web UI or UX design improvement
|
||||
|
||||
- type: textarea
|
||||
id: current_problem
|
||||
attributes:
|
||||
label: "What is the problem that your feature request solves?"
|
||||
description: |
|
||||
Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
|
||||
placeholder: |
|
||||
e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: proposed_solution
|
||||
attributes:
|
||||
label: "What is your proposed solution?"
|
||||
description: |
|
||||
Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
|
||||
placeholder: |
|
||||
e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.
|
||||
The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: workarounds_tried
|
||||
attributes:
|
||||
label: "What hacks or alternative solutions have you tried to solve the problem?"
|
||||
description: |
|
||||
A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
|
||||
placeholder: |
|
||||
e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside
|
||||
the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Share the entire output of the `archivebox version` command for the current version you are using.
|
||||
description: |
|
||||
DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.
|
||||
We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.
|
||||
Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
|
||||
|
||||
Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).
|
||||
For Docker Compose run: `docker compose run archivebox version`
|
||||
For plain Docker run: `docker run -v $PWD:/data archivebox/archivebox version`
|
||||
render: shell
|
||||
placeholder: |
|
||||
0.8.6
|
||||
ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
|
||||
IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
|
||||
EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
|
||||
DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
|
||||
|
||||
Binary Dependencies:
|
||||
√ python 3.14.0 venv_pip ~/.venv/bin/python
|
||||
√ django 6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/__init__.py
|
||||
√ sqlite 2.6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
|
||||
√ pip 24.3.1 venv_pip ~/.venv/bin/pip
|
||||
...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
id: priority
|
||||
attributes:
|
||||
label: "How badly do you want this new feature?"
|
||||
options:
|
||||
- label: "It's an urgent deal-breaker, I can't live without it"
|
||||
required: false
|
||||
- label: "It's important to add it in the near-mid term future"
|
||||
required: false
|
||||
- label: "It would be nice to have eventually"
|
||||
required: false
|
||||
- label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
|
||||
required: false
|
||||
- label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
|
||||
required: false
|
||||
|
||||
- type: checkboxes
|
||||
id: satisfaction_survey
|
||||
attributes:
|
||||
label: Mini Survey
|
||||
description: How do you like ArchiveBox so far?
|
||||
options:
|
||||
- label: "I like ArchiveBox so far / would recommend it to a friend"
|
||||
required: false
|
||||
- label: "I've had a lot of difficulty getting ArchiveBox set up"
|
||||
required: false
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
|
||||
We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.
|
||||
Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.
|
||||
|
||||
In the meantime please consider:
|
||||
|
||||
- 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
|
||||
- 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
|
||||
- ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
|
||||
@@ -1,51 +0,0 @@
|
||||
name: 📑 Documentation improvement
|
||||
description: Submit an idea or correction for the Wiki documentation
|
||||
title: "Documentation: ..."
|
||||
labels: 'touches: docs'
|
||||
type: 'Enhancement'
|
||||
assignees:
|
||||
- pirate
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
|
||||
|
||||
- type: input
|
||||
id: page_url
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: "What is the URL of the page you'd like to see improved?"
|
||||
placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
|
||||
|
||||
- type: input
|
||||
id: section_title
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: "What is the title of the relevant section?"
|
||||
placeholder: e.g. Option B. Automatic Setup Script
|
||||
|
||||
- type: textarea
|
||||
id: suggested_edit
|
||||
attributes:
|
||||
label: "What is the suggested edit?"
|
||||
placeholder: |
|
||||
e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.
|
||||
Attach images, screenshots, code snippets, etc. anything you think would help.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
|
||||
We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.
|
||||
|
||||
In the meantime please consider:
|
||||
|
||||
- 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
|
||||
- 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
|
||||
- ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
|
||||
@@ -1,8 +0,0 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: ❓ Ask a question or start a discussion
|
||||
url: https://github.com/ArchiveBox/ArchiveBox/discussions
|
||||
about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
|
||||
- name: 💬 Chat with the dev team & community on Zulip
|
||||
url: https://zulip.archivebox.io
|
||||
about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
|
||||
@@ -1,18 +0,0 @@
|
||||
<!-- IMPORTANT: Do not submit PRs with only formatting / PEP8 / line length changes. -->
|
||||
|
||||
# Summary
|
||||
|
||||
<!--e.g. This PR fixes ABC or adds the ability to do XYZ...-->
|
||||
|
||||
# Related issues
|
||||
|
||||
<!-- e.g. #123 or Roadmap goal # https://github.com/pirate/ArchiveBox/wiki/Roadmap -->
|
||||
|
||||
# Changes these areas
|
||||
|
||||
- [ ] Bugfixes
|
||||
- [ ] Feature behavior
|
||||
- [ ] Command line interface
|
||||
- [ ] Configuration options
|
||||
- [ ] Internal architecture
|
||||
- [ ] Snapshot data layout on disk
|
||||
@@ -1,68 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
---
|
||||
|
||||
## Security Documentation
|
||||
|
||||
Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
|
||||
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
|
||||
- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
|
||||
- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
|
||||
- https://github.com/ArchiveBox/ArchiveBox#archive-layout
|
||||
- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
|
||||
|
||||
---
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
|
||||
|
||||
You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
|
||||
|
||||
---
|
||||
|
||||
## CVEs and Report Credits
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**
|
||||
> Reporters **will still receive credit in release notes** however.
|
||||
|
||||
> [!TIP]
|
||||
> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
|
||||
|
||||
Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
|
||||
|
||||
---
|
||||
|
||||
## Threat Model
|
||||
|
||||
### All admin users are assumed to be trusted
|
||||
|
||||
ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
|
||||
|
||||
- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
|
||||
- non-logged in anonymous guest users
|
||||
|
||||
Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
|
||||
|
||||
### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
|
||||
|
||||
Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
|
||||
|
||||
Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
|
||||
|
||||
### XSS, CSRF, CORS, CSP
|
||||
|
||||
`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
|
||||
|
||||
[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
|
||||
|
||||
### SSRF / URL Validation
|
||||
|
||||
Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
|
||||
It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
|
||||
@@ -1,42 +0,0 @@
|
||||
# Stable releases require live dev acceptance
|
||||
|
||||
The order is **dev CI → dev image publication → automatic Cabbage deployment →
|
||||
real capture and browser acceptance → stable publication**. Never
|
||||
substitute a published PyPI wheel, healthy container, or local build for this
|
||||
acceptance. The release workflow blocks *before* a stable upload unless Cabbage
|
||||
has successful evidence from `staging-acceptance.yml` matching all tracked
|
||||
source and dependency pins. Only ArchiveBox's own version fields may change
|
||||
between dev and stable.
|
||||
|
||||
The staging workflow observes the existing image watcher; it cannot deploy or
|
||||
restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
|
||||
as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
|
||||
is checked against the candidate. A dedicated SSH key must use
|
||||
`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
|
||||
keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
|
||||
run caller-provided commands or archive arbitrary URLs.
|
||||
|
||||
The `staging` GitHub environment is restricted to dev/main branches. It contains
|
||||
`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
|
||||
Only trusted release events use the private ugNAS runner route. Never expose
|
||||
these credentials to PR jobs or copy an operator's existing root key.
|
||||
|
||||
Acceptance creates one tagged background crawl per source through the normal
|
||||
add/runner path, enabling every snapshot extractor, including TLSNotary. It
|
||||
checks installed pins, completed outputs and their files, unexpected/missing
|
||||
plugin results, host OOM kills, and real rendered snapshot pages. For this
|
||||
release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
|
||||
diagnostics are retained in the report's `limitations` list; they do not disable
|
||||
those plugins or excuse missing/truncated successful outputs. Remove the
|
||||
TLSNotary exception when its follow-up fix ships. Logs, result inventories and
|
||||
screenshots are retained as Actions artifacts. Other failures remain
|
||||
failures; do not publish a success deployment record to bypass them. No code
|
||||
mounts are allowed. The Cabbage job must pass before its record authorizes a
|
||||
stable upload. Queued capture progress is polled; failed tests are not retried.
|
||||
|
||||
Browser acceptance follows the default WACZ viewer into the captured document,
|
||||
matching its replay base and original URL and checking that its body is visible
|
||||
and contains substantial text. An attached viewer iframe can still be
|
||||
`about:blank`, so its presence alone cannot prove replay works. Extracted display
|
||||
titles and the captured document's `<title>` may legitimately differ; neither
|
||||
title equality nor output thumbnails substitute for checking the actual page.
|
||||
@@ -1,3 +0,0 @@
|
||||
self-hosted-runner:
|
||||
labels:
|
||||
- ugnas-linux
|
||||
@@ -1,81 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"CC_BINARY": {
|
||||
"default": "cc"
|
||||
},
|
||||
"LDAPSEARCH_BINARY": {
|
||||
"default": "ldapsearch"
|
||||
},
|
||||
"INITDB_BINARY": {
|
||||
"default": "initdb"
|
||||
},
|
||||
"PG_CTL_BINARY": {
|
||||
"default": "pg_ctl"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CC_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"build-essential"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{LDAPSEARCH_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
},
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"ldap-utils",
|
||||
"python3-dev",
|
||||
"python3-setuptools",
|
||||
"libssl-dev",
|
||||
"libldap2-dev",
|
||||
"libsasl2-dev",
|
||||
"zlib1g-dev",
|
||||
"libatomic1"
|
||||
],
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"postgres_binaries": [
|
||||
{
|
||||
"name": "{INITDB_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"postgresql"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{PG_CTL_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"postgresql"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"BREW_BINARY": {
|
||||
"default": "brew"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{BREW_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"CLANG_BINARY": {
|
||||
"default": "clang"
|
||||
},
|
||||
"LDAPVC_BINARY": {
|
||||
"default": "ldapvc"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CLANG_BINARY}",
|
||||
"binproviders": "env,brew",
|
||||
"overrides": {
|
||||
"brew": {
|
||||
"install_args": [
|
||||
"llvm"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{LDAPVC_BINARY}",
|
||||
"binproviders": "env,brew",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
},
|
||||
"brew": {
|
||||
"install_args": [
|
||||
"openldap"
|
||||
],
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"LDAPVC_BINARY": {
|
||||
"default": "ldapvc"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{LDAPVC_BINARY}",
|
||||
"binproviders": "env",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,349 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"CURL_BINARY": {
|
||||
"default": "curl"
|
||||
},
|
||||
"BASH_BINARY": {
|
||||
"default": "bash"
|
||||
},
|
||||
"CHMOD_BINARY": {
|
||||
"default": "chmod"
|
||||
},
|
||||
"DOCKER_BINARY": {
|
||||
"default": "docker"
|
||||
},
|
||||
"GIT_BINARY": {
|
||||
"default": "git"
|
||||
},
|
||||
"GREP_BINARY": {
|
||||
"default": "grep"
|
||||
},
|
||||
"GH_BINARY": {
|
||||
"default": "gh"
|
||||
},
|
||||
"EXPORTFS_BINARY": {
|
||||
"default": "exportfs"
|
||||
},
|
||||
"JQ_BINARY": {
|
||||
"default": "jq"
|
||||
},
|
||||
"MKDIR_BINARY": {
|
||||
"default": "mkdir"
|
||||
},
|
||||
"MOUNT_CIFS_BINARY": {
|
||||
"default": "mount.cifs"
|
||||
},
|
||||
"MOUNT_NFS_BINARY": {
|
||||
"default": "mount.nfs"
|
||||
},
|
||||
"NODE_BINARY": {
|
||||
"default": "node"
|
||||
},
|
||||
"PS_BINARY": {
|
||||
"default": "ps"
|
||||
},
|
||||
"PYTHON_BINARY": {
|
||||
"default": "python"
|
||||
},
|
||||
"PV_BINARY": {
|
||||
"default": "pv"
|
||||
},
|
||||
"SSH_BINARY": {
|
||||
"default": "ssh"
|
||||
},
|
||||
"SED_BINARY": {
|
||||
"default": "sed"
|
||||
},
|
||||
"SMBD_BINARY": {
|
||||
"default": "smbd"
|
||||
},
|
||||
"SUDO_BINARY": {
|
||||
"default": "sudo"
|
||||
},
|
||||
"SYSTEMCTL_BINARY": {
|
||||
"default": "systemctl"
|
||||
},
|
||||
"TAR_BINARY": {
|
||||
"default": "tar"
|
||||
},
|
||||
"TREE_BINARY": {
|
||||
"default": "tree"
|
||||
},
|
||||
"TR_BINARY": {
|
||||
"default": "tr"
|
||||
},
|
||||
"RM_BINARY": {
|
||||
"default": "rm"
|
||||
},
|
||||
"UMOUNT_BINARY": {
|
||||
"default": "umount"
|
||||
},
|
||||
"UV_BINARY": {
|
||||
"default": "uv"
|
||||
},
|
||||
"UNAME_BINARY": {
|
||||
"default": "uname"
|
||||
}
|
||||
},
|
||||
"ci_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{JQ_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{UV_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"bsd_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env,pyinfra"
|
||||
}
|
||||
],
|
||||
"freebsd_binaries": [
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env,pyinfra"
|
||||
}
|
||||
],
|
||||
"docker_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UNAME_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"deploy_binaries": [
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SSH_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docker_debug_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{PV_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{TAR_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{TREE_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
],
|
||||
"docker_validation_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GREP_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{MKDIR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{RM_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SED_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SSH_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{TR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UNAME_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"mount_validation_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{CHMOD_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{EXPORTFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"nfs-kernel-server"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{MKDIR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{MOUNT_CIFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"cifs-utils"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{MOUNT_NFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"nfs-common"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{SMBD_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"samba"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SYSTEMCTL_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UMOUNT_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"process_binaries": [
|
||||
{
|
||||
"name": "{PS_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docs_binaries": [
|
||||
{
|
||||
"name": "{CURL_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"git_binaries": [
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
],
|
||||
"github_binaries": [
|
||||
{
|
||||
"name": "{GH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
],
|
||||
"node_binaries": [
|
||||
{
|
||||
"name": "{NODE_BINARY}",
|
||||
"binproviders": "env,npm",
|
||||
"min_version": "22.23.1",
|
||||
"overrides": {
|
||||
"npm": {
|
||||
"install_args": [
|
||||
"node@22.23.1"
|
||||
],
|
||||
"postinstall_scripts": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"release_binaries": [
|
||||
{
|
||||
"name": "{UV_BINARY}",
|
||||
"binproviders": "env,pip,brew"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{JQ_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{CURL_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
target-branch: "dev"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
pip:
|
||||
patterns:
|
||||
- "*"
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/etc"
|
||||
target-branch: "dev"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
groups:
|
||||
npm:
|
||||
patterns:
|
||||
- "*"
|
||||
@@ -0,0 +1,25 @@
|
||||
# Historical 0.0.3 documentation
|
||||
|
||||
This documentation-only branch starts at the immutable `v0.0.3` tag.
|
||||
The original README, application files, and pinned wiki (when present) remain unchanged.
|
||||
Sphinx reads them without installing or importing the historical application.
|
||||
The render-time adapter fixes historical link and heading syntax only; the visible warning
|
||||
explains that these old installation commands and external services may no longer work.
|
||||
|
||||
Prerequisites: [uv](https://docs.astral.sh/uv/getting-started/installation/) 0.11.3 and Python 3.13, matching CI and Read the Docs. `uv` can install Python when needed.
|
||||
|
||||
```sh
|
||||
READTHEDOCS=True uv run --no-project --python 3.13 --with-requirements .github/docs/requirements.txt \
|
||||
sphinx-build -E -a -W --keep-going -b html -c .github/docs . /tmp/archivebox-historical-docs
|
||||
```
|
||||
|
||||
Read the Docs must use `.github/.readthedocs.yaml`. The pinned theme supplies
|
||||
its version selector and listens for Read the Docs addon version data.
|
||||
|
||||
`requirements.txt` pins the complete documentation dependency set. To update it,
|
||||
edit `requirements.in` and run:
|
||||
|
||||
```sh
|
||||
uv pip compile --universal --python-version 3.13 .github/docs/requirements.in \
|
||||
--output-file .github/docs/requirements.txt
|
||||
```
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Render this release's historical documentation without importing the application."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from markdown_it import MarkdownIt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
release = "0.0.3"
|
||||
version = release
|
||||
project = "ArchiveBox"
|
||||
author = "ArchiveBox contributors"
|
||||
copyright = "ArchiveBox contributors"
|
||||
extensions = ["myst_parser", "sphinx_rtd_theme"]
|
||||
source_suffix = {".md": "markdown"}
|
||||
root_doc = "index"
|
||||
exclude_patterns = [
|
||||
p.name for p in ROOT.iterdir() if p.name not in {"index.md", "docs"}
|
||||
]
|
||||
exclude_patterns += ["docs/_Sidebar.md", "docs/_Footer.md", "docs/.git"]
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
html_theme_options = {
|
||||
"navigation_depth": 3,
|
||||
"collapse_navigation": False,
|
||||
"version_selector": True,
|
||||
"language_selector": False,
|
||||
}
|
||||
html_context = {"current_version": os.environ.get("READTHEDOCS_VERSION", release)}
|
||||
html_title = f"ArchiveBox {release} historical documentation"
|
||||
html_show_sphinx = False
|
||||
myst_heading_anchors = 6
|
||||
|
||||
|
||||
def rewrite_prose(text, transform):
|
||||
"""Rewrite navigation outside fenced and indented code blocks."""
|
||||
lines = text.splitlines(keepends=True)
|
||||
output = []
|
||||
start = 0
|
||||
for token in MarkdownIt().parse(text):
|
||||
if token.type not in {"fence", "code_block"}:
|
||||
continue
|
||||
first, last = token.map
|
||||
output.append(transform("".join(lines[start:first])))
|
||||
output.append("".join(lines[first:last]))
|
||||
start = last
|
||||
output.append(transform("".join(lines[start:])))
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def prepare_source(app, docname, source):
|
||||
text = (ROOT / "README.md").read_text() if docname == "index" else source[0]
|
||||
directory = ROOT if docname == "index" else ROOT / "docs"
|
||||
|
||||
def wiki(match):
|
||||
parts = match.group(1).split("|", 1)
|
||||
return f"[{parts[0]}]({parts[-1].replace(' ', '-')}.md)"
|
||||
|
||||
text = rewrite_prose(text, lambda prose: re.sub(r"\[\[([^\]\n]+)\]\]", wiki, prose))
|
||||
|
||||
def resource(target):
|
||||
if urlsplit(target).scheme or target.startswith(("#", "//")):
|
||||
return target
|
||||
page, sep, anchor = target.partition("#")
|
||||
path = directory / page
|
||||
if path.suffix == ".md" and path.parent == ROOT / "docs":
|
||||
return target
|
||||
if path.is_file():
|
||||
return (
|
||||
f"https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/v{release}/{path.relative_to(ROOT)}"
|
||||
+ ("#" + anchor if sep else "")
|
||||
)
|
||||
return target
|
||||
|
||||
def rewrite_links(prose):
|
||||
prose = re.sub(
|
||||
r'((?:src|href)=["\'])([^"\']+)', lambda m: m[1] + resource(m[2]), prose
|
||||
)
|
||||
prose = re.sub(r"\]\(([^)\s]+)\)", lambda m: "](" + resource(m[1]) + ")", prose)
|
||||
return prose
|
||||
|
||||
text = rewrite_prose(text, rewrite_links)
|
||||
text = (
|
||||
f"# ArchiveBox {release}"
|
||||
+ (
|
||||
" historical guide"
|
||||
if docname == "index"
|
||||
else " — " + Path(docname).name.replace("-", " ")
|
||||
)
|
||||
+ "\n\n"
|
||||
+ text
|
||||
)
|
||||
lines = text.splitlines()
|
||||
stack = []
|
||||
for token in MarkdownIt().parse(text):
|
||||
if token.type != "heading_open":
|
||||
continue
|
||||
original = int(token.tag[1])
|
||||
while stack and stack[-1] >= original:
|
||||
stack.pop()
|
||||
stack.append(original)
|
||||
lines[token.map[0]] = re.sub(
|
||||
r"^#{1,6}(?=\s)", "#" * len(stack), lines[token.map[0]]
|
||||
)
|
||||
text = "\n".join(lines) + "\n"
|
||||
if docname == "index":
|
||||
notice = f"\n```{{warning}}\nThis is the original guide shipped with ArchiveBox {release}, preserved for historical reference. Commands, dependencies, and external services describe that old release and may no longer work. Use the version selector for current documentation.\n```\n"
|
||||
first, rest = text.split("\n", 1)
|
||||
text = first + "\n" + notice + rest
|
||||
pages = sorted(
|
||||
p.stem for p in (ROOT / "docs").glob("*.md") if not p.name.startswith("_")
|
||||
)
|
||||
if pages:
|
||||
text += (
|
||||
"\n```{toctree}\n:maxdepth: 2\n\n"
|
||||
+ "\n".join("docs/" + p for p in pages)
|
||||
+ "\n```\n"
|
||||
)
|
||||
source[0] = text
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.connect("source-read", prepare_source)
|
||||
@@ -0,0 +1,3 @@
|
||||
Sphinx==8.2.3
|
||||
myst-parser==4.0.1
|
||||
sphinx-rtd-theme==3.1.0
|
||||
@@ -0,0 +1,75 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile --universal --python-version 3.13 .github/docs/requirements.in --output-file .github/docs/requirements.txt
|
||||
alabaster==1.0.0
|
||||
# via sphinx
|
||||
babel==2.18.0
|
||||
# via sphinx
|
||||
certifi==2026.7.22
|
||||
# via requests
|
||||
charset-normalizer==3.5.1
|
||||
# via requests
|
||||
colorama==0.4.6 ; sys_platform == 'win32'
|
||||
# via sphinx
|
||||
docutils==0.21.2
|
||||
# via
|
||||
# myst-parser
|
||||
# sphinx
|
||||
# sphinx-rtd-theme
|
||||
idna==3.20
|
||||
# via requests
|
||||
imagesize==2.0.1
|
||||
# via sphinx
|
||||
jinja2==3.1.6
|
||||
# via
|
||||
# myst-parser
|
||||
# sphinx
|
||||
markdown-it-py==3.0.0
|
||||
# via
|
||||
# mdit-py-plugins
|
||||
# myst-parser
|
||||
markupsafe==3.0.3
|
||||
# via jinja2
|
||||
mdit-py-plugins==0.6.1
|
||||
# via myst-parser
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
myst-parser==4.0.1
|
||||
# via -r .github/docs/requirements.in
|
||||
packaging==26.3
|
||||
# via sphinx
|
||||
pygments==2.21.0
|
||||
# via sphinx
|
||||
pyyaml==6.0.3
|
||||
# via myst-parser
|
||||
requests==2.34.2
|
||||
# via sphinx
|
||||
roman-numerals==4.1.0
|
||||
# via roman-numerals-py
|
||||
roman-numerals-py==4.1.0
|
||||
# via sphinx
|
||||
snowballstemmer==3.1.1
|
||||
# via sphinx
|
||||
sphinx==8.2.3
|
||||
# via
|
||||
# -r .github/docs/requirements.in
|
||||
# myst-parser
|
||||
# sphinx-rtd-theme
|
||||
# sphinxcontrib-jquery
|
||||
sphinx-rtd-theme==3.1.0
|
||||
# via -r .github/docs/requirements.in
|
||||
sphinxcontrib-applehelp==2.0.0
|
||||
# via sphinx
|
||||
sphinxcontrib-devhelp==2.0.0
|
||||
# via sphinx
|
||||
sphinxcontrib-htmlhelp==2.1.0
|
||||
# via sphinx
|
||||
sphinxcontrib-jquery==4.1
|
||||
# via sphinx-rtd-theme
|
||||
sphinxcontrib-jsmath==1.0.1
|
||||
# via sphinx
|
||||
sphinxcontrib-qthelp==2.0.0
|
||||
# via sphinx
|
||||
sphinxcontrib-serializinghtml==2.0.0
|
||||
# via sphinx
|
||||
urllib3==2.8.0
|
||||
# via requests
|
||||
@@ -1,38 +0,0 @@
|
||||
# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
|
||||
# Manual run from the repository root with Lychee 0.24.2:
|
||||
# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
|
||||
# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
|
||||
# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
|
||||
include_fragments = "anchor-only"
|
||||
include_verbatim = true
|
||||
max_retries = 0
|
||||
max_concurrency = 4
|
||||
timeout = 20
|
||||
no_progress = true
|
||||
hidden = true
|
||||
extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
|
||||
exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
|
||||
# Includes override this exclude; this is a documentation-link allowlist.
|
||||
exclude = ['.*']
|
||||
include = [
|
||||
'^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
|
||||
'^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
|
||||
'^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
|
||||
'^https?://docs\.archivebox\.io(?:/|$)',
|
||||
'^file:.*\.md(?:#|$)',
|
||||
'^file:.*/docs/[^/.#]+(?:#|$)',
|
||||
]
|
||||
fallback_extensions = ["md"]
|
||||
remap = [
|
||||
# Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
|
||||
|
||||
'^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
|
||||
'^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
|
||||
# The wiki accepts this case alias, but raw file paths are case-sensitive.
|
||||
'^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
|
||||
|
||||
# Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
|
||||
# Keep fragment URLs on the rendered wiki to catch published heading drift.
|
||||
'^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
|
||||
]
|
||||
preprocess = { command = "bin/lychee_preprocess.py" }
|
||||
@@ -1,124 +0,0 @@
|
||||
# archivebox.io website
|
||||
|
||||
This repository owns its complete website. Build and deploy it without checking
|
||||
out another repository's website, fetching shared snippets, or calling a shared
|
||||
workflow. Common files are ordinary committed copies; update them deliberately.
|
||||
|
||||
## Repository-specific files
|
||||
|
||||
- Content: `publicsite/index.html`.
|
||||
- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
|
||||
- Application capture: `bin/collect_ui_screenshots.sh`.
|
||||
- Deployment branch: `dev`.
|
||||
|
||||
## Common publication contract
|
||||
|
||||
1. Build pull requests without deploying them.
|
||||
2. Build website pushes immediately, independently of application capture CI.
|
||||
3. Restore validated screenshots from trusted default-branch capture jobs.
|
||||
Keep the original capture revision, timestamp, coverage and image checksums.
|
||||
4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
|
||||
published image set. Never replace a complete gallery with an empty one when
|
||||
restoration fails. Apple device groups are independently validated artifacts;
|
||||
Android's checked-in bootstrap captures retain their local provenance.
|
||||
5. Render current content and layout around those captures. Site revision and
|
||||
capture revision are separate facts; reusing captures must not relabel them.
|
||||
6. Serialize production restore/build/deploy under one concurrency group and
|
||||
check out the latest default-branch revision when each build starts. Keep the
|
||||
built revision in provenance; later release-bot commits must not block deploys.
|
||||
7. Keep platform-specific build, installation, capture, and completeness checks
|
||||
in the application's existing capture workflow. Site-only edits do not need
|
||||
to rerun those jobs.
|
||||
|
||||
## Common presentation contract
|
||||
|
||||
The base header, Apps menu and icons, footer, brand styles and navigation behavior
|
||||
should remain identical copies across the seven product sites. Put this site's
|
||||
name, navigation, CTA and extra footer resources outside the common base files.
|
||||
Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
|
||||
mobile layout, reduced-motion preferences and existing product-specific content.
|
||||
Screenshots stay in English. Translation content, when added, lives in separate
|
||||
HTML files; no runtime translation service or cross-repository dependency.
|
||||
|
||||
## Editing and building
|
||||
|
||||
`base/` contains identical independent copies across the seven product sites.
|
||||
Common Python scripts share the same behavior; repository formatters may change
|
||||
line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
|
||||
behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
|
||||
links and CTA. Product content and additional footer resources stay in the native
|
||||
HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
|
||||
slots; the renderer fills them with static HTML, never browser-side fetches.
|
||||
|
||||
The existing builder calls `site.py render` after generating its content. This
|
||||
copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
|
||||
HTML, validates local resources and writes the site revision to root `build.json`.
|
||||
Screenshot manifests are left untouched. No build reads another repository.
|
||||
|
||||
Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
|
||||
another product repository when updating them; do not add a synchronization job,
|
||||
package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
|
||||
and the marked workflow build section are intentionally repository-specific.
|
||||
|
||||
Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
|
||||
|
||||
Check the built site with real Chromium (also run by Pages CI):
|
||||
|
||||
```sh
|
||||
export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
|
||||
CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
|
||||
uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
|
||||
export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
|
||||
uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
|
||||
```
|
||||
|
||||
The browser comes from the normal abxpkg provider/cache path; Playwright drives
|
||||
that resolved executable instead of installing a second Chromium build.
|
||||
Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
|
||||
layout, keyboard dismissal, no-JavaScript links, local resources and visible
|
||||
images, and uploads its screenshots as `site-verification` for 14 days.
|
||||
|
||||
## Branding and future translations
|
||||
|
||||
`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
|
||||
including the existing `.button` and `.ab-btn` classes. Product styles own their
|
||||
layouts, feature sections and platform-specific button states. Shared system
|
||||
fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
|
||||
Twitter tags everywhere while retaining each native template's title, description,
|
||||
canonical URL, locale and social artwork; edit those values with that page's copy.
|
||||
|
||||
For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
|
||||
and list their output paths in `site.json` alongside the English pages. Share CSS,
|
||||
JS, social artwork and the English screenshot gallery. Translate each page's title
|
||||
and description, set its own canonical and `lang` (`zh` for Chinese), and add
|
||||
reciprocal `hreflang` links only for translations that actually exist. The renderer
|
||||
preserves these tags and computes shared asset paths for nested pages. Localized
|
||||
header/footer copy and the small language selector can be added with those pages;
|
||||
there is no language redirect, placeholder translation or translation runtime now.
|
||||
|
||||
## Screenshot strips
|
||||
|
||||
`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
|
||||
Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
|
||||
The renderer takes real images, captions, dimensions and links from the generated
|
||||
`screenshots/index.html`; there is no second screenshot list or browser fetch.
|
||||
Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
|
||||
`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
|
||||
Catalog sites without a slot do not load the strip assets.
|
||||
|
||||
The strip scrolls once, pauses on hover, focus or manual interaction, and has an
|
||||
explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
|
||||
normal image links and horizontal scrolling remain available. Screenshot capture
|
||||
manifests and provenance are never rewritten by this presentation step.
|
||||
|
||||
Standalone one- or two-image illustrations remain in their authored source.
|
||||
Gallery strips with three or more images use the latest restored captures.
|
||||
|
||||
## Keeping the common layout in sync
|
||||
|
||||
The comments in `base/header.html`, `base/footer-start.html`,
|
||||
`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
|
||||
repositories. Update those committed copies together when shared branding,
|
||||
Apps links, keyboard behavior, or footer resources change. Preserve each site's
|
||||
local navigation and content. The former monorepo `site-chrome` generator is
|
||||
retired; builds must use only this repository's files.
|
||||
@@ -1,102 +0,0 @@
|
||||
"""Common screenshot artifact transport; coverage checks belong to each app."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import PurePosixPath
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
def api(repo, path):
|
||||
return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
|
||||
|
||||
|
||||
def trusted(run, repo, workflow, branch, successful=True):
|
||||
return (
|
||||
(run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
|
||||
and run["head_branch"] == branch
|
||||
and run["event"] in ("push", "workflow_dispatch", "schedule")
|
||||
and run["path"] == f".github/workflows/{workflow}"
|
||||
and (not successful or run["conclusion"] == "success")
|
||||
)
|
||||
|
||||
|
||||
def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
|
||||
# Ask for capture artifacts directly instead of checking every historical
|
||||
# workflow run (most app runs never produce a complete screenshot set).
|
||||
# Artifact creation time also includes fresh captures from old rerun jobs.
|
||||
candidates = []
|
||||
for name in artifact_names:
|
||||
rows = subprocess.check_output(
|
||||
[
|
||||
"gh",
|
||||
"api",
|
||||
f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
|
||||
"--paginate",
|
||||
"--jq",
|
||||
".artifacts[] | select(.expired == false) | @json",
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
candidates.extend(json.loads(row) for row in rows.splitlines())
|
||||
seen = set()
|
||||
for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
|
||||
source = artifact.get("workflow_run") or {}
|
||||
run_id = source.get("id")
|
||||
if (
|
||||
artifact["name"] not in artifact_names
|
||||
or not run_id
|
||||
or run_id in seen
|
||||
or source.get("head_branch") != branch
|
||||
or source.get("head_repository_id") != source.get("repository_id")
|
||||
):
|
||||
continue
|
||||
seen.add(run_id)
|
||||
run = api(repo, f"actions/runs/{run_id}")
|
||||
if trusted(run, repo, workflow, branch, successful):
|
||||
yield run
|
||||
|
||||
|
||||
def names(repo, run):
|
||||
rows = subprocess.check_output(
|
||||
[
|
||||
"gh",
|
||||
"api",
|
||||
f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
|
||||
"--paginate",
|
||||
"--jq",
|
||||
".artifacts[] | @json",
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
|
||||
|
||||
|
||||
def download(repo, run, name, destination):
|
||||
subprocess.run(
|
||||
[
|
||||
"gh",
|
||||
"run",
|
||||
"download",
|
||||
str(run["id"]),
|
||||
"--repo",
|
||||
repo,
|
||||
"--name",
|
||||
name,
|
||||
"--dir",
|
||||
str(destination),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def relative_path(name):
|
||||
path = PurePosixPath(name)
|
||||
if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
|
||||
raise ValueError(f"Unsafe screenshot path: {name}")
|
||||
return path
|
||||
|
||||
|
||||
def fetch(base, name):
|
||||
relative_path(name)
|
||||
with urlopen(base + name, timeout=60) as response:
|
||||
return response.read()
|
||||
@@ -1,156 +0,0 @@
|
||||
/*
|
||||
Shared ArchiveBox website navigation styles and behavior.
|
||||
Keep this file in sync with .github/pages/base/chrome.css in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
*/
|
||||
/* Common brand tokens. Native templates own product layout and content. */
|
||||
:root {
|
||||
--abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
--abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
|
||||
--abx-accent: #9b2854;
|
||||
--abx-accent-hover: #782040;
|
||||
--abx-paper: #fcfaf7;
|
||||
--abx-ink: #28252a;
|
||||
--abx-muted: #716c74;
|
||||
--abx-line: #e9e2e3;
|
||||
--font-sans: var(--abx-font-sans);
|
||||
--font-mono: var(--abx-font-mono);
|
||||
--ab-font-sans: var(--abx-font-sans);
|
||||
--ab-font-serif: var(--abx-font-sans);
|
||||
--ab-font-label: var(--abx-font-sans);
|
||||
--ab-font-mono: var(--abx-font-mono);
|
||||
--ab-accent: var(--abx-accent);
|
||||
--ab-accent-hover: var(--abx-accent-hover);
|
||||
--ab-on-accent: #fff;
|
||||
}
|
||||
body { font-family: var(--abx-font-sans); }
|
||||
code, pre, kbd, samp { font-family: var(--abx-font-mono); }
|
||||
/* Reuse existing CTA classes; platform selection continues to control colors. */
|
||||
.button, .ab-btn {
|
||||
display: inline-flex; align-items: center; justify-content: center; gap: 10px;
|
||||
min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
|
||||
border-radius: 24px; font-family: var(--abx-font-sans);
|
||||
font-size: 14px; font-weight: 650; line-height: 1.4;
|
||||
text-align: center; white-space: normal; text-decoration: none;
|
||||
}
|
||||
.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
|
||||
.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
|
||||
.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
|
||||
.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
|
||||
@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
|
||||
|
||||
/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
|
||||
/* Native anchor navigation leaves room for the heading and its surrounding content. */
|
||||
html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
|
||||
@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
|
||||
.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
|
||||
.abx-header *, .abx-footer * { box-sizing: border-box; }
|
||||
.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
|
||||
.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
|
||||
.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
|
||||
.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
|
||||
.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
|
||||
.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
|
||||
.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
|
||||
.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
|
||||
.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
|
||||
.abx-header .abx-nav a:hover { color: #9b2854; }
|
||||
.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
|
||||
.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
|
||||
.abx-header .abx-apps { position: relative; }
|
||||
.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
|
||||
.abx-apps > summary::-webkit-details-marker { display: none; }
|
||||
.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
|
||||
.abx-apps[open] > summary::before { transform: rotate(90deg); }
|
||||
.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
|
||||
.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
|
||||
.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
|
||||
.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
|
||||
.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
|
||||
.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
|
||||
.abx-header .abx-app-links .abx-platform { width: 30px; }
|
||||
.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
|
||||
.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
|
||||
.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
|
||||
.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
|
||||
.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
|
||||
.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
|
||||
.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
|
||||
.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
|
||||
.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
|
||||
.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
|
||||
.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
|
||||
.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
|
||||
.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
|
||||
.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
|
||||
.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
|
||||
.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
|
||||
.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
|
||||
.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
|
||||
.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
|
||||
@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
|
||||
@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
|
||||
.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
|
||||
.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
|
||||
.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
|
||||
.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
|
||||
.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
|
||||
.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
|
||||
.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
|
||||
.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
|
||||
.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
|
||||
.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
|
||||
.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
|
||||
.abx-footer .abx-footer-column li::before { content: none; }
|
||||
.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
|
||||
.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
|
||||
.abx-footer a:hover { color: #fff; text-decoration: underline; }
|
||||
.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
|
||||
.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
|
||||
.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
|
||||
.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
|
||||
.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
|
||||
.abx-footer .abx-footer-bottom a { color: #cebbc9; }
|
||||
.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
|
||||
.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
|
||||
.abx-footer-local > div { padding-top: 18px; }
|
||||
.abx-footer-local a { color: #e5aabd; }
|
||||
.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
|
||||
.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
|
||||
.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
|
||||
.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
|
||||
.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
|
||||
@media (max-width: 1100px) {
|
||||
.abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
|
||||
.abx-header .abx-nav { justify-content: flex-start; }
|
||||
.abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
|
||||
}
|
||||
@media (max-width: 700px) {
|
||||
.abx-header { position: relative; padding: 14px 18px; }
|
||||
.abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
|
||||
.abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
|
||||
.abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
|
||||
.abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
|
||||
.abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
|
||||
.abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
|
||||
.abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
|
||||
.abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
|
||||
.abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
|
||||
.abx-footer .abx-footer-column li { min-height: 40px; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
|
||||
|
||||
.abx-footer-local > div > a + a { margin-left: 12px; }
|
||||
|
||||
@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }
|
||||
@@ -1,28 +0,0 @@
|
||||
/*
|
||||
Shared ArchiveBox website navigation styles and behavior.
|
||||
Keep this file in sync with .github/pages/base/chrome.js in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
*/
|
||||
(() => {
|
||||
const menu = document.querySelector('.abx-apps');
|
||||
document.addEventListener('keydown', event => {
|
||||
if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
|
||||
});
|
||||
document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
|
||||
function revealAnchor() {
|
||||
let target;
|
||||
try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
|
||||
if (!target) return;
|
||||
let details = target.closest('details');
|
||||
let opened = false;
|
||||
while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
|
||||
if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
|
||||
}
|
||||
window.addEventListener('hashchange', revealAnchor);
|
||||
revealAnchor();
|
||||
})();
|
||||
@@ -1,12 +0,0 @@
|
||||
<!--
|
||||
Shared ArchiveBox website footer.
|
||||
Keep this file in sync with .github/pages/base/footer-end.html in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
-->
|
||||
<div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security & privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
|
||||
</footer>
|
||||
@@ -1,13 +0,0 @@
|
||||
<!--
|
||||
Shared ArchiveBox website footer.
|
||||
Keep this file in sync with .github/pages/base/footer-start.html in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
-->
|
||||
<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps & tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS & iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS & iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
|
||||
@@ -1,24 +0,0 @@
|
||||
<!--
|
||||
Shared ArchiveBox website navbar and Apps menu.
|
||||
Keep this file in sync with .github/pages/base/header.html in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
-->
|
||||
<header class="abx-header">
|
||||
<a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
|
||||
<nav class="abx-nav" aria-label="Main navigation">
|
||||
<details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
|
||||
<div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
|
||||
<div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
|
||||
</div></details>
|
||||
__SITE_NAV__
|
||||
</nav>
|
||||
</header>
|
||||
@@ -1,11 +0,0 @@
|
||||
/* Identical standalone gallery strip across ArchiveBox product sites. */
|
||||
/* The root query container measures the usable width, excluding scrollbars. */
|
||||
html { container-type: inline-size; }
|
||||
.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
|
||||
.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
|
||||
.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
|
||||
.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
|
||||
.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
|
||||
.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
|
||||
.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
|
||||
@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }
|
||||
@@ -1,44 +0,0 @@
|
||||
/* Scroll real gallery links once; keep them usable without JavaScript. */
|
||||
(() => {
|
||||
document.querySelectorAll('.abx-marquee').forEach(gallery => {
|
||||
const viewport = gallery.querySelector('.abx-marquee-viewport');
|
||||
const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
|
||||
let paused = reducedMotion.matches;
|
||||
let hovering = false;
|
||||
let focused = false;
|
||||
let visible = false;
|
||||
let frame;
|
||||
let previous;
|
||||
let offset = viewport.scrollLeft;
|
||||
function tick(time) {
|
||||
const distance = viewport.scrollWidth - viewport.clientWidth;
|
||||
if (distance <= 0) return;
|
||||
if (previous !== undefined) {
|
||||
offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
|
||||
viewport.scrollTo({ left: offset, behavior: 'instant' });
|
||||
}
|
||||
if (offset >= distance - 1) { paused = true; return; }
|
||||
previous = time;
|
||||
frame = window.requestAnimationFrame(tick);
|
||||
}
|
||||
function sync() {
|
||||
window.cancelAnimationFrame(frame);
|
||||
previous = undefined;
|
||||
offset = viewport.scrollLeft;
|
||||
if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
|
||||
}
|
||||
viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
|
||||
viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
|
||||
viewport.addEventListener('focusin', () => { focused = true; sync(); });
|
||||
viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
|
||||
function pauseForInteraction() { paused = true; sync(); }
|
||||
viewport.addEventListener('pointerdown', pauseForInteraction);
|
||||
viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
|
||||
viewport.addEventListener('keydown', pauseForInteraction);
|
||||
reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
|
||||
document.addEventListener('visibilitychange', sync);
|
||||
new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
|
||||
new window.ResizeObserver(sync).observe(viewport);
|
||||
viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
|
||||
});
|
||||
})();
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Render homepage screenshot strips from the site's existing gallery HTML."""
|
||||
|
||||
import re
|
||||
from html import escape
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin
|
||||
|
||||
|
||||
class Gallery(HTMLParser):
|
||||
"""Only gallery figures are captures; navigation and decorative images are not."""
|
||||
|
||||
def __init__(self, base):
|
||||
super().__init__()
|
||||
self.base = base
|
||||
self.stack = []
|
||||
self.captures = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == "img" and any(name == "figure" for name, _ in self.stack):
|
||||
context = [item for _, item in self.stack]
|
||||
profile = next(
|
||||
(
|
||||
item.get("data-profile") or item.get("data-viewport")
|
||||
for item in reversed(context)
|
||||
if item.get("data-profile") or item.get("data-viewport")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if profile not in (None, "desktop"):
|
||||
return
|
||||
source = attrs.get("src", "")
|
||||
if not source or not attrs.get("alt"):
|
||||
return
|
||||
anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
|
||||
server = any(item.get("data-platform") == "server" for item in context)
|
||||
self.captures.append(
|
||||
{
|
||||
"src": urljoin(self.base, source),
|
||||
"href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
|
||||
"title": attrs["alt"],
|
||||
"width": attrs.get("width", ""),
|
||||
"height": attrs.get("height", ""),
|
||||
"product": "server" if server else "client",
|
||||
},
|
||||
)
|
||||
if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
|
||||
self.stack.append((tag, attrs))
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
for index in range(len(self.stack) - 1, -1, -1):
|
||||
if self.stack[index][0] == tag:
|
||||
del self.stack[index:]
|
||||
break
|
||||
|
||||
|
||||
def render(output, baseurl="", featured=()):
|
||||
homepage = output / "index.html"
|
||||
text = homepage.read_text()
|
||||
slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
|
||||
if not slots:
|
||||
return False
|
||||
base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
|
||||
gallery = Gallery(base + "screenshots/")
|
||||
gallery.feed((output / "screenshots/index.html").read_text())
|
||||
for slot in slots:
|
||||
product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
|
||||
captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
|
||||
priority = {title: index for index, title in enumerate(featured)}
|
||||
captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
|
||||
cards = []
|
||||
for capture in captures:
|
||||
dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
|
||||
cards.append(
|
||||
f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
|
||||
)
|
||||
viewport_id = slot.lower()
|
||||
content = (
|
||||
f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
|
||||
if cards
|
||||
else ""
|
||||
)
|
||||
start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
|
||||
replacement = start + "\n" + content + "\n" + end
|
||||
if end in text:
|
||||
text = re.sub(
|
||||
re.escape(start) + r".*?" + re.escape(end),
|
||||
lambda _, replacement=replacement: replacement,
|
||||
text,
|
||||
count=1,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
else:
|
||||
text = text.replace(start, replacement, 1)
|
||||
homepage.write_text(text)
|
||||
return True
|
||||
@@ -1,4 +0,0 @@
|
||||
<!-- Repository-specific navigation and primary CTA. -->
|
||||
<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout & security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
|
||||
<a class="abx-cta" href="__SITE_BASE__#install">Get started</a>
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"name": "Web Archive",
|
||||
"url": "https://archivebox.io/",
|
||||
"pages": [
|
||||
"index.html",
|
||||
"screenshots/index.html"
|
||||
],
|
||||
"marquee_featured": [
|
||||
"Snapshots grid — desktop",
|
||||
"Snapshot View (header collapsed) — desktop",
|
||||
"AI agent — desktop",
|
||||
"Archive results — desktop",
|
||||
"Add URLs — desktop",
|
||||
"Snapshots table — desktop"
|
||||
]
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
"""Assemble and verify this repository's static Pages site (standard library only)."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from html import escape, unescape
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from marquee import render as render_marquee
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
|
||||
|
||||
def replace_block(text, name, content):
|
||||
start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
|
||||
pattern = re.escape(start) + r".*?" + re.escape(end)
|
||||
if re.search(pattern, text, re.DOTALL):
|
||||
return re.sub(
|
||||
pattern,
|
||||
lambda _: f"{start}\n{content}\n{end}",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
if text.count(start) != 1:
|
||||
raise ValueError(f"Expected one {name} placeholder")
|
||||
return text.replace(start, f"{start}\n{content}\n{end}", 1)
|
||||
|
||||
|
||||
def render_metadata(text):
|
||||
"""Keep product copy in native templates; render one consistent SEO block."""
|
||||
page = Page()
|
||||
page.feed(text)
|
||||
title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
|
||||
meta = page.metadata
|
||||
description = meta.get("description", meta.get("og:description", ""))
|
||||
image = meta.get("og:image", meta.get("twitter:image", ""))
|
||||
if not description or not image or not page.canonical:
|
||||
raise ValueError("Each page needs a description, social image and canonical")
|
||||
values = {
|
||||
"description": description,
|
||||
"robots": "index,follow,max-image-preview:large",
|
||||
"theme-color": "#9b2854",
|
||||
"og:type": "website",
|
||||
"og:site_name": meta.get("og:site_name", "ArchiveBox"),
|
||||
"og:locale": meta.get("og:locale", "en_US"),
|
||||
"og:url": page.canonical,
|
||||
"og:title": title,
|
||||
"og:description": description,
|
||||
"og:image": image,
|
||||
"og:image:type": meta.get("og:image:type", "image/png"),
|
||||
"og:image:width": meta.get("og:image:width", "1200"),
|
||||
"og:image:height": meta.get("og:image:height", "630"),
|
||||
"og:image:alt": meta.get("og:image:alt", title),
|
||||
"twitter:card": "summary_large_image",
|
||||
"twitter:site": "@ArchiveBoxApp",
|
||||
"twitter:title": title,
|
||||
"twitter:description": description,
|
||||
"twitter:image": image,
|
||||
"twitter:image:alt": meta.get("og:image:alt", title),
|
||||
}
|
||||
|
||||
def strip_managed_tag(match):
|
||||
tag = Page()
|
||||
tag.feed(match[0])
|
||||
if tag.canonicals or any(key in values for key in tag.metadata):
|
||||
return ""
|
||||
# System fonts are shared across sites, without a network font dependency.
|
||||
if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
|
||||
return ""
|
||||
return match[0]
|
||||
|
||||
text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
|
||||
text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
|
||||
tags = [
|
||||
f"<title>{escape(title)}</title>",
|
||||
f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
|
||||
]
|
||||
for key, value in values.items():
|
||||
attr = "property" if key.startswith("og:") else "name"
|
||||
tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
|
||||
if "<!-- ARCHIVEBOX:METADATA -->" not in text:
|
||||
text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
|
||||
return replace_block(text, "METADATA", "\n".join(tags))
|
||||
|
||||
|
||||
def render(output, source=None, baseurl=""):
|
||||
config = json.loads((HERE / "site.json").read_text())
|
||||
output = output.resolve()
|
||||
if source:
|
||||
source = source.resolve()
|
||||
if output == source or output in source.parents or source in output.parents:
|
||||
raise ValueError("Build output must be separate from the source directory")
|
||||
if output == ROOT or output in ROOT.parents:
|
||||
raise ValueError("Build output must not replace a repository")
|
||||
if output.exists():
|
||||
shutil.rmtree(output)
|
||||
shutil.copytree(source, output)
|
||||
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
assets = output / "site-base"
|
||||
assets.mkdir(parents=True, exist_ok=True)
|
||||
has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
|
||||
common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
|
||||
for name in common_assets:
|
||||
shutil.copyfile(HERE / "base" / name, assets / name)
|
||||
for name in config["pages"]:
|
||||
page = output / name
|
||||
text = render_metadata(page.read_text())
|
||||
base = "../" * (len(Path(name).parts) - 1) or "./"
|
||||
header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
|
||||
header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
|
||||
text = replace_block(text, "HEADER", header)
|
||||
for kind in ["FOOTER-START", "FOOTER-END"]:
|
||||
text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
|
||||
text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
|
||||
resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
|
||||
if has_marquee and name == "index.html":
|
||||
resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
|
||||
if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
|
||||
text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
|
||||
text = replace_block(text, "ASSETS", resources)
|
||||
page.write_text(text)
|
||||
# Site build provenance is independent of all original screenshot manifests.
|
||||
provenance_path = output / "build.json"
|
||||
provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
|
||||
provenance.update(
|
||||
revision=revision,
|
||||
generatedAt=datetime.now(UTC).isoformat(),
|
||||
site=config["url"],
|
||||
)
|
||||
provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
|
||||
(output / ".nojekyll").touch()
|
||||
verify(output, baseurl)
|
||||
print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
|
||||
|
||||
|
||||
class Page(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.ids = set()
|
||||
self.metadata = {}
|
||||
self.canonical = ""
|
||||
self.resources = []
|
||||
self.headers = self.footers = self.canonicals = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == "meta":
|
||||
for key in ["name", "property"]:
|
||||
if attrs.get(key):
|
||||
self.metadata[attrs[key]] = attrs.get("content", "")
|
||||
if tag == "link" and attrs.get("rel") == "canonical":
|
||||
self.canonical = attrs["href"]
|
||||
if attrs.get("id"):
|
||||
self.ids.add(attrs["id"])
|
||||
classes = attrs.get("class", "").split()
|
||||
self.headers += "abx-header" in classes
|
||||
self.footers += "abx-footer" in classes
|
||||
self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
|
||||
if tag in ["img", "script"] and attrs.get("src"):
|
||||
self.resources.append(attrs["src"])
|
||||
if tag == "link" and attrs.get("rel") in [
|
||||
"stylesheet",
|
||||
"icon",
|
||||
"apple-touch-icon",
|
||||
]:
|
||||
self.resources.append(attrs["href"])
|
||||
|
||||
|
||||
def verify(output, baseurl=""):
|
||||
config = json.loads((HERE / "site.json").read_text())
|
||||
output = output.resolve()
|
||||
prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
|
||||
for name in config["pages"]:
|
||||
path = output / name
|
||||
text = path.read_text()
|
||||
page = Page()
|
||||
page.feed(text)
|
||||
if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
|
||||
raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
|
||||
if "__SITE_" in text:
|
||||
raise ValueError(f"{name}: unresolved site placeholder")
|
||||
for resource in page.resources:
|
||||
url = urlsplit(resource)
|
||||
if url.scheme or url.netloc or not url.path:
|
||||
continue
|
||||
resource_path = unquote(url.path)
|
||||
if prefix and resource_path.startswith(prefix + "/"):
|
||||
resource_path = resource_path[len(prefix) :]
|
||||
local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
|
||||
if not local.is_relative_to(output) or not local.is_file():
|
||||
raise ValueError(f"{name}: missing local resource {resource}")
|
||||
for asset in ["chrome.css", "chrome.js"]:
|
||||
if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
|
||||
raise ValueError(f"{name}: stale common asset {asset}")
|
||||
print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=["render", "verify"])
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("--source", type=Path)
|
||||
parser.add_argument("--baseurl", default="")
|
||||
args = parser.parse_args()
|
||||
if args.command == "render":
|
||||
render(args.output, args.source, args.baseurl)
|
||||
else:
|
||||
verify(args.output, args.baseurl)
|
||||
@@ -1,256 +0,0 @@
|
||||
"""Check the built site in real Chromium at desktop and phone widths."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from functools import partial
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from urllib.parse import unquote, urljoin, urlsplit
|
||||
|
||||
from playwright.sync_api import expect, sync_playwright
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
class Handler(SimpleHTTPRequestHandler):
|
||||
def handle(self):
|
||||
# Browser navigation can cancel an in-flight lazy image response.
|
||||
try:
|
||||
super().handle()
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
class ScreenshotReferences(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.references = []
|
||||
self.ids = set()
|
||||
|
||||
def handle_starttag(self, tag, attributes):
|
||||
attrs = dict(attributes)
|
||||
if attrs.get("id"):
|
||||
self.ids.add(attrs["id"])
|
||||
for attribute in ("src", "href"):
|
||||
if attrs.get(attribute):
|
||||
self.references.append(attrs[attribute])
|
||||
|
||||
|
||||
def verify_screenshot_references(output, canonical):
|
||||
"""Check gallery embeds on every static page, including optional translations."""
|
||||
pages = {}
|
||||
for path in output.rglob("*.html"):
|
||||
page = ScreenshotReferences()
|
||||
page.feed(path.read_text())
|
||||
pages[path.resolve()] = page
|
||||
for path, page in pages.items():
|
||||
page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
|
||||
for reference in page.references:
|
||||
url = urlsplit(urljoin(page_url, reference))
|
||||
if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
|
||||
continue
|
||||
target = output / unquote(url.path).lstrip("/")
|
||||
if target.is_dir():
|
||||
target /= "index.html"
|
||||
target = target.resolve()
|
||||
assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
|
||||
assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
|
||||
if url.fragment and target in pages:
|
||||
assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
|
||||
|
||||
|
||||
def verify(output, evidence):
|
||||
config = json.loads((HERE / "site.json").read_text())
|
||||
verify_screenshot_references(output, config["url"])
|
||||
evidence.mkdir(parents=True, exist_ok=True)
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
|
||||
thread = Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
origin = f"http://127.0.0.1:{server.server_port}"
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
|
||||
for name in config["pages"]:
|
||||
route = name.removesuffix("index.html")
|
||||
page = browser.new_page(reduced_motion="reduce")
|
||||
errors = []
|
||||
missing = []
|
||||
page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
|
||||
page.on(
|
||||
"response",
|
||||
lambda response, missing=missing: (
|
||||
missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
|
||||
),
|
||||
)
|
||||
for width in [1716, 2560, 390]:
|
||||
page.set_viewport_size({"width": width, "height": 900})
|
||||
page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
|
||||
header = page.locator(".abx-header")
|
||||
expect(header).to_have_count(1)
|
||||
expect(header).to_be_visible()
|
||||
expect(header).to_be_in_viewport()
|
||||
brand = header.locator(".abx-brand")
|
||||
expect(brand).to_be_visible()
|
||||
expect(brand).to_contain_text("ArchiveBox")
|
||||
expect(brand.locator(".abx-logo")).to_be_visible()
|
||||
expect(brand.locator(".abx-logo")).to_be_in_viewport()
|
||||
expect(header).to_have_css("display", "flex")
|
||||
expect(page.locator(".abx-footer")).to_have_count(1)
|
||||
expect(page.locator(".abx-footer-column")).to_have_count(3)
|
||||
expect(header.locator(".abx-cta")).to_be_visible()
|
||||
menu = header.locator(".abx-apps")
|
||||
menu.locator("summary").focus()
|
||||
page.keyboard.press("Enter")
|
||||
expect(menu).to_have_attribute("open", "")
|
||||
expect(menu.locator("a").first).to_be_visible()
|
||||
page.keyboard.press("Escape")
|
||||
expect(menu).not_to_have_attribute("open", "")
|
||||
menu.locator("summary").click()
|
||||
box = header.bounding_box()
|
||||
assert box
|
||||
page.mouse.click(box["x"] + 2, box["y"] + 5)
|
||||
expect(menu).not_to_have_attribute("open", "")
|
||||
assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
|
||||
f"{name}: overflow at {width}"
|
||||
)
|
||||
if not route and "screenshots/index.html" in config["pages"]:
|
||||
strips = page.locator(".abx-marquee")
|
||||
assert strips.count() > 0, "Homepage must use its screenshot gallery"
|
||||
for strip in strips.all():
|
||||
strip.scroll_into_view_if_needed()
|
||||
bounds = strip.bounding_box()
|
||||
usable_width = page.evaluate("document.documentElement.clientWidth")
|
||||
assert bounds and abs(bounds["x"]) < 1
|
||||
assert abs(bounds["width"] - usable_width) < 1
|
||||
assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
|
||||
cards = strip.locator(".abx-marquee-card")
|
||||
assert cards.count() > 0, "Screenshot strip is empty"
|
||||
expect(strip.locator("button")).to_have_count(0)
|
||||
expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
|
||||
viewport = strip.locator(".abx-marquee-viewport")
|
||||
overflowing = viewport.evaluate(
|
||||
"node => node.scrollWidth > node.clientWidth",
|
||||
)
|
||||
if overflowing:
|
||||
page.mouse.move(0, 0)
|
||||
page.emulate_media(reduced_motion="no-preference")
|
||||
initial = viewport.evaluate("node => node.scrollLeft")
|
||||
page.wait_for_function(
|
||||
"([node, initial]) => node.scrollLeft > initial",
|
||||
arg=[viewport.element_handle(), initial],
|
||||
)
|
||||
viewport.hover()
|
||||
stopped = viewport.evaluate("node => node.scrollLeft")
|
||||
viewport.evaluate(
|
||||
"node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
|
||||
)
|
||||
assert viewport.evaluate("node => node.scrollLeft") == stopped
|
||||
page.emulate_media(reduced_motion="reduce")
|
||||
page.mouse.move(0, 0)
|
||||
stopped = viewport.evaluate("node => node.scrollLeft")
|
||||
viewport.evaluate(
|
||||
"node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
|
||||
)
|
||||
assert viewport.evaluate("node => node.scrollLeft") == stopped
|
||||
viewport.focus()
|
||||
page.keyboard.press("ArrowRight")
|
||||
assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
|
||||
else:
|
||||
centered = viewport.evaluate(
|
||||
"""node => {
|
||||
const view = node.getBoundingClientRect();
|
||||
const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
|
||||
return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
|
||||
}""",
|
||||
)
|
||||
assert centered, "Short screenshot strips must be centered"
|
||||
for card in cards.all():
|
||||
href = card.get_attribute("href")
|
||||
assert href and "/screenshots/" in href
|
||||
response = page.request.get(urljoin(page.url, href))
|
||||
assert response.ok, f"Broken screenshot link: {href}"
|
||||
anchor = urlsplit(href).fragment
|
||||
if anchor:
|
||||
assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
|
||||
# Decode every local content image using the real browser loader,
|
||||
# including images in offscreen carousel tracks and lazy galleries.
|
||||
images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
|
||||
local_images = [url for url in images if url.startswith(origin + "/")]
|
||||
page.evaluate(
|
||||
"""async urls => {
|
||||
for (const url of urls) {
|
||||
const image = new Image();
|
||||
image.src = url;
|
||||
await image.decode();
|
||||
if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
|
||||
}
|
||||
}""",
|
||||
local_images,
|
||||
)
|
||||
page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
|
||||
page.evaluate(
|
||||
"() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
|
||||
)
|
||||
page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
|
||||
page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
|
||||
for link in header.locator(".abx-nav > a").all():
|
||||
url = urlsplit(link.get_attribute("href") or "")
|
||||
if url.scheme or url.netloc:
|
||||
continue
|
||||
response = page.request.get(urljoin(page.url, url.geturl()))
|
||||
assert response.ok, f"Broken navigation: {url.geturl()}"
|
||||
if url.fragment:
|
||||
body = response.text()
|
||||
assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
|
||||
f"Missing navigation anchor: {url.geturl()}"
|
||||
)
|
||||
if route == "" and page.locator("#resources").count():
|
||||
page.goto(f"{origin}/#resources")
|
||||
expect(page.locator("#resources")).to_be_visible()
|
||||
assert not errors, errors
|
||||
assert not missing, missing
|
||||
page.close()
|
||||
context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
|
||||
plain = context.new_page()
|
||||
plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
|
||||
expect(plain.locator(".abx-header")).to_be_visible()
|
||||
expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
|
||||
expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
|
||||
plain.locator(".abx-apps summary").click()
|
||||
expect(plain.locator(".abx-app-links a").first).to_be_visible()
|
||||
expect(plain.locator(".abx-footer-column a").first).to_be_visible()
|
||||
if not route and "screenshots/index.html" in config["pages"]:
|
||||
assert plain.locator(".abx-marquee-card").count() > 0
|
||||
for strip in plain.locator(".abx-marquee").all():
|
||||
bounds = strip.bounding_box()
|
||||
usable_width = plain.evaluate("document.documentElement.clientWidth")
|
||||
assert bounds and abs(bounds["x"]) < 1
|
||||
assert abs(bounds["width"] - usable_width) < 1
|
||||
assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
|
||||
expect(plain.locator(".abx-marquee button")).to_have_count(0)
|
||||
expect(plain.locator(".abx-marquee-card").first).to_be_visible()
|
||||
context.close()
|
||||
print(
|
||||
f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
|
||||
flush=True,
|
||||
)
|
||||
browser.close()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
|
||||
args = parser.parse_args()
|
||||
verify(args.output, args.evidence)
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover every test once, batching measured short files."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
|
||||
|
||||
if not archivebox_tests:
|
||||
raise SystemExit("No ArchiveBox tests discovered")
|
||||
|
||||
# These are observed per-file pytest seconds, including fixture/startup
|
||||
# time. Shared job setup is paid once per batch. Unknown/long files, optional
|
||||
# dependencies and hosted-only requirements keep independent jobs.
|
||||
durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
|
||||
hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
|
||||
requested = os.environ.get("TEST_FILE", "")
|
||||
batches: list[list[str]] = []
|
||||
batch: list[str] = []
|
||||
batch_seconds = 0
|
||||
for path in archivebox_tests:
|
||||
test_path = path.relative_to(root).as_posix()
|
||||
duration = durations.get(test_path, 60)
|
||||
if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
|
||||
batches.append([test_path])
|
||||
continue
|
||||
if batch and (len(batch) == 8 or batch_seconds + duration > 180):
|
||||
batches.append(batch)
|
||||
batch = []
|
||||
batch_seconds = 0
|
||||
batch.append(test_path)
|
||||
batch_seconds += duration
|
||||
if batch:
|
||||
batches.append(batch)
|
||||
|
||||
matrix: list[dict[str, object]] = []
|
||||
for paths in batches:
|
||||
matrix.append(
|
||||
{
|
||||
"name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
|
||||
"paths": paths,
|
||||
"paths_arg": " ".join(paths),
|
||||
"extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
|
||||
"count": len(paths),
|
||||
"ugnas": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Ordinary Linux tests can use either pool. Only genuine hosted-only
|
||||
# requirements need a file header; filenames never enter runner policy.
|
||||
# Keep a bounded NAS share so hosted capacity continues working in parallel.
|
||||
# This is queued work, not a worker count: several waves keep the capped
|
||||
# runners occupied while hosted jobs run in parallel.
|
||||
nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
|
||||
# Keep long jobs early in the hosted queue. Giving the longest jobs to the
|
||||
# CPU-capped NAS instead created a tail after hosted work had finished; use
|
||||
# its extra slots for a bounded share of shorter work without filename rules.
|
||||
matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
|
||||
eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
|
||||
for item in eligible[:nas_job_limit]:
|
||||
item["ugnas"] = True
|
||||
|
||||
discovered_paths = [path for entry in matrix for path in entry["paths"]]
|
||||
if len(discovered_paths) != len(set(discovered_paths)):
|
||||
raise SystemExit("Tests were not discovered exactly once")
|
||||
if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
|
||||
raise SystemExit("Discovered test shard coverage does not match test files")
|
||||
|
||||
print(
|
||||
f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
|
||||
f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
|
||||
)
|
||||
print(json.dumps(matrix, separators=(",", ":")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,176 +0,0 @@
|
||||
{
|
||||
"source_runs": [
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
|
||||
],
|
||||
"measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
|
||||
"seconds": {
|
||||
"archivebox/tests/test_api_discovery_performance.py": 28,
|
||||
"archivebox/tests/test_api_v1_auth_browser_session.py": 17,
|
||||
"archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
|
||||
"archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
|
||||
"archivebox/tests/test_api_v1_cli_add.py": 52,
|
||||
"archivebox/tests/test_api_v1_cli_remove.py": 33,
|
||||
"archivebox/tests/test_api_v1_cli_schedule.py": 29,
|
||||
"archivebox/tests/test_api_v1_cli_search.py": 15,
|
||||
"archivebox/tests/test_api_v1_cli_update.py": 60,
|
||||
"archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
|
||||
"archivebox/tests/test_api_v1_core_any_id.py": 13,
|
||||
"archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
|
||||
"archivebox/tests/test_api_v1_core_archiveresults.py": 19,
|
||||
"archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
|
||||
"archivebox/tests/test_api_v1_core_snapshots.py": 17,
|
||||
"archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tags.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_create.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_search.py": 17,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
|
||||
"archivebox/tests/test_api_v1_crawls_crawls.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_binaries.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_machine_current.py": 15,
|
||||
"archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_machines.py": 15,
|
||||
"archivebox/tests/test_api_v1_personas_personas.py": 15,
|
||||
"archivebox/tests/test_api_v1_personas_sync.py": 16,
|
||||
"archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
|
||||
"archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
|
||||
"archivebox/tests/test_archive_result_service.py": 49,
|
||||
"archivebox/tests/test_auth_ldap.py": 37,
|
||||
"archivebox/tests/test_binary_service.py": 46,
|
||||
"archivebox/tests/test_cli_add_1.py": 75,
|
||||
"archivebox/tests/test_cli_add_2.py": 98,
|
||||
"archivebox/tests/test_cli_add_3.py": 93,
|
||||
"archivebox/tests/test_cli_add_4.py": 94,
|
||||
"archivebox/tests/test_cli_archiveresult.py": 48,
|
||||
"archivebox/tests/test_cli_background_screenshot.py": 49,
|
||||
"archivebox/tests/test_cli_binary.py": 12,
|
||||
"archivebox/tests/test_cli_config.py": 88,
|
||||
"archivebox/tests/test_cli_crawl.py": 81,
|
||||
"archivebox/tests/test_cli_extract.py": 97,
|
||||
"archivebox/tests/test_cli_extract_input.py": 84,
|
||||
"archivebox/tests/test_cli_help.py": 16,
|
||||
"archivebox/tests/test_cli_init.py": 111,
|
||||
"archivebox/tests/test_cli_install.py": 38,
|
||||
"archivebox/tests/test_cli_list.py": 116,
|
||||
"archivebox/tests/test_cli_machine.py": 19,
|
||||
"archivebox/tests/test_cli_manage.py": 32,
|
||||
"archivebox/tests/test_cli_mcp.py": 30,
|
||||
"archivebox/tests/test_cli_oneshot.py": 15,
|
||||
"archivebox/tests/test_cli_persona.py": 56,
|
||||
"archivebox/tests/test_cli_piping.py": 118,
|
||||
"archivebox/tests/test_cli_pluginmap.py": 12,
|
||||
"archivebox/tests/test_cli_process.py": 12,
|
||||
"archivebox/tests/test_cli_remove.py": 46,
|
||||
"archivebox/tests/test_cli_run_1.py": 93,
|
||||
"archivebox/tests/test_cli_run_2.py": 111,
|
||||
"archivebox/tests/test_cli_run_3.py": 51,
|
||||
"archivebox/tests/test_cli_run_4.py": 78,
|
||||
"archivebox/tests/test_cli_schedule.py": 59,
|
||||
"archivebox/tests/test_cli_search.py": 25,
|
||||
"archivebox/tests/test_cli_server.py": 126,
|
||||
"archivebox/tests/test_cli_shell.py": 17,
|
||||
"archivebox/tests/test_cli_snapshot.py": 55,
|
||||
"archivebox/tests/test_cli_status.py": 46,
|
||||
"archivebox/tests/test_cli_tag.py": 12,
|
||||
"archivebox/tests/test_cli_update.py": 62,
|
||||
"archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
|
||||
"archivebox/tests/test_cli_version.py": 81,
|
||||
"archivebox/tests/test_config_DELETE_AFTER.py": 145,
|
||||
"archivebox/tests/test_config_MAX_limits.py": 25,
|
||||
"archivebox/tests/test_config_ONLY_NEW.py": 15,
|
||||
"archivebox/tests/test_config_SAVE_TITLE.py": 59,
|
||||
"archivebox/tests/test_config_URL_filters.py": 15,
|
||||
"archivebox/tests/test_core_config.py": 11,
|
||||
"archivebox/tests/test_core_forms.py": 14,
|
||||
"archivebox/tests/test_crawl_history.py": 60,
|
||||
"archivebox/tests/test_crawl_pause.py": 11,
|
||||
"archivebox/tests/test_crawl_runner.py": 22,
|
||||
"archivebox/tests/test_crawl_runner_2.py": 27,
|
||||
"archivebox/tests/test_crawl_service.py": 19,
|
||||
"archivebox/tests/test_frozen_crawl_config.py": 26,
|
||||
"archivebox/tests/test_hook_result_lifecycle.py": 48,
|
||||
"archivebox/tests/test_hooks.py": 54,
|
||||
"archivebox/tests/test_issue_1896.py": 115,
|
||||
"archivebox/tests/test_machine_models.py": 26,
|
||||
"archivebox/tests/test_machine_service.py": 77,
|
||||
"archivebox/tests/test_migrations_04_to_09.py": 25,
|
||||
"archivebox/tests/test_migrations_07_to_09.py": 86,
|
||||
"archivebox/tests/test_migrations_08_to_09.py": 85,
|
||||
"archivebox/tests/test_migrations_fresh.py": 55,
|
||||
"archivebox/tests/test_misc_checks.py": 12,
|
||||
"archivebox/tests/test_misc_monkey_patches.py": 12,
|
||||
"archivebox/tests/test_opencode_agent_1.py": 73,
|
||||
"archivebox/tests/test_opencode_agent_2.py": 141,
|
||||
"archivebox/tests/test_opencode_agent_3.py": 61,
|
||||
"archivebox/tests/test_opencode_browser_1.py": 139,
|
||||
"archivebox/tests/test_opencode_browser_2.py": 164,
|
||||
"archivebox/tests/test_opencode_browser_3.py": 125,
|
||||
"archivebox/tests/test_orm_helpers.py": 14,
|
||||
"archivebox/tests/test_permissions.py": 12,
|
||||
"archivebox/tests/test_persona_casing.py": 45,
|
||||
"archivebox/tests/test_persona_runtime.py": 30,
|
||||
"archivebox/tests/test_postgres_backend.py": 79,
|
||||
"archivebox/tests/test_process_runtime_paths.py": 20,
|
||||
"archivebox/tests/test_process_service.py": 40,
|
||||
"archivebox/tests/test_proof_previews.py": 24,
|
||||
"archivebox/tests/test_recursive_crawl_1.py": 361,
|
||||
"archivebox/tests/test_recursive_crawl_2.py": 55,
|
||||
"archivebox/tests/test_recursive_crawl_3.py": 67,
|
||||
"archivebox/tests/test_release_notes.py": 12,
|
||||
"archivebox/tests/test_release_source_fingerprint.py": 13,
|
||||
"archivebox/tests/test_release_workflow.py": 15,
|
||||
"archivebox/tests/test_resource_admission.py": 13,
|
||||
"archivebox/tests/test_runner_deletion.py": 91,
|
||||
"archivebox/tests/test_runner_watch.py": 75,
|
||||
"archivebox/tests/test_search.py": 142,
|
||||
"archivebox/tests/test_serve_static.py": 19,
|
||||
"archivebox/tests/test_server_automatic_url.py": 22,
|
||||
"archivebox/tests/test_server_security_browser.py": 121,
|
||||
"archivebox/tests/test_settings_signal_webhooks.py": 13,
|
||||
"archivebox/tests/test_setup_script.py": 12,
|
||||
"archivebox/tests/test_shutdown_util.py": 13,
|
||||
"archivebox/tests/test_snapshot_filesystem_migration.py": 16,
|
||||
"archivebox/tests/test_snapshot_output_stacks.py": 63,
|
||||
"archivebox/tests/test_snapshot_ref_lookup.py": 15,
|
||||
"archivebox/tests/test_snapshot_service.py": 48,
|
||||
"archivebox/tests/test_snapshot_url_length.py": 16,
|
||||
"archivebox/tests/test_tag_service.py": 15,
|
||||
"archivebox/tests/test_takeover_util_1.py": 60,
|
||||
"archivebox/tests/test_takeover_util_2.py": 62,
|
||||
"archivebox/tests/test_takeover_util_3.py": 76,
|
||||
"archivebox/tests/test_takeover_util_4.py": 84,
|
||||
"archivebox/tests/test_test_harness.py": 12,
|
||||
"archivebox/tests/test_ui_add_view.py": 26,
|
||||
"archivebox/tests/test_ui_add_view_runtime.py": 120,
|
||||
"archivebox/tests/test_ui_admin_apitoken.py": 16,
|
||||
"archivebox/tests/test_ui_admin_archiveresult.py": 18,
|
||||
"archivebox/tests/test_ui_admin_config_widget.py": 13,
|
||||
"archivebox/tests/test_ui_admin_crawl.py": 25,
|
||||
"archivebox/tests/test_ui_admin_links.py": 23,
|
||||
"archivebox/tests/test_ui_admin_machine.py": 18,
|
||||
"archivebox/tests/test_ui_admin_persona.py": 19,
|
||||
"archivebox/tests/test_ui_admin_snapshot.py": 138,
|
||||
"archivebox/tests/test_ui_admin_tag.py": 16,
|
||||
"archivebox/tests/test_ui_admin_user.py": 16,
|
||||
"archivebox/tests/test_ui_config_views.py": 20,
|
||||
"archivebox/tests/test_ui_live_progress.py": 121,
|
||||
"archivebox/tests/test_ui_public_snapshot.py": 72,
|
||||
"archivebox/tests/test_ui_savepagenow.py": 31,
|
||||
"archivebox/tests/test_urls.py": 105,
|
||||
"archivebox/tests/test_util.py": 13,
|
||||
"archivebox/tests/test_version_metadata.py": 18,
|
||||
"archivebox/tests/test_zip_preview.py": 29
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [dev, main]
|
||||
push:
|
||||
branches: [dev, main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
security-events: write
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
permissions:
|
||||
contents: write
|
||||
uses: ./.github/workflows/prepare-release.yml
|
||||
secrets: inherit
|
||||
|
||||
lint:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Linters
|
||||
uses: ./.github/workflows/lint.yml
|
||||
secrets: inherit
|
||||
|
||||
install-cli-platform:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Install and CLI platform compatibility
|
||||
uses: ./.github/workflows/test.yml
|
||||
secrets: inherit
|
||||
|
||||
tests:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Discovered test matrix
|
||||
uses: ./.github/workflows/test-parallel.yml
|
||||
secrets: inherit
|
||||
|
||||
codeql:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: CodeQL
|
||||
uses: ./.github/workflows/codeql.yml
|
||||
secrets: inherit
|
||||
|
||||
python-artifacts:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Tested Python artifacts
|
||||
uses: ./.github/workflows/pip.yml
|
||||
secrets: inherit
|
||||
|
||||
docker-digests:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Tested Docker images
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
|
||||
secrets: inherit
|
||||
|
||||
required:
|
||||
name: All required CI lanes
|
||||
if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
|
||||
needs:
|
||||
- prepare
|
||||
- lint
|
||||
- install-cli-platform
|
||||
- tests
|
||||
- codeql
|
||||
- python-artifacts
|
||||
- docker-digests
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Verify every required lane succeeded
|
||||
env:
|
||||
LINT_RESULT: ${{ needs.lint.result }}
|
||||
INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
|
||||
TESTS_RESULT: ${{ needs.tests.result }}
|
||||
CODEQL_RESULT: ${{ needs.codeql.result }}
|
||||
PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
|
||||
DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
|
||||
run: |
|
||||
for result in \
|
||||
"$LINT_RESULT" \
|
||||
"$INSTALL_CLI_PLATFORM_RESULT" \
|
||||
"$TESTS_RESULT" \
|
||||
"$CODEQL_RESULT" \
|
||||
"$PYTHON_ARTIFACTS_RESULT" \
|
||||
"$DOCKER_DIGESTS_RESULT"
|
||||
do
|
||||
[[ "$result" == success ]]
|
||||
done
|
||||
echo 'All required CI lanes and tested release artifacts succeeded.'
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
id-token: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
claude_args: '--allowed-tools Bash(gh pr:*)'
|
||||
@@ -1,42 +0,0 @@
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
schedule:
|
||||
- cron: '33 17 * * 6'
|
||||
|
||||
concurrency:
|
||||
group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
security-events: write
|
||||
packages: read
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: python
|
||||
build-mode: none
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
@@ -1,96 +0,0 @@
|
||||
name: Publish website
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
workflow_run:
|
||||
workflows: ['Capture publicsite screenshots']
|
||||
types: [completed]
|
||||
branches: [dev]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# Repository-specific output; all publication logic below is a local copy.
|
||||
SITE_OUTPUT: _site
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
|
||||
# One production queue covers restore, build and deploy. PRs have separate queues.
|
||||
concurrency:
|
||||
group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
outputs:
|
||||
revision: ${{ steps.source.outputs.revision }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
|
||||
persist-credentials: false
|
||||
- name: Record website source revision
|
||||
id: source
|
||||
run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# BEGIN REPOSITORY-SPECIFIC BUILD
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- name: Restore latest complete screenshots
|
||||
run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
- name: Render current gallery layout without recapturing images
|
||||
run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
|
||||
- name: Assemble website with the local base components
|
||||
run: uv run --no-project python .github/pages/site.py render _site --source publicsite
|
||||
# END REPOSITORY-SPECIFIC BUILD
|
||||
|
||||
- name: Verify website in Chromium
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
|
||||
run: |
|
||||
# Resolve the same supported browser as captures; Playwright is the
|
||||
# driver only and must not provision a separate browser/runtime.
|
||||
ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
|
||||
CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
|
||||
uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
|
||||
CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
|
||||
export CHROME_BINARY
|
||||
test -x "$CHROME_BINARY"
|
||||
uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
|
||||
- name: Save website verification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: site-verification
|
||||
path: ${{ runner.temp }}/site-evidence
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload website
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: ${{ env.SITE_OUTPUT }}
|
||||
|
||||
deploy:
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- uses: actions/configure-pages@v5
|
||||
- uses: actions/deploy-pages@v4
|
||||
id: deployment
|
||||
@@ -1,614 +0,0 @@
|
||||
name: Build Docker image
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
push_digests:
|
||||
description: Push validated per-platform images by digest for the release workflow
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
full_tests:
|
||||
description: Run storage, CLI, and size validation in addition to release smoke checks
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
description: Exact 40-character commit SHA to build
|
||||
required: true
|
||||
type: string
|
||||
push_digests:
|
||||
description: Push validated per-platform images by digest for the release workflow
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
full_tests:
|
||||
description: Run storage, CLI, and size validation in addition to release smoke checks
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-24.04
|
||||
artifact_name: digest-linux-amd64
|
||||
local_tag: archivebox-ci:amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
artifact_name: digest-linux-arm64
|
||||
local_tag: archivebox-ci:arm64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
ref: ${{ inputs.source_ref || github.sha }}
|
||||
|
||||
- name: Verify exact source commit
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.source_ref || github.sha }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Resolve Docker through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package for package in packages if package["name"] == "abxpkg"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
|
||||
wheel = matches[0]["wheels"][0]
|
||||
print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
|
||||
PY
|
||||
)"
|
||||
test -n "$ABXPKG_SPEC"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
docker_env="$(
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
|
||||
)"
|
||||
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$docker_env" >> "$GITHUB_ENV"
|
||||
|
||||
docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
{
|
||||
echo "DOCKER_BINARY=$docker_binary"
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "UV_BINARY=$UV_BINARY"
|
||||
} >> "$GITHUB_ENV"
|
||||
for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
"$docker_binary" --version
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
with:
|
||||
install: true
|
||||
|
||||
- name: Builder instance name
|
||||
run: echo ${{ steps.buildx.outputs.name }}
|
||||
|
||||
- name: Available platforms
|
||||
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||
|
||||
- name: Verify published abx-dl image
|
||||
id: abx_dl_image
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
|
||||
print(matches[0])
|
||||
PY
|
||||
)"
|
||||
test -n "$ABX_DL_VERSION"
|
||||
ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
|
||||
INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
|
||||
printf '%s\n' "$INSPECT_OUTPUT"
|
||||
ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
|
||||
import os
|
||||
|
||||
digest = next(
|
||||
(fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
|
||||
"",
|
||||
)
|
||||
print(digest)
|
||||
PY
|
||||
)"
|
||||
[[ "$ABX_DL_DIGEST" == sha256:* ]]
|
||||
ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
|
||||
echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
if: inputs.push_digests
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
if: inputs.push_digests
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Collect Docker labels
|
||||
id: docker_meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
test -n "$VERSION"
|
||||
|
||||
{
|
||||
echo 'labels<<EOF'
|
||||
echo "org.opencontainers.image.version=${VERSION}"
|
||||
echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
|
||||
echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
|
||||
echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
|
||||
echo 'EOF'
|
||||
echo "version=${VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
|
||||
|
||||
- name: Build and push digest
|
||||
id: docker_build
|
||||
if: inputs.push_digests
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
build-args: |
|
||||
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
|
||||
ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
|
||||
pull: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
# Keep the released image as the one cache authority. A second GHA
|
||||
# cache can hold gzip variants of identical layers, and BuildKit may
|
||||
# prefer those over :dev's zstd blobs. That counter-intuitively makes
|
||||
# a cache hit larger than the released image. :dev advances only
|
||||
# after both architectures pass the release checks.
|
||||
cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
|
||||
# Inline metadata reuses the released compressed blobs directly;
|
||||
# force-compression would recompress the whole ~740 MiB image. The
|
||||
# compressed-size gate below remains the regression guard.
|
||||
cache-to: |
|
||||
${{ inputs.push_digests && 'type=inline' || '' }}
|
||||
outputs: |
|
||||
${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
|
||||
${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
|
||||
|
||||
- name: Build pull request image
|
||||
id: docker_build_pr
|
||||
if: ${{ !inputs.push_digests }}
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
build-args: |
|
||||
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
|
||||
ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
|
||||
pull: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
# Test the same layer lineage users receive. Reusing :dev avoids a
|
||||
# second cache whose compression can drift from the released image.
|
||||
cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
|
||||
outputs: |
|
||||
type=docker,name=${{ matrix.local_tag }}
|
||||
type=oci,dest=${{ runner.temp }}/archivebox-image.tar
|
||||
|
||||
- name: Image digest
|
||||
if: inputs.push_digests
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Verify optional agent install with a read-only browser profile
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
PROFILE_DIR="$(mktemp -d)"
|
||||
# shellcheck disable=SC2016
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
|
||||
--volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
|
||||
"$IMAGE" bash -c '
|
||||
set -Eeuo pipefail
|
||||
test -w "$XDG_CONFIG_HOME"
|
||||
test ! -w "$XDG_CONFIG_HOME/chromium"
|
||||
uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
|
||||
'
|
||||
|
||||
- name: Validate exact built image version and commit
|
||||
if: inputs.full_tests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
|
||||
SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
|
||||
SHORT_SHA="${SHORT_SHA::7}"
|
||||
|
||||
DATA_DIR="$(mktemp -d)"
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
|
||||
ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
|
||||
printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
|
||||
IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
|
||||
ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
|
||||
"$UV_BINARY" run --no-cache --no-project python - \
|
||||
"ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
|
||||
"COMMIT_HASH=${SHORT_SHA}" \
|
||||
"COMMIT_HASH=${GITHUB_SHA}" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
|
||||
image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
|
||||
if sys.argv[1] not in version_output:
|
||||
raise SystemExit(f"Missing version marker: {sys.argv[1]}")
|
||||
if sys.argv[2] not in version_output:
|
||||
raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
|
||||
if sys.argv[3] not in image_version_lines:
|
||||
raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
|
||||
PY
|
||||
|
||||
"$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
|
||||
'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
|
||||
|
||||
- name: Validate compressed candidate size
|
||||
if: inputs.push_digests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
DIGEST='${{ steps.docker_build.outputs.digest }}'
|
||||
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
|
||||
# Provenance wraps the real platform image and its attestation in an
|
||||
# OCI index. Measure the image manifest, not the index/attestation.
|
||||
IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
|
||||
if [[ -n "$IMAGE_DIGEST" ]]; then
|
||||
MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
|
||||
fi
|
||||
COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
|
||||
LIMIT=$((1025 * 1024 * 1024))
|
||||
echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
|
||||
(( COMPRESSED_SIZE < LIMIT ))
|
||||
|
||||
- name: Resolve real Docker CLI test dependencies through abxpkg
|
||||
id: mount_dependencies
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mount_env="$(
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
|
||||
)"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$mount_env" >> "$GITHUB_ENV"
|
||||
|
||||
for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
|
||||
binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
|
||||
- name: Diagnose failed mount dependency installation
|
||||
if: failure() && steps.mount_dependencies.conclusion == 'failure'
|
||||
shell: bash
|
||||
run: |
|
||||
sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
|
||||
sudo journalctl --since '-10 minutes' --no-pager \
|
||||
-u smbd -u nmbd -u samba-ad-dc -u nfs-server
|
||||
ps -eo pid,ppid,stat,etimes,wchan:32,comm
|
||||
|
||||
- name: Validate Docker mounts, ports, CLI and compose concurrently
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
env:
|
||||
ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
# These checks own separate collections, Compose projects and ports.
|
||||
# Subshells keep mount cleanup traps and environment changes isolated.
|
||||
validate_mounts() (
|
||||
set -Eeuo pipefail
|
||||
NFS_SOURCE="/srv/archivebox-ci-nfs"
|
||||
SMB_SOURCE="/srv/archivebox-ci-smb"
|
||||
NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
|
||||
SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
|
||||
mounted_nfs=0
|
||||
mounted_smb=0
|
||||
|
||||
# Invoked by the subshell EXIT trap.
|
||||
# shellcheck disable=SC2329
|
||||
cleanup() {
|
||||
local original_status=$? cleanup_status=0
|
||||
trap - EXIT
|
||||
if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$original_status" != "0" ]]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
exit "$cleanup_status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
|
||||
"$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
|
||||
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet smbd.service
|
||||
|
||||
"$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
|
||||
mounted_nfs=1
|
||||
"$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
|
||||
-o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
|
||||
mounted_smb=1
|
||||
|
||||
NFS_TEST_DIR="$NFS_MOUNT" \
|
||||
SMB_TEST_DIR="$SMB_MOUNT" \
|
||||
IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
|
||||
"$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
|
||||
)
|
||||
validate_cli() (
|
||||
set -Eeuo pipefail
|
||||
"$MKDIR_BINARY" "${{ github.workspace }}/data"
|
||||
"$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
|
||||
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
"$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
|
||||
|
||||
"$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
|
||||
# The argument form above exercises the complete default pipeline.
|
||||
# Keep a real capture for stdin parsing without repeating every plugin.
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
|
||||
|
||||
archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
|
||||
[[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
[[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox init
|
||||
"$DOCKER_BINARY" compose up -d --wait
|
||||
"$DOCKER_BINARY" network connect archivebox_default www-nginx
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
|
||||
[[ "$index_html" == *ArchiveBox* ]]
|
||||
[[ "$admin_js" == *window.django* ]]
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
[[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
|
||||
"$DOCKER_BINARY" compose down
|
||||
)
|
||||
validate_ports() (
|
||||
IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
|
||||
)
|
||||
|
||||
checks=(mounts ports cli)
|
||||
pids=()
|
||||
validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
status=0
|
||||
for index in "${!checks[@]}"; do
|
||||
check="${checks[$index]}"
|
||||
echo "::group::Docker $check validation"
|
||||
if wait "${pids[$index]}"; then
|
||||
echo "PASS: Docker $check validation"
|
||||
else
|
||||
echo "::error::Docker $check validation failed"
|
||||
status=1
|
||||
fi
|
||||
cat "$RUNNER_TEMP/docker-$check.log"
|
||||
echo "::endgroup::"
|
||||
done
|
||||
exit "$status"
|
||||
|
||||
- name: Validate Docker port compatibility
|
||||
if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
|
||||
|
||||
- name: Validate compressed image size
|
||||
if: inputs.full_tests && !inputs.push_digests
|
||||
env:
|
||||
BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
|
||||
"$UV_BINARY" run --no-cache --no-project python - \
|
||||
'${{ runner.temp }}/archivebox-image.tar' \
|
||||
'${{ steps.docker_build_pr.outputs.digest }}' \
|
||||
"$LOCAL_IMAGE_ID" \
|
||||
"$((1025 * 1024 * 1024))" <<'PY'
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
|
||||
limit = int(limit_text)
|
||||
metadata = json.loads(os.environ["BUILD_METADATA"])
|
||||
if metadata["containerimage.digest"] != pushed_digest:
|
||||
raise SystemExit("Build push transaction digest does not match the action output")
|
||||
if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
|
||||
raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
|
||||
|
||||
with tarfile.open(archive_path) as archive:
|
||||
def read_blob(descriptor):
|
||||
algorithm, digest = descriptor["digest"].split(":", 1)
|
||||
if algorithm != "sha256":
|
||||
raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
|
||||
member = archive.extractfile(f"blobs/sha256/{digest}")
|
||||
if member is None:
|
||||
raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
|
||||
blob = member.read()
|
||||
if len(blob) != descriptor["size"]:
|
||||
raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
|
||||
if sha256(blob).hexdigest() != digest:
|
||||
raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
|
||||
return blob
|
||||
|
||||
index_member = archive.extractfile("index.json")
|
||||
if index_member is None:
|
||||
raise SystemExit("OCI export is missing index.json")
|
||||
index = json.load(index_member)
|
||||
|
||||
manifests = []
|
||||
|
||||
def collect(descriptor):
|
||||
document = json.loads(read_blob(descriptor))
|
||||
media_type = descriptor["mediaType"]
|
||||
if "image.index" in media_type or "manifest.list" in media_type:
|
||||
for child in document["manifests"]:
|
||||
collect(child)
|
||||
elif "image.manifest" in media_type:
|
||||
config_type = document["config"]["mediaType"]
|
||||
if "image.config" in config_type or "container.image" in config_type:
|
||||
manifests.append(document)
|
||||
|
||||
for root in index["manifests"]:
|
||||
collect(root)
|
||||
if len(manifests) != 1:
|
||||
raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
|
||||
|
||||
manifest = manifests[0]
|
||||
config_digest = manifest["config"]["digest"]
|
||||
if metadata["containerimage.config.digest"] != config_digest:
|
||||
raise SystemExit("OCI config digest does not match the pushed build result")
|
||||
if local_image_id != config_digest:
|
||||
raise SystemExit("Locally tested image does not match the OCI build result")
|
||||
descriptors = [manifest["config"], *manifest["layers"]]
|
||||
for descriptor in descriptors:
|
||||
read_blob(descriptor)
|
||||
total = sum(descriptor["size"] for descriptor in descriptors)
|
||||
|
||||
print(
|
||||
f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
|
||||
f"limit={limit // 1024 // 1024} MiB",
|
||||
)
|
||||
if total > limit:
|
||||
raise SystemExit(f"{pushed_digest} is over the compressed size limit")
|
||||
PY
|
||||
|
||||
- name: Export digest
|
||||
if: inputs.push_digests
|
||||
env:
|
||||
ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.docker_build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
|
||||
printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
|
||||
|
||||
- name: Upload digest
|
||||
if: inputs.push_digests
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: ${{ matrix.artifact_name }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
@@ -1,38 +0,0 @@
|
||||
name: Documentation link checks
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
|
||||
jobs:
|
||||
links:
|
||||
name: Documentation links and anchors
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
- name: List tracked documentation and source inputs
|
||||
run: |
|
||||
git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
|
||||
- uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
|
||||
with:
|
||||
lycheeVersion: v0.24.2
|
||||
args: >-
|
||||
--config .github/lychee.toml
|
||||
--root-dir "${{ github.workspace }}"
|
||||
--files-from "${{ runner.temp }}/link-inputs.txt"
|
||||
fail: true
|
||||
failIfEmpty: true
|
||||
@@ -1,195 +0,0 @@
|
||||
name: Documentation tests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '7 5,17 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
USE_COLOR: "False"
|
||||
SHOW_PROGRESS: "False"
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
key: ${{ steps.check.outputs.key }}
|
||||
steps:
|
||||
- id: check
|
||||
uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
|
||||
|
||||
discover:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
name: Discover documentation examples
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
matrix: ${{ steps.inventory.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Install ArchiveBox development dependencies
|
||||
run: uv sync --no-cache --locked --dev
|
||||
- name: Resolve documentation tools through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
|
||||
run: |
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
uv run --no-cache --no-sync abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
|
||||
bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
|
||||
mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
|
||||
nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
|
||||
test -L "$bash_binary"
|
||||
test -L "$mmdc_binary"
|
||||
test -L "$nginx_binary"
|
||||
test -x "$bash_binary"
|
||||
test -x "$mmdc_binary"
|
||||
test -x "$nginx_binary"
|
||||
{
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
echo "BASH_BINARY=$bash_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
- id: inventory
|
||||
name: Validate the complete authored documentation inventory
|
||||
run: |
|
||||
matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
|
||||
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
||||
|
||||
examples:
|
||||
name: docs/${{ matrix.name }}
|
||||
needs: discover
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 8
|
||||
matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Install ArchiveBox
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
run: uv sync --no-cache --locked --dev
|
||||
- name: Resolve documentation runtime through abxpkg
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
|
||||
run: |
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
dependency_sources=(
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
|
||||
)
|
||||
if [[ '${{ matrix.environment }}' != macos ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
|
||||
fi
|
||||
if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
|
||||
fi
|
||||
if [[ '${{ matrix.environment }}' == docker ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
|
||||
elif [[ '${{ matrix.environment }}' == macos ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
|
||||
fi
|
||||
uv run --no-cache --no-sync abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
"${dependency_sources[@]}"
|
||||
bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
|
||||
python_binary="$ABXPKG_LIB_DIR/env/bin/python"
|
||||
uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
test -L "$bash_binary"
|
||||
test -x "$bash_binary"
|
||||
test -L "$python_binary"
|
||||
test -x "$python_binary"
|
||||
test -L "$uv_binary"
|
||||
test -x "$uv_binary"
|
||||
{
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
echo "BASH_BINARY=$bash_binary"
|
||||
echo "PYTHON_BINARY=$python_binary"
|
||||
echo "UV_BINARY=$uv_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
if [[ '${{ matrix.environment }}' != macos ]]; then
|
||||
sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
|
||||
test -L "$sudo_binary"
|
||||
test -x "$sudo_binary"
|
||||
echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
|
||||
fi
|
||||
if [[ '${{ matrix.environment }}' != macos ]]; then
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
fi
|
||||
if [[ '${{ matrix.environment }}' == docker ]]; then
|
||||
docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
test -L "$docker_binary"
|
||||
test -x "$docker_binary"
|
||||
echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
|
||||
elif [[ '${{ matrix.environment }}' == macos ]]; then
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/brew"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/brew"
|
||||
fi
|
||||
- name: Build the exact checkout's documentation image
|
||||
if: ${{ matrix.environment == 'docker' }}
|
||||
run: |
|
||||
ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
|
||||
"$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
|
||||
- name: Run the Ubuntu, Docker, or macOS documentation lane
|
||||
if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
|
||||
run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
|
||||
- name: Run the root documentation lane
|
||||
if: ${{ matrix.environment == 'root' }}
|
||||
run: |
|
||||
"$SUDO_BINARY" \
|
||||
--preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
|
||||
"$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
|
||||
- name: Diagnose failed Linux package installation
|
||||
if: failure() && runner.os == 'Linux'
|
||||
run: |
|
||||
sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
|
||||
apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
|
||||
ps -eo pid,ppid,stat,etimes,wchan:32,comm
|
||||
|
||||
complete:
|
||||
name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
|
||||
needs: [changes, discover, examples]
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- run: echo 'Scheduled work completed successfully.'
|
||||
@@ -1,38 +0,0 @@
|
||||
name: Run linters
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
UV_NO_SOURCES: "1"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Verify checked dependency lock
|
||||
run: uv lock --no-cache --check
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: uv sync --no-cache --locked --dev --extra sonic --extra debug
|
||||
|
||||
- name: Run prek
|
||||
run: uv run --no-cache --no-sync --no-sources prek run --all-files
|
||||
@@ -1,48 +0,0 @@
|
||||
name: macOS compatibility
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '51 2,10,18 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
key: ${{ steps.check.outputs.key }}
|
||||
steps:
|
||||
- id: check
|
||||
uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
|
||||
|
||||
install-cli:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
uses: ./.github/workflows/test.yml
|
||||
with:
|
||||
platform: macos
|
||||
secrets: inherit
|
||||
|
||||
python-artifacts:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
uses: ./.github/workflows/pip.yml
|
||||
with:
|
||||
platforms: '["macos-15"]'
|
||||
secrets: inherit
|
||||
|
||||
complete:
|
||||
name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
|
||||
needs: [changes, install-cli, python-artifacts]
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- run: echo 'Scheduled macOS coverage completed successfully.'
|
||||
@@ -1,248 +0,0 @@
|
||||
name: Build Pip package
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
platforms:
|
||||
description: Runner matrix for package compatibility
|
||||
required: false
|
||||
default: '["ubuntu-24.04"]'
|
||||
type: string
|
||||
full_tests:
|
||||
description: Run the full package compatibility matrix instead of one release smoke check
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.11.3"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build distributions
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Build every distribution
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv build --no-cache --all-packages --no-sources --out-dir dist --clear
|
||||
|
||||
shopt -s nullglob
|
||||
artifacts=(dist/*)
|
||||
wheels=(dist/archivebox-*.whl)
|
||||
sdists=(dist/archivebox-*.tar.gz)
|
||||
[[ "${#artifacts[@]}" -gt 0 ]]
|
||||
[[ "${#wheels[@]}" -eq 1 ]]
|
||||
[[ "${#sdists[@]}" -eq 1 ]]
|
||||
printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
|
||||
uv run --no-cache --no-project python - <<'PY'
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
root = Path("dist")
|
||||
wheels = list(root.glob("archivebox-*.whl"))
|
||||
sdists = list(root.glob("archivebox-*.tar.gz"))
|
||||
if len(wheels) != 1 or len(sdists) != 1:
|
||||
raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
|
||||
artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
|
||||
checksum_lines = []
|
||||
for artifact in artifacts:
|
||||
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
line = f"{digest} {artifact.name}"
|
||||
checksum_lines.append(line)
|
||||
print(line)
|
||||
Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
|
||||
PY
|
||||
|
||||
- name: Release wheel import and CLI smoke
|
||||
if: ${{ !inputs.full_tests }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
shopt -s nullglob
|
||||
wheels=(dist/archivebox-*.whl)
|
||||
[[ "${#wheels[@]}" -eq 1 ]]
|
||||
uv sync --locked --no-cache --no-sources --no-dev --no-install-project
|
||||
uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
|
||||
unset PYTHONPATH
|
||||
cd "$(mktemp -d)"
|
||||
VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
|
||||
VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
|
||||
|
||||
- name: Upload every distribution
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: dist/*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
install-smoke:
|
||||
name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
|
||||
if: inputs.full_tests
|
||||
needs: build
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: ${{ fromJSON(inputs.platforms) }}
|
||||
python: ["3.13", "3.14"]
|
||||
artifact: [wheel, sdist]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Download distributions
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: ${{ runner.temp }}/python-distributions
|
||||
|
||||
- name: Install and smoke-test the built ${{ matrix.artifact }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
shopt -s nullglob
|
||||
|
||||
case "${{ matrix.artifact }}" in
|
||||
wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
|
||||
sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
[[ "${#candidates[@]}" -eq 1 ]]
|
||||
artifact="${candidates[0]}"
|
||||
|
||||
smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
|
||||
trap 'rm -rf "$smoke_root"' EXIT
|
||||
smoke_env="$smoke_root/env"
|
||||
data_dir="$smoke_root/data"
|
||||
mkdir -p "$data_dir"
|
||||
|
||||
dependency_artifacts=()
|
||||
while IFS= read -r dependency_artifact; do
|
||||
dependency_artifacts+=("$dependency_artifact")
|
||||
done < <(uv run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
|
||||
locked = {
|
||||
package["name"]: package
|
||||
for package in tomllib.load(open("uv.lock", "rb"))["package"]
|
||||
}
|
||||
for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
|
||||
package = locked[package_name]
|
||||
wheels = package.get("wheels") or []
|
||||
if len(wheels) != 1:
|
||||
raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
|
||||
print(wheels[0]["url"])
|
||||
PY
|
||||
)
|
||||
|
||||
uv venv --python "${{ matrix.python }}" "$smoke_env"
|
||||
uv pip install --no-cache \
|
||||
--python "$smoke_env" \
|
||||
"${dependency_artifacts[@]}" \
|
||||
"$artifact"
|
||||
|
||||
unset PYTHONPATH
|
||||
cd "$data_dir"
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import archivebox
|
||||
|
||||
installed_path = Path(archivebox.__file__).resolve()
|
||||
environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
|
||||
workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
|
||||
assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
|
||||
assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
|
||||
print(installed_path)
|
||||
PY
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
|
||||
|
||||
if [[ "${{ runner.os }}" == "Linux" ]]; then
|
||||
root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
|
||||
uv_binary="$(command -v uv)"
|
||||
sudo rm -rf "$root_tool_dir"
|
||||
sudo mkdir -p "$root_tool_dir/data"
|
||||
|
||||
root_install_args=()
|
||||
for dependency_artifact in "${dependency_artifacts[@]}"; do
|
||||
root_install_args+=(--with "$dependency_artifact")
|
||||
done
|
||||
sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
|
||||
"$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
|
||||
"${root_install_args[@]}" "$artifact"
|
||||
|
||||
sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
|
||||
sudo rm -rf /var/lib/archivebox
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data' && archivebox init"
|
||||
sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
|
||||
sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
|
||||
if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
|
||||
echo "Root-owned files remain in $root_tool_dir/data" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo mkdir "$root_tool_dir/external"
|
||||
sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data' && archivebox status"
|
||||
sudo stat -c %U "$root_tool_dir/external" | grep -qx root
|
||||
|
||||
sudo mkdir "$root_tool_dir/data-universal-init"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
|
||||
sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
|
||||
|
||||
sudo mkdir "$root_tool_dir/data-install"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
|
||||
sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
|
||||
|
||||
sudo useradd --create-home archivebox-smoke-user
|
||||
smoke_user_group="$(id -gn archivebox-smoke-user)"
|
||||
sudo mkdir "$root_tool_dir/data-mixed-owner"
|
||||
sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
|
||||
sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
|
||||
if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
|
||||
echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo rm -rf "$root_tool_dir"
|
||||
fi
|
||||
@@ -1,55 +0,0 @@
|
||||
name: Prepare release version
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
outputs:
|
||||
run_jobs:
|
||||
description: Whether this revision owns its release version and should run expensive jobs
|
||||
value: ${{ jobs.prepare.outputs.run_jobs }}
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Prepare the version before starting expensive jobs
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- name: Keep synchronized stable branches on the released version
|
||||
id: channel
|
||||
if: github.event_name == 'push'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
|
||||
git fetch --no-tags origin main
|
||||
MAIN_SHA="$(git rev-parse origin/main)"
|
||||
# Stable reconciliation may preserve dev history with a merge commit.
|
||||
# Its tree still matches main; bumping it would undo synchronization
|
||||
# before the stable publisher can advance the Docker :dev tag.
|
||||
if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
|
||||
git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
|
||||
git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
|
||||
echo 'stable=true' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stable=false' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- id: version
|
||||
if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
|
||||
uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
|
||||
with:
|
||||
package: archivebox
|
||||
branch: ${{ github.ref_name }}
|
||||
tag-prefix: v
|
||||
version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
|
||||
@@ -1,595 +0,0 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI]
|
||||
types: [completed]
|
||||
branches: [dev, main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_branch:
|
||||
description: Branch being promoted
|
||||
required: true
|
||||
type: choice
|
||||
options: [main, dev]
|
||||
release_sha:
|
||||
description: Exact tested commit to release
|
||||
required: true
|
||||
type: string
|
||||
ci_run_id:
|
||||
description: Successful CI run containing the tested Python artifacts
|
||||
required: true
|
||||
type: string
|
||||
docker_digest_run_id:
|
||||
description: Manual Docker digest build run containing the tested image digests
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
|
||||
RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
|
||||
|
||||
jobs:
|
||||
candidate:
|
||||
if: >-
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
(github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
outputs:
|
||||
ready: ${{ steps.verified.outputs.ready }}
|
||||
artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
|
||||
digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
- id: owner
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
|
||||
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
|
||||
if [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
|
||||
echo "Refusing to publish non-stable version $VERSION as a main release" >&2
|
||||
exit 1
|
||||
}
|
||||
git fetch --no-tags origin main
|
||||
git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
|
||||
echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
|
||||
exit 1
|
||||
}
|
||||
TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
|
||||
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo 'ready=true' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# A stable release fast-forwards dev to main. Only main publishes it;
|
||||
# the resulting dev workflow is not a new release candidate.
|
||||
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
|
||||
[[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Require successful full CI for this commit
|
||||
id: verified
|
||||
if: steps.owner.outputs.ready == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
|
||||
MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
|
||||
CI_RUN_ID="$MANUAL_CI_RUN_ID"
|
||||
DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
|
||||
CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
|
||||
--jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
|
||||
[[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
|
||||
else
|
||||
CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
|
||||
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
|
||||
--json databaseId,status,conclusion \
|
||||
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
|
||||
[[ -n "$CI_RUN_ID" ]] || exit 0
|
||||
DIGEST_RUN_ID="$CI_RUN_ID"
|
||||
fi
|
||||
ARTIFACT_RUN_ID="$CI_RUN_ID"
|
||||
{
|
||||
echo 'ready=true'
|
||||
echo "artifact_run_id=$ARTIFACT_RUN_ID"
|
||||
echo "digest_run_id=$DIGEST_RUN_ID"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
python-release:
|
||||
name: Publish tested Python artifacts
|
||||
needs: candidate
|
||||
if: needs.candidate.outputs.ready == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
|
||||
DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
|
||||
environment: pypi
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
deployments: read
|
||||
id-token: write
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
release_ready: ${{ steps.release-state.outputs.ready }}
|
||||
steps:
|
||||
- name: Checkout the tested commit
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
# This precedes every stable upload, including workflow_dispatch. A green
|
||||
# build is not evidence that our automatically updated test servers ran it.
|
||||
- name: Download tested Docker base-image metadata
|
||||
if: env.RELEASE_BRANCH == 'main'
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/tested-docker-artifacts
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
|
||||
|
||||
- name: Require matching acceptance on Cabbage
|
||||
if: env.RELEASE_BRANCH == 'main'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
|
||||
run: uv run --no-project python bin/verify-staging-release.py
|
||||
|
||||
- name: Resolve release binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
|
||||
for name in uv gh git jq curl; do
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download exact tested distributions
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: ${{ runner.temp }}/python-distributions
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.CI_RUN_ID }}
|
||||
|
||||
- name: Record release version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate release branch and version
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION='${{ steps.version.outputs.version }}'
|
||||
case "$RELEASE_BRANCH" in
|
||||
main)
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
|
||||
echo "Refusing to publish non-stable version $VERSION from main" >&2
|
||||
exit 1
|
||||
}
|
||||
;;
|
||||
dev)
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
|
||||
echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
|
||||
exit 1
|
||||
}
|
||||
;;
|
||||
*)
|
||||
echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Publish the exact tested distributions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
|
||||
run: ./bin/release.sh
|
||||
|
||||
- name: Confirm this tested commit owns the release tag
|
||||
id: release-state
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
TARGET=""
|
||||
while read -r sha _; do
|
||||
TARGET="$sha"
|
||||
done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
|
||||
if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
docker-release:
|
||||
name: Publish tested multiarch image
|
||||
if: >-
|
||||
(github.event_name == 'workflow_dispatch') ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push')
|
||||
needs: [candidate, python-release]
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout the tested commit
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- name: Reconcile dev to the tested stable release
|
||||
if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
git fetch --no-tags origin main dev
|
||||
MAIN_SHA="$(git rev-parse origin/main)"
|
||||
DEV_SHA="$(git rev-parse origin/dev)"
|
||||
if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
push_dev() {
|
||||
if ! git push origin "$1:refs/heads/dev"; then
|
||||
CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
|
||||
if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
|
||||
echo "Failed to push dev although it still points to $DEV_SHA" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
|
||||
fi
|
||||
}
|
||||
if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
|
||||
push_dev "$RELEASE_SHA"
|
||||
exit 0
|
||||
else
|
||||
STATUS=$?
|
||||
[[ "$STATUS" -eq 1 ]] || exit "$STATUS"
|
||||
fi
|
||||
BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
|
||||
if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
|
||||
:
|
||||
else
|
||||
STATUS=$?
|
||||
[[ "$STATUS" -eq 1 ]] || exit "$STATUS"
|
||||
echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
|
||||
exit 0
|
||||
fi
|
||||
normalized_metadata() {
|
||||
case "$2" in
|
||||
pyproject.toml)
|
||||
git show "$1:$2" | awk '
|
||||
/^\[/ {section = $0}
|
||||
section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
|
||||
section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
|
||||
{print}
|
||||
' ;;
|
||||
uv.lock)
|
||||
git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
|
||||
etc/package.json)
|
||||
git show "$1:$2" | sed -E 's/^( "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
|
||||
esac
|
||||
}
|
||||
for path in pyproject.toml uv.lock etc/package.json; do
|
||||
BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
|
||||
DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
|
||||
if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
|
||||
echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
# A concurrent release bot may bump only dev's own version metadata.
|
||||
# Keep that commit as a parent while preserving the exact tested stable tree.
|
||||
git switch --detach "$MAIN_SHA"
|
||||
git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
|
||||
merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
|
||||
MERGE_SHA="$(git rev-parse HEAD)"
|
||||
push_dev "$MERGE_SHA"
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve release and Docker binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
|
||||
for name in uv gh git jq docker; do
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
with:
|
||||
# These images were already built and tested; only publish manifests.
|
||||
driver: docker
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Download exact tested image digests
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
|
||||
|
||||
- name: Verify exact successful digest build run
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
|
||||
RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
|
||||
[[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
|
||||
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
|
||||
RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
|
||||
[[ "$RUN_SHA" == "$RELEASE_SHA" ]]
|
||||
fi
|
||||
|
||||
- name: Verify digest set and collect tags
|
||||
id: docker_meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
[[ "${#DIGESTS[@]}" -eq 2 ]]
|
||||
for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
|
||||
[[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
|
||||
|
||||
VERSION='${{ needs.python-release.outputs.version }}'
|
||||
SHORT_SHA="${RELEASE_SHA::12}"
|
||||
TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
|
||||
TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
|
||||
if [[ -z "$TAG_TARGET" ]]; then
|
||||
TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
|
||||
TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
|
||||
fi
|
||||
if [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
|
||||
echo "Refusing to tag :latest with prerelease $VERSION" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
|
||||
echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
|
||||
exit 1
|
||||
}
|
||||
$GIT_BINARY fetch --no-tags origin main
|
||||
$GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
|
||||
echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
|
||||
exit 1
|
||||
}
|
||||
MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
|
||||
[[ "$MAIN_VERSION" == "$VERSION" ]] || {
|
||||
echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
SYNC_DEV=false
|
||||
if [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
$GIT_BINARY fetch --no-tags origin main dev
|
||||
MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
|
||||
DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
|
||||
# Recovery can add CI/docs fixes after the immutable release tag.
|
||||
# Both branches and every shipped application file must agree.
|
||||
if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
|
||||
$GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
|
||||
$GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
|
||||
SYNC_DEV=true
|
||||
fi
|
||||
fi
|
||||
{
|
||||
echo 'dockerhub_tags<<EOF'
|
||||
if [[ "$RELEASE_BRANCH" == dev ]]; then
|
||||
echo "${DOCKERHUB_IMAGE}:dev"
|
||||
elif [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
echo "${DOCKERHUB_IMAGE}:latest"
|
||||
[[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
|
||||
fi
|
||||
echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
|
||||
echo 'EOF'
|
||||
echo 'ghcr_tags<<EOF'
|
||||
if [[ "$RELEASE_BRANCH" == dev ]]; then
|
||||
echo "${GHCR_IMAGE}:dev"
|
||||
elif [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
echo "${GHCR_IMAGE}:latest"
|
||||
[[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
|
||||
fi
|
||||
echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create Docker Hub manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Create GHCR manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Verify published tags contain the exact tested images
|
||||
env:
|
||||
DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
|
||||
GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
# Both architectures already passed CLI/version/capture acceptance by
|
||||
# digest. Verify promotion preserves those exact images and attestations
|
||||
# instead of downloading and running the same image again on one arch.
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
|
||||
EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
|
||||
"$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
|
||||
while IFS= read -r image; do
|
||||
[[ -n "$image" ]] || continue
|
||||
ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
|
||||
[[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
|
||||
echo "Verified exact tested images: $image"
|
||||
done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
|
||||
|
||||
- name: Record published image source
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
# workflow_run can report the default branch's head even when this
|
||||
# run published a different branch/SHA. Preserve the verified target.
|
||||
[[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
|
||||
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
mkdir -p "$RUNNER_TEMP/published-image-source"
|
||||
# shellcheck disable=SC2016
|
||||
"$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
|
||||
'{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
|
||||
|
||||
- name: Upload published image source
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: published-image-source
|
||||
path: ${{ runner.temp }}/published-image-source/source.json
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Update Docker Hub README
|
||||
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: archivebox/archivebox
|
||||
|
||||
cascade:
|
||||
name: Cascade verified release
|
||||
if: needs.python-release.outputs.release_ready == 'true'
|
||||
needs:
|
||||
- candidate
|
||||
- python-release
|
||||
- docker-release
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
|
||||
with:
|
||||
upstream_package: archivebox
|
||||
upstream_version: ${{ needs.python-release.outputs.version }}
|
||||
upstream_artifact_name: python-distributions
|
||||
upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
|
||||
secrets:
|
||||
release_token: ${{ secrets.RELEASE_GH_TOKEN }}
|
||||
@@ -1,81 +0,0 @@
|
||||
name: Capture publicsite screenshots
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '37 5,17 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
changed: ${{ steps.check.outputs.changed }}
|
||||
key: ${{ steps.check.outputs.key }}
|
||||
steps:
|
||||
- id: check
|
||||
uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
|
||||
|
||||
deploy:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Python and uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: false
|
||||
|
||||
- name: Install ArchiveBox and screenshot dependencies
|
||||
run: uv sync --no-cache --frozen
|
||||
|
||||
- name: Build UI screenshot gallery
|
||||
run: bin/collect_ui_screenshots.sh
|
||||
env:
|
||||
BASE_URL: http://archivebox.localhost:5797
|
||||
|
||||
- name: Verify gallery was generated from this exact revision
|
||||
run: |
|
||||
uv run --no-cache --no-sync python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
|
||||
version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
|
||||
assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
|
||||
assert provenance["version"] == version, provenance
|
||||
assert provenance["capture_count"] >= 3 * 35, provenance
|
||||
assert len(provenance["files"]) == provenance["capture_count"], provenance
|
||||
assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
|
||||
PY
|
||||
|
||||
- name: Save validated screenshots
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: site-screenshots
|
||||
path: publicsite/screenshots
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
|
||||
complete:
|
||||
name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
|
||||
needs: [changes, deploy]
|
||||
if: needs.changes.outputs.changed == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- run: echo 'Scheduled work completed successfully.'
|
||||
@@ -1,117 +0,0 @@
|
||||
name: Staging acceptance
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [Release]
|
||||
types: [completed]
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: staging-acceptance
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
published:
|
||||
if: github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
actions: read
|
||||
outputs:
|
||||
ready: ${{ steps.source.outputs.ready }}
|
||||
sha: ${{ steps.source.outputs.sha }}
|
||||
steps:
|
||||
- id: image
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
|
||||
echo "published=$published" >> "$GITHUB_OUTPUT"
|
||||
- name: Download published image source
|
||||
if: steps.image.outputs.published == 'true'
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: published-image-source
|
||||
path: ${{ runner.temp }}/published-image-source
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
- name: Select published dev image
|
||||
id: source
|
||||
if: steps.image.outputs.published == 'true'
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
# The triggering run may report the default branch as its head;
|
||||
# this artifact names the image source verified by the publisher.
|
||||
source="$RUNNER_TEMP/published-image-source/source.json"
|
||||
jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
|
||||
branch=$(jq -er '.branch | select(type == "string")' "$source")
|
||||
sha=$(jq -er '.sha | select(type == "string")' "$source")
|
||||
[[ "$branch" == dev || "$branch" == main ]]
|
||||
[[ "$sha" =~ ^[0-9a-f]{40}$ ]]
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
if [[ "$branch" == dev ]]; then
|
||||
echo 'ready=true' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
acceptance:
|
||||
# The privileged key is never exposed to PRs or arbitrary dispatch refs.
|
||||
needs: published
|
||||
if: needs.published.outputs.ready == 'true'
|
||||
runs-on: [self-hosted, Linux, X64, ugnas-linux]
|
||||
environment: staging
|
||||
timeout-minutes: 110
|
||||
permissions:
|
||||
contents: read
|
||||
deployments: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: [cabbage]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
|
||||
with:
|
||||
ref: ${{ needs.published.outputs.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
|
||||
with:
|
||||
version: '0.11.3'
|
||||
- name: Prepare browser and restricted staging identity
|
||||
env:
|
||||
STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
|
||||
ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
|
||||
printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
|
||||
# Use the candidate's normal provider/cache path, not a second
|
||||
# Playwright-managed browser that bypasses abxpkg validation.
|
||||
ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
|
||||
CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
|
||||
uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
|
||||
CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
|
||||
test -x "$CHROME_BINARY"
|
||||
echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
|
||||
- name: Verify actual deployment, full captures and rendered snapshots
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
STAGING_TARGET: ${{ matrix.target }}
|
||||
STAGING_SHA: ${{ needs.published.outputs.sha }}
|
||||
STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
|
||||
STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
|
||||
run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
|
||||
- name: Remove staging identity
|
||||
if: always()
|
||||
run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
|
||||
if: always()
|
||||
with:
|
||||
name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
|
||||
path: staging-evidence/
|
||||
if-no-files-found: error
|
||||
@@ -1,255 +0,0 @@
|
||||
name: Parallel Tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_file:
|
||||
description: Run one discovered test file
|
||||
required: true
|
||||
default: archivebox/tests/test_cli_list.py
|
||||
type: string
|
||||
workflow_call:
|
||||
inputs:
|
||||
test_file:
|
||||
description: Run one discovered test file; empty runs the complete matrix
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
PYTHONLEGACYWINDOWSSTDIO: utf-8
|
||||
USE_COLOR: False
|
||||
UV_NO_SOURCES: "1"
|
||||
CI_PYTHON_VERSION: "3.13"
|
||||
|
||||
jobs:
|
||||
discover-tests:
|
||||
name: Discover every test
|
||||
# Selection must run on GitHub so an offline NAS cannot block the fallback.
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
tests: ${{ steps.discover.outputs.tests }}
|
||||
runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
|
||||
|
||||
steps:
|
||||
- name: Select available CI capacity
|
||||
id: runner
|
||||
if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
|
||||
uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
|
||||
with:
|
||||
github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
|
||||
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Resolve discovery tools through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
test -n "$ABXPKG_SPEC"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PYTHON_BINARY="$pythonLocation/bin/python"
|
||||
test -x "$PYTHON_BINARY"
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
|
||||
{
|
||||
echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
} >> "$GITHUB_ENV"
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/python"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/python"
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
|
||||
- name: Discover every test exactly once
|
||||
id: discover
|
||||
env:
|
||||
TEST_FILE: ${{ inputs.test_file }}
|
||||
UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
|
||||
test "${#output[@]}" -eq 2
|
||||
if [[ -n "$TEST_FILE" ]]; then
|
||||
output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
|
||||
fi
|
||||
echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
|
||||
echo "${output[0]}"
|
||||
"$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
|
||||
|
||||
tests:
|
||||
name: ${{ matrix.test.name }}
|
||||
needs: discover-tests
|
||||
# Opt in trusted branches only; PRs always use GitHub-hosted runners.
|
||||
# Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
|
||||
# The hosted discovery job falls back when workers are offline or the
|
||||
# availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
|
||||
runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
|
||||
env:
|
||||
CHROME_HEADLESS: "true"
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
|
||||
|
||||
steps:
|
||||
- name: Set job-local runtime paths
|
||||
env:
|
||||
CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
|
||||
run: |
|
||||
workers=2
|
||||
cache_suffix=''
|
||||
if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
|
||||
workers="$(nproc)"
|
||||
cache_suffix="-workers-$workers"
|
||||
# Avoid the hosted image's cold Chromium executable page faults.
|
||||
# abxpkg owns installation and reuse in the existing runtime cache.
|
||||
echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
|
||||
fi
|
||||
{
|
||||
echo "CI_TEST_WORKERS=$workers"
|
||||
echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
|
||||
echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
|
||||
echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
|
||||
echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
|
||||
echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
architecture: x64
|
||||
id: python
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
# Self-hosted tool storage survives jobs. Normal abxpkg validation and
|
||||
# install locks allow reuse across files; cold-install tests use tmp_path.
|
||||
- name: Reuse persistent runner tools
|
||||
if: runner.environment == 'self-hosted'
|
||||
env:
|
||||
PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
|
||||
run: |
|
||||
mkdir -p "$PERSISTENT_ABXPKG_LIB"
|
||||
echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
|
||||
|
||||
# Restored binaries still pass normal abxpkg resolution/version checks.
|
||||
# Installation and isolation tests explicitly choose fresh temporary libs.
|
||||
- name: Identify runtime dependency cache
|
||||
shell: bash
|
||||
run: |
|
||||
uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
|
||||
import hashlib
|
||||
import json
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
|
||||
print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
|
||||
PYTHON
|
||||
|
||||
- name: Reuse validated runtime dependencies
|
||||
if: runner.environment != 'self-hosted'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/archivebox-lib
|
||||
key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
|
||||
restore-keys: |
|
||||
abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
|
||||
|
||||
- name: Resolve optional build dependencies through abxpkg
|
||||
if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
|
||||
env:
|
||||
TEST_EXTRA: ${{ matrix.test.extra }}
|
||||
TEST_PATHS: ${{ matrix.test.paths_arg }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
test -n "$ABXPKG_SPEC"
|
||||
dependency_sources=()
|
||||
if [[ -n "$TEST_EXTRA" ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
|
||||
fi
|
||||
if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
|
||||
fi
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--no-cache \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
"${dependency_sources[@]}"
|
||||
|
||||
- name: Install test dependencies
|
||||
env:
|
||||
TEST_EXTRA: ${{ matrix.test.extra }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
extra_args=()
|
||||
if [[ -n "$TEST_EXTRA" ]]; then
|
||||
extra_args=(--extra "$TEST_EXTRA")
|
||||
fi
|
||||
uv sync --no-cache --locked --dev "${extra_args[@]}"
|
||||
- name: Run ${{ matrix.test.name }}
|
||||
env:
|
||||
TEST_PATHS: ${{ matrix.test.paths_arg }}
|
||||
CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
|
||||
read -r -a test_paths <<< "$TEST_PATHS"
|
||||
test_args=()
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test -f "$test_path"
|
||||
test_args+=("$GITHUB_WORKSPACE/$test_path")
|
||||
done
|
||||
# Each test still gets isolated DB/filesystem state from conftest.
|
||||
# Keep one worker pool per batch instead of starting it for every file.
|
||||
mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
|
||||
uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
|
||||
--basetemp="$RUNNER_TEMP/pytest" \
|
||||
--junitxml="$RUNNER_TEMP/test-results/results.xml"
|
||||
- name: Save test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: test-results-${{ strategy.job-index }}
|
||||
path: ${{ runner.temp }}/test-results/*.xml
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
- name: Save failed runtime diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: runtime-logs-${{ strategy.job-index }}
|
||||
path: ${{ runner.temp }}/pytest/**/*.log
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
@@ -1,60 +0,0 @@
|
||||
name: Install and CLI Compatibility
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
platform:
|
||||
description: Platform for install and CLI compatibility
|
||||
required: false
|
||||
default: linux
|
||||
type: string
|
||||
|
||||
env:
|
||||
DOCKER_IMAGE: archivebox-ci
|
||||
ARCHIVEBOX_IMAGE: archivebox-ci
|
||||
PYTHONIOENCODING: utf-8
|
||||
PYTHONLEGACYWINDOWSSTDIO: utf-8
|
||||
USE_COLOR: False
|
||||
UV_NO_SOURCES: "1"
|
||||
|
||||
jobs:
|
||||
install_cli_platform:
|
||||
name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
### Set up Python
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: uv sync --no-cache --locked --dev
|
||||
|
||||
- name: ArchiveBox full install check
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
DATA_DIR="$PWD/tests/out/data"
|
||||
mkdir -p "$DATA_DIR"
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
|
||||
+15
-68
@@ -1,72 +1,19 @@
|
||||
.DS_Store
|
||||
._*
|
||||
/.heartbeat.json
|
||||
/package.json
|
||||
/package-lock.json
|
||||
*.pyc
|
||||
# Pocket archive output folder
|
||||
pocket/
|
||||
bookmarks/
|
||||
pinboard/
|
||||
html/
|
||||
downloads/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
.mypy_cache/
|
||||
.eggs/
|
||||
tests/out/
|
||||
/docs/screenshots/
|
||||
/publicsite/screenshots/
|
||||
|
||||
# Coverage
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.json
|
||||
coverage/
|
||||
htmlcov/
|
||||
|
||||
# Python and Node dependencies
|
||||
# Virtualenv
|
||||
env/
|
||||
ENV/
|
||||
venv/
|
||||
VENV/
|
||||
.venv/
|
||||
.docker-venv/
|
||||
node_modules/
|
||||
typings/
|
||||
|
||||
# Ignore dev lockfiles (should always be built fresh)
|
||||
pdm.dev.lock
|
||||
requirements-dev.txt
|
||||
|
||||
# Packaging artifacts
|
||||
requirements.txt
|
||||
.pdm-python
|
||||
.pdm-build
|
||||
archivebox.egg-info
|
||||
archivebox-*.tar.gz
|
||||
build/
|
||||
dist/
|
||||
|
||||
# Data folders
|
||||
lib/
|
||||
out/
|
||||
tmp/
|
||||
data/
|
||||
data*/
|
||||
archivebox/tests/data/
|
||||
archive/
|
||||
output/
|
||||
logs/
|
||||
/-
|
||||
/personas/
|
||||
/sources/
|
||||
index.sqlite3
|
||||
queue.sqlite3
|
||||
*.sqlite*
|
||||
data.*
|
||||
.archivebox_id
|
||||
ArchiveBox.conf
|
||||
*.stdout
|
||||
*.stderr
|
||||
*.log
|
||||
.tmp/
|
||||
|
||||
# vim
|
||||
*.sw?
|
||||
.vscode
|
||||
|
||||
# Local Claude Code task locks (not project state)
|
||||
.claude/scheduled_tasks.lock
|
||||
|
||||
/_site/
|
||||
.env/
|
||||
.python-version
|
||||
.env
|
||||
@@ -1,69 +0,0 @@
|
||||
default_language_version:
|
||||
python: python3.13
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/asottile/yesqa
|
||||
rev: v1.5.0
|
||||
hooks:
|
||||
- id: yesqa
|
||||
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies:
|
||||
- tomli
|
||||
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.20.0
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: [--py313-plus]
|
||||
|
||||
- repo: https://github.com/asottile/add-trailing-comma
|
||||
rev: v3.1.0
|
||||
hooks:
|
||||
- id: add-trailing-comma
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: ruff-format
|
||||
name: ruff-format
|
||||
entry: uv run --active --no-sync --no-sources ruff format
|
||||
language: system
|
||||
types_or: [python, pyi]
|
||||
- id: ruff-check
|
||||
name: ruff-check
|
||||
entry: uv run --active --no-sync --no-sources ruff check --fix
|
||||
language: system
|
||||
types_or: [python, pyi]
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: check-ast
|
||||
- id: check-toml
|
||||
- id: check-yaml
|
||||
exclude: ^\.github/workflows/homebrew\.yml$
|
||||
- id: check-json
|
||||
- id: check-merge-conflict
|
||||
- id: check-symlinks
|
||||
- id: destroyed-symlinks
|
||||
- id: check-case-conflict
|
||||
- id: check-illegal-windows-names
|
||||
- id: check-shebang-scripts-are-executable
|
||||
exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
|
||||
- id: mixed-line-ending
|
||||
- id: fix-byte-order-marker
|
||||
- id: end-of-file-fixer
|
||||
# Preserve the exact bytes of captured cryptographic proof fixtures.
|
||||
exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
|
||||
- id: detect-private-key
|
||||
- id: debug-statements
|
||||
- id: forbid-submodules
|
||||
exclude: ^docs$
|
||||
- id: check-added-large-files
|
||||
args: ["--maxkb=600"]
|
||||
- id: name-tests-test
|
||||
args: ["--pytest-test-first"]
|
||||
exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
|
||||
@@ -1,83 +0,0 @@
|
||||
# ArchiveBox Agent Guide
|
||||
|
||||
ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
|
||||
|
||||
## Shared Standards
|
||||
|
||||
- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
|
||||
- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
|
||||
- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
|
||||
- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
|
||||
- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
|
||||
- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
|
||||
- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
|
||||
- Start behavior fixes with a red failing test when a test is requested or practical.
|
||||
- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
|
||||
- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
|
||||
|
||||
## Concurrency Contract
|
||||
|
||||
- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
|
||||
- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
|
||||
- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
|
||||
- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
|
||||
|
||||
## Development Setup
|
||||
|
||||
```bash
|
||||
uv sync --dev --all-extras
|
||||
mkdir -p data
|
||||
cd data
|
||||
uv run --project .. archivebox init --install
|
||||
```
|
||||
|
||||
Run collection commands from inside an initialized data directory:
|
||||
|
||||
```bash
|
||||
cd data
|
||||
uv run --project .. archivebox status
|
||||
uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
|
||||
uv run --project .. archivebox run
|
||||
```
|
||||
|
||||
## User-Facing Setup
|
||||
|
||||
Recommended CLI install:
|
||||
|
||||
```bash
|
||||
uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
|
||||
mkdir -p ~/archivebox/data
|
||||
cd ~/archivebox/data
|
||||
archivebox init --install
|
||||
archivebox add --plugins=parse_txt_urls 'https://example.com/'
|
||||
```
|
||||
|
||||
Alternative install methods:
|
||||
|
||||
- Docker Compose / Docker
|
||||
- Homebrew
|
||||
- Debian package
|
||||
- pip
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
cd ~/archivebox/data
|
||||
archivebox version
|
||||
archivebox help
|
||||
archivebox status
|
||||
archivebox install
|
||||
archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
|
||||
archivebox list --json --with-headers
|
||||
archivebox search 'example'
|
||||
archivebox update --filter-type=domain example.com
|
||||
archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
|
||||
archivebox run
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
|
||||
with `uv run prek run --all-files`. For focused work, select the relevant tests.
|
||||
|
||||
Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
|
||||
@@ -0,0 +1,46 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment include:
|
||||
|
||||
* Using welcoming and inclusive language
|
||||
* Being respectful of differing viewpoints and experiences
|
||||
* Gracefully accepting constructive criticism
|
||||
* Focusing on what is best for the community
|
||||
* Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery and unwelcome sexual attention or advances
|
||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or electronic address, without explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at git@nicksweeting.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
|
||||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
-327
@@ -1,327 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
|
||||
# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
|
||||
# standalone downloader stays independent of ArchiveBox's server features.
|
||||
# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
|
||||
# the layers added by this Dockerfile, never in the shared abx-dl base image.
|
||||
# REQUIRED INSTALLATION CONTRACT — do not work around failures:
|
||||
# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
|
||||
# and ALL their required_binaries, regardless of default enabled state.
|
||||
# Inherit the complete downloader dependencies from the tested abx-dl image;
|
||||
# install every remaining ArchiveBox-only dependency in this image's layers,
|
||||
# including optional server plugins. Never put server dependencies in abx-dl.
|
||||
# Use the normal plugin dependency pipeline: install command -> config.json
|
||||
# required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
|
||||
# failures in that pipeline. OS/interpreter bootstrap is a separate concern,
|
||||
# not a second hand-maintained list of plugin dependencies.
|
||||
# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
|
||||
# must discover everything already present and be NO-OPS offline as UID 911.
|
||||
# No downloads, package installs, sudo, or root repair may be needed to enable
|
||||
# ANY plugin or start a new crawl. Never run verification as root or manually
|
||||
# preinstall packages before it. If the inherited image lacks a downloader
|
||||
# dependency, fix/release abx-dl and consume that tested image; do not patch
|
||||
# over the missing base dependency here or repair deployed containers by SSH.
|
||||
# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
|
||||
# CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
|
||||
# passwordless sudo; abxpkg must escalate automatically when needed. Never
|
||||
# manually preinstall dependencies or wrap the tested command in sudo to make
|
||||
# a failure disappear. This is distinct from the finished-image no-op test.
|
||||
# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
|
||||
# user-facing functionality. Only mechanical/internal
|
||||
# optimizations are allowed. Never remove dependencies, disable features or
|
||||
# plugins, weaken assertions, or skip checks to meet a size limit. If the
|
||||
# complete image exceeds the configured limit, LET CI FAIL and explicitly
|
||||
# ask the maintainer what to do. Do not raise the limit unilaterally.
|
||||
# Build abx-dl first, then point this file at it:
|
||||
# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
|
||||
# --build-context abxbus=../abxbus \
|
||||
# --build-context abxpkg=../abxpkg \
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:main
|
||||
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
ARG ARCHIVEBOX_COMMIT_HASH=""
|
||||
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
LANG=C.UTF-8 \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
|
||||
PYTHONIOENCODING=UTF-8 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_NO_COMPILE=1 \
|
||||
PIP_ONLY_BINARY=aiohttp \
|
||||
npm_config_loglevel=error
|
||||
|
||||
ENV PYTHON_VERSION=3.13 \
|
||||
NODE_VERSION=24
|
||||
|
||||
ENV ARCHIVEBOX_USER=archivebox \
|
||||
DEFAULT_ARCHIVEBOX_UID=911 \
|
||||
DEFAULT_ARCHIVEBOX_GID=911 \
|
||||
IN_DOCKER=True
|
||||
|
||||
ENV CODE_DIR=/app \
|
||||
ABX_RUNTIME=archivebox \
|
||||
DATA_DIR=/data \
|
||||
CONFIG_DIR=/opt/archivebox \
|
||||
ABXPKG_LIB_DIR=/opt/archivebox/lib \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
|
||||
PERSONAS_DIR=/data/personas \
|
||||
CHROME_HEADLESS=true \
|
||||
CHROME_SANDBOX=false \
|
||||
CHROME_ISOLATION=crawl \
|
||||
CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
|
||||
|
||||
ENV TMP_DIR=/tmp/archivebox \
|
||||
PIP_VENV_PYTHON=/venv/bin/python3 \
|
||||
GOOGLE_API_KEY=no \
|
||||
GOOGLE_DEFAULT_CLIENT_ID=no \
|
||||
GOOGLE_DEFAULT_CLIENT_SECRET=no
|
||||
|
||||
ENV HOME=/home/archivebox \
|
||||
XDG_CONFIG_HOME=/home/archivebox/.config \
|
||||
XDG_CACHE_HOME=/opt/archivebox/lib/cache \
|
||||
TIMEOUT=600
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=false \
|
||||
UV_PYTHON_PREFERENCE=managed \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv \
|
||||
VIRTUAL_ENV=/venv \
|
||||
PATH="/venv/bin:/opt/node/bin:$PATH"
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
WORKDIR "$CODE_DIR"
|
||||
|
||||
RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
|
||||
&& (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
FROM archivebox-runtime-base AS archivebox-builder
|
||||
|
||||
WORKDIR "$CODE_DIR"
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
|
||||
--mount=type=bind,source=uv.lock,target=/app/uv.lock \
|
||||
<<'EOF'
|
||||
echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
|
||||
export UV_NO_CACHE=true
|
||||
rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
apt-get update -qq
|
||||
apt-get install -qq -y --no-install-recommends x11-utils
|
||||
# Extend abx-dl's existing venv instead of clearing it. Clearing and later
|
||||
# copying a complete replacement over the base duplicates every venv byte in
|
||||
# the final overlay history even when most packages are unchanged.
|
||||
# Do not rewrite inherited native libraries merely to strip them again.
|
||||
/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
|
||||
|
||||
# The old Docker-only dependency rewrite predated this checkout's lockfile:
|
||||
# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
|
||||
# can itself lag or return 503s. The verified lock already records exact public
|
||||
# artifact URLs and hashes, including extras and platform markers. Consume it
|
||||
# directly, retaining the freshness check instead of resolving a second graph.
|
||||
/usr/bin/uv sync \
|
||||
--project /app \
|
||||
--locked \
|
||||
--no-cache \
|
||||
--no-dev \
|
||||
--inexact \
|
||||
--no-install-project \
|
||||
--no-install-workspace \
|
||||
--no-install-package abxbus \
|
||||
--no-install-package abxpkg \
|
||||
--no-install-package abx-plugins \
|
||||
--no-install-package abx-dl \
|
||||
--no-sources
|
||||
# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
|
||||
# their bytecode. Runtime writes are disabled, so missing caches would make
|
||||
# every hook compile them again. Compile only missing entries: compileall's
|
||||
# timestamp fast path rewrites even valid checked-hash caches, copying the
|
||||
# entire inherited venv into this OCI layer for no runtime benefit.
|
||||
/usr/bin/uv run --no-project --no-sync python - <<'PY'
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import py_compile
|
||||
import sysconfig
|
||||
|
||||
for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
|
||||
cache = Path(importlib.util.cache_from_source(str(source)))
|
||||
if not cache.is_file():
|
||||
py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
|
||||
PY
|
||||
builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
|
||||
ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
|
||||
/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
|
||||
while IFS= read -r -d '' native_library; do
|
||||
if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
|
||||
continue
|
||||
fi
|
||||
magic=''
|
||||
if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
|
||||
"$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
|
||||
fi
|
||||
done < /tmp/archivebox-native-libraries
|
||||
rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
|
||||
rm -f /venv/bin/uv /venv/bin/uvx
|
||||
abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
|
||||
abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
|
||||
rm -rf "$builder_abxpkg_lib_dir"
|
||||
rm -rf /venv/lib/python3.*/site-packages/pip* \
|
||||
/venv/lib/python3.*/site-packages/wheel* \
|
||||
/venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
EOF
|
||||
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
# Compile only ArchiveBox's installed package. Touching all of /venv here would
|
||||
# copy every inherited abx-dl file into a new layer merely to change its mtime.
|
||||
RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
|
||||
&& COMMIT_HASH="$( \
|
||||
if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
|
||||
echo "$ARCHIVEBOX_COMMIT_HASH"; \
|
||||
elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
|
||||
HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
|
||||
if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
|
||||
echo "$HEAD_REF"; \
|
||||
elif [[ "$HEAD_REF" == ref:\ * ]]; then \
|
||||
REF_PATH="${HEAD_REF#ref: }"; \
|
||||
cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
|
||||
fi; \
|
||||
fi)" \
|
||||
&& if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
|
||||
&& /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
|
||||
&& rm -f /venv/bin/uv /venv/bin/uvx \
|
||||
&& cd / \
|
||||
&& ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
|
||||
&& /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
|
||||
&& test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
|
||||
&& /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
|
||||
|
||||
# The builder installs and purges stripping tools in one layer, so its final
|
||||
# filesystem is already runtime-clean. Preserve that ancestry: starting again
|
||||
# from archivebox-runtime-base and COPYing /venv would bake a second full venv
|
||||
# over the inherited one instead of recording only ArchiveBox's package delta.
|
||||
FROM archivebox-builder
|
||||
|
||||
LABEL name="archivebox" \
|
||||
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
|
||||
description="All-in-one self-hosted internet archiving solution" \
|
||||
homepage="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
|
||||
org.opencontainers.image.title="ArchiveBox" \
|
||||
org.opencontainers.image.vendor="ArchiveBox" \
|
||||
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
|
||||
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
com.docker.image.source.entrypoint="Dockerfile"
|
||||
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
|
||||
# The builder invokes abxpkg as root for temporary ELF stripping, which can
|
||||
# rewrite derived state ownership. Restore UID 911 before resolving binaries
|
||||
# as the runtime user; the old copy-based final stage hid this dependency.
|
||||
RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
|
||||
echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
|
||||
&& printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
|
||||
&& ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
|
||||
&& ln -sf /venv/bin/daphne /usr/local/bin/daphne \
|
||||
&& ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
|
||||
&& ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
|
||||
&& getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
|
||||
&& id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
|
||||
&& usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
|
||||
&& [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
|
||||
&& [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
|
||||
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
|
||||
&& chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
|
||||
&& find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
|
||||
&& UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
|
||||
&& openssl rand -hex 16 > /etc/machine-id \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
|
||||
&& echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
|
||||
# Install the complete ArchiveBox catalog, including disabled server plugins.
|
||||
# Explicit plugin names make installation independent of enabled defaults.
|
||||
RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
|
||||
chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
|
||||
&& chmod 1777 /var/tmp/abxpkg-cache \
|
||||
&& export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
|
||||
&& env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
|
||||
setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
|
||||
&& UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
|
||||
# pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
|
||||
# Retaining its download copies made image size depend on whether the build
|
||||
# preserved hardlinks to installed files. Remove the store in this layer;
|
||||
# installed files retain their contents, and the offline check below proves
|
||||
# the complete catalog still works without the disposable package payloads.
|
||||
&& rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
|
||||
RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
|
||||
bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
RUN echo "[+] Initializing image collection..." \
|
||||
&& find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
|
||||
&& HOME="$TMP_DIR" archivebox init \
|
||||
&& (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
|
||||
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
|
||||
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
|
||||
"$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
|
||||
"$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
|
||||
"$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
|
||||
"$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
2>/dev/null || true) \
|
||||
&& find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
|
||||
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
|
||||
|
||||
RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
|
||||
&& test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
|
||||
&& test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
|
||||
&& test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
|
||||
&& stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
|
||||
RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 5797
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
|
||||
CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
|
||||
|
||||
ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
|
||||
CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Nick Sweeting
|
||||
Copyright (c) 2017 Nick Sweeting
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
File renamed without changes.
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
# Bookmark Archiver
|
||||
# Nick Sweeting 2017 | MIT License
|
||||
# https://github.com/pirate/bookmark-archiver
|
||||
|
||||
import sys
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from parse import parse_links
|
||||
from links import validate_links
|
||||
from archive_methods import archive_links, _RESULTS_TOTALS
|
||||
from index import (
|
||||
write_links_index,
|
||||
write_link_index,
|
||||
parse_json_links_index,
|
||||
parse_json_link_index,
|
||||
)
|
||||
from config import (
|
||||
ARCHIVE_PERMISSIONS,
|
||||
HTML_FOLDER,
|
||||
ARCHIVE_FOLDER,
|
||||
ANSI,
|
||||
TIMEOUT,
|
||||
)
|
||||
from util import (
|
||||
download_url,
|
||||
check_dependencies,
|
||||
progress,
|
||||
cleanup_archive,
|
||||
)
|
||||
|
||||
__DESCRIPTION__ = 'Bookmark Archiver: Create a browsable html archive of a list of links.'
|
||||
__DOCUMENTATION__ = 'https://github.com/pirate/bookmark-archiver'
|
||||
|
||||
|
||||
def update_archive(export_path, links, resume=None, append=True):
|
||||
"""update or create index.html+json given a path to an export file containing new links"""
|
||||
|
||||
start_ts = datetime.now().timestamp()
|
||||
|
||||
# loop over links and archive them
|
||||
archive_links(ARCHIVE_FOLDER, links, export_path, resume=resume)
|
||||
|
||||
# print timing information & summary
|
||||
end_ts = datetime.now().timestamp()
|
||||
seconds = round(end_ts - start_ts, 1)
|
||||
duration = '{} min'.format(seconds / 60) if seconds > 60 else '{} sec'.format(seconds)
|
||||
print('{}[√] [{}] Archive update complete ({}){}'.format(
|
||||
ANSI['green'],
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
duration,
|
||||
ANSI['reset'],
|
||||
))
|
||||
print(' - {} entries skipped'.format(_RESULTS_TOTALS['skipped']))
|
||||
print(' - {} entries updated'.format(_RESULTS_TOTALS['succeded']))
|
||||
print(' - {} errors'.format(_RESULTS_TOTALS['failed']))
|
||||
|
||||
|
||||
def update_index(export_path, resume=None, append=True):
|
||||
"""handling parsing new links into the json index, returns a set of clean links"""
|
||||
|
||||
# parse an validate the export file
|
||||
new_links = validate_links(parse_links(export_path))
|
||||
|
||||
# load existing links if archive folder is present
|
||||
existing_links = []
|
||||
if append:
|
||||
existing_links = parse_json_links_index(HTML_FOLDER)
|
||||
links = validate_links(existing_links + new_links)
|
||||
|
||||
|
||||
# merge existing links and new links
|
||||
num_new_links = len(links) - len(existing_links)
|
||||
print('[*] [{}] Adding {} new links from {} to index'.format(
|
||||
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
num_new_links,
|
||||
export_path,
|
||||
))
|
||||
|
||||
# write link index html & json
|
||||
write_links_index(HTML_FOLDER, links)
|
||||
|
||||
return links
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
argc = len(sys.argv)
|
||||
|
||||
if argc < 2 or sys.argv[1] in ('-h', '--help', 'help'):
|
||||
print(__DESCRIPTION__)
|
||||
print("Documentation: {}".format(__DOCUMENTATION__))
|
||||
print("")
|
||||
print("Usage:")
|
||||
print(" ./archive.py ~/Downloads/bookmarks_export.html")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
|
||||
export_path = sys.argv[1] # path to export file
|
||||
resume_from = sys.argv[2] if argc > 2 else None # timestamp to resume dowloading from
|
||||
|
||||
if any(export_path.startswith(s) for s in ('http://', 'https://', 'ftp://')):
|
||||
export_path = download_url(export_path)
|
||||
|
||||
links = update_index(export_path, resume=resume_from, append=True)
|
||||
|
||||
# make sure folder structure is sane
|
||||
cleanup_archive(ARCHIVE_FOLDER, links)
|
||||
update_archive(export_path, links, resume=resume_from, append=True)
|
||||
@@ -0,0 +1,427 @@
|
||||
import os
|
||||
|
||||
from functools import wraps
|
||||
from datetime import datetime
|
||||
from subprocess import run, PIPE, DEVNULL
|
||||
|
||||
from index import html_appended_url, parse_json_link_index, write_link_index
|
||||
from links import links_after_timestamp
|
||||
from config import (
|
||||
ARCHIVE_DIR,
|
||||
CHROME_BINARY,
|
||||
FETCH_WGET,
|
||||
FETCH_WGET_REQUISITES,
|
||||
FETCH_PDF,
|
||||
FETCH_SCREENSHOT,
|
||||
RESOLUTION,
|
||||
SUBMIT_ARCHIVE_DOT_ORG,
|
||||
FETCH_AUDIO,
|
||||
FETCH_VIDEO,
|
||||
FETCH_FAVICON,
|
||||
WGET_USER_AGENT,
|
||||
CHROME_USER_DATA_DIR,
|
||||
TIMEOUT,
|
||||
ANSI,
|
||||
)
|
||||
from util import (
|
||||
check_dependencies,
|
||||
progress,
|
||||
chmod_file,
|
||||
)
|
||||
|
||||
|
||||
_RESULTS_TOTALS = { # globals are bad, mmkay
|
||||
'skipped': 0,
|
||||
'succeded': 0,
|
||||
'failed': 0,
|
||||
}
|
||||
|
||||
def archive_links(out_dir, links, export_path, resume=None):
|
||||
check_dependencies()
|
||||
|
||||
to_archive = links_after_timestamp(links, resume)
|
||||
try:
|
||||
for idx, link in enumerate(to_archive):
|
||||
link_dir = os.path.join(out_dir, link['timestamp'])
|
||||
archive_link(link_dir, link)
|
||||
|
||||
except (KeyboardInterrupt, SystemExit, Exception) as e:
|
||||
print('{red}[X] Archive update stopped on #{idx} out of {total} links{reset}'.format(
|
||||
**ANSI,
|
||||
idx=idx,
|
||||
total=len(list(to_archive)),
|
||||
))
|
||||
print(' Continue where you left off by running:')
|
||||
print(' ./archive.py {} {}'.format(
|
||||
export_path,
|
||||
link['timestamp'],
|
||||
))
|
||||
if not isinstance(e, KeyboardInterrupt):
|
||||
raise e
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def archive_link(out_dir, link, overwrite=False):
|
||||
"""download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
|
||||
|
||||
link = {**parse_json_link_index(out_dir), **link}
|
||||
log_link_archive(out_dir, link)
|
||||
|
||||
if FETCH_WGET:
|
||||
link = fetch_wget(out_dir, link, overwrite=overwrite)
|
||||
|
||||
if FETCH_PDF:
|
||||
link = fetch_pdf(out_dir, link, overwrite=overwrite)
|
||||
|
||||
if FETCH_SCREENSHOT:
|
||||
link = fetch_screenshot(out_dir, link, overwrite=overwrite)
|
||||
|
||||
if SUBMIT_ARCHIVE_DOT_ORG:
|
||||
link = archive_dot_org(out_dir, link, overwrite=overwrite)
|
||||
|
||||
# if FETCH_AUDIO:
|
||||
# link = fetch_audio(out_dir, link, overwrite=overwrite)
|
||||
|
||||
# if FETCH_VIDEO:
|
||||
# link = fetch_video(out_dir, link, overwrite=overwrite)
|
||||
|
||||
if FETCH_FAVICON:
|
||||
link = fetch_favicon(out_dir, link, overwrite=overwrite)
|
||||
|
||||
write_link_index(out_dir, link)
|
||||
|
||||
return link
|
||||
|
||||
def log_link_archive(out_dir, link):
|
||||
update_existing = os.path.exists(out_dir)
|
||||
if not update_existing:
|
||||
os.makedirs(out_dir)
|
||||
|
||||
print('[{symbol_color}{symbol}{reset}] [{timestamp}] "{title}": {blue}{base_url}{reset}'.format(
|
||||
symbol='*' if update_existing else '+',
|
||||
symbol_color=ANSI['black' if update_existing else 'green'],
|
||||
**link,
|
||||
**ANSI,
|
||||
))
|
||||
if link['type']:
|
||||
print(' i Type: {}'.format(link['type']))
|
||||
|
||||
print(' {} ({})'.format(out_dir, 'updating' if update_existing else 'creating'))
|
||||
|
||||
|
||||
|
||||
def attach_result_to_link(method):
|
||||
"""
|
||||
Instead of returning a result={output:'...', status:'success'} object,
|
||||
attach that result to the links's history & latest fields, then return
|
||||
the updated link object.
|
||||
"""
|
||||
def decorator(fetch_func):
|
||||
@wraps(fetch_func)
|
||||
def timed_fetch_func(out_dir, link, overwrite=False, **kwargs):
|
||||
# initialize methods and history json field on link
|
||||
link['latest'] = link.get('latest') or {}
|
||||
link['latest'][method] = link['latest'].get(method) or None
|
||||
link['history'] = link.get('history') or {}
|
||||
link['history'][method] = link['history'].get(method) or []
|
||||
|
||||
start_ts = datetime.now().timestamp()
|
||||
|
||||
# if a valid method output is already present, dont run the fetch function
|
||||
if link['latest'][method] and not overwrite:
|
||||
print(' √ Skipping: {}'.format(method))
|
||||
result = None
|
||||
else:
|
||||
print(' - Fetching: {}'.format(method))
|
||||
result = fetch_func(out_dir, link, **kwargs)
|
||||
|
||||
end_ts = datetime.now().timestamp()
|
||||
duration = str(end_ts * 1000 - start_ts * 1000).split('.')[0]
|
||||
|
||||
# append a history item recording fail/success
|
||||
history_entry = {
|
||||
'timestamp': str(start_ts).split('.')[0],
|
||||
}
|
||||
if result is None:
|
||||
history_entry['status'] = 'skipped'
|
||||
elif isinstance(result.get('output'), Exception):
|
||||
history_entry['status'] = 'failed'
|
||||
history_entry['duration'] = duration
|
||||
history_entry.update(result or {})
|
||||
link['history'][method].append(history_entry)
|
||||
else:
|
||||
history_entry['status'] = 'succeded'
|
||||
history_entry['duration'] = duration
|
||||
history_entry.update(result or {})
|
||||
link['history'][method].append(history_entry)
|
||||
link['latest'][method] = result['output']
|
||||
|
||||
_RESULTS_TOTALS[history_entry['status']] += 1
|
||||
|
||||
return link
|
||||
return timed_fetch_func
|
||||
return decorator
|
||||
|
||||
|
||||
@attach_result_to_link('wget')
|
||||
def fetch_wget(out_dir, link, requisites=FETCH_WGET_REQUISITES, timeout=TIMEOUT):
|
||||
"""download full site using wget"""
|
||||
|
||||
if os.path.exists(os.path.join(out_dir, link['domain'])):
|
||||
return {'output': html_appended_url(link), 'status': 'skipped'}
|
||||
|
||||
CMD = [
|
||||
*'wget --timestamping --adjust-extension --no-parent'.split(' '), # Docs: https://www.gnu.org/software/wget/manual/wget.html
|
||||
*(('--page-requisites', '--convert-links') if FETCH_WGET_REQUISITES else ()),
|
||||
*(('--user-agent="{}"'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()),
|
||||
link['url'],
|
||||
]
|
||||
end = progress(timeout, prefix=' ')
|
||||
try:
|
||||
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # index.html
|
||||
end()
|
||||
output = html_appended_url(link)
|
||||
if result.returncode > 0:
|
||||
print(' got wget response code {}:'.format(result.returncode))
|
||||
print('\n'.join(' ' + line for line in (result.stderr or result.stdout).decode().rsplit('\n', 10)[-10:] if line.strip()))
|
||||
# raise Exception('Failed to wget download')
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
'cmd': CMD,
|
||||
'output': output,
|
||||
}
|
||||
|
||||
|
||||
@attach_result_to_link('pdf')
|
||||
def fetch_pdf(out_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR):
|
||||
"""print PDF of site to file using chrome --headless"""
|
||||
|
||||
if link['type'] in ('PDF', 'image'):
|
||||
return {'output': html_appended_url(link)}
|
||||
|
||||
if os.path.exists(os.path.join(out_dir, 'output.pdf')):
|
||||
return {'output': 'output.pdf', 'status': 'skipped'}
|
||||
|
||||
CMD = [
|
||||
CHROME_BINARY,
|
||||
*'--headless --disable-gpu --print-to-pdf'.split(' '),
|
||||
*chrome_data_dir_args(user_data_dir),
|
||||
link['url']
|
||||
]
|
||||
end = progress(timeout, prefix=' ')
|
||||
try:
|
||||
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # output.pdf
|
||||
end()
|
||||
if result.returncode:
|
||||
print(' ', (result.stderr or result.stdout).decode())
|
||||
raise Exception('Failed to print PDF')
|
||||
output = 'output.pdf'
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
'cmd': CMD,
|
||||
'output': output,
|
||||
}
|
||||
|
||||
|
||||
@attach_result_to_link('screenshot')
|
||||
def fetch_screenshot(out_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR, resolution=RESOLUTION):
|
||||
"""take screenshot of site using chrome --headless"""
|
||||
|
||||
if link['type'] in ('PDF', 'image'):
|
||||
return {'output': html_appended_url(link)}
|
||||
|
||||
if os.path.exists(os.path.join(out_dir, 'screenshot.png')):
|
||||
return {'output': 'screenshot.png', 'status': 'skipped'}
|
||||
|
||||
CMD = [
|
||||
CHROME_BINARY,
|
||||
*'--headless --disable-gpu --screenshot'.split(' '),
|
||||
*chrome_data_dir_args(user_data_dir),
|
||||
'--window-size={}'.format(resolution),
|
||||
link['url']
|
||||
]
|
||||
end = progress(timeout, prefix=' ')
|
||||
try:
|
||||
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=out_dir, timeout=timeout + 1) # sreenshot.png
|
||||
end()
|
||||
if result.returncode:
|
||||
print(' ', (result.stderr or result.stdout).decode())
|
||||
raise Exception('Failed to take screenshot')
|
||||
chmod_file('screenshot.png', cwd=out_dir)
|
||||
output = 'screenshot.png'
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
'cmd': CMD,
|
||||
'output': output,
|
||||
}
|
||||
|
||||
|
||||
@attach_result_to_link('archive_org')
|
||||
def archive_dot_org(out_dir, link, timeout=TIMEOUT):
|
||||
"""submit site to archive.org for archiving via their service, save returned archive url"""
|
||||
|
||||
path = os.path.join(out_dir, 'archive.org.txt')
|
||||
if os.path.exists(path):
|
||||
archive_org_url = open(path, 'r').read().strip()
|
||||
return {'output': archive_org_url, 'status': 'skipped'}
|
||||
|
||||
submit_url = 'https://web.archive.org/save/{}'.format(link['url'].split('?', 1)[0])
|
||||
|
||||
success = False
|
||||
CMD = ['curl', '-I', submit_url]
|
||||
end = progress(timeout, prefix=' ')
|
||||
try:
|
||||
result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # archive.org.txt
|
||||
end()
|
||||
|
||||
# Parse archive.org response headers
|
||||
headers = result.stdout.splitlines()
|
||||
content_location = [h for h in headers if b'Content-Location: ' in h]
|
||||
errors = [h for h in headers if h and b'X-Archive-Wayback-Runtime-Error: ' in h]
|
||||
|
||||
if content_location:
|
||||
archive_path = content_location[0].split(b'Content-Location: ', 1)[-1].decode('utf-8')
|
||||
saved_url = 'https://web.archive.org{}'.format(archive_path)
|
||||
success = True
|
||||
|
||||
elif len(errors) == 1 and b'RobotAccessControlException' in errors[0]:
|
||||
output = submit_url
|
||||
# raise Exception('Archive.org denied by {}/robots.txt'.format(link['domain']))
|
||||
elif errors:
|
||||
raise Exception(', '.join(e.decode() for e in errors))
|
||||
else:
|
||||
raise Exception('Failed to find "Content-Location" URL header in Archive.org response.')
|
||||
except Exception as e:
|
||||
end()
|
||||
print(' Visit url to see output:', ' '.join(CMD))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
if success:
|
||||
with open(os.path.join(out_dir, 'archive.org.txt'), 'w', encoding='utf-8') as f:
|
||||
f.write(saved_url)
|
||||
chmod_file('archive.org.txt', cwd=out_dir)
|
||||
output = saved_url
|
||||
|
||||
return {
|
||||
'cmd': CMD,
|
||||
'output': output,
|
||||
}
|
||||
|
||||
@attach_result_to_link('favicon')
|
||||
def fetch_favicon(out_dir, link, timeout=TIMEOUT):
|
||||
"""download site favicon from google's favicon api"""
|
||||
|
||||
if os.path.exists(os.path.join(out_dir, 'favicon.ico')):
|
||||
return {'output': 'favicon.ico', 'status': 'skipped'}
|
||||
|
||||
CMD = ['curl', 'https://www.google.com/s2/favicons?domain={domain}'.format(**link)]
|
||||
fout = open('{}/favicon.ico'.format(out_dir), 'w')
|
||||
end = progress(timeout, prefix=' ')
|
||||
try:
|
||||
run(CMD, stdout=fout, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # favicon.ico
|
||||
fout.close()
|
||||
end()
|
||||
chmod_file('favicon.ico', cwd=out_dir)
|
||||
output = 'favicon.ico'
|
||||
except Exception as e:
|
||||
fout.close()
|
||||
end()
|
||||
print(' Run to see full output:', ' '.join(CMD))
|
||||
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
output = e
|
||||
|
||||
return {
|
||||
'cmd': CMD,
|
||||
'output': output,
|
||||
}
|
||||
|
||||
# @attach_result_to_link('audio')
|
||||
# def fetch_audio(out_dir, link, timeout=TIMEOUT):
|
||||
# """Download audio rip using youtube-dl"""
|
||||
|
||||
# if link['type'] not in ('soundcloud',)\
|
||||
# and 'audio' not in link['tags']:
|
||||
# return
|
||||
|
||||
# path = os.path.join(out_dir, 'audio')
|
||||
|
||||
# if not os.path.exists(path) or overwrite:
|
||||
# print(' - Downloading audio')
|
||||
# CMD = [
|
||||
# "youtube-dl -x --audio-format mp3 --audio-quality 0 -o '%(title)s.%(ext)s'",
|
||||
# link['url'],
|
||||
# ]
|
||||
# end = progress(timeout, prefix=' ')
|
||||
# try:
|
||||
# result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # audio/audio.mp3
|
||||
# end()
|
||||
# if result.returncode:
|
||||
# print(' ', result.stderr.decode())
|
||||
# raise Exception('Failed to download audio')
|
||||
# chmod_file('audio.mp3', cwd=out_dir)
|
||||
# return 'audio.mp3'
|
||||
# except Exception as e:
|
||||
# end()
|
||||
# print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
|
||||
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
# raise
|
||||
# else:
|
||||
# print(' √ Skipping audio download')
|
||||
|
||||
# @attach_result_to_link('video')
|
||||
# def fetch_video(out_dir, link, timeout=TIMEOUT):
|
||||
# """Download video rip using youtube-dl"""
|
||||
|
||||
# if link['type'] not in ('youtube', 'youku', 'vimeo')\
|
||||
# and 'video' not in link['tags']:
|
||||
# return
|
||||
|
||||
# path = os.path.join(out_dir, 'video')
|
||||
|
||||
# if not os.path.exists(path) or overwrite:
|
||||
# print(' - Downloading video')
|
||||
# CMD = [
|
||||
# "youtube-dl -x --video-format mp4 --audio-quality 0 -o '%(title)s.%(ext)s'",
|
||||
# link['url'],
|
||||
# ]
|
||||
# end = progress(timeout, prefix=' ')
|
||||
# try:
|
||||
# result = run(CMD, stdout=DEVNULL, stderr=DEVNULL, cwd=out_dir, timeout=timeout + 1) # video/movie.mp4
|
||||
# end()
|
||||
# if result.returncode:
|
||||
# print(' ', result.stderr.decode())
|
||||
# raise Exception('Failed to download video')
|
||||
# chmod_file('video.mp4', cwd=out_dir)
|
||||
# return 'video.mp4'
|
||||
# except Exception as e:
|
||||
# end()
|
||||
# print(' Run to see full output:', 'cd {}; {}'.format(out_dir, ' '.join(CMD)))
|
||||
# print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
|
||||
# raise
|
||||
# else:
|
||||
# print(' √ Skipping video download')
|
||||
|
||||
|
||||
def chrome_data_dir_args(user_data_dir=CHROME_USER_DATA_DIR):
|
||||
default = os.path.expanduser('~/Library/Application Support/Google/Chrome/Default')
|
||||
if user_data_dir:
|
||||
return ('--user-data-dir={}'.format(user_data_dir),)
|
||||
elif os.path.exists(default):
|
||||
return ('--user-data-dir={}'.format(default),)
|
||||
return ()
|
||||
@@ -1,6 +0,0 @@
|
||||
[flake8]
|
||||
ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391
|
||||
select = F,E9,W
|
||||
max-line-length = 130
|
||||
max-complexity = 10
|
||||
exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
|
||||
@@ -1 +0,0 @@
|
||||
../README.md
|
||||
@@ -1,150 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Welcome to the ArchiveBox source code! Thanks for checking it out!
|
||||
#
|
||||
# "We are swimming upstream against a great torrent of disorganization.
|
||||
# In this, our main obligation is to establish arbitrary enclaves of order and system.
|
||||
# It is the greatest possible victory to be, to continue to be, and to have been.
|
||||
# No defeat can deprive us of the success of having existed for some moment of time
|
||||
# in a universe that seems indifferent to us."
|
||||
# --Norber Weiner
|
||||
|
||||
__package__ = "archivebox"
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Protocol, cast
|
||||
|
||||
|
||||
class _ReconfigurableStream(Protocol):
|
||||
def reconfigure(self, *, line_buffering: bool) -> object: ...
|
||||
|
||||
|
||||
# Force unbuffered output for real-time logs
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
|
||||
cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
|
||||
os.environ["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
ASCII_LOGO = """
|
||||
█████╗ ██████╗ ██████╗██╗ ██╗██╗██╗ ██╗███████╗ ██████╗ ██████╗ ██╗ ██╗
|
||||
██╔══██╗██╔══██╗██╔════╝██║ ██║██║██║ ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
|
||||
███████║██████╔╝██║ ███████║██║██║ ██║█████╗ ██████╔╝██║ ██║ ╚███╔╝
|
||||
██╔══██║██╔══██╗██║ ██╔══██║██║╚██╗ ██╔╝██╔══╝ ██╔══██╗██║ ██║ ██╔██╗
|
||||
██║ ██║██║ ██║╚██████╗██║ ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
|
||||
╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═══╝ ╚══════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝
|
||||
"""
|
||||
|
||||
PACKAGE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
|
||||
os.environ["TZ"] = "UTC"
|
||||
|
||||
# detect ArchiveBox user's UID/GID based on data dir ownership
|
||||
from .config.permissions import drop_privileges # noqa
|
||||
|
||||
drop_privileges()
|
||||
|
||||
from .misc.checks import check_not_root, check_io_encoding # noqa
|
||||
|
||||
check_not_root()
|
||||
check_io_encoding()
|
||||
|
||||
from .config.version import VERSION # noqa
|
||||
|
||||
|
||||
__version__ = VERSION
|
||||
__author__ = "ArchiveBox"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "CONSTANTS":
|
||||
from .config.constants import CONSTANTS
|
||||
|
||||
os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
|
||||
return CONSTANTS
|
||||
if name == "DATA_DIR":
|
||||
from .config.paths import DATA_DIR
|
||||
|
||||
return DATA_DIR
|
||||
if name == "VERSION":
|
||||
return VERSION
|
||||
if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
|
||||
from abx_plugins import get_plugins_dir
|
||||
from .config.constants import CONSTANTS
|
||||
|
||||
builtin_plugins_dir = Path(get_plugins_dir()).resolve()
|
||||
user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
|
||||
plugins = {
|
||||
"builtin": builtin_plugins_dir,
|
||||
"user": user_plugins_dir,
|
||||
}
|
||||
values = {
|
||||
"BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
|
||||
"USER_PLUGINS_DIR": user_plugins_dir,
|
||||
"ALL_PLUGINS": plugins,
|
||||
"LOADED_PLUGINS": plugins,
|
||||
}
|
||||
return values[name]
|
||||
raise AttributeError(name)
|
||||
|
||||
|
||||
__all__ = (
|
||||
"ASCII_LOGO",
|
||||
"ASCII_ICON",
|
||||
"PACKAGE_DIR",
|
||||
"DATA_DIR",
|
||||
"CONSTANTS",
|
||||
"VERSION",
|
||||
"BUILTIN_PLUGINS_DIR",
|
||||
"USER_PLUGINS_DIR",
|
||||
"ALL_PLUGINS",
|
||||
"LOADED_PLUGINS",
|
||||
)
|
||||
|
||||
ASCII_ICON = """
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ████████████████████████████████████ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ████████████████████████████████████ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██████████████████████████████████████████ ██
|
||||
██ ██████████████████████████████████████████ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
████████████████████████████████████████████████████████████████████████████████
|
||||
"""
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""This is the entrypoint for python -m archivebox ..."""
|
||||
|
||||
__package__ = "archivebox"
|
||||
|
||||
import archivebox # noqa # make sure monkey patches are applied before anything else
|
||||
import sys
|
||||
|
||||
from .cli import main
|
||||
|
||||
ASCII_LOGO_MINI = r"""
|
||||
_ _ _ ____
|
||||
/ \ _ __ ___| |__ (_)_ _____| __ ) _____ __
|
||||
/ _ \ | '__/ __| '_ \| \ \ / / _ \ _ \ / _ \ \/ /
|
||||
/ ___ \| | | (__| | | | |\ V / __/ |_) | (_) > <
|
||||
/_/ \_\_| \___|_| |_|_| \_/ \___|____/ \___/_/\_\
|
||||
"""
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(args=sys.argv[1:])
|
||||
@@ -1 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
@@ -1,120 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from django import forms
|
||||
from django.contrib import admin
|
||||
from django.http import HttpRequest
|
||||
from django.utils.text import capfirst
|
||||
from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
|
||||
from signal_webhooks.settings import webhook_settings
|
||||
from signal_webhooks.utils import get_webhook_model, model_from_reference
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
|
||||
from archivebox.api.models import APIToken
|
||||
|
||||
|
||||
def _webhook_fields(*names: str) -> tuple[str, ...]:
|
||||
model_fields = {field.name for field in get_webhook_model()._meta.fields}
|
||||
return tuple(name for name in names if name in model_fields)
|
||||
|
||||
|
||||
class APITokenAdmin(BaseModelAdmin):
|
||||
list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
|
||||
sort_fields = ("id", "created_at", "created_by", "expires")
|
||||
readonly_fields = ("created_at", "modified_at")
|
||||
search_fields = ("id", "created_by__username", "token")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Token",
|
||||
{
|
||||
"fields": ("token", "expires"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Owner",
|
||||
{
|
||||
"fields": ("created_by",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
list_filter = ("created_by",)
|
||||
ordering = ["-created_at"]
|
||||
list_per_page = 100
|
||||
|
||||
|
||||
class OutboundWebhookAdminForm(WebhookModelForm):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["ref"] = forms.ChoiceField(
|
||||
label=self.fields["ref"].label,
|
||||
help_text=self.fields["ref"].help_text,
|
||||
choices=[
|
||||
(ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
|
||||
for ref in sorted(webhook_settings.HOOKS)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
|
||||
form = OutboundWebhookAdminForm
|
||||
list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
|
||||
sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
|
||||
readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Webhook",
|
||||
{
|
||||
"fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Authentication",
|
||||
{
|
||||
"fields": _webhook_fields("auth_token"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Status",
|
||||
{
|
||||
"fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Owner",
|
||||
{
|
||||
"fields": _webhook_fields("created_by"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": _webhook_fields("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
|
||||
"""Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
|
||||
return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
|
||||
|
||||
|
||||
def register_admin(admin_site: admin.AdminSite) -> None:
|
||||
admin_site.register(APIToken, APITokenAdmin)
|
||||
admin_site.register(get_webhook_model(), CustomWebhookAdmin)
|
||||
@@ -1,19 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class APIConfig(AppConfig):
|
||||
name = "archivebox.api"
|
||||
label = "api"
|
||||
|
||||
def ready(self):
|
||||
from archivebox.api.webhooks import register_delete_webhook_field_loader
|
||||
|
||||
register_delete_webhook_field_loader()
|
||||
|
||||
|
||||
def register_admin(admin_site):
|
||||
from archivebox.api.admin import register_admin
|
||||
|
||||
register_admin(admin_site)
|
||||
@@ -1,137 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
from django.http import HttpRequest
|
||||
from django.contrib.auth import authenticate
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
|
||||
from ninja.errors import HttpError
|
||||
|
||||
|
||||
def get_or_create_api_token(user: User | None):
|
||||
from archivebox.api.models import APIToken
|
||||
|
||||
if user and user.is_superuser:
|
||||
api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
|
||||
if api_tokens.exists():
|
||||
# unexpired token exists, use it
|
||||
api_token = api_tokens.last()
|
||||
else:
|
||||
# does not exist, create a new one
|
||||
api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
|
||||
|
||||
if api_token is None:
|
||||
return None
|
||||
assert api_token.is_valid(), f"API token is not valid {api_token}"
|
||||
|
||||
return api_token
|
||||
return None
|
||||
|
||||
|
||||
def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
|
||||
"""Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
|
||||
from archivebox.api.models import APIToken # lazy import model to avoid loading it at urls.py import time
|
||||
|
||||
user: User | None = None
|
||||
|
||||
submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
|
||||
if not submitted_empty_form:
|
||||
try:
|
||||
api_token = APIToken.objects.get(token=token)
|
||||
if api_token.is_valid() and isinstance(api_token.created_by, User):
|
||||
user = api_token.created_by
|
||||
if request is not None:
|
||||
setattr(request, "_api_token", api_token)
|
||||
except APIToken.DoesNotExist:
|
||||
pass
|
||||
|
||||
return user
|
||||
|
||||
|
||||
def token_from_request(request: HttpRequest) -> str:
|
||||
token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
if not token and auth_header.lower().startswith("bearer "):
|
||||
token = auth_header.split(None, 1)[1].strip()
|
||||
return token
|
||||
|
||||
|
||||
def authenticated_user_from_request(request: HttpRequest) -> User | None:
|
||||
user = request.user
|
||||
if user.is_authenticated and user.is_active:
|
||||
return user
|
||||
|
||||
token = token_from_request(request)
|
||||
token_user = auth_using_token(token=token, request=request) if token else None
|
||||
if token_user and token_user.is_active:
|
||||
request.user = token_user
|
||||
return token_user
|
||||
return None
|
||||
|
||||
|
||||
def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
|
||||
"""Given a username and password, check if they are valid and return the corresponding user"""
|
||||
user: User | None = None
|
||||
|
||||
submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
|
||||
if not submitted_empty_form:
|
||||
authenticated_user = authenticate(
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
if isinstance(authenticated_user, User):
|
||||
user = authenticated_user
|
||||
return user
|
||||
|
||||
|
||||
### Base Auth Types
|
||||
|
||||
|
||||
def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
|
||||
if user and user.pk:
|
||||
request.user = user
|
||||
setattr(request, "_api_auth_method", auth_method)
|
||||
if not user.is_superuser:
|
||||
raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
|
||||
return user
|
||||
|
||||
|
||||
### Django-Ninja-Provided Auth Methods
|
||||
|
||||
|
||||
class HeaderTokenAuth(APIKeyHeader):
|
||||
"""Allow authenticating by passing X-API-Key=xyz as a request header"""
|
||||
|
||||
param_name = "X-ArchiveBox-API-Key"
|
||||
|
||||
def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
|
||||
return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
|
||||
|
||||
|
||||
class BearerTokenAuth(HttpBearer):
|
||||
"""Allow authenticating by passing Bearer=xyz as a request header"""
|
||||
|
||||
def authenticate(self, request: HttpRequest, token: str) -> User | None:
|
||||
return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
|
||||
|
||||
|
||||
class QueryParamTokenAuth(APIKeyQuery):
|
||||
"""Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
|
||||
|
||||
param_name = "api_key"
|
||||
|
||||
def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
|
||||
return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
|
||||
|
||||
|
||||
### Enabled Auth Methods
|
||||
|
||||
API_AUTH_METHODS = [
|
||||
HeaderTokenAuth(),
|
||||
BearerTokenAuth(),
|
||||
QueryParamTokenAuth(),
|
||||
# django_auth_superuser, # django admin cookie auth, not secure to use with csrf=False
|
||||
]
|
||||
@@ -1,32 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from django.http import HttpResponse
|
||||
|
||||
|
||||
class ApiCorsMiddleware:
|
||||
"""Attach permissive CORS headers for API routes (token-based auth)."""
|
||||
|
||||
def __init__(self, get_response):
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request):
|
||||
if request.path.startswith("/api/"):
|
||||
if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
|
||||
response = HttpResponse(status=204)
|
||||
return self._add_cors_headers(request, response)
|
||||
|
||||
response = self.get_response(request)
|
||||
return self._add_cors_headers(request, response)
|
||||
|
||||
return self.get_response(request)
|
||||
|
||||
def _add_cors_headers(self, request, response):
|
||||
origin = request.META.get("HTTP_ORIGIN")
|
||||
if not origin:
|
||||
return response
|
||||
|
||||
response["Access-Control-Allow-Origin"] = "*"
|
||||
response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
|
||||
response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
|
||||
response["Access-Control-Max-Age"] = "600"
|
||||
return response
|
||||
@@ -1,281 +0,0 @@
|
||||
# Generated by hand on 2025-12-29
|
||||
# Creates APIToken and OutboundWebhook tables using raw SQL
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.conf import settings
|
||||
from archivebox.uuid_compat import uuid7
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
import archivebox.api.models
|
||||
import signal_webhooks.fields
|
||||
import signal_webhooks.utils
|
||||
|
||||
|
||||
# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
|
||||
# RunSQL (identical statement splitting → byte-for-byte identical behavior).
|
||||
INITIAL_SQL = """
|
||||
-- Create api_apitoken table
|
||||
CREATE TABLE IF NOT EXISTS api_apitoken (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
modified_at DATETIME NOT NULL,
|
||||
|
||||
token VARCHAR(32) NOT NULL UNIQUE,
|
||||
expires DATETIME,
|
||||
|
||||
created_by_id INTEGER NOT NULL,
|
||||
|
||||
FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
|
||||
CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
|
||||
CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
|
||||
|
||||
-- Create api_outboundwebhook table
|
||||
CREATE TABLE IF NOT EXISTS api_outboundwebhook (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
modified_at DATETIME NOT NULL,
|
||||
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
signal VARCHAR(255) NOT NULL,
|
||||
ref VARCHAR(1024) NOT NULL,
|
||||
endpoint VARCHAR(2048) NOT NULL,
|
||||
headers TEXT NOT NULL DEFAULT '{}',
|
||||
auth_token TEXT NOT NULL DEFAULT '',
|
||||
enabled BOOLEAN NOT NULL DEFAULT 1,
|
||||
keep_last_response BOOLEAN NOT NULL DEFAULT 0,
|
||||
created DATETIME NOT NULL,
|
||||
updated DATETIME NOT NULL,
|
||||
last_response TEXT NOT NULL DEFAULT '',
|
||||
last_success DATETIME,
|
||||
last_failure DATETIME,
|
||||
|
||||
created_by_id INTEGER NOT NULL,
|
||||
|
||||
FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
|
||||
"""
|
||||
|
||||
INITIAL_REVERSE_SQL = """
|
||||
DROP TABLE IF EXISTS api_outboundwebhook;
|
||||
DROP TABLE IF EXISTS api_apitoken;
|
||||
"""
|
||||
|
||||
|
||||
def _run_sqlite_only_sql(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "sqlite":
|
||||
return
|
||||
migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
|
||||
"api",
|
||||
schema_editor,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _run_sqlite_only_sql_reverse(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "sqlite":
|
||||
return
|
||||
migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
|
||||
"api",
|
||||
schema_editor,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _pg_sync_schema(apps, schema_editor):
|
||||
from archivebox.misc.db import rebuild_models_from_migration_state
|
||||
|
||||
rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
|
||||
|
||||
|
||||
def _pg_drop_schema(apps, schema_editor):
|
||||
from archivebox.misc.db import drop_models_on_postgres
|
||||
|
||||
drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.SeparateDatabaseAndState(
|
||||
database_operations=[
|
||||
migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
|
||||
],
|
||||
state_operations=[
|
||||
migrations.CreateModel(
|
||||
name="APIToken",
|
||||
fields=[
|
||||
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
||||
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
||||
("modified_at", models.DateTimeField(auto_now=True)),
|
||||
("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
|
||||
("expires", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"created_by",
|
||||
models.ForeignKey(
|
||||
default=get_or_create_system_user_pk,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "API Key",
|
||||
"verbose_name_plural": "API Keys",
|
||||
"app_label": "api",
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="OutboundWebhook",
|
||||
fields=[
|
||||
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
||||
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
||||
("modified_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"name",
|
||||
models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
|
||||
),
|
||||
(
|
||||
"signal",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("CREATE", "Create"),
|
||||
("UPDATE", "Update"),
|
||||
("DELETE", "Delete"),
|
||||
("M2M", "M2M changed"),
|
||||
("CREATE_OR_UPDATE", "Create or Update"),
|
||||
("CREATE_OR_DELETE", "Create or Delete"),
|
||||
("CREATE_OR_M2M", "Create or M2M changed"),
|
||||
("UPDATE_OR_DELETE", "Update or Delete"),
|
||||
("UPDATE_OR_M2M", "Update or M2M changed"),
|
||||
("DELETE_OR_M2M", "Delete or M2M changed"),
|
||||
("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
|
||||
("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
|
||||
("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
|
||||
("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
|
||||
("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
|
||||
],
|
||||
help_text="Signal the webhook fires to.",
|
||||
max_length=255,
|
||||
verbose_name="signal",
|
||||
),
|
||||
),
|
||||
(
|
||||
"ref",
|
||||
models.CharField(
|
||||
db_index=True,
|
||||
help_text="Dot import notation to the model the webhook is for.",
|
||||
max_length=1023,
|
||||
validators=[signal_webhooks.utils.model_from_reference],
|
||||
verbose_name="referenced model",
|
||||
),
|
||||
),
|
||||
(
|
||||
"endpoint",
|
||||
models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
|
||||
),
|
||||
(
|
||||
"headers",
|
||||
models.JSONField(
|
||||
blank=True,
|
||||
default=dict,
|
||||
help_text="Headers to send with the webhook request.",
|
||||
validators=[signal_webhooks.utils.is_dict],
|
||||
verbose_name="headers",
|
||||
),
|
||||
),
|
||||
(
|
||||
"auth_token",
|
||||
signal_webhooks.fields.TokenField(
|
||||
blank=True,
|
||||
default="",
|
||||
help_text="Authentication token to use in an Authorization header.",
|
||||
max_length=8000,
|
||||
validators=[signal_webhooks.utils.decode_cipher_key],
|
||||
verbose_name="authentication token",
|
||||
),
|
||||
),
|
||||
("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
|
||||
(
|
||||
"keep_last_response",
|
||||
models.BooleanField(
|
||||
default=False,
|
||||
help_text="Should the webhook keep a log of the latest response it got?",
|
||||
verbose_name="keep last response",
|
||||
),
|
||||
),
|
||||
(
|
||||
"created",
|
||||
models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
|
||||
),
|
||||
(
|
||||
"updated",
|
||||
models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
|
||||
),
|
||||
(
|
||||
"last_response",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
default="",
|
||||
help_text="Latest response to this webhook.",
|
||||
max_length=8000,
|
||||
verbose_name="last response",
|
||||
),
|
||||
),
|
||||
(
|
||||
"last_success",
|
||||
models.DateTimeField(
|
||||
default=None,
|
||||
help_text="When the webhook last succeeded.",
|
||||
null=True,
|
||||
verbose_name="last success",
|
||||
),
|
||||
),
|
||||
(
|
||||
"last_failure",
|
||||
models.DateTimeField(
|
||||
default=None,
|
||||
help_text="When the webhook last failed.",
|
||||
null=True,
|
||||
verbose_name="last failure",
|
||||
),
|
||||
),
|
||||
(
|
||||
"created_by",
|
||||
models.ForeignKey(
|
||||
default=get_or_create_system_user_pk,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "API Outbound Webhook",
|
||||
"app_label": "api",
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="outboundwebhook",
|
||||
constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
|
||||
),
|
||||
],
|
||||
),
|
||||
# On non-sqlite backends the raw DDL above is skipped, so the real
|
||||
# schema diverges from Django state. Rebuild both tables (including the
|
||||
# unique constraint) from the post-migration state (no-op on sqlite).
|
||||
migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
|
||||
]
|
||||
@@ -1,54 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
import secrets
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
from signal_webhooks.models import WebhookBase
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
|
||||
|
||||
def generate_secret_token() -> str:
|
||||
return secrets.token_hex(16)
|
||||
|
||||
|
||||
class APIToken(models.Model):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
|
||||
expires = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
app_label = "api"
|
||||
verbose_name = "API Key"
|
||||
verbose_name_plural = "API Keys"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.token
|
||||
|
||||
@property
|
||||
def token_redacted(self):
|
||||
return f"************{self.token[-4:]}"
|
||||
|
||||
def is_valid(self, for_date=None):
|
||||
return not self.expires or self.expires >= (for_date or timezone.now())
|
||||
|
||||
|
||||
class OutboundWebhook(WebhookBase):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta(WebhookBase.Meta):
|
||||
app_label = "api"
|
||||
verbose_name = "API Outbound Webhook"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"[{self.id}] {self.ref} -> {self.endpoint}"
|
||||
@@ -1,26 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from urllib.parse import quote
|
||||
|
||||
from django.http import HttpRequest, HttpResponseRedirect
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import path
|
||||
from django.views.generic.base import RedirectView
|
||||
|
||||
from archivebox.core.routes_util import build_web_url
|
||||
from .v1_api import urls as v1_api_urls
|
||||
|
||||
|
||||
def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
|
||||
if request.META.get("QUERY_STRING"):
|
||||
url = f"{url}?{request.META['QUERY_STRING']}"
|
||||
return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
|
||||
|
||||
|
||||
urlpatterns = [
|
||||
path("", RedirectView.as_view(url="/api/v1/docs")),
|
||||
path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
|
||||
path("v1/", RedirectView.as_view(url="/api/v1/docs")),
|
||||
path("v1/", v1_api_urls),
|
||||
path("v1", RedirectView.as_view(url="/api/v1/docs")),
|
||||
]
|
||||
@@ -1,127 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
|
||||
from io import StringIO
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
from traceback import format_exception
|
||||
from contextlib import redirect_stdout, redirect_stderr
|
||||
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from ninja import NinjaAPI, Swagger
|
||||
from ninja.openapi.schema import OpenAPISchema
|
||||
|
||||
# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
|
||||
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.version import get_COMMIT_HASH
|
||||
|
||||
from archivebox.api.auth import API_AUTH_METHODS
|
||||
from archivebox.api.models import APIToken
|
||||
|
||||
|
||||
COMMIT_HASH = get_COMMIT_HASH() or "unknown"
|
||||
|
||||
html_description = f"""
|
||||
<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
|
||||
<br/>
|
||||
<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
|
||||
<br/>
|
||||
<ul>
|
||||
<li>⬅️ Manage your server: <a href="/admin/api/"><b>Setup API Keys</b></a>, <a href="/admin/">Go to your Server Admin UI</a>, <a href="/">Go to your Snapshots list</a>
|
||||
<li>💬 Ask questions and get help here: <a href="https://zulip.archivebox.io">ArchiveBox Chat Forum</a></li>
|
||||
<li>🐞 Report API bugs here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues">Github Issues</a></li>
|
||||
<li>📚 ArchiveBox Documentation: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Github Wiki</a></li>
|
||||
<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
|
||||
</ul>
|
||||
<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
|
||||
"""
|
||||
|
||||
|
||||
def register_urls(api: NinjaAPI) -> NinjaAPI:
|
||||
api.add_router("/auth/", "archivebox.api.v1_auth.router")
|
||||
api.add_router("/core/", "archivebox.api.v1_core.router")
|
||||
api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
|
||||
api.add_router("/cli/", "archivebox.api.v1_cli.router")
|
||||
api.add_router("/machine/", "archivebox.api.v1_machine.router")
|
||||
api.add_router("/personas/", "archivebox.api.v1_personas.router")
|
||||
return api
|
||||
|
||||
|
||||
class NinjaAPIWithIOCapture(NinjaAPI):
|
||||
def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
|
||||
prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
|
||||
return self._schema_for_prefix(prefix)
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
|
||||
# Routes are registered at startup. Server discovery can reuse their
|
||||
# schema while preserving distinct URL prefixes for mounted instances.
|
||||
return super().get_openapi_schema(path_prefix=path_prefix)
|
||||
|
||||
def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
|
||||
stdout, stderr = StringIO(), StringIO()
|
||||
|
||||
with redirect_stderr(stderr):
|
||||
with redirect_stdout(stdout):
|
||||
setattr(request, "stdout", stdout)
|
||||
setattr(request, "stderr", stderr)
|
||||
|
||||
response = super().create_temporal_response(request)
|
||||
|
||||
# Disable caching of API responses entirely
|
||||
response["Cache-Control"] = "no-store"
|
||||
|
||||
# Add debug stdout and stderr headers to response
|
||||
response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
|
||||
response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
|
||||
# response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
|
||||
|
||||
# Add Auth Headers to response
|
||||
api_token_attr = request.__dict__.get("_api_token")
|
||||
api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
|
||||
token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
|
||||
|
||||
response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
|
||||
response["X-ArchiveBox-Auth-Expires"] = token_expiry
|
||||
response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
|
||||
response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
|
||||
response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
|
||||
|
||||
return response
|
||||
|
||||
|
||||
api = NinjaAPIWithIOCapture(
|
||||
title="ArchiveBox API",
|
||||
description=html_description,
|
||||
version=VERSION,
|
||||
auth=API_AUTH_METHODS,
|
||||
urls_namespace="api-1",
|
||||
docs=Swagger(settings={"persistAuthorization": True}),
|
||||
)
|
||||
api = register_urls(api)
|
||||
urls = api.urls
|
||||
|
||||
|
||||
@api.exception_handler(Exception)
|
||||
def generic_exception_handler(request, err):
|
||||
status = 503
|
||||
if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
|
||||
status = 404
|
||||
|
||||
print("".join(format_exception(err)))
|
||||
|
||||
return api.create_response(
|
||||
request,
|
||||
{
|
||||
"succeeded": False,
|
||||
"message": f"{err.__class__.__name__}: {err}",
|
||||
"errors": [
|
||||
"".join(format_exception(err)),
|
||||
],
|
||||
},
|
||||
status=status,
|
||||
)
|
||||
@@ -1,106 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from django.http import HttpRequest
|
||||
|
||||
from ninja import Router, Schema
|
||||
|
||||
from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
|
||||
|
||||
|
||||
router = Router(tags=["Authentication"], auth=None)
|
||||
|
||||
|
||||
class PasswordAuthSchema(Schema):
|
||||
"""Schema for a /get_api_token request"""
|
||||
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
|
||||
|
||||
@router.post(
|
||||
"/get_api_token",
|
||||
auth=None,
|
||||
summary="Generate an API token for a given username & password (or currently logged-in user)",
|
||||
) # auth=None because they are not authed yet
|
||||
def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
|
||||
user = auth_using_password(
|
||||
username=auth_data.username,
|
||||
password=auth_data.password,
|
||||
request=request,
|
||||
)
|
||||
|
||||
if user and user.is_superuser:
|
||||
api_token = get_or_create_api_token(user)
|
||||
assert api_token is not None, "Failed to create API token"
|
||||
return {
|
||||
"success": True,
|
||||
"user_id": str(user.pk),
|
||||
"username": user.username,
|
||||
"token": api_token.token,
|
||||
"expires": api_token.expires.isoformat() if api_token.expires else None,
|
||||
}
|
||||
|
||||
return {"success": False, "errors": ["Invalid credentials"]}
|
||||
|
||||
|
||||
class TokenAuthSchema(Schema):
|
||||
"""Schema for a /check_api_token request"""
|
||||
|
||||
token: str
|
||||
|
||||
|
||||
@router.post(
|
||||
"/check_api_token",
|
||||
auth=None,
|
||||
summary="Validate an API token to make sure its valid and non-expired",
|
||||
) # auth=None because they are not authed yet
|
||||
def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
|
||||
user = auth_using_token(
|
||||
token=token_data.token,
|
||||
request=request,
|
||||
)
|
||||
if user:
|
||||
return {"success": True, "user_id": str(user.pk)}
|
||||
|
||||
return {"success": False, "user_id": None}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/browser_session",
|
||||
auth=[HeaderTokenAuth(), BearerTokenAuth()],
|
||||
summary="Exchange an administrator API key for a normal browser session",
|
||||
)
|
||||
def browser_session(request: HttpRequest):
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import login
|
||||
from django.http import JsonResponse
|
||||
from django.utils import timezone
|
||||
from ninja.errors import HttpError
|
||||
from archivebox.core.routes_util import get_admin_base_url
|
||||
|
||||
user = request.auth
|
||||
if not user.is_active or not user.is_superuser:
|
||||
raise HttpError(403, "An active administrator is required")
|
||||
login(request, user, backend="django.contrib.auth.backends.ModelBackend")
|
||||
# A browser session must not outlive the key used to establish it.
|
||||
token = request._api_token
|
||||
lifetime = settings.SESSION_COOKIE_AGE
|
||||
if token.expires:
|
||||
lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
|
||||
request.session.set_expiry(lifetime)
|
||||
request.session.save()
|
||||
response = JsonResponse(
|
||||
{
|
||||
"admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
|
||||
"cookie": {
|
||||
"name": settings.SESSION_COOKIE_NAME,
|
||||
"value": request.session.session_key,
|
||||
"expires": request.session.get_expiry_date().timestamp(),
|
||||
"secure": settings.SESSION_COOKIE_SECURE,
|
||||
},
|
||||
},
|
||||
)
|
||||
# The native client installs a host-only HttpOnly cookie on the admin origin;
|
||||
# API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
|
||||
response["Cache-Control"] = "no-store"
|
||||
return response
|
||||
@@ -1,329 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
import json
|
||||
from io import StringIO
|
||||
from typing import Any
|
||||
from enum import Enum
|
||||
|
||||
from django.http import HttpRequest
|
||||
|
||||
from ninja import Router, Schema
|
||||
from ninja.errors import HttpError
|
||||
from pydantic import Field
|
||||
|
||||
from archivebox.misc.util import ansi_to_html
|
||||
from archivebox.core.models import SnapshotQuerySet
|
||||
|
||||
|
||||
# router for API that exposes archivebox cli subcommands as REST endpoints
|
||||
router = Router(tags=["ArchiveBox CLI Sub-Commands"])
|
||||
|
||||
|
||||
# Schemas
|
||||
|
||||
JSONType = list[Any] | dict[str, Any] | bool | int | str | None
|
||||
FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
|
||||
|
||||
|
||||
class CLICommandResponseSchema(Schema):
|
||||
success: bool
|
||||
errors: list[str]
|
||||
result: JSONType
|
||||
result_format: str = "str"
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
FilterTypeChoices = Enum(
|
||||
"FilterTypeChoices",
|
||||
{filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
|
||||
type=str,
|
||||
)
|
||||
|
||||
|
||||
class AddCommandSchema(Schema):
|
||||
urls: list[str]
|
||||
snapshot_ids: list[str] | None = None
|
||||
tag: str = ""
|
||||
depth: int = 0
|
||||
max_urls: int = 0
|
||||
crawl_max_size: int = 0
|
||||
crawl_timeout: int = 0
|
||||
snapshot_max_size: int = 0
|
||||
parser: str = "auto"
|
||||
plugins: str = ""
|
||||
persona: str = "Default"
|
||||
only_new: bool | None = None
|
||||
update: bool = False
|
||||
overwrite: bool = False
|
||||
index_only: bool = False
|
||||
|
||||
|
||||
class SnapshotFilterCommandSchema(Schema):
|
||||
after: float | None = 0
|
||||
before: float | None = None
|
||||
filter_type: str | None = FilterTypeChoices.substring
|
||||
filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
|
||||
status: str | None = None
|
||||
url__icontains: str | None = None
|
||||
url__istartswith: str | None = None
|
||||
tag: str | None = None
|
||||
crawl_id: str | None = None
|
||||
limit: int | None = None
|
||||
sort: str | None = None
|
||||
search: str | None = None
|
||||
|
||||
|
||||
class UpdateCommandSchema(SnapshotFilterCommandSchema):
|
||||
resume: str | None = None
|
||||
batch_size: int = 100
|
||||
continuous: bool = False
|
||||
index_only: bool = False
|
||||
migrate_only: bool = False
|
||||
|
||||
|
||||
class ScheduleCommandSchema(Schema):
|
||||
import_path: str | None = None
|
||||
add: bool = False
|
||||
show: bool = False
|
||||
foreground: bool = False
|
||||
run_all: bool = False
|
||||
quiet: bool = False
|
||||
every: str | None = None
|
||||
tag: str = ""
|
||||
depth: int = 0
|
||||
only_new: bool | None = None
|
||||
update: bool = False
|
||||
overwrite: bool = False
|
||||
clear: bool = False
|
||||
|
||||
|
||||
class ListCommandSchema(SnapshotFilterCommandSchema):
|
||||
as_json: bool = True
|
||||
as_html: bool = False
|
||||
as_csv: str | None = "timestamp,url"
|
||||
with_headers: bool = False
|
||||
|
||||
|
||||
class RemoveCommandSchema(SnapshotFilterCommandSchema):
|
||||
filter_type: str = FilterTypeChoices.exact
|
||||
timeout: float = 60.0
|
||||
|
||||
|
||||
def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
|
||||
kwargs = args.dict()
|
||||
kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
|
||||
kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
|
||||
return kwargs
|
||||
|
||||
|
||||
@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
|
||||
def cli_add(request: HttpRequest, args: AddCommandSchema):
|
||||
from archivebox.cli.archivebox_add import add
|
||||
from archivebox.misc.util import validate_url
|
||||
|
||||
config_overrides: dict[str, object] = {}
|
||||
if args.only_new is not None:
|
||||
config_overrides["ONLY_NEW"] = bool(args.only_new)
|
||||
if args.update or args.overwrite:
|
||||
config_overrides["ONLY_NEW"] = False
|
||||
submitted_urls: str | list[str] = args.urls
|
||||
if len(args.urls) == 1:
|
||||
try:
|
||||
validate_url(args.urls[0])
|
||||
except ValueError:
|
||||
submitted_urls = args.urls[0]
|
||||
else:
|
||||
submitted_urls = []
|
||||
for url in args.urls:
|
||||
try:
|
||||
submitted_urls.append(validate_url(url))
|
||||
except ValueError:
|
||||
continue
|
||||
if not submitted_urls:
|
||||
raise HttpError(400, "No valid URLs were submitted")
|
||||
crawl, snapshots = add(
|
||||
urls=submitted_urls,
|
||||
snapshot_ids=args.snapshot_ids,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
max_urls=args.max_urls,
|
||||
crawl_max_size=args.crawl_max_size,
|
||||
crawl_timeout=args.crawl_timeout,
|
||||
snapshot_max_size=args.snapshot_max_size,
|
||||
index_only=args.index_only,
|
||||
plugins=args.plugins,
|
||||
persona=args.persona,
|
||||
parser=args.parser,
|
||||
bg=True, # Always run in background for API calls
|
||||
created_by_id=request.user.pk,
|
||||
config=config_overrides or None,
|
||||
)
|
||||
|
||||
snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
|
||||
result_payload = {
|
||||
"crawl_id": str(crawl.id),
|
||||
"num_snapshots": len(snapshot_ids),
|
||||
"snapshot_ids": snapshot_ids,
|
||||
"queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
|
||||
}
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result_payload,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
|
||||
def cli_update(request: HttpRequest, args: UpdateCommandSchema):
|
||||
from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
|
||||
from archivebox.core.snapshot_status import normalize_snapshot_status
|
||||
|
||||
try:
|
||||
status = normalize_snapshot_status(args.status)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
|
||||
update_kwargs["status"] = status
|
||||
update_kwargs["stop_daemon_stack"] = False
|
||||
|
||||
is_filtered_update = any(
|
||||
(update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
|
||||
)
|
||||
matched_snapshot_ids = []
|
||||
if is_filtered_update:
|
||||
matched_snapshot_ids = [
|
||||
str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
|
||||
]
|
||||
|
||||
update(**update_kwargs)
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": {
|
||||
"matched_count": len(matched_snapshot_ids),
|
||||
"snapshot_ids": matched_snapshot_ids,
|
||||
}
|
||||
if is_filtered_update
|
||||
else None,
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
|
||||
def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
|
||||
from archivebox.cli.archivebox_schedule import schedule
|
||||
|
||||
config_overrides: dict[str, object] = {}
|
||||
if args.only_new is not None:
|
||||
config_overrides["ONLY_NEW"] = bool(args.only_new)
|
||||
if args.update or args.overwrite:
|
||||
config_overrides["ONLY_NEW"] = False
|
||||
result = schedule(
|
||||
import_path=args.import_path,
|
||||
add=args.add,
|
||||
show=args.show,
|
||||
foreground=args.foreground,
|
||||
run_all=args.run_all,
|
||||
quiet=args.quiet,
|
||||
clear=args.clear,
|
||||
every=args.every,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
config=config_overrides or None,
|
||||
)
|
||||
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
|
||||
def cli_search(request: HttpRequest, args: ListCommandSchema):
|
||||
from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
|
||||
|
||||
search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
|
||||
as_json = search_kwargs.pop("as_json")
|
||||
as_html = search_kwargs.pop("as_html")
|
||||
as_csv = search_kwargs.pop("as_csv")
|
||||
with_headers = search_kwargs.pop("with_headers")
|
||||
try:
|
||||
snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
result_format = "txt"
|
||||
if as_json:
|
||||
result_format = "json"
|
||||
result = [
|
||||
json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
|
||||
for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
|
||||
]
|
||||
elif as_html:
|
||||
result_format = "html"
|
||||
result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
|
||||
elif as_csv:
|
||||
result_format = "csv"
|
||||
cols = [col.strip() for col in as_csv.split(",") if col.strip()]
|
||||
rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
|
||||
result = "\n".join((",".join(cols), *rows) if with_headers else rows)
|
||||
else:
|
||||
result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
|
||||
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": result_format,
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
|
||||
def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
|
||||
from archivebox.cli.archivebox_remove import remove
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
|
||||
timeout_arg = remove_kwargs.pop("timeout")
|
||||
timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
|
||||
try:
|
||||
snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
result = remove(
|
||||
yes=True, # no way to interactively ask for confirmation via API, so we force yes
|
||||
snapshots=snapshots_to_remove,
|
||||
timeout=timeout,
|
||||
)
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": bool(result["success"]),
|
||||
"errors": [str(result["error"])] if result["error"] else [],
|
||||
"result": result,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,278 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
from datetime import datetime
|
||||
from django.http import FileResponse, HttpRequest
|
||||
from django.shortcuts import redirect
|
||||
from django.utils import timezone
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from ninja import Router, Schema
|
||||
from ninja.errors import HttpError
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_PUBLIC,
|
||||
PERMISSIONS_UNLISTED,
|
||||
is_admin_user,
|
||||
normalize_permissions,
|
||||
)
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
|
||||
|
||||
from .auth import API_AUTH_METHODS, authenticated_user_from_request
|
||||
|
||||
router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
|
||||
|
||||
|
||||
class CrawlSchema(Schema):
|
||||
TYPE: str = "crawls.models.Crawl"
|
||||
|
||||
id: UUID
|
||||
|
||||
modified_at: datetime
|
||||
created_at: datetime
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
|
||||
status: str
|
||||
retry_at: datetime | None
|
||||
is_paused: bool
|
||||
|
||||
urls: str
|
||||
max_depth: int
|
||||
tags_str: str
|
||||
config: dict
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by_id)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj):
|
||||
user_model = get_user_model()
|
||||
user = user_model.objects.get(id=obj.created_by_id)
|
||||
username = user.username
|
||||
return username if isinstance(username, str) else str(user)
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses can never leak the raw
|
||||
# token/secret/api-key that the operator stored in Crawl.config.
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return redact_sensitive_config(obj.config)
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshots(obj, context):
|
||||
if bool(context["request"].__dict__.get("with_snapshots", False)):
|
||||
return obj.snapshot_set.all().distinct()
|
||||
return Snapshot.objects.none()
|
||||
|
||||
|
||||
class CrawlUpdateSchema(Schema):
|
||||
action: str | None = None
|
||||
status: str | None = None
|
||||
retry_at: datetime | None = None
|
||||
tags: list[str] | None = None
|
||||
tags_str: str | None = None
|
||||
|
||||
|
||||
class CrawlCreateSchema(Schema):
|
||||
urls: list[str]
|
||||
max_depth: int = 0
|
||||
tags: list[str] | None = None
|
||||
tags_str: str = ""
|
||||
label: str = ""
|
||||
notes: str = ""
|
||||
config: dict = {}
|
||||
|
||||
|
||||
class CrawlDeleteResponseSchema(Schema):
|
||||
success: bool
|
||||
crawl_id: str
|
||||
deleted_count: int
|
||||
deleted_snapshots: int
|
||||
|
||||
|
||||
def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
|
||||
if tags is not None:
|
||||
return [tag.strip() for tag in tags if tag and tag.strip()]
|
||||
return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
|
||||
|
||||
|
||||
@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
|
||||
def get_crawls(request: HttpRequest):
|
||||
return Crawl.objects.all().distinct()
|
||||
|
||||
|
||||
@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
|
||||
def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
|
||||
try:
|
||||
urls = [validate_url(url) for url in data.urls if url and url.strip()]
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
if not urls:
|
||||
raise HttpError(400, "At least one URL is required")
|
||||
if data.max_depth not in (0, 1, 2, 3, 4):
|
||||
raise HttpError(400, "max_depth must be between 0 and 4")
|
||||
|
||||
tags = normalize_tag_list(data.tags, data.tags_str)
|
||||
config = dict(data.config or {})
|
||||
config.setdefault("PARSER", "url_list")
|
||||
config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
|
||||
crawl = Crawl.objects.create(
|
||||
urls="\n".join(urls),
|
||||
max_depth=data.max_depth,
|
||||
tags_str=",".join(tags),
|
||||
label=data.label,
|
||||
notes=data.notes,
|
||||
config=config,
|
||||
status=Crawl.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
created_by=request.user if isinstance(request.user, User) else None,
|
||||
)
|
||||
return crawl
|
||||
|
||||
|
||||
def get_crawl_by_ref(crawl_id: str):
|
||||
return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
|
||||
def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
|
||||
"""Get a specific Crawl by id."""
|
||||
setattr(request, "with_snapshots", with_snapshots)
|
||||
setattr(request, "with_archiveresults", with_archiveresults)
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
|
||||
if crawl and as_rss:
|
||||
query = request.GET.copy()
|
||||
query.pop("as_rss", None)
|
||||
query["crawl_id"] = str(crawl.id)
|
||||
return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
|
||||
|
||||
return crawl
|
||||
|
||||
|
||||
def crawl_file(request: HttpRequest, crawl_id: str, path: str):
|
||||
# Try to resolve the crawl first; if it doesn't exist, return 404.
|
||||
try:
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
except Crawl.DoesNotExist:
|
||||
raise HttpError(404, "Crawl not found")
|
||||
|
||||
user = authenticated_user_from_request(request)
|
||||
|
||||
# Gate access using the same model as SnapshotView/can_view_snapshot:
|
||||
# admins always pass; owners can see their own crawls; otherwise the crawl
|
||||
# must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
|
||||
if not is_admin_user(request):
|
||||
permissions = normalize_permissions(crawl.permissions)
|
||||
is_owner = bool(user and crawl.created_by_id == user.id)
|
||||
if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
|
||||
raise HttpError(404, "Crawl not found")
|
||||
|
||||
crawl_root = Path(crawl.output_dir).resolve()
|
||||
file_path = (crawl_root / path).resolve()
|
||||
if not file_path.is_file() or crawl_root not in file_path.parents:
|
||||
raise HttpError(404, "Crawl file not found")
|
||||
|
||||
if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
|
||||
try:
|
||||
started_at = float(request.GET["after"])
|
||||
except ValueError:
|
||||
raise HttpError(400, "Invalid screencast start time")
|
||||
if file_path.stat().st_mtime < started_at:
|
||||
raise HttpError(404, "No frame for this capture yet")
|
||||
|
||||
response = FileResponse(file_path.open("rb"))
|
||||
response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
|
||||
response["Pragma"] = "no-cache"
|
||||
response["Expires"] = "0"
|
||||
response["X-Content-Type-Options"] = "nosniff"
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
|
||||
def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
|
||||
return crawl_file(request, crawl_id, filename)
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
|
||||
def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
|
||||
return crawl_file(request, crawl_id, f"{folder}/{filename}")
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
|
||||
def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
|
||||
return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
|
||||
|
||||
|
||||
@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
|
||||
def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
|
||||
"""Update a crawl (e.g., set status=sealed to cancel queued work)."""
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
payload = data.dict(exclude_unset=True)
|
||||
update_fields = ["modified_at"]
|
||||
|
||||
action = payload.pop("action", None)
|
||||
if action:
|
||||
if action == "pause":
|
||||
crawl.pause()
|
||||
return crawl
|
||||
if action in ("resume", "unpause"):
|
||||
crawl.resume()
|
||||
return crawl
|
||||
if action == "cancel":
|
||||
crawl.cancel()
|
||||
return crawl
|
||||
raise HttpError(400, f"Invalid action: {action}")
|
||||
|
||||
tags = payload.pop("tags", None)
|
||||
tags_str = payload.pop("tags_str", None)
|
||||
if tags is not None or tags_str is not None:
|
||||
crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
|
||||
update_fields.append("tags_str")
|
||||
|
||||
if "status" in payload:
|
||||
if payload["status"] not in Crawl.StatusChoices.values:
|
||||
raise HttpError(400, f"Invalid status: {payload['status']}")
|
||||
if payload["status"] == Crawl.StatusChoices.SEALED:
|
||||
crawl.cancel()
|
||||
return crawl
|
||||
crawl.status = payload["status"]
|
||||
update_fields.append("status")
|
||||
|
||||
if "retry_at" in payload:
|
||||
crawl.retry_at = payload["retry_at"]
|
||||
update_fields.append("retry_at")
|
||||
|
||||
crawl.save(update_fields=update_fields)
|
||||
return crawl
|
||||
|
||||
|
||||
@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
|
||||
def delete_crawl(request: HttpRequest, crawl_id: str):
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
crawl_id_str = str(crawl.id)
|
||||
crawl.cancel()
|
||||
|
||||
from archivebox.crawls.locks import crawl_lifecycle_lock
|
||||
from archivebox.services.runner import run_pending_crawls
|
||||
|
||||
with crawl_lifecycle_lock(crawl_id_str):
|
||||
run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
|
||||
crawl = get_crawl_by_ref(crawl_id_str)
|
||||
snapshot_count = crawl.snapshot_set.count()
|
||||
deleted_count, _ = crawl.delete()
|
||||
return {
|
||||
"success": True,
|
||||
"crawl_id": crawl_id_str,
|
||||
"deleted_count": deleted_count,
|
||||
"deleted_snapshots": snapshot_count,
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from uuid import UUID
|
||||
from typing import Annotated
|
||||
from datetime import datetime
|
||||
|
||||
from django.http import HttpRequest
|
||||
|
||||
from ninja import FilterLookup, FilterSchema, Query, Router, Schema
|
||||
from ninja.pagination import paginate
|
||||
|
||||
from archivebox.api.v1_core import CustomPagination
|
||||
|
||||
|
||||
router = Router(tags=["Machine and Dependencies"])
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Machine Schemas
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class MachineSchema(Schema):
|
||||
"""Schema for Machine model."""
|
||||
|
||||
TYPE: str = "machine.Machine"
|
||||
id: UUID
|
||||
created_at: datetime
|
||||
modified_at: datetime
|
||||
guid: str
|
||||
hostname: str
|
||||
hw_in_docker: bool
|
||||
hw_in_vm: bool
|
||||
hw_manufacturer: str
|
||||
hw_product: str
|
||||
hw_uuid: str
|
||||
os_arch: str
|
||||
os_family: str
|
||||
os_platform: str
|
||||
os_release: str
|
||||
os_kernel: str
|
||||
stats: dict
|
||||
num_uses_succeeded: int
|
||||
num_uses_failed: int
|
||||
|
||||
|
||||
class MachineFilterSchema(FilterSchema):
|
||||
id: Annotated[str | None, FilterLookup("id__startswith")] = None
|
||||
hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
|
||||
os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
|
||||
os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
|
||||
hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
|
||||
hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
|
||||
bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Binary Schemas
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BinarySchema(Schema):
|
||||
"""Schema for Binary model."""
|
||||
|
||||
TYPE: str = "machine.Binary"
|
||||
id: UUID
|
||||
created_at: datetime
|
||||
modified_at: datetime
|
||||
machine_id: UUID
|
||||
machine_hostname: str
|
||||
name: str
|
||||
binproviders: str
|
||||
binprovider: str
|
||||
abspath: str
|
||||
version: str
|
||||
sha256: str
|
||||
status: str
|
||||
is_valid: bool
|
||||
num_uses_succeeded: int
|
||||
num_uses_failed: int
|
||||
|
||||
@staticmethod
|
||||
def resolve_machine_hostname(obj) -> str:
|
||||
return obj.machine.hostname
|
||||
|
||||
@staticmethod
|
||||
def resolve_is_valid(obj) -> bool:
|
||||
return obj.is_valid
|
||||
|
||||
|
||||
class BinaryFilterSchema(FilterSchema):
|
||||
id: Annotated[str | None, FilterLookup("id__startswith")] = None
|
||||
name: Annotated[str | None, FilterLookup("name__icontains")] = None
|
||||
binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
|
||||
status: Annotated[str | None, FilterLookup("status")] = None
|
||||
machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
|
||||
version: Annotated[str | None, FilterLookup("version__icontains")] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Machine Endpoints
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
|
||||
@paginate(CustomPagination)
|
||||
def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
|
||||
"""List all machines."""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
return filters.filter(Machine.objects.all()).distinct()
|
||||
|
||||
|
||||
@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
|
||||
def get_current_machine(request: HttpRequest):
|
||||
"""Get the current machine."""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
return Machine.current()
|
||||
|
||||
|
||||
@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
|
||||
def get_machine(request: HttpRequest, machine_id: str):
|
||||
"""Get a specific machine by ID."""
|
||||
from archivebox.machine.models import Machine
|
||||
from django.db.models import Q
|
||||
|
||||
return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Binary Endpoints
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
|
||||
@paginate(CustomPagination)
|
||||
def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
|
||||
"""List all binaries."""
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
return filters.filter(Binary.objects.all().select_related("machine")).distinct()
|
||||
|
||||
|
||||
@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
|
||||
def get_binary(request: HttpRequest, binary_id: str):
|
||||
"""Get a specific binary by ID."""
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
return Binary.objects.select_related("machine").get(id__startswith=binary_id)
|
||||
|
||||
|
||||
@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
|
||||
def get_binaries_by_name(request: HttpRequest, name: str):
|
||||
"""Get all binaries with the given name."""
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
|
||||
@@ -1,176 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db.models import Q
|
||||
from django.http import HttpRequest
|
||||
from ninja import Router, Schema
|
||||
from ninja.errors import HttpError
|
||||
from ninja.pagination import paginate
|
||||
from pydantic import Field
|
||||
|
||||
from archivebox.api.v1_core import CustomPagination
|
||||
from archivebox.personas.importers import validate_persona_name
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
|
||||
router = Router(tags=["Personas"])
|
||||
|
||||
|
||||
class PersonaBrowserSettingsSchema(Schema):
|
||||
user_agent: str = ""
|
||||
viewport_size: str = ""
|
||||
viewport_device_scale_factor: float | None = None
|
||||
color_scheme: Literal["", "light", "dark"] = ""
|
||||
language: str = ""
|
||||
timezone: str = ""
|
||||
geolocation: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class PersonaSyncSchema(Schema):
|
||||
extension_persona_id: str
|
||||
name: str
|
||||
settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
|
||||
cookies_txt: str = ""
|
||||
auth_json: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class PersonaSchema(Schema):
|
||||
TYPE: str = "personas.models.Persona"
|
||||
id: UUID
|
||||
name: str
|
||||
created_at: datetime
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
config: dict[str, Any] | None
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by.pk)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj) -> str:
|
||||
return obj.created_by.username
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses don't leak the raw
|
||||
# token/secret/api-key the operator stored in Persona.config.
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return redact_sensitive_config(obj.config)
|
||||
|
||||
|
||||
class PersonaSyncResponseSchema(Schema):
|
||||
success: bool
|
||||
created: bool
|
||||
persona: PersonaSchema
|
||||
cookies_file_written: bool
|
||||
auth_file_written: bool
|
||||
|
||||
|
||||
def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
|
||||
config: dict[str, Any] = {
|
||||
"BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
|
||||
"BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
|
||||
if settings.user_agent:
|
||||
config.update(
|
||||
{
|
||||
"USER_AGENT": settings.user_agent,
|
||||
"CHROME_USER_AGENT": settings.user_agent,
|
||||
"WGET_USER_AGENT": settings.user_agent,
|
||||
"CURL_USER_AGENT": settings.user_agent,
|
||||
},
|
||||
)
|
||||
if settings.viewport_size:
|
||||
config.update(
|
||||
{
|
||||
"RESOLUTION": settings.viewport_size,
|
||||
"CHROME_RESOLUTION": settings.viewport_size,
|
||||
},
|
||||
)
|
||||
if settings.viewport_device_scale_factor is not None:
|
||||
config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
|
||||
if "color_scheme" in settings.model_fields_set:
|
||||
config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
|
||||
if settings.language:
|
||||
config["BROWSER_LANGUAGE"] = settings.language
|
||||
if settings.timezone:
|
||||
config["BROWSER_TIMEZONE"] = settings.timezone
|
||||
if settings.geolocation:
|
||||
config["BROWSER_GEOLOCATION"] = settings.geolocation
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def find_persona(extension_persona_id: str, name: str) -> Persona | None:
|
||||
named = Persona.find_named(name)
|
||||
return (
|
||||
Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
|
||||
.order_by("created_at")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
|
||||
@paginate(CustomPagination)
|
||||
def get_personas(request: HttpRequest):
|
||||
"""List personas available on this ArchiveBox server."""
|
||||
return Persona.objects.all().order_by("name")
|
||||
|
||||
|
||||
@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
|
||||
def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
|
||||
"""
|
||||
Create or update a Persona from a browser extension profile export.
|
||||
|
||||
The extension sends browser settings plus portable auth artifacts. The server
|
||||
keeps browser override settings in Persona.config and writes cookies.txt /
|
||||
auth.json into the persona directory for extractors to consume.
|
||||
"""
|
||||
name = payload.name.strip()
|
||||
is_valid, error_message = validate_persona_name(name)
|
||||
if not is_valid:
|
||||
raise ValueError(error_message)
|
||||
|
||||
try:
|
||||
persona = find_persona(payload.extension_persona_id, name)
|
||||
(persona or Persona(name=name)).validate_name(persona.name if persona else name)
|
||||
except ValidationError as err:
|
||||
raise HttpError(409, "; ".join(err.messages)) from err
|
||||
created = persona is None
|
||||
if persona is None:
|
||||
persona = Persona(name=name)
|
||||
if request.user.is_authenticated:
|
||||
persona.created_by = request.user
|
||||
|
||||
persona.config = {
|
||||
**(persona.config or {}),
|
||||
**browser_settings_to_config(payload.extension_persona_id, payload.settings),
|
||||
}
|
||||
persona.save()
|
||||
persona.ensure_dirs()
|
||||
|
||||
cookies_written = False
|
||||
if "cookies_txt" in payload.model_fields_set:
|
||||
(persona.path / "cookies.txt").write_text(payload.cookies_txt)
|
||||
cookies_written = True
|
||||
|
||||
auth_written = False
|
||||
if "auth_json" in payload.model_fields_set:
|
||||
(persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
|
||||
auth_written = True
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"created": created,
|
||||
"persona": persona,
|
||||
"cookies_file_written": cookies_written,
|
||||
"auth_file_written": auth_written,
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from typing import Any
|
||||
from collections.abc import Callable
|
||||
import logging
|
||||
|
||||
from django.db import transaction
|
||||
from django.db.models.signals import pre_delete
|
||||
from signal_webhooks.handlers import sync_task_handler
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
|
||||
"""Materialize deferred fields while DELETE webhook rows still exist.
|
||||
|
||||
Admin lists defer large fields, but signal_webhooks serializes all concrete
|
||||
fields in post_delete. Loading one then would query an already-deleted row
|
||||
and drop the event. Hydrate only deferred fields on webhook-enabled models
|
||||
here, preserving the normal list-query optimization.
|
||||
"""
|
||||
from signal_webhooks.handlers import find_hook_handler
|
||||
from signal_webhooks.utils import reference_for_model
|
||||
|
||||
ref = reference_for_model(type(instance))
|
||||
deferred_fields = instance.get_deferred_fields()
|
||||
if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
|
||||
return
|
||||
|
||||
instance.refresh_from_db(using=using, fields=deferred_fields)
|
||||
|
||||
|
||||
def register_delete_webhook_field_loader() -> None:
|
||||
pre_delete.connect(
|
||||
load_deferred_fields_for_delete_webhook,
|
||||
dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
|
||||
weak=False,
|
||||
)
|
||||
|
||||
|
||||
def warning_error_handler(hook: Any, error: Exception | None) -> None:
|
||||
if error is not None:
|
||||
logger.warning("Outbound webhook %r failed: %s", hook.name, error)
|
||||
return
|
||||
|
||||
logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
|
||||
|
||||
|
||||
def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
|
||||
def run_webhook() -> None:
|
||||
try:
|
||||
sync_task_handler(hook, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
|
||||
|
||||
try:
|
||||
transaction.on_commit(run_webhook)
|
||||
except Exception:
|
||||
logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
|
||||
@@ -1 +0,0 @@
|
||||
__package__ = "archivebox.base_models"
|
||||
@@ -1,930 +0,0 @@
|
||||
"""Base admin classes for models using UUIDv7."""
|
||||
|
||||
__package__ = "archivebox.base_models"
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from typing import ClassVar, NotRequired, TypedDict, cast
|
||||
|
||||
from django import forms
|
||||
from django.contrib import admin
|
||||
from django.db import DatabaseError, models
|
||||
from django.forms.renderers import BaseRenderer
|
||||
from django.http import HttpRequest, QueryDict
|
||||
from django.urls import path, register_converter
|
||||
from django.utils.safestring import SafeString, mark_safe
|
||||
from django_object_actions import DjangoObjectActions
|
||||
|
||||
|
||||
class HexUUIDConverter:
|
||||
"""URL path converter that canonicalizes UUIDs to their 32-char hex form.
|
||||
|
||||
Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
|
||||
(``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
|
||||
parses either), but ``to_url`` always emits the bare-hex form. This is
|
||||
what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
|
||||
``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
|
||||
hyphenated rendering — admin links throughout the app reverse through
|
||||
this converter once ``BaseModelAdmin.get_urls`` swaps in
|
||||
``<hexuuid:object_id>`` below.
|
||||
"""
|
||||
|
||||
regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
|
||||
def to_python(self, value: str) -> str:
|
||||
# Strip hyphens but stay as a string — Django admin treats the
|
||||
# captured object_id as a string and calls ``model._meta.pk.to_python``
|
||||
# itself, so we don't want to short-circuit that.
|
||||
return value.replace("-", "")
|
||||
|
||||
def to_url(self, value) -> str:
|
||||
if isinstance(value, uuid.UUID):
|
||||
return value.hex
|
||||
return str(value).replace("-", "")
|
||||
|
||||
|
||||
register_converter(HexUUIDConverter, "hexuuid")
|
||||
|
||||
|
||||
class ConfigOption(TypedDict):
|
||||
plugin: str
|
||||
type: str | list[str]
|
||||
default: object
|
||||
description: str
|
||||
enum: NotRequired[list[object]]
|
||||
pattern: NotRequired[str]
|
||||
minimum: NotRequired[int | float]
|
||||
maximum: NotRequired[int | float]
|
||||
|
||||
|
||||
class KeyValueWidget(forms.Widget):
|
||||
"""
|
||||
A widget that renders JSON dict as editable key-value input fields
|
||||
with + and - buttons to add/remove rows.
|
||||
Includes autocomplete for available config keys from the plugin system.
|
||||
"""
|
||||
|
||||
template_name = "" # We render manually
|
||||
|
||||
class Media:
|
||||
css: ClassVar[dict[str, list[str]]] = {
|
||||
"all": [],
|
||||
}
|
||||
js: ClassVar[list[str]] = []
|
||||
|
||||
def _get_config_options(self) -> dict[str, ConfigOption]:
|
||||
"""Get available config options from plugins."""
|
||||
try:
|
||||
from archivebox.config.common import config_field_metadata
|
||||
|
||||
options: dict[str, ConfigOption] = {}
|
||||
for key, metadata in config_field_metadata().items():
|
||||
option_type = metadata.get("type", "string")
|
||||
option: ConfigOption = {
|
||||
"plugin": str(metadata.get("plugin", "archivebox")),
|
||||
"type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
|
||||
"default": metadata.get("default", ""),
|
||||
"description": str(metadata.get("description", "")),
|
||||
}
|
||||
schema = metadata.get("schema")
|
||||
if isinstance(schema, Mapping):
|
||||
for schema_key in ("enum", "pattern", "minimum", "maximum"):
|
||||
if schema_key in schema:
|
||||
option[schema_key] = schema[schema_key]
|
||||
options[key] = option
|
||||
return options
|
||||
except (ImportError, KeyError, TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
def _parse_value(self, value: object) -> dict[str, object]:
|
||||
# Parse JSON value to dict
|
||||
if value is None:
|
||||
return {}
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value) if value else {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): item for key, item in value.items()}
|
||||
return {}
|
||||
|
||||
def render(
|
||||
self,
|
||||
name: str,
|
||||
value: object,
|
||||
attrs: Mapping[str, str] | None = None,
|
||||
renderer: BaseRenderer | None = None,
|
||||
) -> SafeString:
|
||||
data = self._parse_value(value)
|
||||
|
||||
widget_id = attrs.get("id", name) if attrs else name
|
||||
config_options = self._get_config_options()
|
||||
|
||||
# Build datalist options
|
||||
datalist_options = "\n".join(
|
||||
f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
|
||||
for key, opt in sorted(config_options.items())
|
||||
)
|
||||
|
||||
# Build config metadata as JSON for JS
|
||||
config_meta_json = json.dumps(config_options)
|
||||
|
||||
html = f'''
|
||||
<div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
|
||||
<datalist id="{widget_id}_keys">
|
||||
{datalist_options}
|
||||
</datalist>
|
||||
<div id="{widget_id}_rows" class="key-value-rows">
|
||||
'''
|
||||
|
||||
# Render existing key-value pairs
|
||||
for key, val in data.items():
|
||||
val_str = json.dumps(val) if not isinstance(val, str) else val
|
||||
html += self._render_row(widget_id, key, val_str)
|
||||
|
||||
# Always add one empty row for new entries
|
||||
html += self._render_row(widget_id, "", "")
|
||||
|
||||
html += f'''
|
||||
</div>
|
||||
<div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
|
||||
<button type="button" onclick="addKeyValueRow_{widget_id}()"
|
||||
style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
|
||||
+ Add Row
|
||||
</button>
|
||||
</div>
|
||||
<input type="hidden" name="{name}" id="{widget_id}" value="">
|
||||
<script>
|
||||
(function() {{
|
||||
var configMeta_{widget_id} = {config_meta_json};
|
||||
var rowCounter_{widget_id} = 0;
|
||||
|
||||
function stringifyValue_{widget_id}(value) {{
|
||||
return typeof value === 'string' ? value : JSON.stringify(value);
|
||||
}}
|
||||
|
||||
function getTypes_{widget_id}(meta) {{
|
||||
if (!meta || meta.type === undefined || meta.type === null) {{
|
||||
return [];
|
||||
}}
|
||||
return Array.isArray(meta.type) ? meta.type : [meta.type];
|
||||
}}
|
||||
|
||||
function getMetaForKey_{widget_id}(key) {{
|
||||
if (!key) {{
|
||||
return null;
|
||||
}}
|
||||
|
||||
var explicitMeta = configMeta_{widget_id}[key];
|
||||
if (explicitMeta) {{
|
||||
return Object.assign({{ key: key }}, explicitMeta);
|
||||
}}
|
||||
|
||||
if (key.endsWith('_BINARY')) {{
|
||||
return {{
|
||||
key: key,
|
||||
plugin: 'custom',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Path to binary executable',
|
||||
}};
|
||||
}}
|
||||
|
||||
if (isRegexConfigKey_{widget_id}(key)) {{
|
||||
return {{
|
||||
key: key,
|
||||
plugin: 'custom',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Regex pattern list',
|
||||
}};
|
||||
}}
|
||||
|
||||
return null;
|
||||
}}
|
||||
|
||||
function describeMeta_{widget_id}(meta) {{
|
||||
if (!meta) {{
|
||||
return '';
|
||||
}}
|
||||
|
||||
var details = '';
|
||||
if (Array.isArray(meta.enum) && meta.enum.length) {{
|
||||
details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
|
||||
}} else {{
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (types.length) {{
|
||||
details = 'Expected: ' + types.join(' or ');
|
||||
}}
|
||||
}}
|
||||
|
||||
if (meta.minimum !== undefined || meta.maximum !== undefined) {{
|
||||
var bounds = [];
|
||||
if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
|
||||
if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
|
||||
details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
|
||||
}}
|
||||
|
||||
return [meta.description || '', details].filter(Boolean).join(' ');
|
||||
}}
|
||||
|
||||
function getExampleInput_{widget_id}(key, meta) {{
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (key.endsWith('_BINARY')) {{
|
||||
return 'Example: wget or /usr/bin/wget';
|
||||
}}
|
||||
if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
|
||||
return 'Example: ["--extra-arg"]';
|
||||
}}
|
||||
if (types.includes('array')) {{
|
||||
return 'Example: ["value"]';
|
||||
}}
|
||||
if (types.includes('object')) {{
|
||||
return 'Example: {{"key": "value"}}';
|
||||
}}
|
||||
return '';
|
||||
}}
|
||||
|
||||
function isRegexConfigKey_{widget_id}(key) {{
|
||||
return key === 'URL_ALLOWLIST' ||
|
||||
key === 'URL_DENYLIST' ||
|
||||
key.endsWith('_PATTERN') ||
|
||||
key.includes('REGEX');
|
||||
}}
|
||||
|
||||
function isSimpleFilterPattern_{widget_id}(pattern) {{
|
||||
return /^[\\w.*:-]+$/.test(pattern);
|
||||
}}
|
||||
|
||||
function validateRegexPattern_{widget_id}(pattern) {{
|
||||
if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
|
||||
return '';
|
||||
}}
|
||||
|
||||
try {{
|
||||
new RegExp(pattern);
|
||||
}} catch (error) {{
|
||||
return error && error.message ? error.message : 'Invalid regex';
|
||||
}}
|
||||
return '';
|
||||
}}
|
||||
|
||||
function validateRegexConfig_{widget_id}(key, raw, typeName) {{
|
||||
if (typeName === 'object') {{
|
||||
var parsed;
|
||||
try {{
|
||||
parsed = JSON.parse(raw);
|
||||
}} catch (error) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
|
||||
}}
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
|
||||
return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
|
||||
}}
|
||||
for (var regexKey in parsed) {{
|
||||
var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
|
||||
if (objectRegexError) {{
|
||||
return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
|
||||
}}
|
||||
}}
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
|
||||
var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
|
||||
return pattern.trim();
|
||||
}}).filter(Boolean);
|
||||
for (var i = 0; i < patterns.length; i++) {{
|
||||
var regexError = validateRegexPattern_{widget_id}(patterns[i]);
|
||||
if (regexError) {{
|
||||
return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
|
||||
}}
|
||||
}}
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
function validateBinaryValue_{widget_id}(raw) {{
|
||||
if (!raw) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
if (/['"`]/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
|
||||
}}
|
||||
|
||||
if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
|
||||
}}
|
||||
|
||||
if (raw.startsWith('/')) {{
|
||||
if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
|
||||
}}
|
||||
|
||||
if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
|
||||
}}
|
||||
|
||||
function parseValue_{widget_id}(raw) {{
|
||||
try {{
|
||||
if (raw === 'true') return true;
|
||||
if (raw === 'false') return false;
|
||||
if (raw === 'null') return null;
|
||||
if (raw !== '' && !isNaN(raw)) return Number(raw);
|
||||
if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
|
||||
(raw.startsWith('[') && raw.endsWith(']')) ||
|
||||
(raw.startsWith('"') && raw.endsWith('"'))) {{
|
||||
return JSON.parse(raw);
|
||||
}}
|
||||
}} catch (error) {{
|
||||
return raw;
|
||||
}}
|
||||
return raw;
|
||||
}}
|
||||
|
||||
function sameValue_{widget_id}(left, right) {{
|
||||
return left === right || JSON.stringify(left) === JSON.stringify(right);
|
||||
}}
|
||||
|
||||
function parseTypedValue_{widget_id}(raw, typeName, meta) {{
|
||||
var numberValue;
|
||||
var parsed;
|
||||
|
||||
if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
|
||||
return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
|
||||
}}
|
||||
|
||||
if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
|
||||
return validateBinaryValue_{widget_id}(raw);
|
||||
}}
|
||||
|
||||
if (typeName === 'string') {{
|
||||
if (meta.pattern) {{
|
||||
try {{
|
||||
if (!(new RegExp(meta.pattern)).test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
|
||||
}}
|
||||
}} catch (error) {{}}
|
||||
}}
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'integer') {{
|
||||
if (!/^-?\\d+$/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be an integer' }};
|
||||
}}
|
||||
numberValue = Number(raw);
|
||||
if (meta.minimum !== undefined && numberValue < meta.minimum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
|
||||
}}
|
||||
if (meta.maximum !== undefined && numberValue > meta.maximum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
|
||||
}}
|
||||
return {{ ok: true, value: numberValue, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'number') {{
|
||||
if (raw === '' || isNaN(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be a number' }};
|
||||
}}
|
||||
numberValue = Number(raw);
|
||||
if (meta.minimum !== undefined && numberValue < meta.minimum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
|
||||
}}
|
||||
if (meta.maximum !== undefined && numberValue > meta.maximum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
|
||||
}}
|
||||
return {{ ok: true, value: numberValue, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'boolean') {{
|
||||
var lowered = raw.toLowerCase();
|
||||
if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
|
||||
if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
|
||||
return {{ ok: false, value: raw, message: 'Must be true or false' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'null') {{
|
||||
return raw === 'null'
|
||||
? {{ ok: true, value: null, message: '' }}
|
||||
: {{ ok: false, value: raw, message: 'Must be null' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'array' || typeName === 'object') {{
|
||||
try {{
|
||||
parsed = JSON.parse(raw);
|
||||
}} catch (error) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'array' && Array.isArray(parsed)) {{
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
|
||||
return {{
|
||||
ok: false,
|
||||
value: parsed,
|
||||
message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
|
||||
}};
|
||||
}}
|
||||
|
||||
return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
|
||||
}}
|
||||
|
||||
function validateValueAgainstMeta_{widget_id}(raw, meta) {{
|
||||
if (!meta || raw === '') {{
|
||||
return {{ state: 'neutral', value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (!types.length) {{
|
||||
types = ['string'];
|
||||
}}
|
||||
|
||||
var error = 'Invalid value';
|
||||
for (var i = 0; i < types.length; i++) {{
|
||||
var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
|
||||
if (!candidate.ok) {{
|
||||
error = candidate.message || error;
|
||||
continue;
|
||||
}}
|
||||
if (enumValues.length && !enumValues.some(function(enumValue) {{
|
||||
return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
|
||||
}})) {{
|
||||
error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
|
||||
continue;
|
||||
}}
|
||||
return {{ state: 'valid', value: candidate.value, message: '' }};
|
||||
}}
|
||||
|
||||
return {{ state: 'invalid', value: raw, message: error }};
|
||||
}}
|
||||
|
||||
function ensureRowId_{widget_id}(row) {{
|
||||
if (!row.dataset.rowId) {{
|
||||
row.dataset.rowId = String(rowCounter_{widget_id}++);
|
||||
}}
|
||||
return row.dataset.rowId;
|
||||
}}
|
||||
|
||||
function setRowHelp_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var help = row.querySelector('.kv-help');
|
||||
if (!keyInput || !help) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var key = keyInput.value.trim();
|
||||
if (!key) {{
|
||||
help.textContent = '';
|
||||
return;
|
||||
}}
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (meta) {{
|
||||
var extra = isRegexConfigKey_{widget_id}(key)
|
||||
? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
|
||||
? ' Expected: JSON object with regex keys.'
|
||||
: ' Expected: valid regex.')
|
||||
: '';
|
||||
var example = getExampleInput_{widget_id}(key, meta);
|
||||
help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
|
||||
}} else {{
|
||||
help.textContent = 'Custom key';
|
||||
}}
|
||||
}}
|
||||
|
||||
function configureValueInput_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valueInput = row.querySelector('.kv-value');
|
||||
var datalist = row.querySelector('.kv-value-options');
|
||||
if (!keyInput || !valueInput || !datalist) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var rowId = ensureRowId_{widget_id}(row);
|
||||
datalist.id = '{widget_id}_value_options_' + rowId;
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
|
||||
var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (!enumValues.length && types.includes('boolean')) {{
|
||||
enumValues = ['True', 'False'];
|
||||
}}
|
||||
if (enumValues.length) {{
|
||||
datalist.innerHTML = enumValues.map(function(enumValue) {{
|
||||
return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '"') + '"></option>';
|
||||
}}).join('');
|
||||
valueInput.setAttribute('list', datalist.id);
|
||||
}} else {{
|
||||
datalist.innerHTML = '';
|
||||
valueInput.removeAttribute('list');
|
||||
}}
|
||||
}}
|
||||
|
||||
function setValueValidationState_{widget_id}(input, state, message) {{
|
||||
if (!input) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
if (state === 'valid') {{
|
||||
input.style.borderColor = '#2da44e';
|
||||
input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
|
||||
input.style.backgroundColor = '#f6ffed';
|
||||
}} else if (state === 'invalid') {{
|
||||
input.style.borderColor = '#cf222e';
|
||||
input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
|
||||
input.style.backgroundColor = '#fff8f8';
|
||||
}} else {{
|
||||
input.style.borderColor = '#ccc';
|
||||
input.style.boxShadow = 'none';
|
||||
input.style.backgroundColor = '';
|
||||
}}
|
||||
input.title = message || '';
|
||||
}}
|
||||
|
||||
function applyValueValidation_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valueInput = row.querySelector('.kv-value');
|
||||
if (!keyInput || !valueInput) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var key = keyInput.value.trim();
|
||||
if (!key) {{
|
||||
setValueValidationState_{widget_id}(valueInput, 'neutral', '');
|
||||
return;
|
||||
}}
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (!meta) {{
|
||||
setValueValidationState_{widget_id}(valueInput, 'neutral', '');
|
||||
return;
|
||||
}}
|
||||
|
||||
var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
|
||||
setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
|
||||
}}
|
||||
|
||||
function coerceValueForStorage_{widget_id}(key, raw) {{
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (!meta) {{
|
||||
return parseValue_{widget_id}(raw);
|
||||
}}
|
||||
|
||||
var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
|
||||
return validation.state === 'valid' ? validation.value : raw;
|
||||
}}
|
||||
|
||||
function initializeRows_{widget_id}() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
container.querySelectorAll('.key-value-row').forEach(function(row) {{
|
||||
ensureRowId_{widget_id}(row);
|
||||
configureValueInput_{widget_id}(row);
|
||||
setRowHelp_{widget_id}(row);
|
||||
applyValueValidation_{widget_id}(row);
|
||||
}});
|
||||
}}
|
||||
|
||||
function updateHiddenField_{widget_id}() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
var rows = container.querySelectorAll('.key-value-row');
|
||||
var result = {{}};
|
||||
rows.forEach(function(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valInput = row.querySelector('.kv-value');
|
||||
if (keyInput && valInput && keyInput.value.trim()) {{
|
||||
var key = keyInput.value.trim();
|
||||
var val = valInput.value.trim();
|
||||
result[key] = coerceValueForStorage_{widget_id}(key, val);
|
||||
}}
|
||||
}});
|
||||
document.getElementById('{widget_id}').value = JSON.stringify(result);
|
||||
}}
|
||||
|
||||
window.addKeyValueRow_{widget_id} = function() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
var newRow = document.createElement('div');
|
||||
newRow.className = 'key-value-row';
|
||||
newRow.style.cssText = 'margin-bottom: 6px;';
|
||||
newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
|
||||
'<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
|
||||
'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
|
||||
'<input type="text" class="kv-value" placeholder="value" ' +
|
||||
'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
|
||||
'<datalist class="kv-value-options"></datalist>' +
|
||||
'<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
|
||||
'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
|
||||
'</div>' +
|
||||
'<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
|
||||
container.appendChild(newRow);
|
||||
ensureRowId_{widget_id}(newRow);
|
||||
configureValueInput_{widget_id}(newRow);
|
||||
setRowHelp_{widget_id}(newRow);
|
||||
applyValueValidation_{widget_id}(newRow);
|
||||
updateHiddenField_{widget_id}();
|
||||
newRow.querySelector('.kv-key').focus();
|
||||
}};
|
||||
|
||||
window.removeKeyValueRow_{widget_id} = function(btn) {{
|
||||
var row = btn.closest('.key-value-row');
|
||||
row.remove();
|
||||
updateHiddenField_{widget_id}();
|
||||
}};
|
||||
|
||||
window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
|
||||
|
||||
function configRowForKey_{widget_id}(key) {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
if (!container) {{
|
||||
return null;
|
||||
}}
|
||||
var match = null;
|
||||
container.querySelectorAll('.key-value-row').forEach(function(row) {{
|
||||
if (match) {{ return; }}
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
if (keyInput && keyInput.value.trim() === key) {{
|
||||
match = row;
|
||||
}}
|
||||
}});
|
||||
if (!match) {{
|
||||
window.addKeyValueRow_{widget_id}();
|
||||
var rows = container.querySelectorAll('.key-value-row');
|
||||
match = rows[rows.length - 1];
|
||||
var keyInput = match.querySelector('.kv-key');
|
||||
if (keyInput) {{
|
||||
keyInput.value = key;
|
||||
keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
}}
|
||||
}}
|
||||
return match;
|
||||
}}
|
||||
|
||||
function prefillConfigFromQuery_{widget_id}() {{
|
||||
var params = new URLSearchParams(window.location.search);
|
||||
var consumedConfigKeys = [];
|
||||
params.forEach(function(value, key) {{
|
||||
if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
|
||||
return;
|
||||
}}
|
||||
consumedConfigKeys.push(key);
|
||||
var match = configRowForKey_{widget_id}(key);
|
||||
var valueInput = match && match.querySelector('.kv-value');
|
||||
if (valueInput) {{
|
||||
valueInput.value = value;
|
||||
valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
}}
|
||||
}});
|
||||
updateHiddenField_{widget_id}();
|
||||
if (consumedConfigKeys.length) {{
|
||||
var cleanUrl = new URL(window.location.href);
|
||||
consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
|
||||
window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
|
||||
}}
|
||||
}}
|
||||
|
||||
function focusConfigKeyFromHash_{widget_id}() {{
|
||||
// Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
|
||||
// to (or creates) the matching row in this editor. Used by
|
||||
// the setup wizard and live-config detail pages.
|
||||
var hash = (window.location.hash || '').replace(/^#/, '').trim();
|
||||
if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
|
||||
return;
|
||||
}}
|
||||
var match = configRowForKey_{widget_id}(hash);
|
||||
if (!match) {{
|
||||
return;
|
||||
}}
|
||||
match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
|
||||
var prevOutline = match.style.outline;
|
||||
match.style.outline = '2px solid #f59e0b';
|
||||
match.style.outlineOffset = '2px';
|
||||
match.style.transition = 'outline 1.2s ease-out';
|
||||
setTimeout(function() {{
|
||||
match.style.outline = prevOutline || 'none';
|
||||
}}, 1400);
|
||||
var valueInput = match.querySelector('.kv-value');
|
||||
if (valueInput) {{
|
||||
valueInput.focus();
|
||||
try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
|
||||
}}
|
||||
}}
|
||||
|
||||
// Initialize on load
|
||||
document.addEventListener('DOMContentLoaded', function() {{
|
||||
initializeRows_{widget_id}();
|
||||
prefillConfigFromQuery_{widget_id}();
|
||||
updateHiddenField_{widget_id}();
|
||||
focusConfigKeyFromHash_{widget_id}();
|
||||
}});
|
||||
// Also run immediately in case DOM is already ready
|
||||
if (document.readyState !== 'loading') {{
|
||||
initializeRows_{widget_id}();
|
||||
prefillConfigFromQuery_{widget_id}();
|
||||
updateHiddenField_{widget_id}();
|
||||
focusConfigKeyFromHash_{widget_id}();
|
||||
}}
|
||||
|
||||
window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
|
||||
|
||||
// Update on any input change
|
||||
var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
|
||||
|
||||
rowsEl_{widget_id}.addEventListener('input', function(event) {{
|
||||
var row = event.target.closest('.key-value-row');
|
||||
if (!row) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
if (event.target.classList.contains('kv-key')) {{
|
||||
configureValueInput_{widget_id}(row);
|
||||
setRowHelp_{widget_id}(row);
|
||||
}}
|
||||
|
||||
if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
|
||||
applyValueValidation_{widget_id}(row);
|
||||
updateHiddenField_{widget_id}();
|
||||
}}
|
||||
}});
|
||||
}})();
|
||||
</script>
|
||||
</div>
|
||||
'''
|
||||
return mark_safe(html)
|
||||
|
||||
def _render_row(self, widget_id: str, key: str, value: str) -> str:
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
# Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
|
||||
# rendered write-only: the input is a password field with a placeholder
|
||||
# showing the value is set, but the raw value is NEVER sent to the browser.
|
||||
# When the user submits the form with the field left blank, the
|
||||
# ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
|
||||
# value so leaving it untouched is a no-op rather than a destructive clear.
|
||||
is_sensitive = is_sensitive_config_key(key)
|
||||
has_value = bool(value)
|
||||
if is_sensitive:
|
||||
input_type = "password"
|
||||
rendered_value = ""
|
||||
placeholder = (
|
||||
"•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
|
||||
)
|
||||
extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
|
||||
else:
|
||||
input_type = "text"
|
||||
rendered_value = self._escape(value)
|
||||
placeholder = "value"
|
||||
extra_attrs = ""
|
||||
return f'''
|
||||
<div class="key-value-row" style="margin-bottom: 6px;">
|
||||
<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
|
||||
<input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
|
||||
style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
|
||||
<input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
|
||||
style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
|
||||
<datalist class="kv-value-options"></datalist>
|
||||
<button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
|
||||
style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
|
||||
</div>
|
||||
<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
|
||||
</div>
|
||||
'''
|
||||
|
||||
def _escape(self, s: object) -> str:
|
||||
"""Escape HTML special chars in attribute values."""
|
||||
if not s:
|
||||
return ""
|
||||
return str(s).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||
|
||||
def value_from_datadict(
|
||||
self,
|
||||
data: QueryDict | Mapping[str, object],
|
||||
files: object,
|
||||
name: str,
|
||||
) -> str:
|
||||
value = data.get(name, "{}")
|
||||
return value if isinstance(value, str) else "{}"
|
||||
|
||||
|
||||
class ConfigEditorMixin(admin.ModelAdmin):
|
||||
"""
|
||||
Mixin for admin classes with a config JSON field.
|
||||
|
||||
Provides a key-value editor widget with autocomplete for available config keys.
|
||||
"""
|
||||
|
||||
def formfield_for_dbfield(
|
||||
self,
|
||||
db_field: models.Field,
|
||||
request: HttpRequest,
|
||||
**kwargs: object,
|
||||
) -> forms.Field | None:
|
||||
"""Use KeyValueWidget for the config JSON field."""
|
||||
if db_field.name == "config":
|
||||
kwargs["widget"] = KeyValueWidget()
|
||||
return super().formfield_for_dbfield(db_field, request, **kwargs)
|
||||
|
||||
def save_model(self, request: HttpRequest, obj, form, change):
|
||||
"""Preserve write-only redacted credentials on save.
|
||||
|
||||
The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
|
||||
``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
|
||||
the real value never leaves the server. On submit, an empty value
|
||||
for a sensitive key that was previously set means "leave untouched",
|
||||
not "clear it." We honor that here by re-merging the stored value
|
||||
before the row is written. Explicitly removing the row in the UI
|
||||
still clears it (the key is gone from the submitted JSON, so there's
|
||||
nothing to merge over).
|
||||
"""
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
if change and obj.pk and obj.config is not None:
|
||||
try:
|
||||
stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
|
||||
except (AttributeError, DatabaseError, TypeError, ValueError):
|
||||
stored = {}
|
||||
if isinstance(stored, dict):
|
||||
new_config = dict(obj.config or {})
|
||||
for key, new_value in list(new_config.items()):
|
||||
if not is_sensitive_config_key(key):
|
||||
continue
|
||||
if new_value not in (None, "") and new_value != "********":
|
||||
continue
|
||||
if key in stored:
|
||||
new_config[key] = stored[key]
|
||||
obj.config = new_config
|
||||
super().save_model(request, obj, form, change)
|
||||
|
||||
|
||||
class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
|
||||
list_display = ("id", "created_at", "created_by")
|
||||
readonly_fields = ("id", "created_at", "modified_at")
|
||||
show_search_mode_selector = False
|
||||
change_form_template = "admin/archivebox_change_form.html"
|
||||
|
||||
def get_admin_toolbar_actions(self, request, obj):
|
||||
"""Return extra action button dicts for the shared change-form toolbar.
|
||||
|
||||
See ``templates/admin/includes/archivebox_toolbar.html`` for the
|
||||
accepted keys. Default: no extras (toolbar renders Save/History/Delete
|
||||
plus any ``django-object-actions`` change_actions).
|
||||
"""
|
||||
return []
|
||||
|
||||
def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
|
||||
context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
|
||||
return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
|
||||
|
||||
def get_default_search_mode(self) -> str:
|
||||
# The shared changelist template always asks every admin for a default
|
||||
# search mode, even when the search-mode toggle is hidden.
|
||||
return "meta"
|
||||
|
||||
def get_form(
|
||||
self,
|
||||
request: HttpRequest,
|
||||
obj: models.Model | None = None,
|
||||
change: bool = False,
|
||||
**kwargs: object,
|
||||
):
|
||||
form = super().get_form(request, obj, change=change, **kwargs)
|
||||
if "created_by" in form.base_fields:
|
||||
form.base_fields["created_by"].initial = request.user
|
||||
return form
|
||||
|
||||
def get_urls(self):
|
||||
"""Swap the per-object admin URLs from ``<path:object_id>`` to
|
||||
``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
|
||||
32-char hex form. The hyphenated form still resolves because the
|
||||
converter's regex accepts both — Django reverses through ``to_url``
|
||||
which always emits hex, so links in templates / changelists / inline
|
||||
formsets all canonicalize automatically.
|
||||
|
||||
Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
|
||||
won't match the converter's regex and fall back to the default
|
||||
``<path:object_id>`` patterns we still include after our swap.
|
||||
"""
|
||||
info = self.opts.app_label, self.opts.model_name
|
||||
object_routes = [
|
||||
path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
|
||||
path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
|
||||
path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
|
||||
]
|
||||
# Append after super().get_urls() so our patterns are the
|
||||
# *last-registered* ones with the canonical admin URL names — Django's
|
||||
# reverse() picks the later registration when names collide, which is
|
||||
# how we make ``reverse("admin:app_model_change", args=[obj.pk])``
|
||||
# emit the hex form. The original ``<path:object_id>`` routes stay in
|
||||
# place as a fallback for non-UUID PKs and for resolving inbound
|
||||
# hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
|
||||
return super().get_urls() + object_routes
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -1,430 +0,0 @@
|
||||
"""Base models using UUIDv7 for all id fields."""
|
||||
|
||||
__package__ = "archivebox.base_models"
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
from pathlib import Path
|
||||
|
||||
from django.core.exceptions import FieldDoesNotExist
|
||||
from django.db import models
|
||||
from django.db.models import F
|
||||
from django.db import transaction
|
||||
from django.db.models.signals import pre_delete
|
||||
from django.utils import timezone
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.urls import reverse_lazy
|
||||
from django.conf import settings
|
||||
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
|
||||
|
||||
class PagedQuerySet(models.QuerySet):
|
||||
def paged_iterator(self, chunk_size: int = 500):
|
||||
"""
|
||||
Iterate rows using bounded keyset pages instead of one streaming cursor.
|
||||
|
||||
Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
|
||||
cursor open until the full queryset is exhausted. That is fine for
|
||||
read-only exports, but update/migration code does filesystem work and
|
||||
writes while iterating; a long-lived read cursor there can stretch lock
|
||||
waits across thousands of rows. This respects the queryset's existing
|
||||
filters, order_by(), select_related(), and prefetch_related() state; if
|
||||
no ordering is defined, it falls back to primary-key order.
|
||||
"""
|
||||
pk_field = self.model._meta.pk.name
|
||||
raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
|
||||
|
||||
if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
|
||||
offset = 0
|
||||
while True:
|
||||
batch = list(self[offset : offset + chunk_size])
|
||||
if not batch:
|
||||
break
|
||||
yield from batch
|
||||
offset += chunk_size
|
||||
return
|
||||
|
||||
ordering = []
|
||||
for term in raw_ordering:
|
||||
descending = term.startswith("-")
|
||||
field_name = term[1:] if descending else term
|
||||
if field_name == "pk":
|
||||
field_name = pk_field
|
||||
ordering.append(f"-{field_name}" if descending else field_name)
|
||||
|
||||
ordered_field_names = [term.removeprefix("-") for term in ordering]
|
||||
try:
|
||||
if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
|
||||
offset = 0
|
||||
while True:
|
||||
batch = list(self[offset : offset + chunk_size])
|
||||
if not batch:
|
||||
break
|
||||
yield from batch
|
||||
offset += chunk_size
|
||||
return
|
||||
except (AttributeError, FieldDoesNotExist):
|
||||
offset = 0
|
||||
while True:
|
||||
batch = list(self[offset : offset + chunk_size])
|
||||
if not batch:
|
||||
break
|
||||
yield from batch
|
||||
offset += chunk_size
|
||||
return
|
||||
|
||||
unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
|
||||
if not any(field_name in unique_field_names for field_name in ordered_field_names):
|
||||
offset = 0
|
||||
while True:
|
||||
batch = list(self[offset : offset + chunk_size])
|
||||
if not batch:
|
||||
break
|
||||
yield from batch
|
||||
offset += chunk_size
|
||||
return
|
||||
|
||||
last_values = None
|
||||
value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
|
||||
while True:
|
||||
batch_qs = self.order_by(*ordering)
|
||||
if last_values is not None:
|
||||
page_filter = models.Q()
|
||||
for idx, term in enumerate(ordering):
|
||||
descending = term.startswith("-")
|
||||
field_name = term[1:] if descending else term
|
||||
prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
|
||||
comparison = "lt" if descending else "gt"
|
||||
page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
|
||||
# Anchor the OR tie-breaks with an explicit leading range so
|
||||
# SQLite/PG can seek the existing order index instead of
|
||||
# rescanning its prefix for every keyset page.
|
||||
first_comparison = "lte" if ordering[0].startswith("-") else "gte"
|
||||
batch_qs = batch_qs.filter(
|
||||
page_filter,
|
||||
**{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
|
||||
)
|
||||
|
||||
batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
|
||||
if not batch_rows:
|
||||
break
|
||||
|
||||
pk_idx = value_field_names.index(pk_field)
|
||||
row_ids = [row[pk_idx] for row in batch_rows]
|
||||
rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
|
||||
|
||||
for row in batch_rows:
|
||||
row_id = row[pk_idx]
|
||||
obj = rows_by_id.get(row_id)
|
||||
if obj is not None:
|
||||
yield obj
|
||||
|
||||
last_values = batch_rows[-1][: len(ordered_field_names)]
|
||||
|
||||
|
||||
def normalize_config_json_values(config: Any) -> Any:
|
||||
if not isinstance(config, dict):
|
||||
return config
|
||||
|
||||
normalized = dict(config)
|
||||
for key, value in list(normalized.items()):
|
||||
if not isinstance(value, str) or len(value) < 2:
|
||||
continue
|
||||
if value[:1] != '"' or value[-1:] != '"':
|
||||
continue
|
||||
try:
|
||||
decoded = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(decoded, str):
|
||||
normalized[key] = decoded
|
||||
return normalized
|
||||
|
||||
|
||||
def get_or_create_system_user_pk(username="system"):
|
||||
User = get_user_model()
|
||||
# If there's exactly one superuser, use that for all system operations
|
||||
if User.objects.filter(is_superuser=True).count() == 1:
|
||||
return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
|
||||
# Otherwise get or create the system user
|
||||
user, _ = User.objects.get_or_create(
|
||||
username=username,
|
||||
defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
|
||||
)
|
||||
return user.pk
|
||||
|
||||
|
||||
class AutoDateTimeField(models.DateTimeField):
|
||||
"""DateTimeField that automatically updates on save (legacy compatibility)."""
|
||||
|
||||
def pre_save(self, model_instance, add):
|
||||
if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
|
||||
value = timezone.now()
|
||||
setattr(model_instance, self.attname, value)
|
||||
return value
|
||||
return super().pre_save(model_instance, add)
|
||||
|
||||
|
||||
class ModelWithUUID(models.Model):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
created_by = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
default=get_or_create_system_user_pk,
|
||||
null=False,
|
||||
db_index=True,
|
||||
)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"[{self.id}] {self.__class__.__name__}"
|
||||
|
||||
@property
|
||||
def admin_change_url(self) -> str:
|
||||
return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
|
||||
|
||||
@property
|
||||
def api_url(self) -> str:
|
||||
return str(reverse_lazy("api-1:get_any", args=[self.id]))
|
||||
|
||||
@property
|
||||
def api_docs_url(self) -> str:
|
||||
return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
|
||||
|
||||
|
||||
class ModelWithNotes(models.Model):
|
||||
"""Mixin for models with a notes field."""
|
||||
|
||||
notes = models.TextField(blank=True, null=False, default="")
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
|
||||
class ModelWithHealthStats(models.Model):
|
||||
"""Mixin for models with health tracking fields."""
|
||||
|
||||
num_uses_failed = models.PositiveIntegerField(default=0)
|
||||
num_uses_succeeded = models.PositiveIntegerField(default=0)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
@property
|
||||
def admin_change_url(self) -> str:
|
||||
return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
|
||||
|
||||
@property
|
||||
def health(self) -> int:
|
||||
total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
|
||||
return round((self.num_uses_succeeded / total) * 100)
|
||||
|
||||
def increment_health_stats(self, success: bool):
|
||||
"""Atomically increment success or failure counter using F() expression."""
|
||||
field = "num_uses_succeeded" if success else "num_uses_failed"
|
||||
type(self).objects.filter(pk=self.pk).update(
|
||||
**{
|
||||
field: F(field) + 1,
|
||||
"modified_at": timezone.now(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class ModelWithConfig(models.Model):
|
||||
"""Mixin for models with a JSON config field."""
|
||||
|
||||
config = models.JSONField(default=dict, null=True, blank=True, editable=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
normalized_config = normalize_config_json_values(self.config)
|
||||
if normalized_config != self.config:
|
||||
self.config = normalized_config
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if update_fields is not None:
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
|
||||
class ModelWithDeleteAfter(models.Model):
|
||||
delete_after_final_statuses: tuple[str, ...] = ()
|
||||
delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if self.delete_at is None:
|
||||
self.set_delete_at_from_config()
|
||||
if self.delete_at is not None and update_fields is not None:
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
def get_delete_after_config_value(self):
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
|
||||
|
||||
def set_delete_at_from_config(self, config_value=None) -> bool:
|
||||
if self.delete_at is not None:
|
||||
return False
|
||||
|
||||
from archivebox.config.common import parse_delete_after
|
||||
|
||||
duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
|
||||
if duration is None:
|
||||
return False
|
||||
|
||||
self.delete_at = (self.created_at or timezone.now()) + duration
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def missing_delete_at_candidates(cls):
|
||||
return cls.objects.none()
|
||||
|
||||
@classmethod
|
||||
def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
|
||||
if backfill_missing:
|
||||
# Bound the scanned rows BEFORE evaluating JSON policy/parent
|
||||
# joins. LIMIT on the policy query only bounded its output: finding
|
||||
# no matches could scan the entire history with a SQLite read lock.
|
||||
# The runner keeps a keyset cursor between idle passes, so disabled
|
||||
# policies cannot starve older rows. Restarting safely repeats work.
|
||||
page = cls.objects.order_by("-pk")
|
||||
before = backfill_cursors.get(cls) if backfill_cursors is not None else None
|
||||
if before is not None:
|
||||
page = page.filter(pk__lt=before)
|
||||
page_ids = list(page.values_list("pk", flat=True)[:batch_size])
|
||||
if backfill_cursors is not None:
|
||||
backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
|
||||
missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
|
||||
for obj in missing_delete_at:
|
||||
if obj.set_delete_at_from_config():
|
||||
cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
|
||||
delete_at=obj.delete_at,
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
|
||||
# Keep the expiration sweep anchored on delete_at. Some large tables
|
||||
# have millions of final-status rows but almost no retained rows; adding
|
||||
# the status filter before SQLite has narrowed by delete_at can make the
|
||||
# runner scan the hot status index before it claims new work.
|
||||
due_pks = list(
|
||||
cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
|
||||
.order_by("delete_at", "pk")
|
||||
.values_list("pk", flat=True)[:batch_size],
|
||||
)
|
||||
if not due_pks:
|
||||
return 0
|
||||
|
||||
queryset = cls.objects.filter(pk__in=due_pks)
|
||||
if cls.delete_after_final_statuses:
|
||||
queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
|
||||
|
||||
count = 0
|
||||
expired = list(queryset.order_by("delete_at", "pk"))
|
||||
for obj in expired:
|
||||
obj.delete()
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
class ModelWithOutputDir(ModelWithUUID):
|
||||
class Meta(ModelWithUUID.Meta):
|
||||
abstract = True
|
||||
|
||||
_delete_signal_registered = False
|
||||
|
||||
# Saving metadata must not resolve or create output paths. The runner and
|
||||
# explicit artifact writers create their own directories when writing files;
|
||||
# on_commit would still make HTTP requests wait on remote archive storage.
|
||||
|
||||
@property
|
||||
def output_dir_parent(self) -> str:
|
||||
return f"{self._meta.model_name}s"
|
||||
|
||||
@property
|
||||
def output_dir_name(self) -> str:
|
||||
return str(self.id)
|
||||
|
||||
@property
|
||||
def output_dir_str(self) -> str:
|
||||
return f"{self.output_dir_parent}/{self.output_dir_name}"
|
||||
|
||||
@property
|
||||
def output_dir(self) -> Path:
|
||||
raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
|
||||
|
||||
def output_paths_for_delete(self) -> tuple[Path, ...]:
|
||||
return (Path(self.output_dir),)
|
||||
|
||||
@classmethod
|
||||
def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
|
||||
data_dir = CONSTANTS.DATA_DIR.resolve()
|
||||
safe_paths = []
|
||||
for raw_path in paths:
|
||||
path = Path(raw_path)
|
||||
is_safe = False
|
||||
for candidate in (path.absolute(), path.resolve()):
|
||||
try:
|
||||
candidate.relative_to(data_dir)
|
||||
is_safe = True
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
if not is_safe:
|
||||
raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
|
||||
safe_paths.append(path)
|
||||
return tuple(safe_paths)
|
||||
|
||||
@classmethod
|
||||
def delete_output_paths(cls, paths) -> None:
|
||||
for path in cls.validate_output_paths_for_delete(paths):
|
||||
if path.is_symlink() or path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
elif path.is_dir():
|
||||
shutil.rmtree(path)
|
||||
# lstat also catches dangling symlinks and propagates mount and
|
||||
# permission errors rather than mistaking them for absence.
|
||||
try:
|
||||
path.lstat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
raise OSError(f"Output path still exists after deletion: {path}")
|
||||
|
||||
def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
|
||||
"""Capture output paths before DB deletion and remove them after commit."""
|
||||
paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
|
||||
transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
|
||||
|
||||
@classmethod
|
||||
def register_delete_signal(cls) -> None:
|
||||
if cls._delete_signal_registered:
|
||||
return
|
||||
|
||||
def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
|
||||
if getattr(origin, "_output_files_deleted", False):
|
||||
return
|
||||
if not isinstance(instance, ModelWithOutputDir):
|
||||
return
|
||||
instance.schedule_delete_cleanup(using=using)
|
||||
|
||||
pre_delete.connect(
|
||||
schedule_output_dir_cleanup,
|
||||
dispatch_uid="archivebox.output_dir_cleanup_on_delete",
|
||||
weak=False,
|
||||
)
|
||||
cls._delete_signal_registered = True
|
||||
@@ -1,232 +0,0 @@
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox"
|
||||
import os
|
||||
import sys
|
||||
from contextlib import redirect_stdout
|
||||
from importlib import import_module
|
||||
|
||||
import rich_click as click
|
||||
from rich.console import Console
|
||||
|
||||
from archivebox.config.version import VERSION
|
||||
|
||||
STDERR = Console(stderr=True)
|
||||
|
||||
|
||||
if "--debug" in sys.argv:
|
||||
os.environ["DEBUG"] = "True"
|
||||
sys.argv.remove("--debug")
|
||||
|
||||
# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
|
||||
# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
|
||||
# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
|
||||
# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
|
||||
# `init` themselves.
|
||||
if "--init" in sys.argv or "--quick-init" in sys.argv:
|
||||
sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
|
||||
os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
|
||||
|
||||
|
||||
class ArchiveBoxGroup(click.Group):
|
||||
"""lazy loading click group for archivebox commands"""
|
||||
|
||||
meta_commands = {
|
||||
"help": "archivebox.cli.archivebox_help.main",
|
||||
"version": "archivebox.cli.archivebox_version.main",
|
||||
"mcp": "archivebox.cli.archivebox_mcp.main",
|
||||
"oneshot": "archivebox.cli.archivebox_oneshot.main",
|
||||
}
|
||||
setup_commands = {
|
||||
"init": "archivebox.cli.archivebox_init.main",
|
||||
"install": "archivebox.cli.archivebox_install.main",
|
||||
}
|
||||
# Model commands (CRUD operations via subcommands)
|
||||
model_commands = {
|
||||
"crawl": "archivebox.cli.archivebox_crawl.main",
|
||||
"snapshot": "archivebox.cli.archivebox_snapshot.main",
|
||||
"archiveresult": "archivebox.cli.archivebox_archiveresult.main",
|
||||
"tag": "archivebox.cli.archivebox_tag.main",
|
||||
"binary": "archivebox.cli.archivebox_binary.main",
|
||||
"process": "archivebox.cli.archivebox_process.main",
|
||||
"machine": "archivebox.cli.archivebox_machine.main",
|
||||
"persona": "archivebox.cli.archivebox_persona.main",
|
||||
}
|
||||
archive_commands = {
|
||||
# High-level commands
|
||||
"add": "archivebox.cli.archivebox_add.main",
|
||||
"extract": "archivebox.cli.archivebox_extract.main",
|
||||
"list": "archivebox.cli.archivebox_list.main",
|
||||
"remove": "archivebox.cli.archivebox_remove.main",
|
||||
"run": "archivebox.cli.archivebox_run.main",
|
||||
"update": "archivebox.cli.archivebox_update.main",
|
||||
"status": "archivebox.cli.archivebox_status.main",
|
||||
"search": "archivebox.cli.archivebox_search.main",
|
||||
"config": "archivebox.cli.archivebox_config.main",
|
||||
"schedule": "archivebox.cli.archivebox_schedule.main",
|
||||
"server": "archivebox.cli.archivebox_server.main",
|
||||
"shell": "archivebox.cli.archivebox_shell.main",
|
||||
"manage": "archivebox.cli.archivebox_manage.main",
|
||||
# Introspection commands
|
||||
"pluginmap": "archivebox.cli.archivebox_pluginmap.main",
|
||||
}
|
||||
all_subcommands = {
|
||||
**meta_commands,
|
||||
**setup_commands,
|
||||
**model_commands,
|
||||
**archive_commands,
|
||||
}
|
||||
renamed_commands = {
|
||||
"setup": "install",
|
||||
"import": "add",
|
||||
"archive": "add",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_canonical_name(cls, cmd_name):
|
||||
return cls.renamed_commands.get(cmd_name, cmd_name)
|
||||
|
||||
@classmethod
|
||||
def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
|
||||
wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
|
||||
return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
|
||||
|
||||
@classmethod
|
||||
def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
|
||||
if not cls._needs_django_for_lazy_import(cmd_name):
|
||||
return
|
||||
|
||||
from django.apps import apps
|
||||
|
||||
if apps.ready:
|
||||
return
|
||||
|
||||
from archivebox.config.django import setup_django
|
||||
|
||||
setup_django()
|
||||
|
||||
def get_command(self, ctx, cmd_name):
|
||||
# handle renamed commands
|
||||
if cmd_name in self.renamed_commands:
|
||||
new_name = self.renamed_commands[cmd_name]
|
||||
STDERR.print(
|
||||
f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
|
||||
)
|
||||
cmd_name = new_name
|
||||
ctx.invoked_subcommand = cmd_name
|
||||
|
||||
# handle lazy loading of commands
|
||||
if cmd_name in self.all_subcommands:
|
||||
self._setup_django_for_lazy_import(cmd_name)
|
||||
return self._lazy_load(cmd_name)
|
||||
|
||||
# fall-back to using click's default command lookup
|
||||
return super().get_command(ctx, cmd_name)
|
||||
|
||||
@classmethod
|
||||
def _lazy_load(cls, cmd_name_or_path):
|
||||
import_path = cls.all_subcommands.get(cmd_name_or_path)
|
||||
if import_path is None:
|
||||
import_path = cmd_name_or_path
|
||||
modname, funcname = import_path.rsplit(".", 1)
|
||||
|
||||
mod = import_module(modname)
|
||||
func = vars(mod)[funcname]
|
||||
|
||||
if func.__doc__ is None:
|
||||
raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
|
||||
|
||||
return func
|
||||
|
||||
|
||||
@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
|
||||
@click.option("--help", "-h", is_flag=True, help="Show help")
|
||||
@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
|
||||
@click.pass_context
|
||||
def cli(ctx, help=False):
|
||||
"""ArchiveBox: The self-hosted internet archive"""
|
||||
|
||||
subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
|
||||
|
||||
# if --help is passed or no subcommand is given, show custom help message
|
||||
if help or ctx.invoked_subcommand is None:
|
||||
ctx.invoke(ctx.command.get_command(ctx, "help"))
|
||||
|
||||
# if the subcommand is in archive_commands or model_commands,
|
||||
# then we need to set up the django environment and check that we're in a valid data folder
|
||||
wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
|
||||
if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
|
||||
try:
|
||||
# RUNSERVER_WORKER sets the web-worker markers on its own child.
|
||||
# Setting them on the server parent leaks them through supervisord
|
||||
# into runners waiting for takeover, which then register themselves
|
||||
# as web workers and trigger spurious runner_watch reloads.
|
||||
from archivebox.config.django import setup_django
|
||||
from archivebox.misc.checks import check_data_folder, check_migrations
|
||||
|
||||
setup_django()
|
||||
if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
|
||||
# Universal `--init` was passed: build/upgrade the data folder before
|
||||
# the regular preflight runs, so it succeeds on a fresh dir and an
|
||||
# out-of-date schema both. Drop the env var afterwards so spawned
|
||||
# subprocesses (supervisord workers, daphne, runner, etc.) inherit
|
||||
# a clean env and don't re-trigger init in every child.
|
||||
from archivebox.cli.archivebox_init import init as archivebox_init
|
||||
|
||||
archivebox_init(quick=True)
|
||||
os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
|
||||
check_data_folder()
|
||||
if subcommand != "update" and not check_migrations(auto_apply=True):
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
# Host registration and config reconciliation belong to CLI
|
||||
# startup; get_config() is also used by read-only HTTP views.
|
||||
Machine.current()
|
||||
except Exception as e:
|
||||
STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
|
||||
if subcommand not in ("manage", "shell"): # not all management commands need django to be setup beforehand
|
||||
raise
|
||||
|
||||
|
||||
def main(args=None, prog_name=None):
|
||||
from archivebox.config.constants import CONSTANTS
|
||||
|
||||
os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
|
||||
|
||||
# show `docker run archivebox xyz` in help messages if running in docker
|
||||
IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
|
||||
IS_TTY = sys.stdin.isatty()
|
||||
prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
|
||||
|
||||
previous_unraisablehook = sys.unraisablehook
|
||||
|
||||
def ignore_shutdown_unraisable(unraisable):
|
||||
if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
|
||||
return
|
||||
previous_unraisablehook(unraisable)
|
||||
|
||||
sys.unraisablehook = ignore_shutdown_unraisable
|
||||
try:
|
||||
cli(args=args, prog_name=prog_name, standalone_mode=False)
|
||||
except click.Abort:
|
||||
STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
|
||||
raise SystemExit(130) from None
|
||||
except click.ClickException as err:
|
||||
err.show()
|
||||
raise SystemExit(err.exit_code) from None
|
||||
except click.exceptions.Exit as err:
|
||||
raise SystemExit(err.exit_code) from None
|
||||
except KeyboardInterrupt:
|
||||
STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
|
||||
raise SystemExit(130) from None
|
||||
finally:
|
||||
sys.unraisablehook = previous_unraisablehook
|
||||
supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
|
||||
if supervisor_util is not None:
|
||||
# Short commands can start a foreground-owned search daemon too.
|
||||
# Finish its cleanup before another CLI can borrow a dying stack.
|
||||
with redirect_stdout(sys.stderr):
|
||||
supervisor_util.stop_own_supervisord_process()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,446 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox add"
|
||||
|
||||
import sys
|
||||
import os
|
||||
import signal
|
||||
from pathlib import Path
|
||||
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
import rich_click as click
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.misc.util import enforce_types, docstring
|
||||
from archivebox.misc.util import parse_filesize_to_bytes
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import QuerySet
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
|
||||
def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
|
||||
from archivebox.misc.jsonl import read_args_or_stdin
|
||||
from archivebox.misc.util import validate_url
|
||||
|
||||
urls: list[str] = []
|
||||
for record in read_args_or_stdin(args):
|
||||
url = record.get("url")
|
||||
if isinstance(url, str) and url:
|
||||
try:
|
||||
urls.append(validate_url(url))
|
||||
except ValueError as err:
|
||||
raise click.BadParameter(str(err), param_hint="URL") from err
|
||||
|
||||
urls_field = record.get("urls")
|
||||
if isinstance(urls_field, str):
|
||||
for line in urls_field.splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
try:
|
||||
urls.append(validate_url(line))
|
||||
except ValueError as err:
|
||||
raise click.BadParameter(str(err), param_hint="URL") from err
|
||||
|
||||
return urls
|
||||
|
||||
|
||||
@enforce_types
|
||||
def add(
|
||||
urls: str | list[str],
|
||||
snapshot_ids: list[str] | None = None,
|
||||
depth: int | str = 0,
|
||||
max_urls: int = 0,
|
||||
crawl_max_size: int | str = 0,
|
||||
crawl_timeout: int = 0,
|
||||
snapshot_max_size: int | str = 0,
|
||||
crawl_max_concurrent_snapshots: int | None = None,
|
||||
tag: str = "",
|
||||
url_allowlist: str = "",
|
||||
url_denylist: str = "",
|
||||
parser: str = "auto",
|
||||
plugins: str = "",
|
||||
persona: str = "Default",
|
||||
index_only: bool = False,
|
||||
bg: bool = False,
|
||||
created_by_id: int | None = None,
|
||||
config: dict[str, Any] | None = None,
|
||||
) -> tuple[Crawl, QuerySet[Snapshot]]:
|
||||
"""Add a URL list or imported URL document to a new Crawl."""
|
||||
|
||||
from rich import print
|
||||
|
||||
depth = int(depth)
|
||||
max_urls = int(max_urls or 0)
|
||||
crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
|
||||
crawl_timeout = int(crawl_timeout or 0)
|
||||
snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
|
||||
from archivebox.config.permissions import USER, HOSTNAME
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
config_overrides = dict(config or {})
|
||||
runtime_config = get_config()
|
||||
crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
|
||||
if crawl_max_concurrent_snapshots is None:
|
||||
crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
|
||||
crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
|
||||
|
||||
if depth not in (0, 1, 2, 3, 4):
|
||||
raise ValueError("Depth must be 0-4")
|
||||
if max_urls < 0:
|
||||
raise ValueError("max_urls must be >= 0")
|
||||
if crawl_max_size < 0:
|
||||
raise ValueError("crawl_max_size must be >= 0")
|
||||
if crawl_timeout < 0:
|
||||
raise ValueError("crawl_timeout must be >= 0")
|
||||
if snapshot_max_size < 0:
|
||||
raise ValueError("snapshot_max_size must be >= 0")
|
||||
if crawl_max_concurrent_snapshots < 1:
|
||||
raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
|
||||
|
||||
# import models once django is set up
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.personas.models import Persona
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.misc.system import get_dir_size
|
||||
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
|
||||
from django.utils import timezone
|
||||
|
||||
created_by_id = created_by_id or get_or_create_system_user_pk()
|
||||
started_at = timezone.now()
|
||||
|
||||
if isinstance(urls, str):
|
||||
source_text = urls
|
||||
else:
|
||||
from archivebox.misc.util import validate_url
|
||||
|
||||
source_text = "\n".join(validate_url(str(url)) for url in urls)
|
||||
|
||||
# 2. Create a new Crawl with inline URLs
|
||||
# Foreground add must claim runner ownership before publishing runnable
|
||||
# work. Otherwise an existing server runner can lease the new Crawl in the
|
||||
# gap between Crawl.objects.create() and current_command(), then be killed
|
||||
# by the newer add process halfway through its first Snapshot hook.
|
||||
command = None
|
||||
if not bg and not index_only:
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.core.takeover_util import current_command
|
||||
|
||||
command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
|
||||
|
||||
cli_args = [*sys.argv]
|
||||
if cli_args[0].lower().endswith("archivebox"):
|
||||
cli_args[0] = "archivebox"
|
||||
cmd_str = " ".join(cli_args)
|
||||
|
||||
timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
|
||||
|
||||
persona_name = (persona or "Default").strip() or "Default"
|
||||
plugins = plugins or ""
|
||||
persona_obj = Persona.get_or_create_named(persona_name)
|
||||
effective_persona_config = get_config(persona=persona_obj)
|
||||
|
||||
crawl_config = {
|
||||
"PERMISSIONS": str(effective_persona_config.PERMISSIONS),
|
||||
**({"INDEX_ONLY": True} if index_only else {}),
|
||||
**({"PLUGINS": plugins} if plugins else {}),
|
||||
**(
|
||||
{"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
|
||||
if crawl_max_concurrent_snapshots_override
|
||||
and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
|
||||
else {}
|
||||
),
|
||||
**({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
|
||||
**({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
|
||||
**({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
|
||||
**({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
|
||||
**({"PARSER": parser} if parser != "auto" else {}),
|
||||
**({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
|
||||
**({"URL_DENYLIST": url_denylist} if url_denylist else {}),
|
||||
}
|
||||
# Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
|
||||
# priority for crawl-frozen keys. Runtime-derived execution keys are
|
||||
# stripped by Crawl.save() and rederived when hooks run.
|
||||
crawl_config.update(config_overrides)
|
||||
|
||||
# Crawl.urls is the user's submitted source, not a derived work queue for
|
||||
# this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
|
||||
# callers audit or resume imports without losing RSS/Netscape/JSON metadata
|
||||
# that is not representable as one plain URL per line.
|
||||
try:
|
||||
crawl = Crawl.objects.create(
|
||||
urls=source_text,
|
||||
max_depth=depth,
|
||||
tags_str=tag,
|
||||
persona_id=persona_obj.id,
|
||||
label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
|
||||
created_by_id=created_by_id,
|
||||
status=Crawl.StatusChoices.QUEUED,
|
||||
retry_at=None if index_only else timezone.now(),
|
||||
config=crawl_config,
|
||||
)
|
||||
first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
|
||||
except BaseException:
|
||||
if command is not None:
|
||||
command.mark_exited(exit_code=1)
|
||||
raise
|
||||
if command is not None:
|
||||
command.url = first_url
|
||||
command.save(update_fields=["url", "modified_at"])
|
||||
|
||||
print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
|
||||
print(f" [dim]First URL: {first_url}[/dim]")
|
||||
|
||||
# The runner parses Crawl.urls after claiming the Crawl, then persists the
|
||||
# resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
|
||||
|
||||
if index_only:
|
||||
print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
|
||||
return crawl, crawl.snapshot_set.none()
|
||||
|
||||
# 5. Start the crawl runner to process the queue
|
||||
# The runner will:
|
||||
# - Process Crawl -> create Snapshots from all URLs
|
||||
# - Process Snapshots -> run extractors
|
||||
# - Parser extractors discover new URLs -> create child Snapshots
|
||||
# - Repeat until max_depth reached
|
||||
|
||||
if bg:
|
||||
# Background mode: just queue work and return (background runner via server will pick it up).
|
||||
print(
|
||||
"[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
|
||||
)
|
||||
else:
|
||||
# Foreground mode: run full crawl runner until all work is done
|
||||
print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
|
||||
from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
|
||||
|
||||
assert command is not None
|
||||
exit_code = 0
|
||||
interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
|
||||
|
||||
def interrupt_hook() -> None:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
def crawl_is_complete() -> bool:
|
||||
crawl.refresh_from_db(fields=["status"])
|
||||
return crawl.status not in crawl.RUNNABLE_STATES
|
||||
|
||||
try:
|
||||
try:
|
||||
with (
|
||||
foreground_shutdown_signals(
|
||||
first_signal_message=None,
|
||||
interrupt_handlers={signal.SIGINT: interrupt_hook},
|
||||
),
|
||||
foreground_parent_watchdog(),
|
||||
):
|
||||
while True:
|
||||
standby = standby_until_foreground_runner_needed(
|
||||
command,
|
||||
data_dir=CONSTANTS.DATA_DIR,
|
||||
work_is_complete=crawl_is_complete,
|
||||
)
|
||||
if standby["work_completed"]:
|
||||
break
|
||||
exit_code = run_runner_worker(
|
||||
["--crawl-id", str(crawl.id)],
|
||||
name=f"worker_runner_add_{os.getpid()}",
|
||||
interactive_interrupts=interactive_interrupts,
|
||||
config=get_config(crawl=crawl),
|
||||
)
|
||||
crawl.refresh_from_db(fields=["status", "retry_at"])
|
||||
if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
|
||||
break
|
||||
# A shared supervisord can be stopped by another
|
||||
# foreground owner (for example `archivebox server`
|
||||
# shutting down) and stop the one-shot add runner while
|
||||
# this add's crawl is still runnable. Keep the add
|
||||
# process alive so it can re-enter the normal
|
||||
# foreground-runner ownership loop and finish its crawl.
|
||||
supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
|
||||
if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
|
||||
continue
|
||||
if exit_code == 0:
|
||||
break
|
||||
if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
|
||||
continue
|
||||
raise SystemExit(exit_code)
|
||||
except KeyboardInterrupt:
|
||||
exit_code = 130
|
||||
print("\n[red][X] archivebox add interrupted.[/red]")
|
||||
print("[yellow]Hint: resume this crawl with:[/yellow]")
|
||||
print(f" [green]archivebox run --crawl-id={crawl.id}[/green]")
|
||||
raise SystemExit(exit_code)
|
||||
finally:
|
||||
command.mark_exited(exit_code=exit_code)
|
||||
stop_own_supervisord_process()
|
||||
|
||||
# Print summary for foreground runs
|
||||
try:
|
||||
crawl.refresh_from_db()
|
||||
try:
|
||||
from django.db.models import Count, Sum
|
||||
|
||||
totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
|
||||
snapshots_count = int(totals["snapshot_count"] or 0)
|
||||
total_bytes = int(totals["total_bytes"] or 0)
|
||||
except Exception:
|
||||
snapshots_count = crawl.snapshot_set.count()
|
||||
total_bytes, _, _ = get_dir_size(crawl.output_dir)
|
||||
total_size = printable_filesize(total_bytes)
|
||||
total_time = timezone.now() - started_at
|
||||
total_seconds = int(total_time.total_seconds())
|
||||
mins, secs = divmod(total_seconds, 60)
|
||||
hours, mins = divmod(mins, 60)
|
||||
if hours:
|
||||
duration_str = f"{hours}h {mins}m {secs}s"
|
||||
elif mins:
|
||||
duration_str = f"{mins}m {secs}s"
|
||||
else:
|
||||
duration_str = f"{secs}s"
|
||||
|
||||
# Output dir relative to DATA_DIR
|
||||
try:
|
||||
rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
|
||||
rel_output_str = f"./{rel_output}"
|
||||
except Exception:
|
||||
rel_output_str = str(crawl.output_dir)
|
||||
|
||||
from archivebox.core.routes_util import build_admin_url
|
||||
|
||||
admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
|
||||
|
||||
if snapshots_count:
|
||||
print("\n[bold]snapshot output saved to:[/bold]")
|
||||
for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
|
||||
click.echo(f" {snapshot.output_dir}")
|
||||
print("\n[bold]crawl logs and setup:[/bold]")
|
||||
click.echo(f" {rel_output_str}")
|
||||
click.echo(f" {admin_url}")
|
||||
print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
|
||||
print(f"[bold]total size:[/bold] {total_size}")
|
||||
print(f"[bold]total time:[/bold] {duration_str}")
|
||||
except Exception:
|
||||
# Summary is best-effort; avoid failing the command if something goes wrong
|
||||
pass
|
||||
|
||||
# 6. Return the list of Snapshots in this crawl
|
||||
snapshots = crawl.snapshot_set.all()
|
||||
return crawl, snapshots
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--depth",
|
||||
"-d",
|
||||
type=click.Choice([str(i) for i in range(5)]),
|
||||
default="0",
|
||||
help="Recursively archive linked pages up to N hops away",
|
||||
)
|
||||
@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
|
||||
@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
|
||||
@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
|
||||
@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
|
||||
@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
|
||||
@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
|
||||
@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
|
||||
@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
|
||||
@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
|
||||
@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
|
||||
@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
|
||||
@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
|
||||
@click.option(
|
||||
"--only-new/--no-only-new",
|
||||
"only_new",
|
||||
default=None,
|
||||
help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
|
||||
"Pass --no-only-new to force re-archive of URLs that already exist.",
|
||||
)
|
||||
@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
|
||||
@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
|
||||
@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
|
||||
@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
|
||||
@click.argument("urls", nargs=-1, type=click.Path())
|
||||
@docstring(add.__doc__)
|
||||
def main(**kwargs):
|
||||
"""Add a new URL or list of URLs to your archive"""
|
||||
|
||||
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
|
||||
|
||||
with foreground_shutdown_signals(), foreground_parent_watchdog():
|
||||
raw_urls = kwargs.pop("urls")
|
||||
if raw_urls:
|
||||
urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
|
||||
elif not sys.stdin.isatty():
|
||||
urls = sys.stdin.read()
|
||||
else:
|
||||
urls = []
|
||||
if not urls:
|
||||
raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
|
||||
if int(kwargs.get("max_urls") or 0) < 0:
|
||||
raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
|
||||
if int(kwargs.get("crawl_timeout") or 0) < 0:
|
||||
raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
|
||||
try:
|
||||
kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
|
||||
except ValueError as err:
|
||||
raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
|
||||
try:
|
||||
kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
|
||||
except ValueError as err:
|
||||
raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
|
||||
if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
|
||||
raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
|
||||
|
||||
# Translate --only-new/--no-only-new into a crawl config override.
|
||||
# add() takes config overrides as a dict; no per-flag kwargs.
|
||||
overwrite = kwargs.pop("overwrite", False)
|
||||
update = kwargs.pop("update", False)
|
||||
only_new = kwargs.pop("only_new", None)
|
||||
extract = kwargs.pop("extract", "")
|
||||
if overwrite or update:
|
||||
only_new = False
|
||||
if only_new is not None:
|
||||
kwargs["config"] = {"ONLY_NEW": bool(only_new)}
|
||||
if extract:
|
||||
from archivebox.plugins.discovery import get_plugin_catalog
|
||||
|
||||
catalog = get_plugin_catalog()
|
||||
tokens = [token.strip() for token in extract.split(",") if token.strip()]
|
||||
plugin_names = {name.lower(): name for name in catalog}
|
||||
selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
|
||||
selected += catalog.matching_output(tokens)
|
||||
if not selected:
|
||||
raise click.UsageError(f"No plugins found matching extract types: {extract}")
|
||||
existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
|
||||
kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
|
||||
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
# CLI commands own their process record and logs. Queue model saves
|
||||
# must not register a process implicitly: HTTP add() shares that path.
|
||||
process = Process.current()
|
||||
exit_code = 0
|
||||
try:
|
||||
add(urls=urls, **kwargs)
|
||||
except BaseException:
|
||||
exit_code = 1
|
||||
raise
|
||||
finally:
|
||||
# --bg/--index-only return without the foreground runner's cleanup.
|
||||
# Leaving them RUNNING makes the active-process index accumulate
|
||||
# historical CLI commands forever, defeating bounded recovery.
|
||||
if process.status == Process.StatusChoices.RUNNING:
|
||||
process.mark_exited(exit_code=exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,383 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
archivebox archiveresult <action> [args...] [--filters]
|
||||
|
||||
Manage ArchiveResult records (plugin extraction results).
|
||||
|
||||
Actions:
|
||||
create - Emit plugin extraction request records for Snapshots
|
||||
list - List ArchiveResults as JSONL (with optional filters)
|
||||
update - Update ArchiveResults from stdin JSONL
|
||||
delete - Delete ArchiveResults from stdin JSONL
|
||||
|
||||
Examples:
|
||||
# Emit extraction requests; `archivebox run` schedules their parent snapshots
|
||||
archivebox snapshot list --status=queued | archivebox archiveresult create
|
||||
archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
|
||||
|
||||
# List with filters
|
||||
archivebox archiveresult list --status=failed
|
||||
archivebox archiveresult list --plugin=screenshot --status=succeeded
|
||||
|
||||
# Delete
|
||||
archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
|
||||
|
||||
# Re-run failed extractions
|
||||
archivebox archiveresult list --status=failed | archivebox run
|
||||
"""
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox archiveresult"
|
||||
|
||||
import sys
|
||||
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
|
||||
return {
|
||||
"type": "ArchiveResult",
|
||||
"snapshot_id": str(snapshot_id),
|
||||
"plugin": plugin,
|
||||
"hook_name": hook_name,
|
||||
"status": status,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CREATE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def create_archiveresults(
|
||||
snapshot_id: str | None = None,
|
||||
plugin: str | None = None,
|
||||
status: str = "queued",
|
||||
) -> int:
|
||||
"""
|
||||
Create ArchiveResult request records for Snapshots.
|
||||
|
||||
Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
|
||||
Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
|
||||
If --plugin is specified, only emits requests for that plugin.
|
||||
Otherwise, emits requests for all enabled snapshot hooks.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: Failure
|
||||
"""
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.plugins.hooks import discover_hooks
|
||||
from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
# If snapshot_id provided directly, use that
|
||||
if snapshot_id:
|
||||
try:
|
||||
snapshots = [Snapshot.objects.get(id=snapshot_id)]
|
||||
pass_through_records = []
|
||||
except Snapshot.DoesNotExist:
|
||||
rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
|
||||
return 1
|
||||
else:
|
||||
# Read from stdin
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Separate snapshot records from pass-through records
|
||||
snapshot_ids = []
|
||||
pass_through_records = []
|
||||
|
||||
for record in records:
|
||||
record_type = record.get("type", "")
|
||||
|
||||
if record_type == TYPE_SNAPSHOT:
|
||||
# Pass through the Snapshot record itself
|
||||
pass_through_records.append(record)
|
||||
if record.get("id"):
|
||||
snapshot_ids.append(record["id"])
|
||||
|
||||
elif record_type == TYPE_ARCHIVERESULT:
|
||||
# ArchiveResult records: pass through if they have an id
|
||||
if record.get("id"):
|
||||
pass_through_records.append(record)
|
||||
# If no id, we could create it, but for now just pass through
|
||||
else:
|
||||
pass_through_records.append(record)
|
||||
|
||||
elif record_type:
|
||||
# Other typed records (Crawl, Tag, etc): pass through
|
||||
pass_through_records.append(record)
|
||||
|
||||
elif record.get("id"):
|
||||
# Untyped record with id - assume it's a snapshot ID
|
||||
snapshot_ids.append(record["id"])
|
||||
|
||||
# Output pass-through records first
|
||||
if not is_tty:
|
||||
for record in pass_through_records:
|
||||
write_record(record)
|
||||
|
||||
if not snapshot_ids:
|
||||
if pass_through_records:
|
||||
rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
|
||||
|
||||
if not snapshots:
|
||||
rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
|
||||
return 0 if pass_through_records else 1
|
||||
|
||||
created_count = 0
|
||||
for snapshot in snapshots:
|
||||
config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
|
||||
hooks = [
|
||||
hook
|
||||
for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
|
||||
if not plugin or hook.parent.name == plugin
|
||||
]
|
||||
for hook_path in hooks:
|
||||
hook_name = hook_path.stem
|
||||
plugin_name = hook_path.parent.name
|
||||
if not is_tty:
|
||||
write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
|
||||
created_count += 1
|
||||
|
||||
rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LIST
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def list_archiveresults(
|
||||
status: str | None = None,
|
||||
plugin: str | None = None,
|
||||
snapshot_id: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List ArchiveResults as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = ArchiveResult.objects.all().order_by("-start_ts")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"status": status,
|
||||
"plugin": plugin,
|
||||
"snapshot_id": snapshot_id,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for result in queryset:
|
||||
if is_tty:
|
||||
status_color = {
|
||||
"queued": "yellow",
|
||||
"started": "blue",
|
||||
"succeeded": "green",
|
||||
"failed": "red",
|
||||
"skipped": "dim",
|
||||
"noresults": "dim",
|
||||
"backoff": "magenta",
|
||||
}.get(result.status, "dim")
|
||||
rprint(
|
||||
f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
|
||||
)
|
||||
else:
|
||||
write_record(result.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# UPDATE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_archiveresults(
|
||||
status: str | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update ArchiveResults from stdin JSONL.
|
||||
|
||||
Reads ArchiveResult records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
result_id = record.get("id")
|
||||
if not result_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
result = ArchiveResult.objects.get(id=result_id)
|
||||
|
||||
# Apply updates from CLI flags
|
||||
if status:
|
||||
result.status = status
|
||||
|
||||
result.save()
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
write_record(result.to_json())
|
||||
|
||||
except ArchiveResult.DoesNotExist:
|
||||
rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DELETE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete ArchiveResults from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
result_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not result_ids:
|
||||
rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results = ArchiveResult.objects.filter(id__in=result_ids)
|
||||
count = results.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
|
||||
for result in results[:10]:
|
||||
rprint(f" [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
|
||||
if count > 10:
|
||||
rprint(f" ... and {count - 10} more", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = results.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Commands
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@click.group()
|
||||
def main():
|
||||
"""Manage ArchiveResult records (plugin extraction results)."""
|
||||
pass
|
||||
|
||||
|
||||
@main.command("create")
|
||||
@click.option("--snapshot-id", help="Snapshot ID to create results for")
|
||||
@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
|
||||
@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
|
||||
def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
|
||||
"""Emit Snapshot plugin extraction requests as JSONL."""
|
||||
sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
|
||||
|
||||
|
||||
@main.command("list")
|
||||
@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
|
||||
@click.option("--plugin", "-p", help="Filter by plugin name")
|
||||
@click.option("--snapshot-id", help="Filter by snapshot ID")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(
|
||||
status: str | None,
|
||||
plugin: str | None,
|
||||
snapshot_id: str | None,
|
||||
limit: int | None,
|
||||
):
|
||||
"""List ArchiveResults as JSONL."""
|
||||
sys.exit(
|
||||
list_archiveresults(
|
||||
status=status,
|
||||
plugin=plugin,
|
||||
snapshot_id=snapshot_id,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--status", "-s", help="Set status")
|
||||
def update_cmd(status: str | None):
|
||||
"""Update ArchiveResults from stdin JSONL."""
|
||||
sys.exit(update_archiveresults(status=status))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
"""Delete ArchiveResults from stdin JSONL."""
|
||||
sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,319 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
archivebox binary <action> [args...] [--filters]
|
||||
|
||||
Manage Binary records (detected executables like chrome, wget, etc.).
|
||||
|
||||
Actions:
|
||||
create - Create/register a Binary
|
||||
list - List Binaries as JSONL (with optional filters)
|
||||
update - Update Binaries from stdin JSONL
|
||||
delete - Delete Binaries from stdin JSONL
|
||||
|
||||
Examples:
|
||||
# List all binaries
|
||||
archivebox binary list
|
||||
|
||||
# List specific binary
|
||||
archivebox binary list --name=chrome
|
||||
|
||||
# List binaries with specific version
|
||||
archivebox binary list --version__icontains=120
|
||||
|
||||
# Delete old binary entries
|
||||
archivebox binary list --name=chrome | archivebox binary delete --yes
|
||||
"""
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox binary"
|
||||
|
||||
import sys
|
||||
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CREATE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def create_binary(
|
||||
name: str,
|
||||
abspath: str,
|
||||
version: str = "",
|
||||
) -> int:
|
||||
"""
|
||||
Create/register a Binary.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: Failure
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
if not name or not abspath:
|
||||
rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
machine = Machine.current()
|
||||
created = not Binary.objects.filter(
|
||||
machine=machine,
|
||||
name=name,
|
||||
abspath=abspath,
|
||||
version=version,
|
||||
).exists()
|
||||
|
||||
# Mirror the Binary model lifecycle used elsewhere in the system so CLI
|
||||
# records are owned by the current machine and can be safely piped into
|
||||
# `archivebox run` without creating invalid rows missing machine_id.
|
||||
binary = Binary.from_json(
|
||||
{
|
||||
"name": name,
|
||||
"abspath": abspath,
|
||||
"version": version,
|
||||
"binproviders": "env",
|
||||
"binprovider": "env",
|
||||
},
|
||||
)
|
||||
if binary is None:
|
||||
raise ValueError("failed to create binary record")
|
||||
|
||||
if not is_tty:
|
||||
write_record(binary.to_json())
|
||||
|
||||
if created:
|
||||
rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
|
||||
else:
|
||||
rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LIST
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def list_binaries(
|
||||
name: str | None = None,
|
||||
abspath__icontains: str | None = None,
|
||||
version__icontains: str | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Binaries as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"name": name,
|
||||
"abspath__icontains": abspath__icontains,
|
||||
"version__icontains": version__icontains,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for binary in queryset:
|
||||
if is_tty:
|
||||
rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
|
||||
else:
|
||||
write_record(binary.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# UPDATE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_binaries(
|
||||
version: str | None = None,
|
||||
abspath: str | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update Binaries from stdin JSONL.
|
||||
|
||||
Reads Binary records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
binary_id = record.get("id")
|
||||
if not binary_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
binary = Binary.objects.get(id=binary_id)
|
||||
|
||||
# Apply updates from CLI flags
|
||||
if version:
|
||||
binary.version = version
|
||||
if abspath:
|
||||
binary.abspath = abspath
|
||||
|
||||
binary.save()
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
write_record(binary.to_json())
|
||||
|
||||
except Binary.DoesNotExist:
|
||||
rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DELETE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete Binaries from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
binary_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not binary_ids:
|
||||
rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
binaries = Binary.objects.filter(id__in=binary_ids)
|
||||
count = binaries.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
|
||||
for binary in binaries:
|
||||
rprint(f" {binary.name} {binary.abspath}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = binaries.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Commands
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@click.group()
|
||||
def main():
|
||||
"""Manage Binary records (detected executables)."""
|
||||
pass
|
||||
|
||||
|
||||
@main.command("create")
|
||||
@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
|
||||
@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
|
||||
@click.option("--version", "-v", default="", help="Binary version")
|
||||
def create_cmd(name: str, abspath: str, version: str):
|
||||
"""Create/register a Binary."""
|
||||
sys.exit(create_binary(name=name, abspath=abspath, version=version))
|
||||
|
||||
|
||||
@main.command("list")
|
||||
@click.option("--name", "-n", help="Filter by name")
|
||||
@click.option("--abspath__icontains", help="Filter by path contains")
|
||||
@click.option("--version__icontains", help="Filter by version contains")
|
||||
@click.option("--limit", type=int, help="Limit number of results")
|
||||
def list_cmd(
|
||||
name: str | None,
|
||||
abspath__icontains: str | None,
|
||||
version__icontains: str | None,
|
||||
limit: int | None,
|
||||
):
|
||||
"""List Binaries as JSONL."""
|
||||
sys.exit(
|
||||
list_binaries(
|
||||
name=name,
|
||||
abspath__icontains=abspath__icontains,
|
||||
version__icontains=version__icontains,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--version", "-v", help="Set version")
|
||||
@click.option("--abspath", "-p", help="Set path")
|
||||
def update_cmd(version: str | None, abspath: str | None):
|
||||
"""Update Binaries from stdin JSONL."""
|
||||
sys.exit(update_binaries(version=version, abspath=abspath))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
"""Delete Binaries from stdin JSONL."""
|
||||
sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,199 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
|
||||
import sys
|
||||
import toml
|
||||
import rich_click as click
|
||||
from rich import print
|
||||
from pathlib import Path
|
||||
|
||||
from archivebox.misc.util import docstring, enforce_types
|
||||
from archivebox.misc.toml_util import CustomTOMLEncoder
|
||||
|
||||
|
||||
def _format_toml(config: dict) -> str:
|
||||
return toml.dumps(config, encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n")
|
||||
|
||||
|
||||
@enforce_types
|
||||
def config(
|
||||
*keys,
|
||||
get: bool = False,
|
||||
set: bool = False,
|
||||
search: bool = False,
|
||||
reset: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Get and set your ArchiveBox project configuration values"""
|
||||
|
||||
from archivebox.misc.checks import check_data_folder
|
||||
from archivebox.misc.logging_util import printable_config
|
||||
from abx_plugins.plugins.base.utils import resolve_alias
|
||||
from archivebox.config.collection import write_config_file
|
||||
from archivebox.config import CONSTANTS_CONFIG
|
||||
from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
|
||||
from archivebox.plugins.discovery import discover_plugin_configs
|
||||
|
||||
check_data_folder()
|
||||
|
||||
FLAT_CONFIG = get_config().as_dict()
|
||||
runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
|
||||
readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
|
||||
writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
|
||||
readable_config = {**writable_config, **readonly_config}
|
||||
CONFIGS = get_all_configs()
|
||||
plugin_schemas = {
|
||||
plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
|
||||
}
|
||||
core_config_aliases = {
|
||||
alias.upper(): field_name
|
||||
for field_name, field in ArchiveBoxConfig.model_fields.items()
|
||||
for alias in (field_name, str(field.alias or ""))
|
||||
if alias
|
||||
}
|
||||
|
||||
config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
|
||||
no_args = not (get or set or reset or config_options)
|
||||
|
||||
matching_config = {}
|
||||
if search:
|
||||
if config_options:
|
||||
search_terms = [key.strip().lower() for key in config_options]
|
||||
|
||||
for existing_key, value in readable_config.items():
|
||||
if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
|
||||
matching_config[existing_key] = value
|
||||
|
||||
for alias, key in core_config_aliases.items():
|
||||
if key in readable_config and any(term in alias.lower() for term in search_terms):
|
||||
matching_config[key] = readable_config[key]
|
||||
|
||||
for schema in plugin_schemas.values():
|
||||
for key, metadata in schema.items():
|
||||
if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
|
||||
matching_config[key] = readable_config[key]
|
||||
|
||||
print(printable_config(matching_config))
|
||||
raise SystemExit(not matching_config)
|
||||
|
||||
elif get or no_args:
|
||||
if config_options:
|
||||
config_options = [
|
||||
core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
|
||||
]
|
||||
matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
|
||||
failed_config = [key for key in config_options if key not in readable_config]
|
||||
if failed_config:
|
||||
print("\n[red][X] These options failed to get[/red]")
|
||||
print(" {}".format("\n ".join(config_options)))
|
||||
raise SystemExit(1)
|
||||
else:
|
||||
matching_config = readable_config
|
||||
|
||||
# Display core config sections
|
||||
for config_section in CONFIGS.values():
|
||||
section_header = config_section.toml_section_header
|
||||
if isinstance(section_header, str) and section_header:
|
||||
print(f"[grey53]\\[{section_header}][/grey53]")
|
||||
else:
|
||||
print("[grey53]\\[CONSTANTS] # (read-only)[/grey53]")
|
||||
|
||||
kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
|
||||
print(_format_toml(kv_in_section))
|
||||
print("[grey53]################################################################[/grey53]")
|
||||
|
||||
readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
|
||||
if readonly_keys:
|
||||
print("[grey53]\\[CONSTANTS] # (read-only)[/grey53]")
|
||||
print(_format_toml(readonly_keys))
|
||||
print("[grey53]################################################################[/grey53]")
|
||||
|
||||
plugin_keys = {}
|
||||
|
||||
# Collect all plugin config keys
|
||||
for schema in plugin_schemas.values():
|
||||
for key in schema.keys():
|
||||
if key in matching_config and key in writable_config:
|
||||
plugin_keys[key] = matching_config[key]
|
||||
|
||||
# Display all plugin config in single [PLUGINS] section
|
||||
if plugin_keys:
|
||||
print("[grey53]\\[PLUGINS][/grey53]")
|
||||
print(_format_toml(plugin_keys))
|
||||
print("[grey53]################################################################[/grey53]")
|
||||
|
||||
raise SystemExit(not matching_config)
|
||||
|
||||
elif set:
|
||||
new_config = {}
|
||||
failed_options = []
|
||||
for line in config_options:
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
if "=" not in line:
|
||||
print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
|
||||
print(f" {line}")
|
||||
raise SystemExit(2)
|
||||
|
||||
raw_key, val = line.split("=", 1)
|
||||
raw_key = raw_key.upper().strip()
|
||||
key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
|
||||
if key != raw_key:
|
||||
print(
|
||||
f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
|
||||
)
|
||||
|
||||
if key in writable_config:
|
||||
new_config[key] = val.strip()
|
||||
else:
|
||||
failed_options.append(line)
|
||||
|
||||
if new_config:
|
||||
before = FLAT_CONFIG
|
||||
matching_config = write_config_file(new_config)
|
||||
after = get_config().as_dict()
|
||||
print(printable_config(matching_config))
|
||||
|
||||
side_effect_changes = {}
|
||||
for key, val in after.items():
|
||||
if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
|
||||
side_effect_changes[key] = after[key]
|
||||
|
||||
if side_effect_changes:
|
||||
print(file=sys.stderr)
|
||||
print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
|
||||
print(" {}".format(printable_config(side_effect_changes, prefix=" ")), file=sys.stderr)
|
||||
|
||||
if failed_options:
|
||||
print()
|
||||
print("[red][X] These options failed to set (check for typos):[/red]")
|
||||
print(" {}".format("\n ".join(failed_options)))
|
||||
raise SystemExit(1)
|
||||
|
||||
elif reset:
|
||||
print("[red][X] This command is not implemented yet.[/red]")
|
||||
print(" Please manually remove the relevant lines from your config file:")
|
||||
raise SystemExit(2)
|
||||
|
||||
else:
|
||||
print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
|
||||
print(" archivebox config")
|
||||
print(" archivebox config --get SOME_KEY")
|
||||
print(" archivebox config --set SOME_KEY=SOME_VALUE")
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
|
||||
@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
|
||||
@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
|
||||
@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
|
||||
@click.argument("KEY=VALUE", nargs=-1, type=str)
|
||||
@docstring(config.__doc__)
|
||||
def main(**kwargs) -> None:
|
||||
config(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,406 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
archivebox crawl <action> [args...] [--filters]
|
||||
|
||||
Manage Crawl records.
|
||||
|
||||
Actions:
|
||||
create - Create Crawl jobs from URLs
|
||||
list - List Crawls as JSONL (with optional filters)
|
||||
update - Update Crawls from stdin JSONL
|
||||
delete - Delete Crawls from stdin JSONL
|
||||
|
||||
Examples:
|
||||
# Create
|
||||
archivebox crawl create https://example.com https://foo.com --depth=1
|
||||
archivebox crawl create --tag=news https://example.com
|
||||
|
||||
# List with filters
|
||||
archivebox crawl list --status=queued
|
||||
archivebox crawl list --urls__icontains=example.com
|
||||
|
||||
# Update
|
||||
archivebox crawl list --status=started | archivebox crawl update --status=queued
|
||||
|
||||
# Delete
|
||||
archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
|
||||
|
||||
# Full pipeline
|
||||
archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
|
||||
"""
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox crawl"
|
||||
|
||||
import sys
|
||||
from collections.abc import Iterable
|
||||
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CREATE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def create_crawl(
|
||||
urls: Iterable[str],
|
||||
depth: int = 0,
|
||||
tag: str = "",
|
||||
status: str = "queued",
|
||||
created_by_id: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Create a Crawl job from URLs.
|
||||
|
||||
Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
|
||||
Pass-through: Records that are not URLs are output unchanged (for piping).
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: Failure
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
|
||||
from archivebox.misc.util import validate_url
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
created_by_id = created_by_id or get_or_create_system_user_pk()
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
# Collect all input records
|
||||
records = list(read_args_or_stdin(urls))
|
||||
|
||||
if not records:
|
||||
rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Separate pass-through records from URL records
|
||||
url_list = []
|
||||
pass_through_records = []
|
||||
|
||||
for record in records:
|
||||
record_type = record.get("type", "")
|
||||
|
||||
# Pass-through: output records that aren't URL/Crawl types
|
||||
if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
|
||||
pass_through_records.append(record)
|
||||
continue
|
||||
|
||||
# Handle existing Crawl records (just pass through with id)
|
||||
if record_type == TYPE_CRAWL and record.get("id"):
|
||||
pass_through_records.append(record)
|
||||
continue
|
||||
|
||||
# Collect URLs
|
||||
url = record.get("url")
|
||||
if url:
|
||||
try:
|
||||
url_list.append(validate_url(str(url)))
|
||||
except ValueError as err:
|
||||
rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Handle 'urls' field (newline-separated)
|
||||
urls_field = record.get("urls")
|
||||
if urls_field:
|
||||
for line in urls_field.split("\n"):
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
try:
|
||||
url_list.append(validate_url(line))
|
||||
except ValueError as err:
|
||||
rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Output pass-through records first
|
||||
if not is_tty:
|
||||
for record in pass_through_records:
|
||||
write_record(record)
|
||||
|
||||
if not url_list:
|
||||
if pass_through_records:
|
||||
# If we had pass-through records but no URLs, that's OK
|
||||
rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
rprint("[red]No valid URLs found[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
# Build crawl record with all URLs as newline-separated string
|
||||
crawl_record = {
|
||||
"urls": "\n".join(url_list),
|
||||
"max_depth": depth,
|
||||
"tags_str": tag,
|
||||
"status": status,
|
||||
"label": "",
|
||||
}
|
||||
|
||||
crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
|
||||
if not crawl:
|
||||
rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Output JSONL record (only when piped)
|
||||
if not is_tty:
|
||||
write_record(crawl.to_json())
|
||||
|
||||
rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
|
||||
|
||||
# If TTY, show human-readable output
|
||||
if is_tty:
|
||||
rprint(f" [dim]{crawl.id}[/dim]", file=sys.stderr)
|
||||
for url in url_list[:5]: # Show first 5 URLs
|
||||
rprint(f" {url[:70]}", file=sys.stderr)
|
||||
if len(url_list) > 5:
|
||||
rprint(f" ... and {len(url_list) - 5} more", file=sys.stderr)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LIST
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def list_crawls(
|
||||
status: str | None = None,
|
||||
urls__icontains: str | None = None,
|
||||
max_depth: int | None = None,
|
||||
limit: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
List Crawls as JSONL with optional filters.
|
||||
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Crawl.objects.all().order_by("-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {
|
||||
"status": status,
|
||||
"urls__icontains": urls__icontains,
|
||||
"max_depth": max_depth,
|
||||
}
|
||||
queryset = apply_filters(queryset, filter_kwargs, limit=limit)
|
||||
|
||||
count = 0
|
||||
for crawl in queryset:
|
||||
if is_tty:
|
||||
status_color = {
|
||||
"queued": "yellow",
|
||||
"started": "blue",
|
||||
"sealed": "green",
|
||||
}.get(crawl.status, "dim")
|
||||
url_preview = crawl.urls[:50].replace("\n", " ")
|
||||
rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
|
||||
else:
|
||||
write_record(crawl.to_json())
|
||||
count += 1
|
||||
|
||||
rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# UPDATE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def update_crawls(
|
||||
status: str | None = None,
|
||||
max_depth: int | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Update Crawls from stdin JSONL.
|
||||
|
||||
Reads Crawl records from stdin and applies updates.
|
||||
Uses PATCH semantics - only specified fields are updated.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or error
|
||||
"""
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.misc.jsonl import read_stdin, write_record
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
updated_count = 0
|
||||
for record in records:
|
||||
crawl_id = record.get("id")
|
||||
if not crawl_id:
|
||||
continue
|
||||
|
||||
try:
|
||||
crawl = Crawl.objects.get(id=crawl_id)
|
||||
|
||||
if status:
|
||||
if status not in Crawl.StatusChoices.values:
|
||||
rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
|
||||
continue
|
||||
if status == Crawl.StatusChoices.SEALED:
|
||||
crawl.cancel()
|
||||
elif status == Crawl.StatusChoices.PAUSED:
|
||||
crawl.pause()
|
||||
elif status == Crawl.StatusChoices.QUEUED:
|
||||
if crawl.status == Crawl.StatusChoices.PAUSED:
|
||||
crawl.resume()
|
||||
else:
|
||||
crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
|
||||
elif status == Crawl.StatusChoices.STARTED:
|
||||
crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
|
||||
if max_depth is not None:
|
||||
crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
|
||||
crawl.max_depth = max_depth
|
||||
elif not status:
|
||||
crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
|
||||
updated_count += 1
|
||||
|
||||
if not is_tty:
|
||||
crawl.refresh_from_db()
|
||||
write_record(crawl.to_json())
|
||||
|
||||
except Crawl.DoesNotExist:
|
||||
rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
|
||||
continue
|
||||
|
||||
rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DELETE
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
|
||||
"""
|
||||
Delete Crawls from stdin JSONL.
|
||||
|
||||
Requires --yes flag to confirm deletion.
|
||||
|
||||
Exit codes:
|
||||
0: Success
|
||||
1: No input or missing --yes flag
|
||||
"""
|
||||
from archivebox.misc.jsonl import read_stdin
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
records = list(read_stdin())
|
||||
if not records:
|
||||
rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
crawl_ids = [r.get("id") for r in records if r.get("id")]
|
||||
|
||||
if not crawl_ids:
|
||||
rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
crawls = Crawl.objects.filter(id__in=crawl_ids)
|
||||
count = crawls.count()
|
||||
|
||||
if count == 0:
|
||||
rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if dry_run:
|
||||
rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
|
||||
for crawl in crawls:
|
||||
url_preview = crawl.urls[:50].replace("\n", " ")
|
||||
rprint(f" [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
if not yes:
|
||||
rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Perform deletion
|
||||
deleted_count, _ = crawls.delete()
|
||||
rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Commands
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@click.group()
|
||||
def main():
|
||||
"""Manage Crawl records."""
|
||||
pass
|
||||
|
||||
|
||||
@main.command("create")
|
||||
@click.argument("urls", nargs=-1)
|
||||
@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
|
||||
@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
|
||||
@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
|
||||
def create_cmd(urls: tuple, depth: int, tag: str, status: str):
|
||||
"""Create a Crawl job from URLs or stdin."""
|
||||
sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
|
||||
|
||||
|
||||
@main.command("list")
|
||||
@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
|
||||
@click.option("--urls__icontains", help="Filter by URLs contains")
|
||||
@click.option("--max-depth", type=int, help="Filter by max depth")
|
||||
@click.option("--limit", "-n", type=int, help="Limit number of results")
|
||||
def list_cmd(
|
||||
status: str | None,
|
||||
urls__icontains: str | None,
|
||||
max_depth: int | None,
|
||||
limit: int | None,
|
||||
):
|
||||
"""List Crawls as JSONL."""
|
||||
sys.exit(
|
||||
list_crawls(
|
||||
status=status,
|
||||
urls__icontains=urls__icontains,
|
||||
max_depth=max_depth,
|
||||
limit=limit,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@main.command("update")
|
||||
@click.option("--status", "-s", help="Set status")
|
||||
@click.option("--max-depth", type=int, help="Set max depth")
|
||||
def update_cmd(status: str | None, max_depth: int | None):
|
||||
"""Update Crawls from stdin JSONL."""
|
||||
sys.exit(update_crawls(status=status, max_depth=max_depth))
|
||||
|
||||
|
||||
@main.command("delete")
|
||||
@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
|
||||
def delete_cmd(yes: bool, dry_run: bool):
|
||||
"""Delete Crawls from stdin JSONL."""
|
||||
sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,203 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox extract"
|
||||
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
import rich_click as click
|
||||
|
||||
|
||||
def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
|
||||
"""Resolve CLI records to snapshot-level execution requests.
|
||||
|
||||
ArchiveResult input is accepted as a convenient reference to its parent
|
||||
Snapshot and plugin. It is never reset or converted into queued work.
|
||||
"""
|
||||
from archivebox.api.v1_core import _uuid_ref_query
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
|
||||
|
||||
explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
|
||||
requested: dict[str, set[str]] = defaultdict(set)
|
||||
missing: set[str] = set()
|
||||
|
||||
for record in records:
|
||||
record_type = record.get("type")
|
||||
record_id = str(record.get("id") or "")
|
||||
|
||||
if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
|
||||
requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
|
||||
requested[str(record["snapshot_id"])].discard("")
|
||||
continue
|
||||
|
||||
if record_type == TYPE_ARCHIVERESULT:
|
||||
result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
|
||||
if result is not None:
|
||||
requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
|
||||
continue
|
||||
|
||||
snapshot_id = str(record.get("snapshot_id") or record_id or "")
|
||||
snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
|
||||
if snapshot is None and record.get("url"):
|
||||
snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
|
||||
# Bare UUID CLI arguments are parsed as Snapshot records because the
|
||||
# input layer cannot know which model owns them. Preserve the public
|
||||
# convenience of passing an ArchiveResult ID by trying that reference
|
||||
# only after the Snapshot lookup misses.
|
||||
if snapshot is None and record_id and not record.get("url"):
|
||||
result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
|
||||
if result is not None:
|
||||
requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
|
||||
continue
|
||||
if snapshot is None:
|
||||
missing.add(snapshot_id or str(record.get("url") or ""))
|
||||
continue
|
||||
requested[str(snapshot.id)].update(explicit_plugins)
|
||||
|
||||
return requested, missing
|
||||
|
||||
|
||||
def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
|
||||
from django.utils import timezone
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.services.runner import run_crawl
|
||||
|
||||
snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
|
||||
if not snapshots:
|
||||
rprint("[red]No snapshots to process[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.plugins.discovery import get_enabled_plugins
|
||||
|
||||
for snapshot_id, plugin_names in requested.items():
|
||||
snapshot = snapshots.get(snapshot_id)
|
||||
if snapshot is not None and not plugin_names:
|
||||
plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
|
||||
|
||||
if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
|
||||
from archivebox.core.takeover_util import ensure_daemon_stack
|
||||
|
||||
with redirect_stdout(sys.stderr):
|
||||
ensure_daemon_stack(reason="Sonic snapshot indexing")
|
||||
|
||||
# Explicit extraction resumes open/paused snapshots at the snapshot level.
|
||||
# Sealed snapshots stay sealed during targeted maintenance backfills.
|
||||
if wait:
|
||||
for snapshot in snapshots.values():
|
||||
if snapshot.status != Snapshot.StatusChoices.SEALED:
|
||||
snapshot.update_and_requeue(
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
|
||||
if not wait:
|
||||
for snapshot_id, plugin_names in requested.items():
|
||||
snapshot = snapshots.get(snapshot_id)
|
||||
if snapshot is None:
|
||||
continue
|
||||
if plugin_names:
|
||||
snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
|
||||
elif snapshot.status == Snapshot.StatusChoices.SEALED:
|
||||
snapshot.update_and_requeue(retry_at=timezone.now())
|
||||
else:
|
||||
snapshot.update_and_requeue(
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
if show_progress:
|
||||
rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
|
||||
for snapshot_id, plugin_names in requested.items():
|
||||
snapshot = snapshots.get(snapshot_id)
|
||||
if snapshot is not None:
|
||||
grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
|
||||
|
||||
for (crawl_id, plugin_names), snapshot_ids in grouped.items():
|
||||
run_crawl(
|
||||
crawl_id,
|
||||
snapshot_ids=sorted(snapshot_ids),
|
||||
selected_plugins=list(plugin_names) or None,
|
||||
show_progress=show_progress,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def run_plugins(
|
||||
args: tuple,
|
||||
records: list[dict] | None = None,
|
||||
plugins: str = "",
|
||||
wait: bool = True,
|
||||
emit_results: bool = True,
|
||||
show_progress: bool = True,
|
||||
) -> int:
|
||||
"""Execute selected plugins through the snapshot-level runner."""
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.misc.jsonl import read_args_or_stdin, write_record
|
||||
|
||||
if records is None:
|
||||
records = list(read_args_or_stdin(args))
|
||||
if not records:
|
||||
rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
requested, missing = _resolve_requests(records, plugins)
|
||||
for value in sorted(missing):
|
||||
rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
|
||||
if not requested:
|
||||
return 1
|
||||
|
||||
exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
|
||||
if exit_code or not emit_results:
|
||||
return exit_code
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
for snapshot in Snapshot.objects.filter(id__in=requested):
|
||||
results = snapshot.archiveresult_set.all()
|
||||
requested_plugins = requested[str(snapshot.id)]
|
||||
if requested_plugins:
|
||||
results = results.filter(plugin__in=requested_plugins)
|
||||
for result in results:
|
||||
if is_tty:
|
||||
color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
|
||||
rprint(f" [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
|
||||
else:
|
||||
write_record(result.to_json())
|
||||
return 0
|
||||
|
||||
|
||||
def process_archiveresult_by_id(archiveresult_id: str) -> int:
|
||||
"""Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
|
||||
return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
|
||||
@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
|
||||
@click.argument("args", nargs=-1)
|
||||
def main(plugins: str, wait: bool, args: tuple):
|
||||
"""Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
|
||||
from archivebox.misc.jsonl import read_args_or_stdin
|
||||
|
||||
records = list(read_args_or_stdin(args))
|
||||
if not records:
|
||||
from rich import print as rprint
|
||||
|
||||
rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
__package__ = "archivebox.cli"
|
||||
__command__ = "archivebox help"
|
||||
|
||||
import ast
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich import print
|
||||
from rich.panel import Panel
|
||||
|
||||
|
||||
def _command_doc(cmd: str, import_path: str) -> str:
|
||||
def first_doc_line(docstring: str | None) -> str:
|
||||
lines = (docstring or "").splitlines()
|
||||
return lines[0] if lines else ""
|
||||
|
||||
modname, _ = import_path.rsplit(".", 1)
|
||||
spec = importlib.util.find_spec(modname)
|
||||
if spec is None or spec.origin is None:
|
||||
return ""
|
||||
|
||||
try:
|
||||
tree = ast.parse(Path(spec.origin).read_text())
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
for name in (cmd, "main"):
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.FunctionDef) and node.name == name:
|
||||
return first_doc_line(ast.get_docstring(node))
|
||||
|
||||
return first_doc_line(ast.get_docstring(tree))
|
||||
|
||||
|
||||
def help() -> None:
|
||||
"""Print the ArchiveBox help message and usage"""
|
||||
|
||||
from archivebox.cli import ArchiveBoxGroup
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.permissions import IN_DOCKER
|
||||
|
||||
version_msg = (
|
||||
f"[dark_magenta]\\[{datetime.now(UTC):%Y-%m-%d %H:%M:%S}][/dark_magenta] "
|
||||
f"[dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]: "
|
||||
"[green4]archivebox [green3]help"
|
||||
)
|
||||
print(Panel(version_msg), file=sys.stderr)
|
||||
|
||||
COMMANDS_HELP_TEXT = (
|
||||
"\n ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
|
||||
+ "\n\n "
|
||||
+ "\n ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
|
||||
+ "\n\n "
|
||||
+ "\n ".join(
|
||||
f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
|
||||
)
|
||||
)
|
||||
|
||||
DOCKER_USAGE = (
|
||||
"""
|
||||
[dodger_blue3]Docker Usage:[/dodger_blue3]
|
||||
[grey53]# using Docker Compose:[/grey53]
|
||||
[blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
|
||||
|
||||
[grey53]# using Docker:[/grey53]
|
||||
[blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
|
||||
"""
|
||||
if IN_DOCKER
|
||||
else ""
|
||||
)
|
||||
DOCKER_DOCS = (
|
||||
"\n [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
|
||||
if IN_DOCKER
|
||||
else ""
|
||||
)
|
||||
DOCKER_OUTSIDE_HINT = "\n [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
|
||||
DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
|
||||
|
||||
print(f"""{DOCKER_USAGE}
|
||||
[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
|
||||
[dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
|
||||
|
||||
[deep_sky_blue4]Commands:[/deep_sky_blue4]
|
||||
{COMMANDS_HELP_TEXT}
|
||||
|
||||
[deep_sky_blue4]Documentation:[/deep_sky_blue4]
|
||||
[link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
|
||||
[link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
|
||||
[link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
|
||||
""")
|
||||
|
||||
if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
|
||||
pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
|
||||
EXAMPLE_USAGE = f"""
|
||||
[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
|
||||
|
||||
[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
|
||||
[dark_green]archivebox[/dark_green] [green]init[/green] [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
|
||||
[dark_green]archivebox[/dark_green] [green]install[/green] [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
|
||||
[dark_green]archivebox[/dark_green] [green]status[/green] [grey53]# get a health checkup report on your collection[/grey53]
|
||||
[dark_green]archivebox[/dark_green] [green]update[/green] [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
|
||||
|
||||
[violet]Hint:[/violet] [i]More example usage:[/i]
|
||||
[dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
|
||||
[dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
|
||||
[dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
|
||||
[dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue] [grey53]# Start the Web UI / API server[/grey53]
|
||||
"""
|
||||
print(
|
||||
Panel(
|
||||
EXAMPLE_USAGE,
|
||||
expand=False,
|
||||
border_style="grey53",
|
||||
title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
|
||||
subtitle="Commands run inside this dir will only apply to this collection.",
|
||||
),
|
||||
)
|
||||
else:
|
||||
DATA_SETUP_HELP = "\n"
|
||||
if IN_DOCKER:
|
||||
DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
|
||||
DATA_SETUP_HELP += " docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
|
||||
DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
|
||||
DATA_SETUP_HELP += " 1. [green]cd[/green] ~/archivebox/data [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
|
||||
DATA_SETUP_HELP += f" 2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green] [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
|
||||
DATA_SETUP_HELP += f" 3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green] [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
|
||||
DATA_SETUP_HELP += f" 4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green] [grey53]# ...get help with next steps... [/grey53]\n\n"
|
||||
DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
|
||||
DATA_SETUP_HELP += " 1. [green]mkdir[/green] ~/archivebox/data [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
|
||||
DATA_SETUP_HELP += " 2. [green]cd[/green] ~/archivebox/data [grey53]# cd into the new directory[/grey53]\n"
|
||||
DATA_SETUP_HELP += f" 3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green] [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
|
||||
DATA_SETUP_HELP += f" 4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green] [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
|
||||
DATA_SETUP_HELP += f" 5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green] [grey53]# ... get help with next steps... [/grey53]\n"
|
||||
print(
|
||||
Panel(
|
||||
DATA_SETUP_HELP,
|
||||
expand=False,
|
||||
border_style="grey53",
|
||||
title="[red]:cross_mark: No collection is currently active[/red]",
|
||||
subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--help", "-h", is_flag=True, help="Show help")
|
||||
def main(**kwargs):
|
||||
"""Print the ArchiveBox help message and usage"""
|
||||
return help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,215 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
__package__ = "archivebox.cli"
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from rich import print
|
||||
import rich_click as click
|
||||
|
||||
from archivebox.misc.util import docstring, enforce_types
|
||||
|
||||
|
||||
def _display_data_path(path: Path, data_dir: Path) -> str:
|
||||
path = Path(path).resolve()
|
||||
data_dir = Path(data_dir).resolve()
|
||||
try:
|
||||
return f"./{path.relative_to(data_dir)}"
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
|
||||
"""Initialize a new ArchiveBox collection in the current directory"""
|
||||
|
||||
from archivebox.misc.checks import check_not_inside_source_dir
|
||||
|
||||
check_not_inside_source_dir()
|
||||
|
||||
from archivebox.config import CONSTANTS, VERSION
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.config.collection import write_config_file
|
||||
from archivebox.misc.db import apply_migrations, database_exists, ensure_database_ready
|
||||
from archivebox.misc.checks import check_migrations
|
||||
|
||||
config = get_config()
|
||||
|
||||
is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
|
||||
existing_index = database_exists()
|
||||
if is_empty and not existing_index:
|
||||
print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
|
||||
print("[green]----------------------------------------------------------------------[/green]")
|
||||
elif existing_index:
|
||||
# TODO: properly detect and print the existing version in current index as well
|
||||
print(f"[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]")
|
||||
print("[green]----------------------------------------------------------------------[/green]")
|
||||
else:
|
||||
if force:
|
||||
print("[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]")
|
||||
print("[red] Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]")
|
||||
else:
|
||||
print(
|
||||
"[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
|
||||
" You must run init in a completely empty directory, or an existing data folder.\n\n"
|
||||
" [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
|
||||
" then run and run 'archivebox init' to pick up where you left off.\n\n"
|
||||
" (Always make sure your data folder is backed up first before updating ArchiveBox)",
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
if existing_index:
|
||||
print("\n[green][*] Verifying archive folder structure...[/green]")
|
||||
else:
|
||||
print("\n[green][+] Building archive folder structure...[/green]")
|
||||
|
||||
archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
|
||||
sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
|
||||
logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
|
||||
print(f" + {archive_path}, {sources_path}, {logs_path}...")
|
||||
Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
|
||||
CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
|
||||
for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
|
||||
path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
|
||||
|
||||
print(f" + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
|
||||
|
||||
# create the .archivebox_id file with a unique ID for this collection
|
||||
from archivebox.config.paths import _get_collection_id
|
||||
|
||||
_get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
|
||||
|
||||
# create the ArchiveBox.conf file
|
||||
write_config_file({"SECRET_KEY": config.SECRET_KEY})
|
||||
|
||||
if existing_index:
|
||||
print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
|
||||
else:
|
||||
print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
|
||||
|
||||
# For postgres, make sure the server is reachable and create the database
|
||||
# if it doesn't exist yet (sqlite creates its file automatically).
|
||||
ensure_database_ready()
|
||||
|
||||
from archivebox.config.django import setup_django
|
||||
|
||||
setup_django()
|
||||
previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
|
||||
os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
|
||||
try:
|
||||
check_migrations(blocking=True, auto_apply=False)
|
||||
|
||||
for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
|
||||
sys.stdout.write(f" {migration_line}\n")
|
||||
finally:
|
||||
if previous_wants_init is None:
|
||||
os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
|
||||
else:
|
||||
os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
|
||||
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
# Registration requires the final schema and reconciles legacy file config.
|
||||
Machine.current()
|
||||
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
# Initialize here, not when /add/ is rendered: a GET must not create profiles.
|
||||
Persona.get_or_create_default().ensure_dirs()
|
||||
|
||||
from archivebox.misc.db import database_display_location, is_postgres
|
||||
|
||||
assert database_exists()
|
||||
print()
|
||||
if is_postgres():
|
||||
print(f" √ {database_display_location()}")
|
||||
else:
|
||||
print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
|
||||
|
||||
print()
|
||||
print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
snapshot_count = 0
|
||||
|
||||
if existing_index:
|
||||
snapshot_count = Snapshot.objects.count()
|
||||
print(f" √ Loaded {snapshot_count} links from existing main index.")
|
||||
|
||||
print(" > Skipping orphan snapshot import during init.")
|
||||
print()
|
||||
print(" [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
|
||||
print(" archivebox update --rescan --migrate-only")
|
||||
|
||||
print("\n[green]----------------------------------------------------------------------[/green]")
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
config = get_config()
|
||||
if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter(
|
||||
username=config.ADMIN_USERNAME,
|
||||
).exists():
|
||||
print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
|
||||
User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD)
|
||||
|
||||
if existing_index:
|
||||
print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
|
||||
else:
|
||||
print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]")
|
||||
|
||||
CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
|
||||
|
||||
config = get_config()
|
||||
create_and_chown_dir(config.TMP_DIR)
|
||||
create_and_chown_dir(config.ABXPKG_LIB_DIR)
|
||||
create_and_chown_dir(config.ABXPKG_LIB_DIR / "bin")
|
||||
|
||||
working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
|
||||
if working_tmp_dir:
|
||||
working_tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
|
||||
if working_lib_dir:
|
||||
working_lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if install:
|
||||
from archivebox.cli.archivebox_install import install as install_method
|
||||
|
||||
install_method()
|
||||
|
||||
if Snapshot.objects.count() < 25: # hide the hints for experienced users
|
||||
from archivebox.core.routes_util import build_admin_url
|
||||
|
||||
admin_url = build_admin_url("/admin/", config=config)
|
||||
print()
|
||||
print(" [violet]Hint:[/violet] To finish web setup, run:")
|
||||
print(
|
||||
f" archivebox server # then visit [deep_sky_blue4][link={admin_url}]{admin_url}[/link][/deep_sky_blue4]",
|
||||
)
|
||||
print()
|
||||
print(" To add new links, you can run:")
|
||||
print(" archivebox add < ~/some/path/to/list_of_links.txt")
|
||||
print()
|
||||
print(" For more usage and examples, run:")
|
||||
print(" archivebox help")
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
|
||||
@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
|
||||
@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
|
||||
@docstring(init.__doc__)
|
||||
def main(**kwargs) -> None:
|
||||
init(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loaded 100 of 937 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user