Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nick Sweeting
|
726bcabd82 | ||
|
Nick Sweeting
|
d18418cc22 |
No files matched your search
+2
-67
@@ -1,63 +1,24 @@
|
||||
.DS_Store
|
||||
._*
|
||||
/.heartbeat.json
|
||||
/package.json
|
||||
/package-lock.json
|
||||
__pycache__
|
||||
**/__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
__pycache__/
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.uv-cache/
|
||||
.cache/
|
||||
.cache*/
|
||||
.github/
|
||||
.pdm-build/
|
||||
.pdm-build/**
|
||||
.pdm-python
|
||||
.eggs/
|
||||
*.egg-info/
|
||||
.git/
|
||||
!.git/
|
||||
.git/*
|
||||
.vscode/
|
||||
!.git/HEAD
|
||||
!.git/packed-refs
|
||||
!.git/refs/
|
||||
!.git/refs/heads/
|
||||
!.git/refs/heads/**
|
||||
!.git/refs/heads/*
|
||||
|
||||
venv/
|
||||
.venv/
|
||||
.venv-old/
|
||||
.docker_venv/
|
||||
.docker-venv/
|
||||
node_modules/
|
||||
abx-dl/
|
||||
abxpkg/
|
||||
abx-plugins/
|
||||
abxbus/
|
||||
chrome/
|
||||
chromeprofile/
|
||||
chrome_profile/
|
||||
lib/
|
||||
out/
|
||||
tests/out/
|
||||
users/
|
||||
archive/
|
||||
crawls/
|
||||
snapshots/
|
||||
logs/
|
||||
archivebox-docker-smoke*/
|
||||
archivebox-compose-smoke*/
|
||||
docker-test/
|
||||
docker-test*/
|
||||
core
|
||||
*.core
|
||||
|
||||
pdm.dev.lock
|
||||
pdm.lock
|
||||
@@ -68,37 +29,11 @@ dist/
|
||||
brew_dist/
|
||||
deb_dist/
|
||||
pip_dist/
|
||||
cache/
|
||||
caches/
|
||||
assets/
|
||||
docker/
|
||||
website/
|
||||
typings/
|
||||
|
||||
# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
|
||||
# development UI screenshots or test trees. PNGs are already compressed, so
|
||||
# including them creates a large image layer that later cleanup cannot remove.
|
||||
publicsite/
|
||||
old/
|
||||
test/
|
||||
tests/
|
||||
archivebox/tests/
|
||||
|
||||
tmp/
|
||||
.tmp/
|
||||
data/
|
||||
data*/
|
||||
-
|
||||
personas/
|
||||
sources/
|
||||
output/
|
||||
index.sqlite3
|
||||
index.sqlite3-wal
|
||||
queue.sqlite3
|
||||
*.sqlite*
|
||||
data.*
|
||||
.archivebox_id
|
||||
ArchiveBox.conf
|
||||
*.stdout
|
||||
*.stderr
|
||||
*.log
|
||||
@@ -0,0 +1,2 @@
|
||||
**/*.lock
|
||||
**/*-lock.json
|
||||
@@ -1,26 +0,0 @@
|
||||
# Read the Docs config for https://docs.archivebox.io
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html
|
||||
|
||||
version: 2
|
||||
|
||||
submodules:
|
||||
include: all
|
||||
recursive: true
|
||||
|
||||
build:
|
||||
os: ubuntu-22.04
|
||||
tools:
|
||||
python: "3.12"
|
||||
#nodejs: "20" # not needed unless we need the full archivebox to run while building docs for some reason
|
||||
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
|
||||
formats:
|
||||
- pdf
|
||||
- epub
|
||||
|
||||
# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
github: ["ArchiveBox", "pirate"]
|
||||
custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
|
||||
custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
|
||||
@@ -1,197 +0,0 @@
|
||||
name: 🐞 Bug report
|
||||
description: Report a bug or error you encountered in ArchiveBox
|
||||
title: "Bug: ..."
|
||||
assignees:
|
||||
- pirate
|
||||
type: 'Bug'
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
*Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).
|
||||
Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Provide a screenshot and describe the bug
|
||||
description: |
|
||||
Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.
|
||||
placeholder: |
|
||||
Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
|
||||
I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: steps_to_reproduce
|
||||
attributes:
|
||||
label: Steps to reproduce
|
||||
description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
|
||||
render: markdown
|
||||
placeholder: |
|
||||
1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
|
||||
2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
|
||||
3. Typed 'https://example.com/xyz' into the 'Add URL' input field
|
||||
4. Clicked the 'Add+' button
|
||||
5. Got a 500 error and saw the errors below in terminal
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs or errors
|
||||
description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
|
||||
placeholder: |
|
||||
╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
|
||||
│ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567 │
|
||||
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
||||
|
||||
[+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
|
||||
> Saved verbatim input to sources/1730577269-import.txt
|
||||
> Parsed 1 URLs from input (Generic TXT)
|
||||
...
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: ArchiveBox Version
|
||||
description: |
|
||||
**REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).
|
||||
For Docker Compose run: `docker compose run archivebox version`
|
||||
For plain Docker run: `docker run -v $PWD:/data archivebox/archivebox version`
|
||||
render: shell
|
||||
placeholder: |
|
||||
0.8.6
|
||||
ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
|
||||
IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
|
||||
EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
|
||||
DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
|
||||
|
||||
Binary Dependencies:
|
||||
√ python 3.14.0 venv_pip ~/.venv/bin/python
|
||||
√ django 6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/__init__.py
|
||||
√ sqlite 2.6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
|
||||
√ pip 24.3.1 venv_pip ~/.venv/bin/pip
|
||||
...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: dropdown
|
||||
id: install_method
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: How did you install the version of ArchiveBox you are using?
|
||||
multiple: false
|
||||
options:
|
||||
- pip
|
||||
- apt
|
||||
- brew
|
||||
- nix
|
||||
- Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
|
||||
- Other
|
||||
|
||||
- type: dropdown
|
||||
id: operating_system
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: What operating system are you running on?
|
||||
description: |
|
||||
Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
|
||||
multiple: false
|
||||
options:
|
||||
- Linux (Ubuntu/Debian/Arch/Alpine/etc.)
|
||||
- macOS (including Docker on macOS)
|
||||
- BSD (FreeBSD/OpenBSD/NetBSD/etc.)
|
||||
- Windows (including WSL, WSL2, Docker Desktop on Windows)
|
||||
- Other
|
||||
|
||||
- type: checkboxes
|
||||
id: filesystem
|
||||
attributes:
|
||||
label: What type of drive are you using to store your ArchiveBox data?
|
||||
description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
|
||||
options:
|
||||
- label: "some of `data/` is on a local SSD or NVMe drive"
|
||||
required: false
|
||||
- label: "some of `data/` is on a spinning hard drive or external USB drive"
|
||||
required: false
|
||||
- label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
|
||||
required: false
|
||||
- label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
|
||||
required: false
|
||||
|
||||
|
||||
- type: textarea
|
||||
id: docker_compose_yml
|
||||
attributes:
|
||||
label: Docker Compose Configuration
|
||||
description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
|
||||
placeholder: |
|
||||
services:
|
||||
archivebox:
|
||||
image: archivebox/archivebox:latest
|
||||
ports:
|
||||
- 5797:5797
|
||||
volumes:
|
||||
- ./data:/data
|
||||
environment:
|
||||
- ADMIN_USERNAME=admin
|
||||
- ADMIN_PASSWORD=****<redact any passwords>****
|
||||
- ALLOWED_HOSTS=*
|
||||
- CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
|
||||
- PUBLIC_INDEX=True
|
||||
- PUBLIC_SNAPSHOTS=True
|
||||
- PUBLIC_ADD_VIEW=False
|
||||
...
|
||||
|
||||
archivebox_scheduler:
|
||||
image: archivebox/archivebox:latest
|
||||
command: schedule --foreground --update --every=day
|
||||
environment:
|
||||
...
|
||||
|
||||
...
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: configuration
|
||||
attributes:
|
||||
label: ArchiveBox Configuration
|
||||
description: "Please share your full `data/ArchiveBox.conf` file here."
|
||||
render: shell
|
||||
placeholder: |
|
||||
[SERVER_CONFIG]
|
||||
SECRET_KEY = "*********<redact any secrets/passwords>************"
|
||||
|
||||
WGET_RESTRICT_FILE_NAMES=windows
|
||||
USE_SYSTEM_WGET=true
|
||||
CHECK_SSL_VALIDITY=false
|
||||
...
|
||||
validations:
|
||||
required: false
|
||||
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
|
||||
We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.
|
||||
Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.
|
||||
|
||||
In the meantime please consider:
|
||||
|
||||
- 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
|
||||
- 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
|
||||
- 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
|
||||
- ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
name: 💡 Feature or enhancement request
|
||||
description: Suggest an idea or improvement for this project
|
||||
title: "Feature Request: ..."
|
||||
assignees:
|
||||
- pirate
|
||||
type: 'Enhancement'
|
||||
labels: 'status: idea phase'
|
||||
body:
|
||||
- type: dropdown
|
||||
id: suggestion_type
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: "What type of suggestion are you making?"
|
||||
multiple: false
|
||||
options:
|
||||
- New extractor / type of content to save
|
||||
- Proposing a new feature
|
||||
- Modification of existing behavior
|
||||
- Web UI or UX design improvement
|
||||
|
||||
- type: textarea
|
||||
id: current_problem
|
||||
attributes:
|
||||
label: "What is the problem that your feature request solves?"
|
||||
description: |
|
||||
Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
|
||||
placeholder: |
|
||||
e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: proposed_solution
|
||||
attributes:
|
||||
label: "What is your proposed solution?"
|
||||
description: |
|
||||
Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
|
||||
placeholder: |
|
||||
e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.
|
||||
The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: workarounds_tried
|
||||
attributes:
|
||||
label: "What hacks or alternative solutions have you tried to solve the problem?"
|
||||
description: |
|
||||
A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
|
||||
placeholder: |
|
||||
e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside
|
||||
the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: Share the entire output of the `archivebox version` command for the current version you are using.
|
||||
description: |
|
||||
DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.
|
||||
We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.
|
||||
Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
|
||||
|
||||
Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).
|
||||
For Docker Compose run: `docker compose run archivebox version`
|
||||
For plain Docker run: `docker run -v $PWD:/data archivebox/archivebox version`
|
||||
render: shell
|
||||
placeholder: |
|
||||
0.8.6
|
||||
ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
|
||||
IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
|
||||
EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
|
||||
DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
|
||||
|
||||
Binary Dependencies:
|
||||
√ python 3.14.0 venv_pip ~/.venv/bin/python
|
||||
√ django 6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/__init__.py
|
||||
√ sqlite 2.6.0 venv_pip ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
|
||||
√ pip 24.3.1 venv_pip ~/.venv/bin/pip
|
||||
...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
id: priority
|
||||
attributes:
|
||||
label: "How badly do you want this new feature?"
|
||||
options:
|
||||
- label: "It's an urgent deal-breaker, I can't live without it"
|
||||
required: false
|
||||
- label: "It's important to add it in the near-mid term future"
|
||||
required: false
|
||||
- label: "It would be nice to have eventually"
|
||||
required: false
|
||||
- label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
|
||||
required: false
|
||||
- label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
|
||||
required: false
|
||||
|
||||
- type: checkboxes
|
||||
id: satisfaction_survey
|
||||
attributes:
|
||||
label: Mini Survey
|
||||
description: How do you like ArchiveBox so far?
|
||||
options:
|
||||
- label: "I like ArchiveBox so far / would recommend it to a friend"
|
||||
required: false
|
||||
- label: "I've had a lot of difficulty getting ArchiveBox set up"
|
||||
required: false
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
|
||||
We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.
|
||||
Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.
|
||||
|
||||
In the meantime please consider:
|
||||
|
||||
- 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
|
||||
- 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
|
||||
- ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
|
||||
@@ -1,51 +0,0 @@
|
||||
name: 📑 Documentation improvement
|
||||
description: Submit an idea or correction for the Wiki documentation
|
||||
title: "Documentation: ..."
|
||||
labels: 'touches: docs'
|
||||
type: 'Enhancement'
|
||||
assignees:
|
||||
- pirate
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
|
||||
|
||||
- type: input
|
||||
id: page_url
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: "What is the URL of the page you'd like to see improved?"
|
||||
placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
|
||||
|
||||
- type: input
|
||||
id: section_title
|
||||
validations:
|
||||
required: true
|
||||
attributes:
|
||||
label: "What is the title of the relevant section?"
|
||||
placeholder: e.g. Option B. Automatic Setup Script
|
||||
|
||||
- type: textarea
|
||||
id: suggested_edit
|
||||
attributes:
|
||||
label: "What is the suggested edit?"
|
||||
placeholder: |
|
||||
e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.
|
||||
Attach images, screenshots, code snippets, etc. anything you think would help.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
|
||||
We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.
|
||||
|
||||
In the meantime please consider:
|
||||
|
||||
- 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
|
||||
- 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
|
||||
- ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
name: 🐞 Bug report
|
||||
about: Create a report to help us improve
|
||||
title: 'Bug: ...'
|
||||
labels: 'bug'
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!--
|
||||
Please fill out the following information,
|
||||
feel free to delete sections if they're not applicable
|
||||
or if long issue templates annoy you.
|
||||
(the only required section is the version information)
|
||||
-->
|
||||
|
||||
#### Describe the bug
|
||||
<!--
|
||||
A description of what the bug is,
|
||||
what you expected to happen,
|
||||
and any relevant context about issue.
|
||||
-->
|
||||
|
||||
#### Steps to reproduce
|
||||
<!--
|
||||
For example:
|
||||
1. Ran ArchiveBox with the following config '...'
|
||||
2. Saw this output during archiving '....'
|
||||
3. UI didn't show the thing I was expecting '....'
|
||||
-->
|
||||
|
||||
#### Screenshots or log output
|
||||
|
||||
<!--
|
||||
If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
|
||||
If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
|
||||
-->
|
||||
|
||||
#### ArchiveBox version
|
||||
|
||||
<!-- Run the `archivebox version` command locally then copy paste the result here: -->
|
||||
```logs
|
||||
replace this line with the *full*, unshortened output of running `archivebox version`
|
||||
```
|
||||
<!-- Tickets without full version info will closed until it is provided,
|
||||
we need the full output here to help you solve your issue -->
|
||||
@@ -1,8 +0,0 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: ❓ Ask a question or start a discussion
|
||||
url: https://github.com/ArchiveBox/ArchiveBox/discussions
|
||||
about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
|
||||
- name: 💬 Chat with the dev team & community on Zulip
|
||||
url: https://zulip.archivebox.io
|
||||
about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
name: 📑 Documentation change
|
||||
about: Submit a suggestion for the Wiki documentation
|
||||
title: 'Documentation: Improvement request ...'
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
|
||||
|
||||
## Wiki Page URL
|
||||
<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
|
||||
|
||||
|
||||
## Suggested Edit
|
||||
<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
name: 💡 Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: 'Feature Request: ...'
|
||||
labels: 'changes: behavior,status: idea phase'
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!--
|
||||
Please fill out the following information,
|
||||
feel free to delete sections if they're not applicable
|
||||
or if long issue templates annoy you :)
|
||||
-->
|
||||
|
||||
## Type
|
||||
|
||||
- [ ] General question or discussion
|
||||
- [ ] Propose a brand new feature
|
||||
- [ ] Request modification of existing behavior or design
|
||||
|
||||
## What is the problem that your feature request solves
|
||||
<!--
|
||||
e.g. I need to be able to archive spanish and french subtitle files
|
||||
from a particular <example.com> movie site that's going down soon.
|
||||
-->
|
||||
|
||||
## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
|
||||
<!--
|
||||
e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.
|
||||
The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
|
||||
-->
|
||||
|
||||
## What hacks or alternative solutions have you tried to solve the problem?
|
||||
<!--
|
||||
A clear and concise description of any alternative solutions,
|
||||
workarounds, or other software you've considered using to fix the problem.
|
||||
-->
|
||||
|
||||
## How badly do you want this new feature?
|
||||
|
||||
- [ ] It's an urgent deal-breaker, I can't live without it
|
||||
- [ ] It's important to add it in the near-mid term future
|
||||
- [ ] It would be nice to have eventually
|
||||
|
||||
---
|
||||
|
||||
- [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
|
||||
- [ ] I like ArchiveBox so far / would recommend it to a friend
|
||||
- [ ] I've had a lot of difficulty getting ArchiveBox set up
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
name: 💬 Question, discussion, or support request
|
||||
about: Start a discussion or ask a question about ArchiveBox
|
||||
title: 'Question: ...'
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
---
|
||||
|
||||
## Security Documentation
|
||||
|
||||
Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
|
||||
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
|
||||
- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
|
||||
- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
|
||||
- https://github.com/ArchiveBox/ArchiveBox#archive-layout
|
||||
- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
|
||||
|
||||
---
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
|
||||
|
||||
You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
|
||||
|
||||
---
|
||||
|
||||
## CVEs and Report Credits
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**
|
||||
> Reporters **will still receive credit in release notes** however.
|
||||
|
||||
> [!TIP]
|
||||
> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
|
||||
|
||||
Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
|
||||
|
||||
---
|
||||
|
||||
## Threat Model
|
||||
|
||||
### All admin users are assumed to be trusted
|
||||
|
||||
ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
|
||||
|
||||
- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
|
||||
- non-logged in anonymous guest users
|
||||
|
||||
Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
|
||||
|
||||
### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
|
||||
|
||||
Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
|
||||
|
||||
Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
|
||||
|
||||
### XSS, CSRF, CORS, CSP
|
||||
|
||||
`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
|
||||
|
||||
[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
|
||||
|
||||
### SSRF / URL Validation
|
||||
|
||||
Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
|
||||
It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
|
||||
@@ -1,81 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"CC_BINARY": {
|
||||
"default": "cc"
|
||||
},
|
||||
"LDAPSEARCH_BINARY": {
|
||||
"default": "ldapsearch"
|
||||
},
|
||||
"INITDB_BINARY": {
|
||||
"default": "initdb"
|
||||
},
|
||||
"PG_CTL_BINARY": {
|
||||
"default": "pg_ctl"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CC_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"build-essential"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{LDAPSEARCH_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
},
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"ldap-utils",
|
||||
"python3-dev",
|
||||
"python3-setuptools",
|
||||
"libssl-dev",
|
||||
"libldap2-dev",
|
||||
"libsasl2-dev",
|
||||
"zlib1g-dev",
|
||||
"libatomic1"
|
||||
],
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"postgres_binaries": [
|
||||
{
|
||||
"name": "{INITDB_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"postgresql"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{PG_CTL_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"postgresql"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"BREW_BINARY": {
|
||||
"default": "brew"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{BREW_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"CLANG_BINARY": {
|
||||
"default": "clang"
|
||||
},
|
||||
"LDAPVC_BINARY": {
|
||||
"default": "ldapvc"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CLANG_BINARY}",
|
||||
"binproviders": "env,brew",
|
||||
"overrides": {
|
||||
"brew": {
|
||||
"install_args": [
|
||||
"llvm"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{LDAPVC_BINARY}",
|
||||
"binproviders": "env,brew",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
},
|
||||
"brew": {
|
||||
"install_args": [
|
||||
"openldap"
|
||||
],
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"LDAPVC_BINARY": {
|
||||
"default": "ldapvc"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{LDAPVC_BINARY}",
|
||||
"binproviders": "env",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,349 +0,0 @@
|
||||
{
|
||||
"properties": {
|
||||
"CURL_BINARY": {
|
||||
"default": "curl"
|
||||
},
|
||||
"BASH_BINARY": {
|
||||
"default": "bash"
|
||||
},
|
||||
"CHMOD_BINARY": {
|
||||
"default": "chmod"
|
||||
},
|
||||
"DOCKER_BINARY": {
|
||||
"default": "docker"
|
||||
},
|
||||
"GIT_BINARY": {
|
||||
"default": "git"
|
||||
},
|
||||
"GREP_BINARY": {
|
||||
"default": "grep"
|
||||
},
|
||||
"GH_BINARY": {
|
||||
"default": "gh"
|
||||
},
|
||||
"EXPORTFS_BINARY": {
|
||||
"default": "exportfs"
|
||||
},
|
||||
"JQ_BINARY": {
|
||||
"default": "jq"
|
||||
},
|
||||
"MKDIR_BINARY": {
|
||||
"default": "mkdir"
|
||||
},
|
||||
"MOUNT_CIFS_BINARY": {
|
||||
"default": "mount.cifs"
|
||||
},
|
||||
"MOUNT_NFS_BINARY": {
|
||||
"default": "mount.nfs"
|
||||
},
|
||||
"NODE_BINARY": {
|
||||
"default": "node"
|
||||
},
|
||||
"PS_BINARY": {
|
||||
"default": "ps"
|
||||
},
|
||||
"PYTHON_BINARY": {
|
||||
"default": "python"
|
||||
},
|
||||
"PV_BINARY": {
|
||||
"default": "pv"
|
||||
},
|
||||
"SSH_BINARY": {
|
||||
"default": "ssh"
|
||||
},
|
||||
"SED_BINARY": {
|
||||
"default": "sed"
|
||||
},
|
||||
"SMBD_BINARY": {
|
||||
"default": "smbd"
|
||||
},
|
||||
"SUDO_BINARY": {
|
||||
"default": "sudo"
|
||||
},
|
||||
"SYSTEMCTL_BINARY": {
|
||||
"default": "systemctl"
|
||||
},
|
||||
"TAR_BINARY": {
|
||||
"default": "tar"
|
||||
},
|
||||
"TREE_BINARY": {
|
||||
"default": "tree"
|
||||
},
|
||||
"TR_BINARY": {
|
||||
"default": "tr"
|
||||
},
|
||||
"RM_BINARY": {
|
||||
"default": "rm"
|
||||
},
|
||||
"UMOUNT_BINARY": {
|
||||
"default": "umount"
|
||||
},
|
||||
"UV_BINARY": {
|
||||
"default": "uv"
|
||||
},
|
||||
"UNAME_BINARY": {
|
||||
"default": "uname"
|
||||
}
|
||||
},
|
||||
"ci_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{JQ_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{UV_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"bsd_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env,pyinfra"
|
||||
}
|
||||
],
|
||||
"freebsd_binaries": [
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env,pyinfra"
|
||||
}
|
||||
],
|
||||
"docker_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UNAME_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"deploy_binaries": [
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SSH_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docker_debug_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{PV_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{TAR_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{TREE_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
],
|
||||
"docker_validation_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GREP_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{MKDIR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{RM_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SED_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SSH_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{TR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UNAME_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"mount_validation_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{CHMOD_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{EXPORTFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"nfs-kernel-server"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{MKDIR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{MOUNT_CIFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"cifs-utils"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{MOUNT_NFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"nfs-common"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{SMBD_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"samba"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SYSTEMCTL_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UMOUNT_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"process_binaries": [
|
||||
{
|
||||
"name": "{PS_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docs_binaries": [
|
||||
{
|
||||
"name": "{CURL_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"git_binaries": [
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
],
|
||||
"github_binaries": [
|
||||
{
|
||||
"name": "{GH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
],
|
||||
"node_binaries": [
|
||||
{
|
||||
"name": "{NODE_BINARY}",
|
||||
"binproviders": "env,npm",
|
||||
"min_version": "22.23.1",
|
||||
"overrides": {
|
||||
"npm": {
|
||||
"install_args": [
|
||||
"node@22.23.1"
|
||||
],
|
||||
"postinstall_scripts": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"release_binaries": [
|
||||
{
|
||||
"name": "{UV_BINARY}",
|
||||
"binproviders": "env,pip,brew"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GH_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{JQ_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
},
|
||||
{
|
||||
"name": "{CURL_BINARY}",
|
||||
"binproviders": "env,brew,apt"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
# archivebox.io website
|
||||
|
||||
This repository owns its complete website. Build and deploy it without checking
|
||||
out another repository's website, fetching shared snippets, or calling a shared
|
||||
workflow. Common files are ordinary committed copies; update them deliberately.
|
||||
|
||||
## Repository-specific files
|
||||
|
||||
- Content: `publicsite/index.html`.
|
||||
- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
|
||||
- Application capture: `bin/collect_ui_screenshots.sh`.
|
||||
- Deployment branch: `dev`.
|
||||
|
||||
## Common publication contract
|
||||
|
||||
1. Build pull requests without deploying them.
|
||||
2. Build website pushes immediately, independently of application capture CI.
|
||||
3. Restore validated screenshots from trusted default-branch capture jobs.
|
||||
Keep the original capture revision, timestamp, coverage and image checksums.
|
||||
4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
|
||||
published image set. Never replace a complete gallery with an empty one when
|
||||
restoration fails. Apple device groups are independently validated artifacts;
|
||||
Android's checked-in bootstrap captures retain their local provenance.
|
||||
5. Render current content and layout around those captures. Site revision and
|
||||
capture revision are separate facts; reusing captures must not relabel them.
|
||||
6. Serialize production restore/build/deploy under one concurrency group and
|
||||
check out the latest default-branch revision when each build starts. Keep the
|
||||
built revision in provenance; later release-bot commits must not block deploys.
|
||||
7. Keep platform-specific build, installation, capture, and completeness checks
|
||||
in the application's existing capture workflow. Site-only edits do not need
|
||||
to rerun those jobs.
|
||||
|
||||
## Common presentation contract
|
||||
|
||||
The base header, Apps menu and icons, footer, brand styles and navigation behavior
|
||||
should remain identical copies across the seven product sites. Put this site's
|
||||
name, navigation, CTA and extra footer resources outside the common base files.
|
||||
Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
|
||||
mobile layout, reduced-motion preferences and existing product-specific content.
|
||||
Screenshots stay in English. Translation content, when added, lives in separate
|
||||
HTML files; no runtime translation service or cross-repository dependency.
|
||||
|
||||
## Editing and building
|
||||
|
||||
`base/` contains identical independent copies across the seven product sites.
|
||||
Common Python scripts share the same behavior; repository formatters may change
|
||||
line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
|
||||
behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
|
||||
links and CTA. Product content and additional footer resources stay in the native
|
||||
HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
|
||||
slots; the renderer fills them with static HTML, never browser-side fetches.
|
||||
|
||||
The existing builder calls `site.py render` after generating its content. This
|
||||
copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
|
||||
HTML, validates local resources and writes the site revision to root `build.json`.
|
||||
Screenshot manifests are left untouched. No build reads another repository.
|
||||
|
||||
Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
|
||||
another product repository when updating them; do not add a synchronization job,
|
||||
package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
|
||||
and the marked workflow build section are intentionally repository-specific.
|
||||
|
||||
Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
|
||||
|
||||
Check the built site with real Chromium (also run by Pages CI):
|
||||
|
||||
```sh
|
||||
uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
|
||||
uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
|
||||
```
|
||||
|
||||
Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
|
||||
layout, keyboard dismissal, no-JavaScript links, local resources and visible
|
||||
images, and uploads its screenshots as `site-verification` for 14 days.
|
||||
|
||||
## Branding and future translations
|
||||
|
||||
`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
|
||||
including the existing `.button` and `.ab-btn` classes. Product styles own their
|
||||
layouts, feature sections and platform-specific button states. Shared system
|
||||
fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
|
||||
Twitter tags everywhere while retaining each native template's title, description,
|
||||
canonical URL, locale and social artwork; edit those values with that page's copy.
|
||||
|
||||
For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
|
||||
and list their output paths in `site.json` alongside the English pages. Share CSS,
|
||||
JS, social artwork and the English screenshot gallery. Translate each page's title
|
||||
and description, set its own canonical and `lang` (`zh` for Chinese), and add
|
||||
reciprocal `hreflang` links only for translations that actually exist. The renderer
|
||||
preserves these tags and computes shared asset paths for nested pages. Localized
|
||||
header/footer copy and the small language selector can be added with those pages;
|
||||
there is no language redirect, placeholder translation or translation runtime now.
|
||||
|
||||
## Screenshot strips
|
||||
|
||||
`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
|
||||
Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
|
||||
The renderer takes real images, captions, dimensions and links from the generated
|
||||
`screenshots/index.html`; there is no second screenshot list or browser fetch.
|
||||
Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
|
||||
`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
|
||||
Catalog sites without a slot do not load the strip assets.
|
||||
|
||||
The strip scrolls once, pauses on hover, focus or manual interaction, and has an
|
||||
explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
|
||||
normal image links and horizontal scrolling remain available. Screenshot capture
|
||||
manifests and provenance are never rewritten by this presentation step.
|
||||
|
||||
Standalone one- or two-image illustrations remain in their authored source.
|
||||
Gallery strips with three or more images use the latest restored captures.
|
||||
|
||||
## Keeping the common layout in sync
|
||||
|
||||
The comments in `base/header.html`, `base/footer-start.html`,
|
||||
`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
|
||||
repositories. Update those committed copies together when shared branding,
|
||||
Apps links, keyboard behavior, or footer resources change. Preserve each site's
|
||||
local navigation and content. The former monorepo `site-chrome` generator is
|
||||
retired; builds must use only this repository's files.
|
||||
@@ -1,102 +0,0 @@
|
||||
"""Common screenshot artifact transport; coverage checks belong to each app."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import PurePosixPath
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
def api(repo, path):
|
||||
return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
|
||||
|
||||
|
||||
def trusted(run, repo, workflow, branch, successful=True):
|
||||
return (
|
||||
(run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
|
||||
and run["head_branch"] == branch
|
||||
and run["event"] in ("push", "workflow_dispatch")
|
||||
and run["path"] == f".github/workflows/{workflow}"
|
||||
and (not successful or run["conclusion"] == "success")
|
||||
)
|
||||
|
||||
|
||||
def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
|
||||
# Ask for capture artifacts directly instead of checking every historical
|
||||
# workflow run (most app runs never produce a complete screenshot set).
|
||||
# Artifact creation time also includes fresh captures from old rerun jobs.
|
||||
candidates = []
|
||||
for name in artifact_names:
|
||||
rows = subprocess.check_output(
|
||||
[
|
||||
"gh",
|
||||
"api",
|
||||
f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
|
||||
"--paginate",
|
||||
"--jq",
|
||||
".artifacts[] | select(.expired == false) | @json",
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
candidates.extend(json.loads(row) for row in rows.splitlines())
|
||||
seen = set()
|
||||
for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
|
||||
source = artifact.get("workflow_run") or {}
|
||||
run_id = source.get("id")
|
||||
if (
|
||||
artifact["name"] not in artifact_names
|
||||
or not run_id
|
||||
or run_id in seen
|
||||
or source.get("head_branch") != branch
|
||||
or source.get("head_repository_id") != source.get("repository_id")
|
||||
):
|
||||
continue
|
||||
seen.add(run_id)
|
||||
run = api(repo, f"actions/runs/{run_id}")
|
||||
if trusted(run, repo, workflow, branch, successful):
|
||||
yield run
|
||||
|
||||
|
||||
def names(repo, run):
|
||||
rows = subprocess.check_output(
|
||||
[
|
||||
"gh",
|
||||
"api",
|
||||
f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
|
||||
"--paginate",
|
||||
"--jq",
|
||||
".artifacts[] | @json",
|
||||
],
|
||||
text=True,
|
||||
)
|
||||
return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
|
||||
|
||||
|
||||
def download(repo, run, name, destination):
|
||||
subprocess.run(
|
||||
[
|
||||
"gh",
|
||||
"run",
|
||||
"download",
|
||||
str(run["id"]),
|
||||
"--repo",
|
||||
repo,
|
||||
"--name",
|
||||
name,
|
||||
"--dir",
|
||||
str(destination),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def relative_path(name):
|
||||
path = PurePosixPath(name)
|
||||
if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
|
||||
raise ValueError(f"Unsafe screenshot path: {name}")
|
||||
return path
|
||||
|
||||
|
||||
def fetch(base, name):
|
||||
relative_path(name)
|
||||
with urlopen(base + name, timeout=60) as response:
|
||||
return response.read()
|
||||
@@ -1,156 +0,0 @@
|
||||
/*
|
||||
Shared ArchiveBox website navigation styles and behavior.
|
||||
Keep this file in sync with .github/pages/base/chrome.css in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
*/
|
||||
/* Common brand tokens. Native templates own product layout and content. */
|
||||
:root {
|
||||
--abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
--abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
|
||||
--abx-accent: #9b2854;
|
||||
--abx-accent-hover: #782040;
|
||||
--abx-paper: #fcfaf7;
|
||||
--abx-ink: #28252a;
|
||||
--abx-muted: #716c74;
|
||||
--abx-line: #e9e2e3;
|
||||
--font-sans: var(--abx-font-sans);
|
||||
--font-mono: var(--abx-font-mono);
|
||||
--ab-font-sans: var(--abx-font-sans);
|
||||
--ab-font-serif: var(--abx-font-sans);
|
||||
--ab-font-label: var(--abx-font-sans);
|
||||
--ab-font-mono: var(--abx-font-mono);
|
||||
--ab-accent: var(--abx-accent);
|
||||
--ab-accent-hover: var(--abx-accent-hover);
|
||||
--ab-on-accent: #fff;
|
||||
}
|
||||
body { font-family: var(--abx-font-sans); }
|
||||
code, pre, kbd, samp { font-family: var(--abx-font-mono); }
|
||||
/* Reuse existing CTA classes; platform selection continues to control colors. */
|
||||
.button, .ab-btn {
|
||||
display: inline-flex; align-items: center; justify-content: center; gap: 10px;
|
||||
min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
|
||||
border-radius: 24px; font-family: var(--abx-font-sans);
|
||||
font-size: 14px; font-weight: 650; line-height: 1.4;
|
||||
text-align: center; white-space: normal; text-decoration: none;
|
||||
}
|
||||
.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
|
||||
.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
|
||||
.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
|
||||
.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
|
||||
@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
|
||||
|
||||
/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
|
||||
/* Native anchor navigation leaves room for the heading and its surrounding content. */
|
||||
html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
|
||||
@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
|
||||
.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
|
||||
.abx-header *, .abx-footer * { box-sizing: border-box; }
|
||||
.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
|
||||
.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
|
||||
.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
|
||||
.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
|
||||
.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
|
||||
.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
|
||||
.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
|
||||
.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
|
||||
.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
|
||||
.abx-header .abx-nav a:hover { color: #9b2854; }
|
||||
.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
|
||||
.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
|
||||
.abx-header .abx-apps { position: relative; }
|
||||
.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
|
||||
.abx-apps > summary::-webkit-details-marker { display: none; }
|
||||
.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
|
||||
.abx-apps[open] > summary::before { transform: rotate(90deg); }
|
||||
.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
|
||||
.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
|
||||
.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
|
||||
.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
|
||||
.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
|
||||
.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
|
||||
.abx-header .abx-app-links .abx-platform { width: 30px; }
|
||||
.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
|
||||
.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
|
||||
.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
|
||||
.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
|
||||
.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
|
||||
.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
|
||||
.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
|
||||
.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
|
||||
.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
|
||||
.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
|
||||
.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
|
||||
.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
|
||||
.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
|
||||
.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
|
||||
.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
|
||||
.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
|
||||
.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
|
||||
.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
|
||||
.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
|
||||
.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
|
||||
@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
|
||||
@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
|
||||
.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
|
||||
.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
|
||||
.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
|
||||
.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
|
||||
.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
|
||||
.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
|
||||
.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
|
||||
.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
|
||||
.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
|
||||
.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
|
||||
.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
|
||||
.abx-footer .abx-footer-column li::before { content: none; }
|
||||
.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
|
||||
.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
|
||||
.abx-footer a:hover { color: #fff; text-decoration: underline; }
|
||||
.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
|
||||
.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
|
||||
.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
|
||||
.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
|
||||
.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
|
||||
.abx-footer .abx-footer-bottom a { color: #cebbc9; }
|
||||
.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
|
||||
.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
|
||||
.abx-footer-local > div { padding-top: 18px; }
|
||||
.abx-footer-local a { color: #e5aabd; }
|
||||
.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
|
||||
.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
|
||||
.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
|
||||
.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
|
||||
.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
|
||||
@media (max-width: 1100px) {
|
||||
.abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
|
||||
.abx-header .abx-nav { justify-content: flex-start; }
|
||||
.abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
|
||||
}
|
||||
@media (max-width: 700px) {
|
||||
.abx-header { position: relative; padding: 14px 18px; }
|
||||
.abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
|
||||
.abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
|
||||
.abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
|
||||
.abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
|
||||
.abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
|
||||
.abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
|
||||
.abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
|
||||
.abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
|
||||
.abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
|
||||
.abx-footer .abx-footer-column li { min-height: 40px; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
|
||||
|
||||
.abx-footer-local > div > a + a { margin-left: 12px; }
|
||||
|
||||
@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }
|
||||
@@ -1,28 +0,0 @@
|
||||
/*
|
||||
Shared ArchiveBox website navigation styles and behavior.
|
||||
Keep this file in sync with .github/pages/base/chrome.js in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
*/
|
||||
(() => {
|
||||
const menu = document.querySelector('.abx-apps');
|
||||
document.addEventListener('keydown', event => {
|
||||
if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
|
||||
});
|
||||
document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
|
||||
function revealAnchor() {
|
||||
let target;
|
||||
try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
|
||||
if (!target) return;
|
||||
let details = target.closest('details');
|
||||
let opened = false;
|
||||
while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
|
||||
if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
|
||||
}
|
||||
window.addEventListener('hashchange', revealAnchor);
|
||||
revealAnchor();
|
||||
})();
|
||||
@@ -1,12 +0,0 @@
|
||||
<!--
|
||||
Shared ArchiveBox website footer.
|
||||
Keep this file in sync with .github/pages/base/footer-end.html in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
-->
|
||||
<div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security & privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
|
||||
</footer>
|
||||
@@ -1,13 +0,0 @@
|
||||
<!--
|
||||
Shared ArchiveBox website footer.
|
||||
Keep this file in sync with .github/pages/base/footer-start.html in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
-->
|
||||
<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps & tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS & iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS & iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
|
||||
@@ -1,24 +0,0 @@
|
||||
<!--
|
||||
Shared ArchiveBox website navbar and Apps menu.
|
||||
Keep this file in sync with .github/pages/base/header.html in:
|
||||
ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
|
||||
electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
|
||||
Each site builds its committed copy independently; there is no central generator.
|
||||
Keep site-specific navigation, CTAs, and extra footer links outside this base.
|
||||
Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
debian-archivebox/website/index.html when shared links or branding change.
|
||||
-->
|
||||
<header class="abx-header">
|
||||
<a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
|
||||
<nav class="abx-nav" aria-label="Main navigation">
|
||||
<details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
|
||||
<div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
|
||||
<div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
|
||||
<div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
|
||||
</div></details>
|
||||
__SITE_NAV__
|
||||
</nav>
|
||||
</header>
|
||||
@@ -1,11 +0,0 @@
|
||||
/* Identical standalone gallery strip across ArchiveBox product sites. */
|
||||
/* The root query container measures the usable width, excluding scrollbars. */
|
||||
html { container-type: inline-size; }
|
||||
.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
|
||||
.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
|
||||
.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
|
||||
.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
|
||||
.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
|
||||
.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
|
||||
.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
|
||||
@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }
|
||||
@@ -1,44 +0,0 @@
|
||||
/* Scroll real gallery links once; keep them usable without JavaScript. */
|
||||
(() => {
|
||||
document.querySelectorAll('.abx-marquee').forEach(gallery => {
|
||||
const viewport = gallery.querySelector('.abx-marquee-viewport');
|
||||
const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
|
||||
let paused = reducedMotion.matches;
|
||||
let hovering = false;
|
||||
let focused = false;
|
||||
let visible = false;
|
||||
let frame;
|
||||
let previous;
|
||||
let offset = viewport.scrollLeft;
|
||||
function tick(time) {
|
||||
const distance = viewport.scrollWidth - viewport.clientWidth;
|
||||
if (distance <= 0) return;
|
||||
if (previous !== undefined) {
|
||||
offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
|
||||
viewport.scrollTo({ left: offset, behavior: 'instant' });
|
||||
}
|
||||
if (offset >= distance - 1) { paused = true; return; }
|
||||
previous = time;
|
||||
frame = window.requestAnimationFrame(tick);
|
||||
}
|
||||
function sync() {
|
||||
window.cancelAnimationFrame(frame);
|
||||
previous = undefined;
|
||||
offset = viewport.scrollLeft;
|
||||
if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
|
||||
}
|
||||
viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
|
||||
viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
|
||||
viewport.addEventListener('focusin', () => { focused = true; sync(); });
|
||||
viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
|
||||
function pauseForInteraction() { paused = true; sync(); }
|
||||
viewport.addEventListener('pointerdown', pauseForInteraction);
|
||||
viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
|
||||
viewport.addEventListener('keydown', pauseForInteraction);
|
||||
reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
|
||||
document.addEventListener('visibilitychange', sync);
|
||||
new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
|
||||
new window.ResizeObserver(sync).observe(viewport);
|
||||
viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
|
||||
});
|
||||
})();
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Render homepage screenshot strips from the site's existing gallery HTML."""
|
||||
|
||||
import re
|
||||
from html import escape
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin
|
||||
|
||||
|
||||
class Gallery(HTMLParser):
|
||||
"""Only gallery figures are captures; navigation and decorative images are not."""
|
||||
|
||||
def __init__(self, base):
|
||||
super().__init__()
|
||||
self.base = base
|
||||
self.stack = []
|
||||
self.captures = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == "img" and any(name == "figure" for name, _ in self.stack):
|
||||
context = [item for _, item in self.stack]
|
||||
profile = next(
|
||||
(
|
||||
item.get("data-profile") or item.get("data-viewport")
|
||||
for item in reversed(context)
|
||||
if item.get("data-profile") or item.get("data-viewport")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if profile not in (None, "desktop"):
|
||||
return
|
||||
source = attrs.get("src", "")
|
||||
if not source or not attrs.get("alt"):
|
||||
return
|
||||
anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
|
||||
server = any(item.get("data-platform") == "server" for item in context)
|
||||
self.captures.append(
|
||||
{
|
||||
"src": urljoin(self.base, source),
|
||||
"href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
|
||||
"title": attrs["alt"],
|
||||
"width": attrs.get("width", ""),
|
||||
"height": attrs.get("height", ""),
|
||||
"product": "server" if server else "client",
|
||||
},
|
||||
)
|
||||
if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
|
||||
self.stack.append((tag, attrs))
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
for index in range(len(self.stack) - 1, -1, -1):
|
||||
if self.stack[index][0] == tag:
|
||||
del self.stack[index:]
|
||||
break
|
||||
|
||||
|
||||
def render(output, baseurl="", featured=()):
|
||||
homepage = output / "index.html"
|
||||
text = homepage.read_text()
|
||||
slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
|
||||
if not slots:
|
||||
return False
|
||||
base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
|
||||
gallery = Gallery(base + "screenshots/")
|
||||
gallery.feed((output / "screenshots/index.html").read_text())
|
||||
for slot in slots:
|
||||
product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
|
||||
captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
|
||||
priority = {title: index for index, title in enumerate(featured)}
|
||||
captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
|
||||
cards = []
|
||||
for capture in captures:
|
||||
dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
|
||||
cards.append(
|
||||
f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
|
||||
)
|
||||
viewport_id = slot.lower()
|
||||
content = (
|
||||
f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
|
||||
if cards
|
||||
else ""
|
||||
)
|
||||
start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
|
||||
replacement = start + "\n" + content + "\n" + end
|
||||
if end in text:
|
||||
text = re.sub(
|
||||
re.escape(start) + r".*?" + re.escape(end),
|
||||
lambda _, replacement=replacement: replacement,
|
||||
text,
|
||||
count=1,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
else:
|
||||
text = text.replace(start, replacement, 1)
|
||||
homepage.write_text(text)
|
||||
return True
|
||||
@@ -1,4 +0,0 @@
|
||||
<!-- Repository-specific navigation and primary CTA. -->
|
||||
<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout & security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
|
||||
<a class="abx-cta" href="__SITE_BASE__#install">Get started</a>
|
||||
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"name": "Web Archive",
|
||||
"url": "https://archivebox.io/",
|
||||
"pages": [
|
||||
"index.html",
|
||||
"screenshots/index.html"
|
||||
],
|
||||
"marquee_featured": [
|
||||
"Snapshots grid — desktop",
|
||||
"Snapshot View (header collapsed) — desktop",
|
||||
"AI agent — desktop",
|
||||
"Archive results — desktop",
|
||||
"Add URLs — desktop",
|
||||
"Snapshots table — desktop"
|
||||
]
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
"""Assemble and verify this repository's static Pages site (standard library only)."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from html import escape, unescape
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from marquee import render as render_marquee
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
|
||||
|
||||
def replace_block(text, name, content):
|
||||
start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
|
||||
pattern = re.escape(start) + r".*?" + re.escape(end)
|
||||
if re.search(pattern, text, re.DOTALL):
|
||||
return re.sub(
|
||||
pattern,
|
||||
lambda _: f"{start}\n{content}\n{end}",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
if text.count(start) != 1:
|
||||
raise ValueError(f"Expected one {name} placeholder")
|
||||
return text.replace(start, f"{start}\n{content}\n{end}", 1)
|
||||
|
||||
|
||||
def render_metadata(text):
|
||||
"""Keep product copy in native templates; render one consistent SEO block."""
|
||||
page = Page()
|
||||
page.feed(text)
|
||||
title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
|
||||
meta = page.metadata
|
||||
description = meta.get("description", meta.get("og:description", ""))
|
||||
image = meta.get("og:image", meta.get("twitter:image", ""))
|
||||
if not description or not image or not page.canonical:
|
||||
raise ValueError("Each page needs a description, social image and canonical")
|
||||
values = {
|
||||
"description": description,
|
||||
"robots": "index,follow,max-image-preview:large",
|
||||
"theme-color": "#9b2854",
|
||||
"og:type": "website",
|
||||
"og:site_name": meta.get("og:site_name", "ArchiveBox"),
|
||||
"og:locale": meta.get("og:locale", "en_US"),
|
||||
"og:url": page.canonical,
|
||||
"og:title": title,
|
||||
"og:description": description,
|
||||
"og:image": image,
|
||||
"og:image:type": meta.get("og:image:type", "image/png"),
|
||||
"og:image:width": meta.get("og:image:width", "1200"),
|
||||
"og:image:height": meta.get("og:image:height", "630"),
|
||||
"og:image:alt": meta.get("og:image:alt", title),
|
||||
"twitter:card": "summary_large_image",
|
||||
"twitter:site": "@ArchiveBoxApp",
|
||||
"twitter:title": title,
|
||||
"twitter:description": description,
|
||||
"twitter:image": image,
|
||||
"twitter:image:alt": meta.get("og:image:alt", title),
|
||||
}
|
||||
|
||||
def strip_managed_tag(match):
|
||||
tag = Page()
|
||||
tag.feed(match[0])
|
||||
if tag.canonicals or any(key in values for key in tag.metadata):
|
||||
return ""
|
||||
# System fonts are shared across sites, without a network font dependency.
|
||||
if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
|
||||
return ""
|
||||
return match[0]
|
||||
|
||||
text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
|
||||
text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
|
||||
tags = [
|
||||
f"<title>{escape(title)}</title>",
|
||||
f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
|
||||
]
|
||||
for key, value in values.items():
|
||||
attr = "property" if key.startswith("og:") else "name"
|
||||
tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
|
||||
if "<!-- ARCHIVEBOX:METADATA -->" not in text:
|
||||
text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
|
||||
return replace_block(text, "METADATA", "\n".join(tags))
|
||||
|
||||
|
||||
def render(output, source=None, baseurl=""):
|
||||
config = json.loads((HERE / "site.json").read_text())
|
||||
output = output.resolve()
|
||||
if source:
|
||||
source = source.resolve()
|
||||
if output == source or output in source.parents or source in output.parents:
|
||||
raise ValueError("Build output must be separate from the source directory")
|
||||
if output == ROOT or output in ROOT.parents:
|
||||
raise ValueError("Build output must not replace a repository")
|
||||
if output.exists():
|
||||
shutil.rmtree(output)
|
||||
shutil.copytree(source, output)
|
||||
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
||||
assets = output / "site-base"
|
||||
assets.mkdir(parents=True, exist_ok=True)
|
||||
has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
|
||||
common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
|
||||
for name in common_assets:
|
||||
shutil.copyfile(HERE / "base" / name, assets / name)
|
||||
for name in config["pages"]:
|
||||
page = output / name
|
||||
text = render_metadata(page.read_text())
|
||||
base = "../" * (len(Path(name).parts) - 1) or "./"
|
||||
header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
|
||||
header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
|
||||
text = replace_block(text, "HEADER", header)
|
||||
for kind in ["FOOTER-START", "FOOTER-END"]:
|
||||
text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
|
||||
text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
|
||||
resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
|
||||
if has_marquee and name == "index.html":
|
||||
resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
|
||||
if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
|
||||
text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
|
||||
text = replace_block(text, "ASSETS", resources)
|
||||
page.write_text(text)
|
||||
# Site build provenance is independent of all original screenshot manifests.
|
||||
provenance_path = output / "build.json"
|
||||
provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
|
||||
provenance.update(
|
||||
revision=revision,
|
||||
generatedAt=datetime.now(UTC).isoformat(),
|
||||
site=config["url"],
|
||||
)
|
||||
provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
|
||||
(output / ".nojekyll").touch()
|
||||
verify(output, baseurl)
|
||||
print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
|
||||
|
||||
|
||||
class Page(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.ids = set()
|
||||
self.metadata = {}
|
||||
self.canonical = ""
|
||||
self.resources = []
|
||||
self.headers = self.footers = self.canonicals = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == "meta":
|
||||
for key in ["name", "property"]:
|
||||
if attrs.get(key):
|
||||
self.metadata[attrs[key]] = attrs.get("content", "")
|
||||
if tag == "link" and attrs.get("rel") == "canonical":
|
||||
self.canonical = attrs["href"]
|
||||
if attrs.get("id"):
|
||||
self.ids.add(attrs["id"])
|
||||
classes = attrs.get("class", "").split()
|
||||
self.headers += "abx-header" in classes
|
||||
self.footers += "abx-footer" in classes
|
||||
self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
|
||||
if tag in ["img", "script"] and attrs.get("src"):
|
||||
self.resources.append(attrs["src"])
|
||||
if tag == "link" and attrs.get("rel") in [
|
||||
"stylesheet",
|
||||
"icon",
|
||||
"apple-touch-icon",
|
||||
]:
|
||||
self.resources.append(attrs["href"])
|
||||
|
||||
|
||||
def verify(output, baseurl=""):
|
||||
config = json.loads((HERE / "site.json").read_text())
|
||||
output = output.resolve()
|
||||
prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
|
||||
for name in config["pages"]:
|
||||
path = output / name
|
||||
text = path.read_text()
|
||||
page = Page()
|
||||
page.feed(text)
|
||||
if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
|
||||
raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
|
||||
if "__SITE_" in text:
|
||||
raise ValueError(f"{name}: unresolved site placeholder")
|
||||
for resource in page.resources:
|
||||
url = urlsplit(resource)
|
||||
if url.scheme or url.netloc or not url.path:
|
||||
continue
|
||||
resource_path = unquote(url.path)
|
||||
if prefix and resource_path.startswith(prefix + "/"):
|
||||
resource_path = resource_path[len(prefix) :]
|
||||
local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
|
||||
if not local.is_relative_to(output) or not local.is_file():
|
||||
raise ValueError(f"{name}: missing local resource {resource}")
|
||||
for asset in ["chrome.css", "chrome.js"]:
|
||||
if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
|
||||
raise ValueError(f"{name}: stale common asset {asset}")
|
||||
print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=["render", "verify"])
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("--source", type=Path)
|
||||
parser.add_argument("--baseurl", default="")
|
||||
args = parser.parse_args()
|
||||
if args.command == "render":
|
||||
render(args.output, args.source, args.baseurl)
|
||||
else:
|
||||
verify(args.output, args.baseurl)
|
||||
@@ -1,255 +0,0 @@
|
||||
"""Check the built site in real Chromium at desktop and phone widths."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from functools import partial
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from urllib.parse import unquote, urljoin, urlsplit
|
||||
|
||||
from playwright.sync_api import expect, sync_playwright
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
class Handler(SimpleHTTPRequestHandler):
|
||||
def handle(self):
|
||||
# Browser navigation can cancel an in-flight lazy image response.
|
||||
try:
|
||||
super().handle()
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
class ScreenshotReferences(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.references = []
|
||||
self.ids = set()
|
||||
|
||||
def handle_starttag(self, tag, attributes):
|
||||
attrs = dict(attributes)
|
||||
if attrs.get("id"):
|
||||
self.ids.add(attrs["id"])
|
||||
for attribute in ("src", "href"):
|
||||
if attrs.get(attribute):
|
||||
self.references.append(attrs[attribute])
|
||||
|
||||
|
||||
def verify_screenshot_references(output, canonical):
|
||||
"""Check gallery embeds on every static page, including optional translations."""
|
||||
pages = {}
|
||||
for path in output.rglob("*.html"):
|
||||
page = ScreenshotReferences()
|
||||
page.feed(path.read_text())
|
||||
pages[path.resolve()] = page
|
||||
for path, page in pages.items():
|
||||
page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
|
||||
for reference in page.references:
|
||||
url = urlsplit(urljoin(page_url, reference))
|
||||
if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
|
||||
continue
|
||||
target = output / unquote(url.path).lstrip("/")
|
||||
if target.is_dir():
|
||||
target /= "index.html"
|
||||
target = target.resolve()
|
||||
assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
|
||||
assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
|
||||
if url.fragment and target in pages:
|
||||
assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
|
||||
|
||||
|
||||
def verify(output, evidence):
|
||||
config = json.loads((HERE / "site.json").read_text())
|
||||
verify_screenshot_references(output, config["url"])
|
||||
evidence.mkdir(parents=True, exist_ok=True)
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
|
||||
thread = Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
origin = f"http://127.0.0.1:{server.server_port}"
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
|
||||
for name in config["pages"]:
|
||||
route = name.removesuffix("index.html")
|
||||
page = browser.new_page(reduced_motion="reduce")
|
||||
errors = []
|
||||
missing = []
|
||||
page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
|
||||
page.on(
|
||||
"response",
|
||||
lambda response, missing=missing: (
|
||||
missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
|
||||
),
|
||||
)
|
||||
for width in [1716, 2560, 390]:
|
||||
page.set_viewport_size({"width": width, "height": 900})
|
||||
page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
|
||||
header = page.locator(".abx-header")
|
||||
expect(header).to_have_count(1)
|
||||
expect(header).to_be_visible()
|
||||
expect(header).to_be_in_viewport()
|
||||
brand = header.locator(".abx-brand")
|
||||
expect(brand).to_be_visible()
|
||||
expect(brand).to_contain_text("ArchiveBox")
|
||||
expect(brand.locator(".abx-logo")).to_be_visible()
|
||||
expect(brand.locator(".abx-logo")).to_be_in_viewport()
|
||||
expect(header).to_have_css("display", "flex")
|
||||
expect(page.locator(".abx-footer")).to_have_count(1)
|
||||
expect(page.locator(".abx-footer-column")).to_have_count(3)
|
||||
expect(header.locator(".abx-cta")).to_be_visible()
|
||||
menu = header.locator(".abx-apps")
|
||||
menu.locator("summary").focus()
|
||||
page.keyboard.press("Enter")
|
||||
expect(menu).to_have_attribute("open", "")
|
||||
expect(menu.locator("a").first).to_be_visible()
|
||||
page.keyboard.press("Escape")
|
||||
expect(menu).not_to_have_attribute("open", "")
|
||||
menu.locator("summary").click()
|
||||
box = header.bounding_box()
|
||||
assert box
|
||||
page.mouse.click(box["x"] + 2, box["y"] + 5)
|
||||
expect(menu).not_to_have_attribute("open", "")
|
||||
assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
|
||||
f"{name}: overflow at {width}"
|
||||
)
|
||||
if not route and "screenshots/index.html" in config["pages"]:
|
||||
strips = page.locator(".abx-marquee")
|
||||
assert strips.count() > 0, "Homepage must use its screenshot gallery"
|
||||
for strip in strips.all():
|
||||
strip.scroll_into_view_if_needed()
|
||||
bounds = strip.bounding_box()
|
||||
usable_width = page.evaluate("document.documentElement.clientWidth")
|
||||
assert bounds and abs(bounds["x"]) < 1
|
||||
assert abs(bounds["width"] - usable_width) < 1
|
||||
assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
|
||||
cards = strip.locator(".abx-marquee-card")
|
||||
assert cards.count() > 0, "Screenshot strip is empty"
|
||||
expect(strip.locator("button")).to_have_count(0)
|
||||
expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
|
||||
viewport = strip.locator(".abx-marquee-viewport")
|
||||
overflowing = viewport.evaluate(
|
||||
"node => node.scrollWidth > node.clientWidth",
|
||||
)
|
||||
if overflowing:
|
||||
page.mouse.move(0, 0)
|
||||
page.emulate_media(reduced_motion="no-preference")
|
||||
initial = viewport.evaluate("node => node.scrollLeft")
|
||||
page.wait_for_function(
|
||||
"([node, initial]) => node.scrollLeft > initial",
|
||||
arg=[viewport.element_handle(), initial],
|
||||
)
|
||||
viewport.hover()
|
||||
stopped = viewport.evaluate("node => node.scrollLeft")
|
||||
viewport.evaluate(
|
||||
"node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
|
||||
)
|
||||
assert viewport.evaluate("node => node.scrollLeft") == stopped
|
||||
page.emulate_media(reduced_motion="reduce")
|
||||
page.mouse.move(0, 0)
|
||||
stopped = viewport.evaluate("node => node.scrollLeft")
|
||||
viewport.evaluate(
|
||||
"node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
|
||||
)
|
||||
assert viewport.evaluate("node => node.scrollLeft") == stopped
|
||||
viewport.focus()
|
||||
page.keyboard.press("ArrowRight")
|
||||
assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
|
||||
else:
|
||||
centered = viewport.evaluate(
|
||||
"""node => {
|
||||
const view = node.getBoundingClientRect();
|
||||
const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
|
||||
return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
|
||||
}""",
|
||||
)
|
||||
assert centered, "Short screenshot strips must be centered"
|
||||
for card in cards.all():
|
||||
href = card.get_attribute("href")
|
||||
assert href and "/screenshots/" in href
|
||||
response = page.request.get(urljoin(page.url, href))
|
||||
assert response.ok, f"Broken screenshot link: {href}"
|
||||
anchor = urlsplit(href).fragment
|
||||
if anchor:
|
||||
assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
|
||||
# Decode every local content image using the real browser loader,
|
||||
# including images in offscreen carousel tracks and lazy galleries.
|
||||
images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
|
||||
local_images = [url for url in images if url.startswith(origin + "/")]
|
||||
page.evaluate(
|
||||
"""async urls => {
|
||||
for (const url of urls) {
|
||||
const image = new Image();
|
||||
image.src = url;
|
||||
await image.decode();
|
||||
if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
|
||||
}
|
||||
}""",
|
||||
local_images,
|
||||
)
|
||||
page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
|
||||
page.evaluate(
|
||||
"() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
|
||||
)
|
||||
page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
|
||||
page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
|
||||
for link in header.locator(".abx-nav > a").all():
|
||||
url = urlsplit(link.get_attribute("href") or "")
|
||||
if url.scheme or url.netloc:
|
||||
continue
|
||||
response = page.request.get(urljoin(page.url, url.geturl()))
|
||||
assert response.ok, f"Broken navigation: {url.geturl()}"
|
||||
if url.fragment:
|
||||
body = response.text()
|
||||
assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
|
||||
f"Missing navigation anchor: {url.geturl()}"
|
||||
)
|
||||
if route == "" and page.locator("#resources").count():
|
||||
page.goto(f"{origin}/#resources")
|
||||
expect(page.locator("#resources")).to_be_visible()
|
||||
assert not errors, errors
|
||||
assert not missing, missing
|
||||
page.close()
|
||||
context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
|
||||
plain = context.new_page()
|
||||
plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
|
||||
expect(plain.locator(".abx-header")).to_be_visible()
|
||||
expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
|
||||
expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
|
||||
plain.locator(".abx-apps summary").click()
|
||||
expect(plain.locator(".abx-app-links a").first).to_be_visible()
|
||||
expect(plain.locator(".abx-footer-column a").first).to_be_visible()
|
||||
if not route and "screenshots/index.html" in config["pages"]:
|
||||
assert plain.locator(".abx-marquee-card").count() > 0
|
||||
for strip in plain.locator(".abx-marquee").all():
|
||||
bounds = strip.bounding_box()
|
||||
usable_width = plain.evaluate("document.documentElement.clientWidth")
|
||||
assert bounds and abs(bounds["x"]) < 1
|
||||
assert abs(bounds["width"] - usable_width) < 1
|
||||
assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
|
||||
expect(plain.locator(".abx-marquee button")).to_have_count(0)
|
||||
expect(plain.locator(".abx-marquee-card").first).to_be_visible()
|
||||
context.close()
|
||||
print(
|
||||
f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
|
||||
flush=True,
|
||||
)
|
||||
browser.close()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
|
||||
args = parser.parse_args()
|
||||
verify(args.output, args.evidence)
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover every ArchiveBox test file and expose it as one CI job."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path.cwd().resolve()
|
||||
archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
|
||||
|
||||
if not archivebox_tests:
|
||||
raise SystemExit("No ArchiveBox tests discovered")
|
||||
|
||||
matrix: list[dict[str, object]] = []
|
||||
for path in archivebox_tests:
|
||||
test_path = path.relative_to(root).as_posix()
|
||||
matrix.append(
|
||||
{
|
||||
"name": f"main/{path.stem}",
|
||||
"paths": [test_path],
|
||||
"paths_arg": test_path,
|
||||
"extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
|
||||
"count": 1,
|
||||
},
|
||||
)
|
||||
|
||||
discovered_paths = [path for entry in matrix for path in entry["paths"]]
|
||||
if len(discovered_paths) != len(set(discovered_paths)):
|
||||
raise SystemExit("Tests were not discovered exactly once")
|
||||
if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
|
||||
raise SystemExit("Discovered test shard coverage does not match test files")
|
||||
|
||||
print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
|
||||
print(json.dumps(matrix, separators=(",", ":")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,92 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [dev, main]
|
||||
push:
|
||||
branches: [dev, main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
security-events: write
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Linters
|
||||
uses: ./.github/workflows/lint.yml
|
||||
secrets: inherit
|
||||
|
||||
install-cli-platform:
|
||||
name: Install and CLI platform compatibility
|
||||
uses: ./.github/workflows/test.yml
|
||||
secrets: inherit
|
||||
|
||||
tests:
|
||||
name: Discovered test matrix
|
||||
uses: ./.github/workflows/test-parallel.yml
|
||||
secrets: inherit
|
||||
|
||||
documentation:
|
||||
name: Documentation and root tests
|
||||
uses: ./.github/workflows/docs.yml
|
||||
secrets: inherit
|
||||
|
||||
codeql:
|
||||
name: CodeQL
|
||||
uses: ./.github/workflows/codeql.yml
|
||||
secrets: inherit
|
||||
|
||||
python-artifacts:
|
||||
name: Tested Python artifacts
|
||||
uses: ./.github/workflows/pip.yml
|
||||
secrets: inherit
|
||||
|
||||
docker-digests:
|
||||
name: Tested Docker images
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
|
||||
secrets: inherit
|
||||
|
||||
required:
|
||||
name: All required CI lanes
|
||||
if: ${{ always() }}
|
||||
needs:
|
||||
- lint
|
||||
- install-cli-platform
|
||||
- tests
|
||||
- documentation
|
||||
- codeql
|
||||
- python-artifacts
|
||||
- docker-digests
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Verify every required lane succeeded
|
||||
env:
|
||||
LINT_RESULT: ${{ needs.lint.result }}
|
||||
INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
|
||||
TESTS_RESULT: ${{ needs.tests.result }}
|
||||
DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
|
||||
CODEQL_RESULT: ${{ needs.codeql.result }}
|
||||
PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
|
||||
DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
|
||||
run: |
|
||||
for result in \
|
||||
"$LINT_RESULT" \
|
||||
"$INSTALL_CLI_PLATFORM_RESULT" \
|
||||
"$TESTS_RESULT" \
|
||||
"$DOCUMENTATION_RESULT" \
|
||||
"$CODEQL_RESULT" \
|
||||
"$PYTHON_ARTIFACTS_RESULT" \
|
||||
"$DOCKER_DIGESTS_RESULT"
|
||||
do
|
||||
[[ "$result" == success ]]
|
||||
done
|
||||
echo 'All required CI lanes and tested release artifacts succeeded.'
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
id-token: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
claude_args: '--allowed-tools Bash(gh pr:*)'
|
||||
@@ -1,21 +1,42 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
push:
|
||||
branches: [ "dev" ]
|
||||
pull_request:
|
||||
branches: [ "dev" ]
|
||||
schedule:
|
||||
- cron: '33 17 * * 6'
|
||||
|
||||
concurrency:
|
||||
group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: ubuntu-24.04
|
||||
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners (GitHub.com only)
|
||||
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
|
||||
permissions:
|
||||
# required for all workflows
|
||||
security-events: write
|
||||
|
||||
# required to fetch internal or private CodeQL packs
|
||||
packages: read
|
||||
|
||||
# only required for workflows in private repositories
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
@@ -25,18 +46,47 @@ jobs:
|
||||
include:
|
||||
- language: python
|
||||
build-mode: none
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
|
||||
# Use `c-cpp` to analyze code written in C, C++ or both
|
||||
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
|
||||
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
||||
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
|
||||
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
|
||||
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
|
||||
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
# If the analyze step fails for one of the languages you are analyzing with
|
||||
# "We were unable to automatically build your code", modify the matrix above
|
||||
# to set the build mode to "manual" for that language. Then modify this step
|
||||
# to build your code.
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
- if: matrix.build-mode == 'manual'
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Build Debian package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
|
||||
env:
|
||||
DEB_BUILD_OPTIONS: nocheck
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Install packaging dependencies
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
python3 python3-dev python3-pip python3-venv python3-all \
|
||||
dh-python debhelper devscripts dput software-properties-common \
|
||||
python3-distutils python3-setuptools python3-wheel python3-stdeb
|
||||
|
||||
# - name: Build Debian/Apt sdist_dsc
|
||||
# run: |
|
||||
# ./bin/build_pip.sh
|
||||
|
||||
# - name: Check ArchiveBox version
|
||||
# run: |
|
||||
# # must create dir needed for snaps to run as non-root on github actions
|
||||
# sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
|
||||
# mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
|
||||
# archivebox --version
|
||||
# archivebox init --setup
|
||||
|
||||
# - name: Add some links to test
|
||||
# run: |
|
||||
# cd "${{ github.workspace }}/data"
|
||||
# archivebox add 'https://example.com'
|
||||
# archivebox status
|
||||
|
||||
# - name: Commit built package
|
||||
# run: |
|
||||
# cd deb_dist/
|
||||
# git config --local user.email "action@github.com"
|
||||
# git config --local user.name "GitHub Action"
|
||||
# git commit -m "Debian package autobuild" -a
|
||||
|
||||
# - name: Push build to Github
|
||||
# uses: ad-m/github-push-action@master
|
||||
# with:
|
||||
# github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# repository: ArchiveBox/debian-archivebox
|
||||
# branch: ${{ github.ref }}
|
||||
# directory: deb_dist
|
||||
|
||||
# - name: Push build to Launchpad PPA
|
||||
# run: |
|
||||
# debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
|
||||
# dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
|
||||
@@ -1,87 +0,0 @@
|
||||
name: Publish website
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
workflow_run:
|
||||
workflows: ['Capture publicsite screenshots']
|
||||
types: [completed]
|
||||
branches: [dev]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# Repository-specific output; all publication logic below is a local copy.
|
||||
SITE_OUTPUT: _site
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
|
||||
# One production queue covers restore, build and deploy. PRs have separate queues.
|
||||
concurrency:
|
||||
group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
outputs:
|
||||
revision: ${{ steps.source.outputs.revision }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
|
||||
persist-credentials: false
|
||||
- name: Record website source revision
|
||||
id: source
|
||||
run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# BEGIN REPOSITORY-SPECIFIC BUILD
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
- name: Restore latest complete screenshots
|
||||
run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
- name: Render current gallery layout without recapturing images
|
||||
run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
|
||||
- name: Assemble website with the local base components
|
||||
run: uv run --no-project python .github/pages/site.py render _site --source publicsite
|
||||
# END REPOSITORY-SPECIFIC BUILD
|
||||
|
||||
- name: Verify website in Chromium
|
||||
run: |
|
||||
uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
|
||||
uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
|
||||
- name: Save website verification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: site-verification
|
||||
path: ${{ runner.temp }}/site-evidence
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload website
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: ${{ env.SITE_OUTPUT }}
|
||||
|
||||
deploy:
|
||||
if: github.event_name != 'pull_request'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- uses: actions/configure-pages@v5
|
||||
- uses: actions/deploy-pages@v4
|
||||
id: deployment
|
||||
+65
-529
@@ -1,143 +1,37 @@
|
||||
name: Build Docker image
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
push_digests:
|
||||
description: Push validated per-platform images by digest for the release workflow
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
full_tests:
|
||||
description: Run storage, CLI, and size validation in addition to release smoke checks
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
description: Exact 40-character commit SHA to build
|
||||
required: true
|
||||
type: string
|
||||
push_digests:
|
||||
description: Push validated per-platform images by digest for the release workflow
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
full_tests:
|
||||
description: Run storage, CLI, and size validation in addition to release smoke checks
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
tags:
|
||||
- 'v*'
|
||||
# pull_request:
|
||||
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
DOCKER_IMAGE: archivebox-ci
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
runner: ubuntu-24.04
|
||||
artifact_name: digest-linux-amd64
|
||||
local_tag: archivebox-ci:amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
artifact_name: digest-linux-arm64
|
||||
local_tag: archivebox-ci:arm64
|
||||
buildx:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.source_ref || github.sha }}
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Verify exact source commit
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.source_ref }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Resolve Docker through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package for package in packages if package["name"] == "abxpkg"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
|
||||
wheel = matches[0]["wheels"][0]
|
||||
print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
|
||||
PY
|
||||
)"
|
||||
test -n "$ABXPKG_SPEC"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
docker_env="$(
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
|
||||
)"
|
||||
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$docker_env" >> "$GITHUB_ENV"
|
||||
|
||||
docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
{
|
||||
echo "DOCKER_BINARY=$docker_binary"
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "UV_BINARY=$UV_BINARY"
|
||||
} >> "$GITHUB_ENV"
|
||||
for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
"$docker_binary" --version
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
version: latest
|
||||
install: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Builder instance name
|
||||
run: echo ${{ steps.buildx.outputs.name }}
|
||||
@@ -145,426 +39,68 @@ jobs:
|
||||
- name: Available platforms
|
||||
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||
|
||||
- name: Verify published abx-dl image
|
||||
id: abx_dl_image
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
|
||||
print(matches[0])
|
||||
PY
|
||||
)"
|
||||
test -n "$ABX_DL_VERSION"
|
||||
ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
|
||||
INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
|
||||
printf '%s\n' "$INSPECT_OUTPUT"
|
||||
ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
|
||||
import os
|
||||
|
||||
digest = next(
|
||||
(fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
|
||||
"",
|
||||
)
|
||||
print(digest)
|
||||
PY
|
||||
)"
|
||||
[[ "$ABX_DL_DIGEST" == sha256:* ]]
|
||||
ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
|
||||
echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
if: inputs.push_digests
|
||||
- name: Docker Login
|
||||
uses: docker/login-action@v3
|
||||
if: github.event_name != 'pull_request'
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
if: inputs.push_digests
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Collect Docker labels
|
||||
- name: Collect Docker tags
|
||||
# https://github.com/docker/metadata-action
|
||||
id: docker_meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: archivebox/archivebox,nikisweeting/archivebox
|
||||
tags: |
|
||||
# :stable
|
||||
type=ref,event=branch
|
||||
# :0.7.3
|
||||
type=semver,pattern={{version}}
|
||||
# :0.7
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
# :sha-463ea54
|
||||
type=sha
|
||||
# :latest
|
||||
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
|
||||
|
||||
print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
test -n "$VERSION"
|
||||
|
||||
{
|
||||
echo 'labels<<EOF'
|
||||
echo "org.opencontainers.image.version=${VERSION}"
|
||||
echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
|
||||
echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
|
||||
echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
|
||||
echo 'EOF'
|
||||
echo "version=${VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
|
||||
|
||||
- name: Build and push digest
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
if: inputs.push_digests
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
build-args: |
|
||||
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
|
||||
ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
|
||||
pull: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
# Keep the released image as the one cache authority. A second GHA
|
||||
# cache can hold gzip variants of identical layers, and BuildKit may
|
||||
# prefer those over :dev's zstd blobs. That counter-intuitively makes
|
||||
# a cache hit larger than the released image. :dev advances only
|
||||
# after both architectures pass the release checks.
|
||||
cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
|
||||
# Inline metadata reuses the released compressed blobs directly;
|
||||
# force-compression would recompress the whole ~740 MiB image. The
|
||||
# compressed-size gate below remains the regression guard.
|
||||
cache-to: |
|
||||
${{ inputs.push_digests && 'type=inline' || '' }}
|
||||
outputs: |
|
||||
${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
|
||||
${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
|
||||
|
||||
- name: Build pull request image
|
||||
id: docker_build_pr
|
||||
if: ${{ !inputs.push_digests }}
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
build-args: |
|
||||
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
|
||||
ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
|
||||
pull: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
# Test the same layer lineage users receive. Reusing :dev avoids a
|
||||
# second cache whose compression can drift from the released image.
|
||||
cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
|
||||
outputs: |
|
||||
type=docker,name=${{ matrix.local_tag }}
|
||||
type=oci,dest=${{ runner.temp }}/archivebox-image.tar
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||
cache-from: type=local,src=/tmp/.buildx-cache
|
||||
cache-to: type=local,dest=/tmp/.buildx-cache-new
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Image digest
|
||||
if: inputs.push_digests
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Verify OpenCode install with a read-only browser profile
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
PROFILE_DIR="$(mktemp -d)"
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
|
||||
--volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
|
||||
"$IMAGE" bash -c '
|
||||
set -Eeuo pipefail
|
||||
test -w "$XDG_CONFIG_HOME"
|
||||
test ! -w "$XDG_CONFIG_HOME/chromium"
|
||||
# Check the shipped executable before the installer can repair
|
||||
# an image that accidentally omitted this opt-in dependency.
|
||||
"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
|
||||
archivebox install opencode
|
||||
"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
|
||||
'
|
||||
|
||||
- name: Validate exact built image version and commit
|
||||
if: ${{ !inputs.push_digests }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
|
||||
SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
|
||||
SHORT_SHA="${SHORT_SHA::7}"
|
||||
|
||||
DATA_DIR="$(mktemp -d)"
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
|
||||
ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
|
||||
printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
|
||||
IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
|
||||
ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
|
||||
"$UV_BINARY" run --no-cache --no-project python - \
|
||||
"ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
|
||||
"COMMIT_HASH=${SHORT_SHA}" \
|
||||
"COMMIT_HASH=${GITHUB_SHA}" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
|
||||
image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
|
||||
if sys.argv[1] not in version_output:
|
||||
raise SystemExit(f"Missing version marker: {sys.argv[1]}")
|
||||
if sys.argv[2] not in version_output:
|
||||
raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
|
||||
if sys.argv[3] not in image_version_lines:
|
||||
raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
|
||||
PY
|
||||
|
||||
"$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
|
||||
'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
|
||||
|
||||
- name: Validate compressed candidate size
|
||||
if: inputs.push_digests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
DIGEST='${{ steps.docker_build.outputs.digest }}'
|
||||
[[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||
MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
|
||||
# Provenance wraps the real platform image and its attestation in an
|
||||
# OCI index. Measure the image manifest, not the index/attestation.
|
||||
IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
|
||||
if [[ -n "$IMAGE_DIGEST" ]]; then
|
||||
MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
|
||||
fi
|
||||
COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
|
||||
LIMIT=$((1025 * 1024 * 1024))
|
||||
echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
|
||||
(( COMPRESSED_SIZE < LIMIT ))
|
||||
|
||||
- name: Resolve real Docker CLI test dependencies through abxpkg
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mount_env="$(
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
|
||||
)"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$mount_env" >> "$GITHUB_ENV"
|
||||
|
||||
for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
|
||||
binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
|
||||
- name: Validate Docker UID and GID behavior on real NFS and SMB mounts
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
NFS_SOURCE="/srv/archivebox-ci-nfs"
|
||||
SMB_SOURCE="/srv/archivebox-ci-smb"
|
||||
NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
|
||||
SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
|
||||
mounted_nfs=0
|
||||
mounted_smb=0
|
||||
|
||||
cleanup() {
|
||||
local original_status=$? cleanup_status=0
|
||||
trap - EXIT
|
||||
if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$original_status" != "0" ]]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
exit "$cleanup_status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
|
||||
"$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
|
||||
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet smbd.service
|
||||
|
||||
"$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
|
||||
mounted_nfs=1
|
||||
"$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
|
||||
-o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
|
||||
mounted_smb=1
|
||||
|
||||
NFS_TEST_DIR="$NFS_MOUNT" \
|
||||
SMB_TEST_DIR="$SMB_MOUNT" \
|
||||
IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
|
||||
"$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
|
||||
|
||||
- name: Validate Docker port compatibility
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
|
||||
|
||||
- name: Validate Docker CLI add/list and compose behavior
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
env:
|
||||
ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
"$MKDIR_BINARY" "${{ github.workspace }}/data"
|
||||
"$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
|
||||
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
"$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
|
||||
|
||||
"$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
|
||||
|
||||
archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
|
||||
[[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
[[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox init
|
||||
"$DOCKER_BINARY" compose up -d --wait
|
||||
"$DOCKER_BINARY" network connect archivebox_default www-nginx
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
|
||||
[[ "$index_html" == *ArchiveBox* ]]
|
||||
[[ "$admin_js" == *window.django* ]]
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
[[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
|
||||
"$DOCKER_BINARY" compose down
|
||||
|
||||
- name: Validate compressed image size
|
||||
if: inputs.full_tests && !inputs.push_digests
|
||||
env:
|
||||
BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
|
||||
"$UV_BINARY" run --no-cache --no-project python - \
|
||||
'${{ runner.temp }}/archivebox-image.tar' \
|
||||
'${{ steps.docker_build_pr.outputs.digest }}' \
|
||||
"$LOCAL_IMAGE_ID" \
|
||||
"$((1025 * 1024 * 1024))" <<'PY'
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
|
||||
limit = int(limit_text)
|
||||
metadata = json.loads(os.environ["BUILD_METADATA"])
|
||||
if metadata["containerimage.digest"] != pushed_digest:
|
||||
raise SystemExit("Build push transaction digest does not match the action output")
|
||||
if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
|
||||
raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
|
||||
|
||||
with tarfile.open(archive_path) as archive:
|
||||
def read_blob(descriptor):
|
||||
algorithm, digest = descriptor["digest"].split(":", 1)
|
||||
if algorithm != "sha256":
|
||||
raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
|
||||
member = archive.extractfile(f"blobs/sha256/{digest}")
|
||||
if member is None:
|
||||
raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
|
||||
blob = member.read()
|
||||
if len(blob) != descriptor["size"]:
|
||||
raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
|
||||
if sha256(blob).hexdigest() != digest:
|
||||
raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
|
||||
return blob
|
||||
|
||||
index_member = archive.extractfile("index.json")
|
||||
if index_member is None:
|
||||
raise SystemExit("OCI export is missing index.json")
|
||||
index = json.load(index_member)
|
||||
|
||||
manifests = []
|
||||
|
||||
def collect(descriptor):
|
||||
document = json.loads(read_blob(descriptor))
|
||||
media_type = descriptor["mediaType"]
|
||||
if "image.index" in media_type or "manifest.list" in media_type:
|
||||
for child in document["manifests"]:
|
||||
collect(child)
|
||||
elif "image.manifest" in media_type:
|
||||
config_type = document["config"]["mediaType"]
|
||||
if "image.config" in config_type or "container.image" in config_type:
|
||||
manifests.append(document)
|
||||
|
||||
for root in index["manifests"]:
|
||||
collect(root)
|
||||
if len(manifests) != 1:
|
||||
raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
|
||||
|
||||
manifest = manifests[0]
|
||||
config_digest = manifest["config"]["digest"]
|
||||
if metadata["containerimage.config.digest"] != config_digest:
|
||||
raise SystemExit("OCI config digest does not match the pushed build result")
|
||||
if local_image_id != config_digest:
|
||||
raise SystemExit("Locally tested image does not match the OCI build result")
|
||||
descriptors = [manifest["config"], *manifest["layers"]]
|
||||
for descriptor in descriptors:
|
||||
read_blob(descriptor)
|
||||
total = sum(descriptor["size"] for descriptor in descriptors)
|
||||
|
||||
print(
|
||||
f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
|
||||
f"limit={limit // 1024 // 1024} MiB",
|
||||
)
|
||||
if total > limit:
|
||||
raise SystemExit(f"{pushed_digest} is over the compressed size limit")
|
||||
PY
|
||||
|
||||
- name: Export digest
|
||||
if: inputs.push_digests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.docker_build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
|
||||
|
||||
- name: Upload digest
|
||||
if: inputs.push_digests
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
- name: Update README
|
||||
uses: peter-evans/dockerhub-description@v4
|
||||
with:
|
||||
name: ${{ matrix.artifact_name }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: archivebox/archivebox
|
||||
|
||||
# This ugly bit is necessary if you don't want your cache to grow forever
|
||||
# until it hits GitHub's limit of 5GB.
|
||||
# Temp fix
|
||||
# https://github.com/docker/build-push-action/issues/252
|
||||
# https://github.com/moby/buildkit/issues/1896
|
||||
- name: Move cache
|
||||
run: |
|
||||
rm -rf /tmp/.buildx-cache
|
||||
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
|
||||
@@ -1,160 +0,0 @@
|
||||
name: Documentation tests
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
USE_COLOR: "False"
|
||||
SHOW_PROGRESS: "False"
|
||||
|
||||
jobs:
|
||||
discover:
|
||||
name: Discover documentation examples
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
matrix: ${{ steps.inventory.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Install ArchiveBox development dependencies
|
||||
run: uv sync --no-cache --locked --dev
|
||||
- name: Resolve documentation tools through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
|
||||
run: |
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
uv run --no-cache --no-sync abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
|
||||
bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
|
||||
mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
|
||||
nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
|
||||
test -L "$bash_binary"
|
||||
test -L "$mmdc_binary"
|
||||
test -L "$nginx_binary"
|
||||
test -x "$bash_binary"
|
||||
test -x "$mmdc_binary"
|
||||
test -x "$nginx_binary"
|
||||
{
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
echo "BASH_BINARY=$bash_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
- id: inventory
|
||||
name: Validate the complete authored documentation inventory
|
||||
run: |
|
||||
matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
|
||||
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
||||
|
||||
examples:
|
||||
name: docs/${{ matrix.name }}
|
||||
needs: discover
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 4
|
||||
matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Install ArchiveBox
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
run: uv sync --no-cache --locked --dev
|
||||
- name: Resolve documentation runtime through abxpkg
|
||||
if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
|
||||
run: |
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
dependency_sources=(
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
|
||||
)
|
||||
if [[ '${{ matrix.environment }}' != macos ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
|
||||
fi
|
||||
if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
|
||||
fi
|
||||
if [[ '${{ matrix.environment }}' == docker ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
|
||||
elif [[ '${{ matrix.environment }}' == macos ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
|
||||
fi
|
||||
uv run --no-cache --no-sync abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
"${dependency_sources[@]}"
|
||||
bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
|
||||
python_binary="$ABXPKG_LIB_DIR/env/bin/python"
|
||||
uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
test -L "$bash_binary"
|
||||
test -x "$bash_binary"
|
||||
test -L "$python_binary"
|
||||
test -x "$python_binary"
|
||||
test -L "$uv_binary"
|
||||
test -x "$uv_binary"
|
||||
{
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
echo "BASH_BINARY=$bash_binary"
|
||||
echo "PYTHON_BINARY=$python_binary"
|
||||
echo "UV_BINARY=$uv_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
if [[ '${{ matrix.environment }}' != macos ]]; then
|
||||
sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
|
||||
test -L "$sudo_binary"
|
||||
test -x "$sudo_binary"
|
||||
echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
|
||||
fi
|
||||
if [[ '${{ matrix.environment }}' != macos ]]; then
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
fi
|
||||
if [[ '${{ matrix.environment }}' == docker ]]; then
|
||||
docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
test -L "$docker_binary"
|
||||
test -x "$docker_binary"
|
||||
echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
|
||||
elif [[ '${{ matrix.environment }}' == macos ]]; then
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/brew"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/brew"
|
||||
fi
|
||||
- name: Build the exact checkout's documentation image
|
||||
if: ${{ matrix.environment == 'docker' }}
|
||||
run: |
|
||||
ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
|
||||
"$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
|
||||
- name: Run the Ubuntu, Docker, or macOS documentation lane
|
||||
if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
|
||||
run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
|
||||
- name: Run the root documentation lane
|
||||
if: ${{ matrix.environment == 'root' }}
|
||||
run: |
|
||||
"$SUDO_BINARY" \
|
||||
--preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
|
||||
"$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
|
||||
@@ -0,0 +1,51 @@
|
||||
name: Build Homebrew package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
# TODO: modify archivebox.rb to update src url, hashes, and dependencies
|
||||
|
||||
- name: Build Homebrew Bottle
|
||||
run: |
|
||||
pip3 install --upgrade pip setuptools wheel
|
||||
cd brew_dist/
|
||||
brew install --build-bottle ./archivebox.rb
|
||||
# brew bottle archivebox
|
||||
archivebox version
|
||||
|
||||
- name: Add some links to test
|
||||
run: |
|
||||
mkdir data && cd data
|
||||
archivebox init --setup
|
||||
archivebox add 'https://example.com'
|
||||
archivebox version
|
||||
archivebox status
|
||||
|
||||
# - name: Commit built package
|
||||
# run: |
|
||||
# cd brew_dist/
|
||||
# git config --local user.email "action@github.com"
|
||||
# git config --local user.name "GitHub Action"
|
||||
# git commit -m "Homebrew package autobuild" -a
|
||||
|
||||
# - name: Push build to Github
|
||||
# uses: ad-m/github-push-action@master
|
||||
# with:
|
||||
# github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# repository: ArchiveBox/homebrew-archivebox
|
||||
# branch: ${{ github.ref }}
|
||||
# directory: brew_dist
|
||||
|
||||
# TODO: push bottle homebrew core PR with latest changes
|
||||
@@ -0,0 +1,58 @@
|
||||
# Sample workflow for building and deploying a Jekyll site to GitHub Pages
|
||||
name: Build GitHub Pages website
|
||||
|
||||
on:
|
||||
# Runs on pushes targeting the default branch
|
||||
push:
|
||||
branches: ["dev"]
|
||||
|
||||
# Allows you to run this workflow manually from the Actions tab
|
||||
workflow_dispatch:
|
||||
|
||||
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
|
||||
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
|
||||
concurrency:
|
||||
group: "pages"
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Build job
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
- name: Copy README.md into place
|
||||
run: |
|
||||
rm ./website/README.md
|
||||
cp ./README.md ./website/README.md
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v3
|
||||
- name: Build with Jekyll
|
||||
uses: actions/jekyll-build-pages@v1
|
||||
with:
|
||||
source: ./website
|
||||
destination: ./_site
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v2
|
||||
|
||||
# Deployment job
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v2
|
||||
+17
-20
@@ -1,38 +1,35 @@
|
||||
name: Run linters
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
|
||||
env:
|
||||
UV_NO_SOURCES: "1"
|
||||
MAX_LINE_LENGTH: 110
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: "3.13"
|
||||
python-version: 3.9
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
- name: Install flake8
|
||||
run: |
|
||||
pip install flake8
|
||||
|
||||
- name: Verify checked dependency lock
|
||||
run: uv lock --no-cache --check
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: uv sync --no-cache --locked --dev --extra sonic --extra debug
|
||||
|
||||
- name: Run prek
|
||||
run: uv run --no-cache --no-sync --no-sources prek run --all-files
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
cd archivebox
|
||||
# one pass for show-stopper syntax errors or undefined names
|
||||
flake8 . --count --show-source --statistics
|
||||
# one pass for small stylistic things
|
||||
flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
|
||||
Executable → Regular
+52
-217
@@ -1,243 +1,78 @@
|
||||
name: Build Pip package
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
full_tests:
|
||||
description: Run the full package compatibility matrix instead of one release smoke check
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
workflow_dispatch:
|
||||
push:
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.11.3"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build distributions
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Build every distribution
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv build --no-cache --all-packages --no-sources --out-dir dist --clear
|
||||
|
||||
shopt -s nullglob
|
||||
artifacts=(dist/*)
|
||||
wheels=(dist/archivebox-*.whl)
|
||||
sdists=(dist/archivebox-*.tar.gz)
|
||||
[[ "${#artifacts[@]}" -gt 0 ]]
|
||||
[[ "${#wheels[@]}" -eq 1 ]]
|
||||
[[ "${#sdists[@]}" -eq 1 ]]
|
||||
printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
|
||||
uv run --no-cache --no-project python - <<'PY'
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
root = Path("dist")
|
||||
wheels = list(root.glob("archivebox-*.whl"))
|
||||
sdists = list(root.glob("archivebox-*.tar.gz"))
|
||||
if len(wheels) != 1 or len(sdists) != 1:
|
||||
raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
|
||||
artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
|
||||
checksum_lines = []
|
||||
for artifact in artifacts:
|
||||
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
line = f"{digest} {artifact.name}"
|
||||
checksum_lines.append(line)
|
||||
print(line)
|
||||
Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
|
||||
PY
|
||||
|
||||
- name: Release wheel import and CLI smoke
|
||||
if: ${{ !inputs.full_tests }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
shopt -s nullglob
|
||||
wheels=(dist/archivebox-*.whl)
|
||||
[[ "${#wheels[@]}" -eq 1 ]]
|
||||
uv sync --locked --no-cache --no-sources --no-dev --no-install-project
|
||||
uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
|
||||
unset PYTHONPATH
|
||||
cd "$(mktemp -d)"
|
||||
VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
|
||||
VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
|
||||
|
||||
- name: Upload every distribution
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: dist/*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
install-smoke:
|
||||
name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
|
||||
if: inputs.full_tests
|
||||
needs: build
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-24.04, macos-15]
|
||||
python: ["3.13", "3.14"]
|
||||
artifact: [wheel, sdist]
|
||||
permissions:
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
python-version: '3.11'
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
- uses: awalsh128/cache-apt-pkgs-action@latest
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
|
||||
version: 1.0
|
||||
|
||||
- name: Download distributions
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
- uses: pdm-project/setup-pdm@v3
|
||||
with:
|
||||
name: python-distributions
|
||||
path: ${{ runner.temp }}/python-distributions
|
||||
python-version: '3.11'
|
||||
cache: true
|
||||
|
||||
- name: Install and smoke-test the built ${{ matrix.artifact }}
|
||||
shell: bash
|
||||
- name: Install dependencies
|
||||
run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
|
||||
|
||||
- name: Build package
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
shopt -s nullglob
|
||||
pdm build
|
||||
|
||||
case "${{ matrix.artifact }}" in
|
||||
wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
|
||||
sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
[[ "${#candidates[@]}" -eq 1 ]]
|
||||
artifact="${candidates[0]}"
|
||||
- name: Install from build
|
||||
run: pip install ./dist/archivebox-*.whl
|
||||
|
||||
smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
|
||||
trap 'rm -rf "$smoke_root"' EXIT
|
||||
smoke_env="$smoke_root/env"
|
||||
data_dir="$smoke_root/data"
|
||||
mkdir -p "$data_dir"
|
||||
- name: Add some links to test
|
||||
run: |
|
||||
mkdir data && cd data
|
||||
archivebox init
|
||||
archivebox add 'https://example.com'
|
||||
archivebox version
|
||||
archivebox status
|
||||
|
||||
dependency_artifacts=()
|
||||
while IFS= read -r dependency_artifact; do
|
||||
dependency_artifacts+=("$dependency_artifact")
|
||||
done < <(uv run --no-cache --no-project python - <<'PY'
|
||||
import tomllib
|
||||
#- name: Publish package distributions to PyPI
|
||||
# run: pdm publish --no-build
|
||||
|
||||
locked = {
|
||||
package["name"]: package
|
||||
for package in tomllib.load(open("uv.lock", "rb"))["package"]
|
||||
}
|
||||
for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
|
||||
package = locked[package_name]
|
||||
wheels = package.get("wheels") or []
|
||||
if len(wheels) != 1:
|
||||
raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
|
||||
print(wheels[0]["url"])
|
||||
PY
|
||||
)
|
||||
# - name: Push build to PyPI
|
||||
# run: |
|
||||
# cd pip_dist/
|
||||
# python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
|
||||
# python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
|
||||
|
||||
uv venv --python "${{ matrix.python }}" "$smoke_env"
|
||||
uv pip install --no-cache \
|
||||
--python "$smoke_env" \
|
||||
"${dependency_artifacts[@]}" \
|
||||
"$artifact"
|
||||
# - name: Commit built package
|
||||
# run: |
|
||||
# cd pip_dist/
|
||||
# git config --local user.email "action@github.com"
|
||||
# git config --local user.name "GitHub Action"
|
||||
# git commit -m "Pip package autobuild" -a
|
||||
|
||||
# - name: Push build to Github
|
||||
# uses: ad-m/github-push-action@master
|
||||
# with:
|
||||
# github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# repository: ArchiveBox/pip-archivebox
|
||||
# branch: ${{ github.ref }}
|
||||
# directory: pip_dist
|
||||
|
||||
unset PYTHONPATH
|
||||
cd "$data_dir"
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import archivebox
|
||||
|
||||
installed_path = Path(archivebox.__file__).resolve()
|
||||
environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
|
||||
workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
|
||||
assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
|
||||
assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
|
||||
print(installed_path)
|
||||
PY
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
|
||||
VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
|
||||
|
||||
if [[ "${{ runner.os }}" == "Linux" ]]; then
|
||||
root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
|
||||
uv_binary="$(command -v uv)"
|
||||
sudo rm -rf "$root_tool_dir"
|
||||
sudo mkdir -p "$root_tool_dir/data"
|
||||
|
||||
root_install_args=()
|
||||
for dependency_artifact in "${dependency_artifacts[@]}"; do
|
||||
root_install_args+=(--with "$dependency_artifact")
|
||||
done
|
||||
sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
|
||||
"$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
|
||||
"${root_install_args[@]}" "$artifact"
|
||||
|
||||
sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
|
||||
sudo rm -rf /var/lib/archivebox
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data' && archivebox init"
|
||||
sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
|
||||
sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
|
||||
if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
|
||||
echo "Root-owned files remain in $root_tool_dir/data" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo mkdir "$root_tool_dir/external"
|
||||
sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data' && archivebox status"
|
||||
sudo stat -c %U "$root_tool_dir/external" | grep -qx root
|
||||
|
||||
sudo mkdir "$root_tool_dir/data-universal-init"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
|
||||
sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
|
||||
|
||||
sudo mkdir "$root_tool_dir/data-install"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
|
||||
sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
|
||||
|
||||
sudo useradd --create-home archivebox-smoke-user
|
||||
smoke_user_group="$(id -gn archivebox-smoke-user)"
|
||||
sudo mkdir "$root_tool_dir/data-mixed-owner"
|
||||
sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
|
||||
sudo env PATH="$root_tool_dir/bin:$PATH" \
|
||||
bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
|
||||
sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
|
||||
if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
|
||||
echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo rm -rf "$root_tool_dir"
|
||||
fi
|
||||
@@ -1,79 +0,0 @@
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: release-candidate-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
action: ${{ steps.version.outputs.action }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN }}
|
||||
|
||||
- name: Keep synchronized stable branches on the released version
|
||||
id: channel
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
|
||||
MAIN_SHA="$(git ls-remote origin refs/heads/main | cut -f1)"
|
||||
if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$MAIN_SHA" == "$GITHUB_SHA" ]]; then
|
||||
echo 'stable=true' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stable=false' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- id: version
|
||||
if: steps.channel.outputs.stable != 'true'
|
||||
uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
|
||||
with:
|
||||
package: archivebox
|
||||
branch: ${{ github.ref_name }}
|
||||
tag-prefix: v
|
||||
version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
|
||||
|
||||
python-artifacts:
|
||||
needs: prepare
|
||||
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
|
||||
uses: ./.github/workflows/pip.yml
|
||||
with:
|
||||
full_tests: false
|
||||
secrets: inherit
|
||||
|
||||
docker-digests:
|
||||
needs: prepare
|
||||
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
push_digests: true
|
||||
full_tests: false
|
||||
secrets: inherit
|
||||
|
||||
opencode:
|
||||
needs: prepare
|
||||
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_file:
|
||||
- archivebox/tests/test_opencode_agent.py
|
||||
- archivebox/tests/test_opencode_browser.py
|
||||
uses: ./.github/workflows/test-parallel.yml
|
||||
with:
|
||||
test_file: ${{ matrix.test_file }}
|
||||
secrets: inherit
|
||||
@@ -1,465 +0,0 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI, Release candidate]
|
||||
types: [completed]
|
||||
branches: [dev, main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_branch:
|
||||
description: Branch being promoted
|
||||
required: true
|
||||
type: choice
|
||||
options: [main, dev]
|
||||
release_sha:
|
||||
description: Exact tested commit to release
|
||||
required: true
|
||||
type: string
|
||||
ci_run_id:
|
||||
description: Successful CI run containing the tested Python artifacts
|
||||
required: true
|
||||
type: string
|
||||
docker_digest_run_id:
|
||||
description: Manual Docker digest build run containing the tested image digests
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
|
||||
RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
|
||||
|
||||
jobs:
|
||||
candidate:
|
||||
if: >-
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
(github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
|
||||
(github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
outputs:
|
||||
ready: ${{ steps.verified.outputs.ready }}
|
||||
artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
|
||||
digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
- id: owner
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
|
||||
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
|
||||
if [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
|
||||
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo 'ready=true' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# A stable release fast-forwards dev to main. Only main publishes it;
|
||||
# the resulting dev workflow is not a new release candidate.
|
||||
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
|
||||
echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
|
||||
[[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Require successful full CI for this commit
|
||||
id: verified
|
||||
if: steps.owner.outputs.ready == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
|
||||
MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
|
||||
CI_RUN_ID="$MANUAL_CI_RUN_ID"
|
||||
DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
|
||||
CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
|
||||
--jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
|
||||
[[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
|
||||
else
|
||||
CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
|
||||
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
|
||||
--json databaseId,status,conclusion \
|
||||
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
|
||||
CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
|
||||
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
|
||||
--json databaseId,status,conclusion \
|
||||
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
|
||||
[[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
|
||||
if [[ "$RELEASE_BRANCH" == dev ]]; then
|
||||
DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
|
||||
else
|
||||
DIGEST_RUN_ID="$CI_RUN_ID"
|
||||
fi
|
||||
fi
|
||||
if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
|
||||
ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
|
||||
else
|
||||
ARTIFACT_RUN_ID="$CI_RUN_ID"
|
||||
fi
|
||||
{
|
||||
echo 'ready=true'
|
||||
echo "artifact_run_id=$ARTIFACT_RUN_ID"
|
||||
echo "digest_run_id=$DIGEST_RUN_ID"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
python-release:
|
||||
name: Publish tested Python artifacts
|
||||
needs: candidate
|
||||
if: needs.candidate.outputs.ready == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
|
||||
environment: pypi
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
id-token: write
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
release_ready: ${{ steps.release-state.outputs.ready }}
|
||||
steps:
|
||||
- name: Checkout the tested commit
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve release binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
|
||||
for name in uv gh git jq curl; do
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download exact tested distributions
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: ${{ runner.temp }}/python-distributions
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.CI_RUN_ID }}
|
||||
|
||||
- name: Record release version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate release branch and version
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION='${{ steps.version.outputs.version }}'
|
||||
case "$RELEASE_BRANCH" in
|
||||
main)
|
||||
;;
|
||||
dev)
|
||||
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
|
||||
echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
|
||||
exit 1
|
||||
}
|
||||
;;
|
||||
*)
|
||||
echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Publish the exact tested distributions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
|
||||
run: ./bin/release.sh
|
||||
|
||||
- name: Confirm this tested commit owns the release tag
|
||||
id: release-state
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
TARGET=""
|
||||
while read -r sha _; do
|
||||
TARGET="$sha"
|
||||
done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
|
||||
if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
|
||||
echo "ready=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
docker-release:
|
||||
name: Publish tested multiarch image
|
||||
if: >-
|
||||
(github.event_name == 'workflow_dispatch') ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push')
|
||||
needs: [candidate, python-release]
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout the tested commit
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- name: Fast-forward dev to the tested stable release
|
||||
if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
git fetch --no-tags origin main dev
|
||||
MAIN_SHA="$(git rev-parse origin/main)"
|
||||
DEV_SHA="$(git rev-parse origin/dev)"
|
||||
if [[ "$MAIN_SHA" == "$RELEASE_SHA" && "$DEV_SHA" != "$RELEASE_SHA" ]] &&
|
||||
git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
|
||||
git push origin "$RELEASE_SHA:refs/heads/dev"
|
||||
fi
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve release and Docker binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
|
||||
for name in uv gh git jq docker; do
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
with:
|
||||
install: true
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Download exact tested image digests
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
|
||||
|
||||
- name: Verify exact successful digest build run
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
|
||||
RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
|
||||
[[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
|
||||
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
|
||||
RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
|
||||
[[ "$RUN_SHA" == "$RELEASE_SHA" ]]
|
||||
fi
|
||||
|
||||
- name: Verify digest set and collect tags
|
||||
id: docker_meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
[[ "${#DIGESTS[@]}" -eq 2 ]]
|
||||
for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
|
||||
[[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
|
||||
|
||||
VERSION='${{ needs.python-release.outputs.version }}'
|
||||
SHORT_SHA="${RELEASE_SHA::12}"
|
||||
TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
|
||||
TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
|
||||
if [[ -z "$TAG_TARGET" ]]; then
|
||||
TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
|
||||
TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
|
||||
fi
|
||||
SYNC_DEV=false
|
||||
if [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
$GIT_BINARY fetch --no-tags origin main dev
|
||||
MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
|
||||
DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
|
||||
# Recovery can add CI/docs fixes after the immutable release tag.
|
||||
# Both branches and every shipped application file must agree.
|
||||
if [[ "$MAIN_TARGET" == "$DEV_TARGET" ]] &&
|
||||
$GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
|
||||
SYNC_DEV=true
|
||||
fi
|
||||
fi
|
||||
{
|
||||
echo 'dockerhub_tags<<EOF'
|
||||
if [[ "$RELEASE_BRANCH" == dev ]]; then
|
||||
echo "${DOCKERHUB_IMAGE}:dev"
|
||||
elif [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
echo "${DOCKERHUB_IMAGE}:latest"
|
||||
[[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
|
||||
fi
|
||||
echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
|
||||
echo 'EOF'
|
||||
echo 'ghcr_tags<<EOF'
|
||||
if [[ "$RELEASE_BRANCH" == dev ]]; then
|
||||
echo "${GHCR_IMAGE}:dev"
|
||||
elif [[ "$RELEASE_BRANCH" == main ]]; then
|
||||
echo "${GHCR_IMAGE}:latest"
|
||||
[[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
|
||||
fi
|
||||
echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create Docker Hub manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Create GHCR manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Verify published Docker images run
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION='${{ needs.python-release.outputs.version }}'
|
||||
SHORT_SHA="${RELEASE_SHA::12}"
|
||||
for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
|
||||
"$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
|
||||
INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
|
||||
test "$INSTALLED_VERSION" = "$VERSION"
|
||||
done
|
||||
|
||||
- name: Update Docker Hub README
|
||||
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: archivebox/archivebox
|
||||
|
||||
cascade:
|
||||
name: Cascade verified release
|
||||
if: needs.python-release.outputs.release_ready == 'true'
|
||||
needs:
|
||||
- candidate
|
||||
- python-release
|
||||
- docker-release
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
|
||||
with:
|
||||
upstream_package: archivebox
|
||||
upstream_version: ${{ needs.python-release.outputs.version }}
|
||||
upstream_artifact_name: python-distributions
|
||||
upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
|
||||
secrets:
|
||||
release_token: ${{ secrets.RELEASE_GH_TOKEN }}
|
||||
@@ -1,71 +0,0 @@
|
||||
name: Capture publicsite screenshots
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
paths:
|
||||
- archivebox/**
|
||||
- bin/collect_ui_screenshots.*
|
||||
- bin/generate_ui_screenshot_gallery.py
|
||||
- bin/take_screenshot.js
|
||||
- docs/Screenshots.md
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
- .github/workflows/screenshots.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Python and uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: false
|
||||
|
||||
- name: Install ArchiveBox and screenshot dependencies
|
||||
run: uv sync --no-cache --frozen
|
||||
|
||||
- name: Build UI screenshot gallery
|
||||
run: bin/collect_ui_screenshots.sh
|
||||
env:
|
||||
BASE_URL: http://archivebox.localhost:5797
|
||||
|
||||
- name: Verify gallery was generated from this exact revision
|
||||
run: |
|
||||
uv run --no-cache --no-sync python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
|
||||
version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
|
||||
assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
|
||||
assert provenance["version"] == version, provenance
|
||||
assert provenance["capture_count"] >= 3 * 35, provenance
|
||||
assert len(provenance["files"]) == provenance["capture_count"], provenance
|
||||
assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
|
||||
PY
|
||||
|
||||
- name: Save validated screenshots
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: site-screenshots
|
||||
path: publicsite/screenshots
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
@@ -1,168 +0,0 @@
|
||||
name: Parallel Tests
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
test_file:
|
||||
description: Run one discovered test file; empty runs the complete matrix
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
PYTHONLEGACYWINDOWSSTDIO: utf-8
|
||||
USE_COLOR: False
|
||||
UV_NO_SOURCES: "1"
|
||||
CI_PYTHON_VERSION: "3.13"
|
||||
|
||||
jobs:
|
||||
discover-tests:
|
||||
name: Discover every test
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
tests: ${{ steps.discover.outputs.tests }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Resolve discovery tools through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
test -n "$ABXPKG_SPEC"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PYTHON_BINARY="$pythonLocation/bin/python"
|
||||
test -x "$PYTHON_BINARY"
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
|
||||
{
|
||||
echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
} >> "$GITHUB_ENV"
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/python"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/python"
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
|
||||
- name: Discover every test exactly once
|
||||
id: discover
|
||||
env:
|
||||
TEST_FILE: ${{ inputs.test_file }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
|
||||
test "${#output[@]}" -eq 2
|
||||
if [[ -n "$TEST_FILE" ]]; then
|
||||
output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
|
||||
fi
|
||||
echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
|
||||
echo "${output[0]}"
|
||||
"$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
|
||||
|
||||
tests:
|
||||
name: ${{ matrix.test.name }}
|
||||
needs: discover-tests
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
ABXPKG_LIB_DIR: /tmp/archivebox-lib
|
||||
CHROME_HEADLESS: "true"
|
||||
DATA_DIR: /tmp/archivebox-test-data
|
||||
PERSONAS_DIR: /tmp/abx-personas
|
||||
CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
architecture: x64
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
- name: Resolve optional build dependencies through abxpkg
|
||||
if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
|
||||
env:
|
||||
TEST_EXTRA: ${{ matrix.test.extra }}
|
||||
TEST_PATHS: ${{ matrix.test.paths_arg }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
|
||||
test -n "$ABXPKG_SPEC"
|
||||
dependency_sources=()
|
||||
if [[ -n "$TEST_EXTRA" ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
|
||||
fi
|
||||
if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
|
||||
dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
|
||||
fi
|
||||
uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
|
||||
--install \
|
||||
--no-cache \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
"${dependency_sources[@]}"
|
||||
|
||||
- name: Install test dependencies
|
||||
env:
|
||||
TEST_EXTRA: ${{ matrix.test.extra }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
extra_args=()
|
||||
if [[ -n "$TEST_EXTRA" ]]; then
|
||||
extra_args=(--extra "$TEST_EXTRA")
|
||||
fi
|
||||
uv sync --no-cache --locked --dev "${extra_args[@]}"
|
||||
- name: Install Playwright Chromium for browser tests
|
||||
if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks') || contains(matrix.test.paths_arg, 'test_ui_admin_snapshot')
|
||||
run: uv run --no-cache --no-sync --no-sources playwright install chromium
|
||||
|
||||
- name: Run ${{ matrix.test.name }}
|
||||
env:
|
||||
TEST_PATHS: ${{ matrix.test.paths_arg }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
read -r -a test_paths <<< "$TEST_PATHS"
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test -f "$test_path"
|
||||
done
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test_name="$(basename "$test_path" .py)"
|
||||
test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
|
||||
export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
|
||||
export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
|
||||
export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
|
||||
mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
|
||||
uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
|
||||
--basetemp="$RUNNER_TEMP/pytest/$test_slug"
|
||||
done
|
||||
Executable → Regular
+132
-37
@@ -1,63 +1,158 @@
|
||||
name: Install and CLI Compatibility
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
name: Run tests
|
||||
on: [push]
|
||||
|
||||
env:
|
||||
DOCKER_IMAGE: archivebox-ci
|
||||
ARCHIVEBOX_IMAGE: archivebox-ci
|
||||
PYTHONIOENCODING: utf-8
|
||||
PYTHONLEGACYWINDOWSSTDIO: utf-8
|
||||
USE_COLOR: False
|
||||
UV_NO_SOURCES: "1"
|
||||
|
||||
jobs:
|
||||
install_cli_platform:
|
||||
name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
|
||||
python_tests:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-24.04
|
||||
os_name: linux
|
||||
python: "3.14"
|
||||
- os: macos-15
|
||||
os_name: macOS
|
||||
python: "3.13"
|
||||
- os: macos-15
|
||||
os_name: macOS
|
||||
python: "3.14"
|
||||
os: [ubuntu-22.04]
|
||||
# os: [ubuntu-22.04, macos-latest, windows-latest]
|
||||
python: [3.11]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
### Set up Python
|
||||
### Setup Python & JS Languages
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
- name: Set up Node JS
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
node-version: 20.10.0
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: uv sync --no-cache --locked --dev
|
||||
- name: Setup PDM
|
||||
uses: pdm-project/setup-pdm@v3
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: true
|
||||
|
||||
- name: ArchiveBox full install check
|
||||
### Install Python & JS Dependencies
|
||||
- name: Get pip cache dir
|
||||
id: pip-cache
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
DATA_DIR="$PWD/tests/out/data"
|
||||
mkdir -p "$DATA_DIR"
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
|
||||
uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
|
||||
echo "::set-output name=dir::$(pip cache dir)"
|
||||
|
||||
- name: Cache pip
|
||||
uses: actions/cache@v3
|
||||
id: cache-pip
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.python }}-venv-
|
||||
|
||||
- uses: awalsh128/cache-apt-pkgs-action@latest
|
||||
with:
|
||||
packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
|
||||
version: 1.0
|
||||
|
||||
- name: Install pip dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip setuptools wheel pytest bottle build
|
||||
python -m pip install -r requirements.txt
|
||||
python -m pip install -e .[sonic,ldap]
|
||||
|
||||
- name: Get npm cache dir
|
||||
id: npm-cache
|
||||
run: |
|
||||
echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
|
||||
|
||||
- name: Cache npm
|
||||
uses: actions/cache@v3
|
||||
id: cache-npm
|
||||
with:
|
||||
path: ${{ steps.npm-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node_modules
|
||||
|
||||
- name: Install npm requirements
|
||||
run: |
|
||||
npm install
|
||||
echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
|
||||
echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
|
||||
echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV
|
||||
|
||||
### Run the tests
|
||||
- name: Directory listing for debugging
|
||||
run: |
|
||||
pwd
|
||||
ls
|
||||
|
||||
- name: Archivebox version
|
||||
run: |
|
||||
archivebox version
|
||||
|
||||
- name: Test built package with pytest
|
||||
# TODO: remove this exception for windows once we get tests passing on that platform
|
||||
if: ${{ !contains(matrix.os, 'windows') }}
|
||||
run: |
|
||||
python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
|
||||
|
||||
docker_tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
# TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
|
||||
# - uses: satackey/action-docker-layer-caching@v0.0.8
|
||||
|
||||
- name: Build image
|
||||
run: |
|
||||
docker build . -t "$DOCKER_IMAGE"
|
||||
|
||||
- name: Init data dir
|
||||
run: |
|
||||
mkdir "${{ github.workspace }}/data"
|
||||
docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
|
||||
|
||||
- name: Run test server
|
||||
run: |
|
||||
sudo bash -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
docker run --name www-nginx -p 80:80 -d nginx
|
||||
|
||||
- name: Add link
|
||||
run: |
|
||||
docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
|
||||
|
||||
- name: Add stdin link
|
||||
run: |
|
||||
echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
|
||||
|
||||
- name: List links
|
||||
run: |
|
||||
docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
|
||||
- name: Start docker-compose stack
|
||||
run: |
|
||||
docker-compose run archivebox init
|
||||
docker-compose up -d
|
||||
sleep 5
|
||||
curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
|
||||
curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
|
||||
|
||||
- name: Check added urls show up in index
|
||||
run: |
|
||||
docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
|
||||
curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
|
||||
docker-compose down || true
|
||||
-35
@@ -1,36 +1,22 @@
|
||||
.DS_Store
|
||||
._*
|
||||
/.heartbeat.json
|
||||
/package.json
|
||||
/package-lock.json
|
||||
*.pyc
|
||||
__pycache__/
|
||||
.mypy_cache/
|
||||
.eggs/
|
||||
tests/out/
|
||||
/docs/screenshots/
|
||||
/publicsite/screenshots/
|
||||
|
||||
# Coverage
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.json
|
||||
coverage/
|
||||
htmlcov/
|
||||
|
||||
# Python and Node dependencies
|
||||
venv/
|
||||
.venv/
|
||||
.docker-venv/
|
||||
node_modules/
|
||||
typings/
|
||||
|
||||
# Ignore dev lockfiles (should always be built fresh)
|
||||
pdm.dev.lock
|
||||
requirements-dev.txt
|
||||
|
||||
# Packaging artifacts
|
||||
requirements.txt
|
||||
.pdm-python
|
||||
.pdm-build
|
||||
archivebox.egg-info
|
||||
@@ -39,34 +25,13 @@ build/
|
||||
dist/
|
||||
|
||||
# Data folders
|
||||
lib/
|
||||
out/
|
||||
tmp/
|
||||
data/
|
||||
data*/
|
||||
archivebox/tests/data/
|
||||
archive/
|
||||
output/
|
||||
logs/
|
||||
/-
|
||||
/personas/
|
||||
/sources/
|
||||
index.sqlite3
|
||||
queue.sqlite3
|
||||
*.sqlite*
|
||||
data.*
|
||||
.archivebox_id
|
||||
ArchiveBox.conf
|
||||
*.stdout
|
||||
*.stderr
|
||||
*.log
|
||||
.tmp/
|
||||
|
||||
# vim
|
||||
*.sw?
|
||||
.vscode
|
||||
|
||||
# Local Claude Code task locks (not project state)
|
||||
.claude/scheduled_tasks.lock
|
||||
|
||||
/_site/
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
[submodule "docs"]
|
||||
path = docs
|
||||
url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
|
||||
|
||||
[submodule "deb_dist"]
|
||||
path = deb_dist
|
||||
url = https://github.com/ArchiveBox/debian-archivebox.git
|
||||
[submodule "brew_dist"]
|
||||
path = brew_dist
|
||||
url = https://github.com/ArchiveBox/homebrew-archivebox.git
|
||||
[submodule "pip_dist"]
|
||||
path = pip_dist
|
||||
url = https://github.com/ArchiveBox/pip-archivebox.git
|
||||
[submodule "docker"]
|
||||
path = docker
|
||||
url = https://github.com/ArchiveBox/docker-archivebox.git
|
||||
[submodule "archivebox/vendor/base32-crockford"]
|
||||
path = archivebox/vendor/base32-crockford
|
||||
url = https://github.com/jbittel/base32-crockford
|
||||
[submodule "archivebox/vendor/pocket"]
|
||||
path = archivebox/vendor/pocket
|
||||
url = https://github.com/tapanpandita/pocket
|
||||
[submodule "archivebox/vendor/django-taggit"]
|
||||
path = archivebox/vendor/django-taggit
|
||||
url = https://github.com/jazzband/django-taggit
|
||||
[submodule "archivebox/vendor/python-atomicwrites"]
|
||||
path = archivebox/vendor/python-atomicwrites
|
||||
url = https://github.com/untitaker/python-atomicwrites
|
||||
[submodule "archivebox/vendor/pydantic-pkgr"]
|
||||
path = archivebox/vendor/pydantic-pkgr
|
||||
url = https://github.com/ArchiveBox/pydantic-pkgr
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
tests/
|
||||
archivebox/
|
||||
archivebox.egg-info/
|
||||
build/
|
||||
dist/
|
||||
docs/
|
||||
etc/
|
||||
.github
|
||||
.gitmodules
|
||||
.dockerignore
|
||||
.flake8
|
||||
CNAME
|
||||
_config.yml
|
||||
docker-compose.yaml
|
||||
docker-compose.yml
|
||||
Dockerfile
|
||||
MANIFEST.in
|
||||
Pipfile
|
||||
setup.py
|
||||
@@ -1,69 +0,0 @@
|
||||
default_language_version:
|
||||
python: python3.13
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/asottile/yesqa
|
||||
rev: v1.5.0
|
||||
hooks:
|
||||
- id: yesqa
|
||||
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies:
|
||||
- tomli
|
||||
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.20.0
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: [--py313-plus]
|
||||
|
||||
- repo: https://github.com/asottile/add-trailing-comma
|
||||
rev: v3.1.0
|
||||
hooks:
|
||||
- id: add-trailing-comma
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: ruff-format
|
||||
name: ruff-format
|
||||
entry: uv run --active --no-sync --no-sources ruff format
|
||||
language: system
|
||||
types_or: [python, pyi]
|
||||
- id: ruff-check
|
||||
name: ruff-check
|
||||
entry: uv run --active --no-sync --no-sources ruff check --fix
|
||||
language: system
|
||||
types_or: [python, pyi]
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
- id: check-ast
|
||||
- id: check-toml
|
||||
- id: check-yaml
|
||||
exclude: ^\.github/workflows/homebrew\.yml$
|
||||
- id: check-json
|
||||
- id: check-merge-conflict
|
||||
- id: check-symlinks
|
||||
- id: destroyed-symlinks
|
||||
- id: check-case-conflict
|
||||
- id: check-illegal-windows-names
|
||||
- id: check-shebang-scripts-are-executable
|
||||
exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
|
||||
- id: mixed-line-ending
|
||||
- id: fix-byte-order-marker
|
||||
- id: end-of-file-fixer
|
||||
# Preserve the exact bytes of captured cryptographic proof fixtures.
|
||||
exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
|
||||
- id: detect-private-key
|
||||
- id: debug-statements
|
||||
- id: forbid-submodules
|
||||
exclude: ^docs$
|
||||
- id: check-added-large-files
|
||||
args: ["--maxkb=600"]
|
||||
- id: name-tests-test
|
||||
args: ["--pytest-test-first"]
|
||||
exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
|
||||
@@ -0,0 +1,33 @@
|
||||
# Read the Docs configuration file for Sphinx projects
|
||||
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
|
||||
|
||||
version: 2
|
||||
|
||||
submodules:
|
||||
include: all
|
||||
recursive: true
|
||||
|
||||
# Set the OS, Python version and other tools you might need
|
||||
build:
|
||||
os: ubuntu-22.04
|
||||
tools:
|
||||
python: "3.12"
|
||||
nodejs: "20"
|
||||
|
||||
# Build documentation in the "docs/" directory with Sphinx
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
# You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
|
||||
# builder: "dirhtml"
|
||||
|
||||
# Optionally build your docs in additional formats such as PDF and ePub
|
||||
formats:
|
||||
- pdf
|
||||
- epub
|
||||
|
||||
# Optional but recommended, declare the Python requirements required
|
||||
# to build your documentation
|
||||
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
@@ -1,87 +0,0 @@
|
||||
# ArchiveBox Agent Guide
|
||||
|
||||
ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
|
||||
|
||||
## Shared Standards
|
||||
|
||||
- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
|
||||
- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
|
||||
- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
|
||||
- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
|
||||
- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
|
||||
- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
|
||||
- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
|
||||
- Start behavior fixes with a red failing test when a test is requested or practical.
|
||||
- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
|
||||
- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
|
||||
|
||||
## Concurrency Contract
|
||||
|
||||
- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
|
||||
- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
|
||||
- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
|
||||
- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
|
||||
|
||||
## Development Setup
|
||||
|
||||
```bash
|
||||
uv sync --dev --all-extras
|
||||
mkdir -p data
|
||||
cd data
|
||||
uv run --project .. archivebox init --install
|
||||
```
|
||||
|
||||
Run collection commands from inside an initialized data directory:
|
||||
|
||||
```bash
|
||||
cd data
|
||||
uv run --project .. archivebox status
|
||||
uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
|
||||
uv run --project .. archivebox run
|
||||
```
|
||||
|
||||
## User-Facing Setup
|
||||
|
||||
Recommended CLI install:
|
||||
|
||||
```bash
|
||||
uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
|
||||
mkdir -p ~/archivebox/data
|
||||
cd ~/archivebox/data
|
||||
archivebox init --install
|
||||
archivebox add --plugins=parse_txt_urls 'https://example.com/'
|
||||
```
|
||||
|
||||
Alternative install methods:
|
||||
|
||||
- Docker Compose / Docker
|
||||
- Homebrew
|
||||
- Debian package
|
||||
- pip
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
cd ~/archivebox/data
|
||||
archivebox version
|
||||
archivebox help
|
||||
archivebox status
|
||||
archivebox install
|
||||
archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
|
||||
archivebox list --json --with-headers
|
||||
archivebox search 'example'
|
||||
archivebox update --filter-type=domain example.com
|
||||
archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
|
||||
archivebox run
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
Use targeted tests for focused work:
|
||||
|
||||
```bash
|
||||
uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
|
||||
uv run prek run --all-files
|
||||
```
|
||||
|
||||
Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
|
||||
+299
-310
@@ -1,247 +1,22 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
|
||||
# python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
|
||||
# Usage:
|
||||
# git submodule update --init --recursive
|
||||
# git pull --recurse-submodules
|
||||
# docker build . -t archivebox --no-cache
|
||||
# docker run -v "$PWD/data":/data archivebox init
|
||||
# docker run -v "$PWD/data":/data archivebox add 'https://example.com'
|
||||
# docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
|
||||
# docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
|
||||
# Multi-arch build:
|
||||
# docker buildx create --use
|
||||
# docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
|
||||
#
|
||||
# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
|
||||
|
||||
# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
|
||||
# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
|
||||
# standalone downloader stays independent of ArchiveBox's server features.
|
||||
# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
|
||||
# the layers added by this Dockerfile, never in the shared abx-dl base image.
|
||||
# REQUIRED INSTALLATION CONTRACT — do not work around failures:
|
||||
# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
|
||||
# and ALL their required_binaries, regardless of default enabled state.
|
||||
# Inherit the complete downloader dependencies from the tested abx-dl image;
|
||||
# install every remaining ArchiveBox-only dependency in this image's layers,
|
||||
# including OpenCode and Sonic. Never put server dependencies in abx-dl.
|
||||
# Use the normal plugin dependency pipeline: install command -> config.json
|
||||
# required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
|
||||
# failures in that pipeline. OS/interpreter bootstrap is a separate concern,
|
||||
# not a second hand-maintained list of plugin dependencies.
|
||||
# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
|
||||
# must discover everything already present and be NO-OPS offline as UID 911.
|
||||
# No downloads, package installs, sudo, or root repair may be needed to enable
|
||||
# ANY plugin or start a new crawl. Never run verification as root or manually
|
||||
# preinstall packages before it. If the inherited image lacks a downloader
|
||||
# dependency, fix/release abx-dl and consume that tested image; do not patch
|
||||
# over the missing base dependency here or repair deployed containers by SSH.
|
||||
# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
|
||||
# CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
|
||||
# passwordless sudo; abxpkg must escalate automatically when needed. Never
|
||||
# manually preinstall dependencies or wrap the tested command in sudo to make
|
||||
# a failure disappear. This is distinct from the finished-image no-op test.
|
||||
# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
|
||||
# user-facing functionality. Only mechanical/internal
|
||||
# optimizations are allowed. Never remove dependencies, disable features or
|
||||
# plugins, weaken assertions, or skip checks to meet a size limit. If the
|
||||
# complete image exceeds the configured limit, LET CI FAIL and explicitly
|
||||
# ask the maintainer what to do. Do not raise the limit unilaterally.
|
||||
# Build abx-dl first, then point this file at it:
|
||||
# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
|
||||
# --build-context abxbus=../abxbus \
|
||||
# --build-context abxpkg=../abxpkg \
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:main
|
||||
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
ARG ARCHIVEBOX_COMMIT_HASH=""
|
||||
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
LANG=C.UTF-8 \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
|
||||
PYTHONIOENCODING=UTF-8 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_NO_COMPILE=1 \
|
||||
PIP_ONLY_BINARY=aiohttp \
|
||||
npm_config_loglevel=error
|
||||
|
||||
ENV PYTHON_VERSION=3.13 \
|
||||
NODE_VERSION=24
|
||||
|
||||
ENV ARCHIVEBOX_USER=archivebox \
|
||||
DEFAULT_ARCHIVEBOX_UID=911 \
|
||||
DEFAULT_ARCHIVEBOX_GID=911 \
|
||||
IN_DOCKER=True
|
||||
|
||||
ENV CODE_DIR=/app \
|
||||
ABX_RUNTIME=archivebox \
|
||||
DATA_DIR=/data \
|
||||
CONFIG_DIR=/opt/archivebox \
|
||||
ABXPKG_LIB_DIR=/opt/archivebox/lib \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
|
||||
PERSONAS_DIR=/data/personas \
|
||||
CHROME_HEADLESS=true \
|
||||
CHROME_SANDBOX=false \
|
||||
CHROME_ISOLATION=crawl \
|
||||
CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
|
||||
|
||||
ENV TMP_DIR=/tmp/archivebox \
|
||||
PIP_VENV_PYTHON=/venv/bin/python3 \
|
||||
GOOGLE_API_KEY=no \
|
||||
GOOGLE_DEFAULT_CLIENT_ID=no \
|
||||
GOOGLE_DEFAULT_CLIENT_SECRET=no
|
||||
|
||||
ENV HOME=/home/archivebox \
|
||||
XDG_CONFIG_HOME=/home/archivebox/.config \
|
||||
XDG_CACHE_HOME=/opt/archivebox/lib/cache \
|
||||
TIMEOUT=600
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=false \
|
||||
UV_PYTHON_PREFERENCE=managed \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv \
|
||||
VIRTUAL_ENV=/venv \
|
||||
PATH="/venv/bin:/opt/node/bin:$PATH"
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
WORKDIR "$CODE_DIR"
|
||||
|
||||
RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
|
||||
&& (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
FROM archivebox-runtime-base AS archivebox-builder
|
||||
|
||||
WORKDIR "$CODE_DIR"
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
|
||||
<<'EOF'
|
||||
echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
|
||||
export UV_NO_CACHE=true
|
||||
rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
apt-get update -qq
|
||||
apt-get install -qq -y --no-install-recommends x11-utils
|
||||
# Extend abx-dl's existing venv instead of clearing it. Clearing and later
|
||||
# copying a complete replacement over the base duplicates every venv byte in
|
||||
# the final overlay history even when most packages are unchanged.
|
||||
# Do not rewrite inherited native libraries merely to strip them again.
|
||||
/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
|
||||
|
||||
mkdir -p /tmp/archivebox-uv-project
|
||||
/venv/bin/python - <<'PY'
|
||||
from pathlib import Path
|
||||
import json
|
||||
import re
|
||||
import urllib.request
|
||||
|
||||
source = Path("/app/pyproject.toml")
|
||||
target = Path("/tmp/archivebox-uv-project/pyproject.toml")
|
||||
text = source.read_text()
|
||||
text = text.replace(
|
||||
'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
|
||||
'environments = ["sys_platform == \'linux\'"]',
|
||||
)
|
||||
|
||||
# Docker builds need the just-published internal abx wheels immediately, but
|
||||
# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
|
||||
# Docker-only dependency view from the version JSON so the published package
|
||||
# metadata stays normal while image builds remain resumable after a release.
|
||||
for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
|
||||
match = re.search(
|
||||
rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
|
||||
text,
|
||||
)
|
||||
if not match:
|
||||
continue
|
||||
version = match.group("version")
|
||||
with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
|
||||
data = json.load(response)
|
||||
wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
|
||||
text = re.sub(
|
||||
rf'"{re.escape(package)}(?:==|>=)[^"]+"',
|
||||
f'"{package} @ {wheel_url}"',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
|
||||
target.write_text(text)
|
||||
PY
|
||||
|
||||
/usr/bin/uv sync \
|
||||
--project /tmp/archivebox-uv-project \
|
||||
--no-cache \
|
||||
--no-dev \
|
||||
--inexact \
|
||||
--no-install-project \
|
||||
--no-install-workspace \
|
||||
--no-install-package abxbus \
|
||||
--no-install-package abxpkg \
|
||||
--no-install-package abx-plugins \
|
||||
--no-install-package abx-dl \
|
||||
--no-sources
|
||||
builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
|
||||
ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
|
||||
/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
|
||||
while IFS= read -r -d '' native_library; do
|
||||
if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
|
||||
continue
|
||||
fi
|
||||
magic=''
|
||||
if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
|
||||
"$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
|
||||
fi
|
||||
done < /tmp/archivebox-native-libraries
|
||||
rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
|
||||
rm -f /venv/bin/uv /venv/bin/uvx
|
||||
abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
|
||||
abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
|
||||
rm -rf "$builder_abxpkg_lib_dir"
|
||||
rm -rf /venv/lib/python3.*/site-packages/pip* \
|
||||
/venv/lib/python3.*/site-packages/wheel* \
|
||||
/venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
|
||||
rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
|
||||
EOF
|
||||
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
# Compile only ArchiveBox's installed package. Touching all of /venv here would
|
||||
# copy every inherited abx-dl file into a new layer merely to change its mtime.
|
||||
RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
|
||||
&& COMMIT_HASH="$( \
|
||||
if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
|
||||
echo "$ARCHIVEBOX_COMMIT_HASH"; \
|
||||
elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
|
||||
HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
|
||||
if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
|
||||
echo "$HEAD_REF"; \
|
||||
elif [[ "$HEAD_REF" == ref:\ * ]]; then \
|
||||
REF_PATH="${HEAD_REF#ref: }"; \
|
||||
cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
|
||||
fi; \
|
||||
fi)" \
|
||||
&& if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
|
||||
&& /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
|
||||
&& rm -f /venv/bin/uv /venv/bin/uvx \
|
||||
&& cd / \
|
||||
&& ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
|
||||
&& /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
|
||||
&& test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
|
||||
&& /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
|
||||
|
||||
# The builder installs and purges stripping tools in one layer, so its final
|
||||
# filesystem is already runtime-clean. Preserve that ancestry: starting again
|
||||
# from archivebox-runtime-base and COPYing /venv would bake a second full venv
|
||||
# over the inherited one instead of recording only ArchiveBox's package delta.
|
||||
FROM archivebox-builder
|
||||
# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
|
||||
FROM python:3.11-slim-bookworm
|
||||
|
||||
LABEL name="archivebox" \
|
||||
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
|
||||
@@ -252,86 +27,300 @@ LABEL name="archivebox" \
|
||||
org.opencontainers.image.vendor="ArchiveBox" \
|
||||
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
|
||||
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
com.docker.image.source.entrypoint="Dockerfile"
|
||||
com.docker.image.source.entrypoint="Dockerfile" \
|
||||
# TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
|
||||
# https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
|
||||
com.docker.desktop.extension.api.version=">= 1.4.7" \
|
||||
com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
|
||||
com.docker.extension.publisher-url="https://archivebox.io" \
|
||||
com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
|
||||
com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
|
||||
com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
|
||||
com.docker.extension.categories='database,utility-tools'
|
||||
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
|
||||
# The builder invokes abxpkg as root for temporary ELF stripping, which can
|
||||
# rewrite derived state ownership. Restore UID 911 before resolving binaries
|
||||
# as the runtime user; the old copy-based final stage hid this dependency.
|
||||
RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
|
||||
echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
|
||||
&& printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
|
||||
&& ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
|
||||
&& ln -sf /venv/bin/daphne /usr/local/bin/daphne \
|
||||
&& ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
|
||||
&& ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
|
||||
&& getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
|
||||
&& id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
|
||||
&& usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
|
||||
&& [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
|
||||
&& [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
|
||||
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
|
||||
&& chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
|
||||
&& find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
|
||||
&& UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
|
||||
&& openssl rand -hex 16 > /etc/machine-id \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
|
||||
&& echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
######### Environment Variables #################################
|
||||
|
||||
# Install the complete ArchiveBox catalog, including disabled server plugins.
|
||||
# Explicit plugin names make installation independent of enabled defaults.
|
||||
RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
|
||||
chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
|
||||
&& chmod 1777 /var/tmp/abxpkg-cache \
|
||||
&& export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
|
||||
&& env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
|
||||
setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
|
||||
# pnpm includes musl variants that Debian's glibc runtime cannot use.
|
||||
&& find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
|
||||
&& find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} +
|
||||
RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
|
||||
bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
|
||||
# Global system-level config
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
LANG=C.UTF-8 \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
|
||||
PYTHONIOENCODING=UTF-8 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
npm_config_loglevel=error
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
RUN echo "[+] Initializing image collection..." \
|
||||
&& find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
|
||||
&& HOME="$TMP_DIR" archivebox init \
|
||||
&& (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
|
||||
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
|
||||
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
|
||||
"$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
|
||||
"$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
|
||||
"$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
|
||||
"$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
2>/dev/null || true) \
|
||||
&& find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
|
||||
# Version config
|
||||
ENV PYTHON_VERSION=3.11 \
|
||||
NODE_VERSION=20
|
||||
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
|
||||
# User config
|
||||
ENV ARCHIVEBOX_USER="archivebox" \
|
||||
DEFAULT_PUID=911 \
|
||||
DEFAULT_PGID=911
|
||||
|
||||
RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
|
||||
&& stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
# Global paths
|
||||
ENV CODE_DIR=/app \
|
||||
DATA_DIR=/data \
|
||||
GLOBAL_VENV=/venv \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/browsers
|
||||
|
||||
RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
# Application-level paths
|
||||
ENV APP_VENV=/app/.venv \
|
||||
NODE_MODULES=/app/node_modules
|
||||
|
||||
# Build shell config
|
||||
ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
|
||||
######### System Environment ####################################
|
||||
|
||||
# Detect ArchiveBox version number by reading package.json
|
||||
COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
|
||||
RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
|
||||
|
||||
# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
|
||||
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
|
||||
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
|
||||
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
|
||||
&& rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
|
||||
# Print debug info about build and save it to disk, for human eyes only, not used by anything else
|
||||
RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& echo \
|
||||
&& echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
|
||||
&& echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
|
||||
&& echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
|
||||
&& echo \
|
||||
&& uname -a \
|
||||
&& cat /etc/os-release | head -n7 \
|
||||
&& which bash && bash --version | head -n1 \
|
||||
&& which dpkg && dpkg --version | head -n1 \
|
||||
&& echo -e '\n\n' && env && echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Create non-privileged user for archivebox and chrome
|
||||
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& groupadd --system $ARCHIVEBOX_USER \
|
||||
&& useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
|
||||
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
|
||||
| tee -a /VERSION.txt
|
||||
# DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
|
||||
# https://docs.linuxserver.io/general/understanding-puid-and-pgid
|
||||
|
||||
# Install system apt dependencies (adding backports to access more recent apt updates)
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
|
||||
&& echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports \
|
||||
# 1. packaging dependencies
|
||||
apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
|
||||
# 2. docker and init system dependencies
|
||||
zlib1g-dev dumb-init gosu cron unzip grep \
|
||||
# 3. frivolous CLI helpers to make debugging failed archiving easier
|
||||
# nano iputils-ping dnsutils htop procps jq yq
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
######### Language Environments ####################################
|
||||
|
||||
# Install Python environment
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
|
||||
# && apt-get update -qq \
|
||||
# && apt-get install -qq -y -t bookworm-backports --no-upgrade \
|
||||
# python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
|
||||
# && rm -rf /var/lib/apt/lists/* \
|
||||
# tell PDM to allow using global system python site packages
|
||||
# && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
|
||||
# create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
|
||||
# && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
|
||||
# && python3 -m venv --system-site-packages $GLOBAL_VENV \
|
||||
# && python3 -m venv $GLOBAL_VENV \
|
||||
# install global dependencies / python build dependencies in GLOBAL_VENV
|
||||
# && pip install --upgrade pip setuptools wheel \
|
||||
# Save version info
|
||||
&& ( \
|
||||
which python3 && python3 --version | grep " $PYTHON_VERSION" \
|
||||
&& which pip && pip --version \
|
||||
# && which pdm && pdm --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
# Install Node environment
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
|
||||
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
|
||||
&& curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
|
||||
&& apt-get install -y -t bookworm-backports --no-upgrade \
|
||||
nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
# Update NPM to latest version
|
||||
&& npm i -g npm --cache /root/.npm \
|
||||
# Save version info
|
||||
&& ( \
|
||||
which node && node --version \
|
||||
&& which npm && npm --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
######### Extractor Dependencies ##################################
|
||||
|
||||
# Install apt dependencies
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing APT extractor dependencies globally using apt..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports \
|
||||
curl wget git yt-dlp ffmpeg ripgrep \
|
||||
# Packages we have also needed in the past:
|
||||
# youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
# Save version info
|
||||
&& ( \
|
||||
which curl && curl --version | head -n1 \
|
||||
&& which wget && wget --version 2>&1 | head -n1 \
|
||||
&& which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
|
||||
&& which git && git --version 2>&1 | head -n1 \
|
||||
&& which rg && rg --version 2>&1 | head -n1 \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Install chromium browser using playwright
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports \
|
||||
fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
|
||||
at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
|
||||
libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
|
||||
libxaw7 libxcomposite1 libxdamage1 libxfont2 \
|
||||
libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
|
||||
# xfonts-scalable xfonts-utils xserver-common xvfb \
|
||||
# chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
|
||||
# libxss1 dbus dbus-x11 upower \
|
||||
# && service dbus start \
|
||||
# install Chromium using playwright
|
||||
&& pip install playwright \
|
||||
&& cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& playwright install chromium \
|
||||
&& export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
|
||||
&& mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
|
||||
&& chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
|
||||
&& mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
# Save version info
|
||||
&& ( \
|
||||
which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Install Node dependencies
|
||||
WORKDIR "$CODE_DIR"
|
||||
COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
|
||||
RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
|
||||
&& npm ci --prefer-offline --no-audit --cache /root/.npm \
|
||||
&& ( \
|
||||
which node && node --version \
|
||||
&& which npm && npm version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
######### Build Dependencies ####################################
|
||||
|
||||
# Install ArchiveBox Python dependencies
|
||||
WORKDIR "$CODE_DIR"
|
||||
COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y -t bookworm-backports \
|
||||
build-essential \
|
||||
libssl-dev libldap2-dev libsasl2-dev \
|
||||
python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
|
||||
# && ln -s "$GLOBAL_VENV" "$APP_VENV" \
|
||||
# && pdm use --venv in-project \
|
||||
# && pdm run python -m ensurepip \
|
||||
# && pdm sync --fail-fast --no-editable --group :all --no-self \
|
||||
# && pdm export -o requirements.txt --without-hashes \
|
||||
# && source $GLOBAL_VENV/bin/activate \
|
||||
&& pip install -r requirements.txt \
|
||||
&& apt-get purge -y \
|
||||
build-essential \
|
||||
&& apt-get autoremove -y \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install ArchiveBox Python package from source
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
|
||||
# && apt-get update -qq \
|
||||
# install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
|
||||
# && apt-get install -qq -y -t bookworm-backports \
|
||||
# build-essential \
|
||||
# INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
|
||||
&& pip install -e "$CODE_DIR"[sonic,ldap] \
|
||||
# save docker image size and always remove compilers / build tools after building is complete
|
||||
# && apt-get purge -y build-essential \
|
||||
# && apt-get autoremove -y \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
####################################################
|
||||
|
||||
# Setup ArchiveBox runtime config
|
||||
WORKDIR "$DATA_DIR"
|
||||
ENV IN_DOCKER=True \
|
||||
DISPLAY=novnc:0.0 \
|
||||
CUSTOM_TEMPLATES_DIR=/data/templates \
|
||||
GOOGLE_API_KEY=no \
|
||||
GOOGLE_DEFAULT_CLIENT_ID=no \
|
||||
GOOGLE_DEFAULT_CLIENT_SECRET=no \
|
||||
ALLOWED_HOSTS=*
|
||||
## No need to set explicitly, these values will be autodetected by archivebox in docker:
|
||||
# WGET_BINARY="wget" \
|
||||
# YOUTUBEDL_BINARY="yt-dlp" \
|
||||
# CHROME_BINARY="/usr/bin/chromium-browser" \
|
||||
# USE_SINGLEFILE=True \
|
||||
# SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
|
||||
# USE_READABILITY=True \
|
||||
# READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
|
||||
# USE_MERCURY=True \
|
||||
# MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
|
||||
|
||||
# Print version for nice docker finish summary
|
||||
RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
|
||||
|
||||
####################################################
|
||||
|
||||
# Open up the interfaces to the outside world
|
||||
WORKDIR "$DATA_DIR"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 5797
|
||||
EXPOSE 8000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
|
||||
CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
|
||||
CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
|
||||
|
||||
ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
|
||||
CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]
|
||||
CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Nick Sweeting
|
||||
Copyright (c) 2020 Nick Sweeting
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# Security Policy
|
||||
|
||||
---
|
||||
|
||||
## Security Information
|
||||
|
||||
Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
|
||||
|
||||
https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
|
||||
|
||||
Also see this section of the README about important caveats when running ArchiveBox:
|
||||
|
||||
https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
|
||||
|
||||
You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
|
||||
|
||||
- https://github.com/ArchiveBox/ArchiveBox#archive-layout
|
||||
- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
|
||||
- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
|
||||
|
||||
---
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
|
||||
|
||||
1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
|
||||
|
||||
2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
|
||||
|
||||
3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
|
||||
|
||||
You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
|
||||
+1
-1
@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
|
||||
select = F,E9,W
|
||||
max-line-length = 130
|
||||
max-complexity = 10
|
||||
exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
|
||||
exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../LICENSE
|
||||
+2
-148
@@ -1,150 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Welcome to the ArchiveBox source code! Thanks for checking it out!
|
||||
#
|
||||
# "We are swimming upstream against a great torrent of disorganization.
|
||||
# In this, our main obligation is to establish arbitrary enclaves of order and system.
|
||||
# It is the greatest possible victory to be, to continue to be, and to have been.
|
||||
# No defeat can deprive us of the success of having existed for some moment of time
|
||||
# in a universe that seems indifferent to us."
|
||||
# --Norber Weiner
|
||||
|
||||
__package__ = "archivebox"
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Protocol, cast
|
||||
__package__ = 'archivebox'
|
||||
|
||||
|
||||
class _ReconfigurableStream(Protocol):
|
||||
def reconfigure(self, *, line_buffering: bool) -> object: ...
|
||||
|
||||
|
||||
# Force unbuffered output for real-time logs
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
|
||||
cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
|
||||
os.environ["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
ASCII_LOGO = """
|
||||
█████╗ ██████╗ ██████╗██╗ ██╗██╗██╗ ██╗███████╗ ██████╗ ██████╗ ██╗ ██╗
|
||||
██╔══██╗██╔══██╗██╔════╝██║ ██║██║██║ ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
|
||||
███████║██████╔╝██║ ███████║██║██║ ██║█████╗ ██████╔╝██║ ██║ ╚███╔╝
|
||||
██╔══██║██╔══██╗██║ ██╔══██║██║╚██╗ ██╔╝██╔══╝ ██╔══██╗██║ ██║ ██╔██╗
|
||||
██║ ██║██║ ██║╚██████╗██║ ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
|
||||
╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚═╝ ╚═══╝ ╚══════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚═╝
|
||||
"""
|
||||
|
||||
PACKAGE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
|
||||
os.environ["TZ"] = "UTC"
|
||||
|
||||
# detect ArchiveBox user's UID/GID based on data dir ownership
|
||||
from .config.permissions import drop_privileges # noqa
|
||||
|
||||
drop_privileges()
|
||||
|
||||
from .misc.checks import check_not_root, check_io_encoding # noqa
|
||||
|
||||
check_not_root()
|
||||
check_io_encoding()
|
||||
|
||||
from .config.version import VERSION # noqa
|
||||
|
||||
|
||||
__version__ = VERSION
|
||||
__author__ = "ArchiveBox"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "CONSTANTS":
|
||||
from .config.constants import CONSTANTS
|
||||
|
||||
os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
|
||||
return CONSTANTS
|
||||
if name == "DATA_DIR":
|
||||
from .config.paths import DATA_DIR
|
||||
|
||||
return DATA_DIR
|
||||
if name == "VERSION":
|
||||
return VERSION
|
||||
if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
|
||||
from abx_plugins import get_plugins_dir
|
||||
from .config.constants import CONSTANTS
|
||||
|
||||
builtin_plugins_dir = Path(get_plugins_dir()).resolve()
|
||||
user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
|
||||
plugins = {
|
||||
"builtin": builtin_plugins_dir,
|
||||
"user": user_plugins_dir,
|
||||
}
|
||||
values = {
|
||||
"BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
|
||||
"USER_PLUGINS_DIR": user_plugins_dir,
|
||||
"ALL_PLUGINS": plugins,
|
||||
"LOADED_PLUGINS": plugins,
|
||||
}
|
||||
return values[name]
|
||||
raise AttributeError(name)
|
||||
|
||||
|
||||
__all__ = (
|
||||
"ASCII_LOGO",
|
||||
"ASCII_ICON",
|
||||
"PACKAGE_DIR",
|
||||
"DATA_DIR",
|
||||
"CONSTANTS",
|
||||
"VERSION",
|
||||
"BUILTIN_PLUGINS_DIR",
|
||||
"USER_PLUGINS_DIR",
|
||||
"ALL_PLUGINS",
|
||||
"LOADED_PLUGINS",
|
||||
)
|
||||
|
||||
ASCII_ICON = """
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██████████████████████████████████████████████████████████████████████████████████████████████████
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ████████████████████████████████████ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ██ █████████████████████████ █ ██
|
||||
██ ████████████████████████████████████ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██████████████████████████████████████████ ██
|
||||
██ ██████████████████████████████████████████ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
██ ██
|
||||
████████████████████████████████████████████████████████████████████████████████
|
||||
"""
|
||||
from .monkey_patches import *
|
||||
+3
-12
@@ -1,20 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""This is the entrypoint for python -m archivebox ..."""
|
||||
|
||||
__package__ = "archivebox"
|
||||
__package__ = 'archivebox'
|
||||
|
||||
import archivebox # noqa # make sure monkey patches are applied before anything else
|
||||
import sys
|
||||
|
||||
from .cli import main
|
||||
|
||||
ASCII_LOGO_MINI = r"""
|
||||
_ _ _ ____
|
||||
/ \ _ __ ___| |__ (_)_ _____| __ ) _____ __
|
||||
/ _ \ | '__/ __| '_ \| \ \ / / _ \ _ \ / _ \ \/ /
|
||||
/ ___ \| | | (__| | | | |\ V / __/ |_) | (_) > <
|
||||
/_/ \_\_| \___|_| |_|_| \_/ \___|____/ \___/_/\_\
|
||||
"""
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(args=sys.argv[1:])
|
||||
if __name__ == '__main__':
|
||||
main(args=sys.argv[1:], stdin=sys.stdin)
|
||||
@@ -0,0 +1 @@
|
||||
__package__ = 'abid_utils'
|
||||
@@ -0,0 +1,202 @@
|
||||
from typing import NamedTuple, Any, Union, Optional
|
||||
|
||||
import ulid
|
||||
import uuid6
|
||||
import hashlib
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from uuid import UUID
|
||||
from typeid import TypeID # type: ignore[import-untyped]
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
|
||||
ABID_PREFIX_LEN = 4
|
||||
ABID_SUFFIX_LEN = 26
|
||||
ABID_LEN = 30
|
||||
ABID_TS_LEN = 10
|
||||
ABID_URI_LEN = 8
|
||||
ABID_SUBTYPE_LEN = 2
|
||||
ABID_RAND_LEN = 6
|
||||
|
||||
DEFAULT_ABID_PREFIX = 'obj_'
|
||||
|
||||
# allows people to keep their uris secret on a per-instance basis by changing the salt.
|
||||
# the default means everyone can share the same namespace for URI hashes,
|
||||
# meaning anyone who has a URI and wants to check if you have it can guess the ABID
|
||||
DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
|
||||
|
||||
|
||||
class ABID(NamedTuple):
|
||||
"""
|
||||
e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
|
||||
"""
|
||||
prefix: str # e.g. obj_
|
||||
ts: str # e.g. 01HX9FPYTR
|
||||
uri: str # e.g. E4A5CCD9
|
||||
subtype: str # e.g. 01
|
||||
rand: str # e.g. ZYEBQE
|
||||
|
||||
# salt: str = DEFAULT_ABID_URI_SALT
|
||||
|
||||
def __getattr__(self, attr: str) -> Any:
|
||||
return getattr(self.ulid, attr)
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
try:
|
||||
return self.ulid == other.ulid
|
||||
except AttributeError:
|
||||
return NotImplemented
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.prefix + self.suffix
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.prefix + self.suffix)
|
||||
|
||||
@classmethod
|
||||
def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
|
||||
assert buffer, f'Attempted to create ABID from null value {buffer}'
|
||||
|
||||
buffer = str(buffer)
|
||||
if '_' in buffer:
|
||||
prefix, suffix = buffer.split('_')
|
||||
else:
|
||||
prefix, suffix = prefix.strip('_'), buffer
|
||||
|
||||
assert len(prefix) == ABID_PREFIX_LEN - 1 # length without trailing _
|
||||
assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
|
||||
|
||||
return cls(
|
||||
prefix=abid_part_from_prefix(prefix),
|
||||
ts=suffix[0:10].upper(),
|
||||
uri=suffix[10:18].upper(),
|
||||
subtype=suffix[18:20].upper(),
|
||||
rand=suffix[20:26].upper(),
|
||||
)
|
||||
|
||||
@property
|
||||
def uri_salt(self) -> str:
|
||||
return DEFAULT_ABID_URI_SALT
|
||||
|
||||
@property
|
||||
def suffix(self):
|
||||
return ''.join((self.ts, self.uri, self.subtype, self.rand))
|
||||
|
||||
@property
|
||||
def ulid(self) -> ulid.ULID:
|
||||
return ulid.parse(self.suffix)
|
||||
|
||||
@property
|
||||
def uuid(self) -> UUID:
|
||||
return self.ulid.uuid
|
||||
|
||||
@property
|
||||
def uuid6(self) -> uuid6.UUID:
|
||||
return uuid6.UUID(hex=self.uuid.hex)
|
||||
|
||||
@property
|
||||
def typeid(self) -> TypeID:
|
||||
return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
|
||||
|
||||
@property
|
||||
def datetime(self) -> datetime:
|
||||
return self.ulid.timestamp().datetime
|
||||
|
||||
|
||||
|
||||
####################################################
|
||||
|
||||
|
||||
def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
|
||||
"""
|
||||
'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
|
||||
"""
|
||||
if isinstance(uri, bytes):
|
||||
uri_str: str = uri.decode()
|
||||
else:
|
||||
uri_str = str(uri)
|
||||
|
||||
# only hash the domain part of URLs
|
||||
if '://' in uri_str:
|
||||
try:
|
||||
domain = urlparse(uri_str).netloc
|
||||
if domain:
|
||||
uri_str = domain
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
|
||||
|
||||
return hashlib.sha256(uri_bytes).hexdigest().upper()
|
||||
|
||||
def abid_part_from_prefix(prefix: Optional[str]) -> str:
|
||||
"""
|
||||
'snp_'
|
||||
"""
|
||||
if prefix is None:
|
||||
return 'obj_'
|
||||
|
||||
prefix = prefix.strip('_').lower()
|
||||
assert len(prefix) == 3
|
||||
return prefix + '_'
|
||||
|
||||
def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
|
||||
"""
|
||||
'E4A5CCD9' # takes first 8 characters of sha256(url)
|
||||
"""
|
||||
uri = str(uri)
|
||||
return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
|
||||
|
||||
def abid_part_from_ts(ts: Optional[datetime]) -> str:
|
||||
"""
|
||||
'01HX9FPYTR' # produces 10 character Timestamp section of ulid based on added date
|
||||
"""
|
||||
return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
|
||||
|
||||
def abid_part_from_subtype(subtype: str) -> str:
|
||||
"""
|
||||
Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
|
||||
Also allows us to change the ulid spec later by putting special sigil values here.
|
||||
"""
|
||||
subtype = str(subtype)
|
||||
if len(subtype) == ABID_SUBTYPE_LEN:
|
||||
return subtype
|
||||
|
||||
return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
|
||||
|
||||
def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
|
||||
"""
|
||||
'ZYEBQE' # takes last 6 characters of randomness from existing legacy uuid db field
|
||||
"""
|
||||
if rand is None:
|
||||
# if it's None we generate a new random 6 character hex string
|
||||
return str(ulid.new())[-ABID_RAND_LEN:]
|
||||
elif isinstance(rand, UUID):
|
||||
# if it's a uuid we take the last 6 characters of the ULID represation of it
|
||||
return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
|
||||
elif isinstance(rand, int):
|
||||
# if it's a BigAutoInteger field we convert it from an int to a 0-padded string
|
||||
rand_str = str(rand)[-ABID_RAND_LEN:]
|
||||
padding_needed = ABID_RAND_LEN - len(rand_str)
|
||||
rand_str = ('0'*padding_needed) + rand_str
|
||||
return rand_str
|
||||
|
||||
# otherwise treat it as a string, take the last 6 characters of it verbatim
|
||||
return str(rand)[-ABID_RAND_LEN:].upper()
|
||||
|
||||
|
||||
def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
|
||||
"""
|
||||
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
||||
"""
|
||||
|
||||
abid = ABID(
|
||||
prefix=abid_part_from_prefix(prefix),
|
||||
ts=abid_part_from_ts(ts),
|
||||
uri=abid_part_from_uri(uri, salt=salt),
|
||||
subtype=abid_part_from_subtype(subtype),
|
||||
rand=abid_part_from_rand(rand),
|
||||
)
|
||||
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
|
||||
return abid
|
||||
@@ -0,0 +1,18 @@
|
||||
from django.contrib import admin
|
||||
|
||||
|
||||
class ABIDModelAdmin(admin.ModelAdmin):
|
||||
list_display = ('created', 'created_by', 'abid', '__str__')
|
||||
sort_fields = ('created', 'created_by', 'abid', '__str__')
|
||||
readonly_fields = ('abid', 'created', '__str__')
|
||||
|
||||
def get_form(self, request, obj=None, **kwargs):
|
||||
form = super().get_form(request, obj, **kwargs)
|
||||
if 'created_by' in form.base_fields:
|
||||
form.base_fields['created_by'].initial = request.user
|
||||
return form
|
||||
|
||||
# def save_model(self, request, obj, form, change):
|
||||
# if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
|
||||
# obj.created_by = request.user
|
||||
# obj.save()
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class AbidUtilsConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
|
||||
name = 'abid_utils'
|
||||
File renamed without changes.
@@ -0,0 +1,344 @@
|
||||
"""
|
||||
This file provides the Django ABIDField and ABIDModel base model to inherit from.
|
||||
|
||||
It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Union, List, Set, NamedTuple, cast
|
||||
|
||||
from ulid import ULID
|
||||
from uuid import uuid4, UUID
|
||||
from typeid import TypeID # type: ignore[import-untyped]
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from charidfield import CharIDField # type: ignore[import-untyped]
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django.db.utils import OperationalError
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
from .abid import (
|
||||
ABID,
|
||||
ABID_LEN,
|
||||
ABID_RAND_LEN,
|
||||
ABID_SUFFIX_LEN,
|
||||
DEFAULT_ABID_PREFIX,
|
||||
DEFAULT_ABID_URI_SALT,
|
||||
abid_part_from_prefix,
|
||||
abid_from_values
|
||||
)
|
||||
|
||||
####################################################
|
||||
|
||||
|
||||
# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
|
||||
ABIDField = partial(
|
||||
CharIDField,
|
||||
max_length=ABID_LEN,
|
||||
help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
|
||||
default=None,
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
unique=True,
|
||||
)
|
||||
|
||||
def get_or_create_system_user_pk(username='system'):
|
||||
"""Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
# if only one user exists total, return that user
|
||||
if User.objects.filter(is_superuser=True).count() == 1:
|
||||
return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
|
||||
|
||||
# otherwise, create a dedicated "system" user
|
||||
user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
|
||||
return user.pk
|
||||
|
||||
|
||||
class ABIDModel(models.Model):
|
||||
"""
|
||||
Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
|
||||
"""
|
||||
abid_prefix: str = DEFAULT_ABID_PREFIX # e.g. 'tag_'
|
||||
abid_ts_src = 'None' # e.g. 'self.created'
|
||||
abid_uri_src = 'None' # e.g. 'self.uri'
|
||||
abid_subtype_src = 'None' # e.g. 'self.extractor'
|
||||
abid_rand_src = 'None' # e.g. 'self.uuid' or 'self.id'
|
||||
|
||||
# id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
|
||||
# uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def save(self, *args: Any, **kwargs: Any) -> None:
|
||||
# when first creating a row, self.ABID is the source of truth
|
||||
# overwrite default prefilled self.id & self.abid with generated self.ABID value
|
||||
if self._state.adding or not self.id:
|
||||
self.id = self.ABID.uuid
|
||||
if self._state.adding or not self.abid:
|
||||
self.abid = str(self.ABID)
|
||||
|
||||
super().save(*args, **kwargs)
|
||||
assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
|
||||
assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
|
||||
|
||||
@property
|
||||
def abid_values(self) -> Dict[str, Any]:
|
||||
return {
|
||||
'prefix': self.abid_prefix,
|
||||
'ts': eval(self.abid_ts_src),
|
||||
'uri': eval(self.abid_uri_src),
|
||||
'subtype': eval(self.abid_subtype_src),
|
||||
'rand': eval(self.abid_rand_src),
|
||||
}
|
||||
|
||||
def generate_abid(self) -> ABID:
|
||||
"""
|
||||
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
||||
"""
|
||||
prefix, ts, uri, subtype, rand = self.abid_values.values()
|
||||
|
||||
if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
|
||||
suggested_abid = self.__class__.__name__[:3].lower()
|
||||
raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
|
||||
|
||||
if not ts:
|
||||
# default to unix epoch with 00:00:00 UTC
|
||||
ts = datetime.fromtimestamp(0, timezone.utc) # equivalent to: ts = datetime.utcfromtimestamp(0)
|
||||
print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
|
||||
|
||||
if not uri:
|
||||
uri = str(self)
|
||||
print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
|
||||
|
||||
if not subtype:
|
||||
subtype = self.__class__.__name__
|
||||
print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
|
||||
|
||||
if not rand:
|
||||
rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
|
||||
print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
|
||||
|
||||
abid = abid_from_values(
|
||||
prefix=prefix,
|
||||
ts=ts,
|
||||
uri=uri,
|
||||
subtype=subtype,
|
||||
rand=rand,
|
||||
salt=DEFAULT_ABID_URI_SALT,
|
||||
)
|
||||
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
|
||||
return abid
|
||||
|
||||
@property
|
||||
def ABID(self) -> ABID:
|
||||
"""
|
||||
ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
|
||||
"""
|
||||
|
||||
# if object is not yet saved to DB, always generate fresh ABID from values
|
||||
if self._state.adding:
|
||||
return self.generate_abid()
|
||||
|
||||
# otherwise DB is single source of truth, load ABID from existing db pk
|
||||
abid: ABID | None = None
|
||||
try:
|
||||
abid = abid or ABID.parse(self.pk)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
abid = abid or ABID.parse(self.id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
abid = abid or ABID.parse(cast(str, self.abid))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
abid = abid or self.generate_abid()
|
||||
|
||||
return abid
|
||||
|
||||
@property
|
||||
def ULID(self) -> ULID:
|
||||
"""
|
||||
Get a ulid.ULID representation of the object's ABID.
|
||||
"""
|
||||
return self.ABID.ulid
|
||||
|
||||
@property
|
||||
def UUID(self) -> UUID:
|
||||
"""
|
||||
Get a uuid.UUID (v4) representation of the object's ABID.
|
||||
"""
|
||||
return self.ABID.uuid
|
||||
|
||||
@property
|
||||
def TypeID(self) -> TypeID:
|
||||
"""
|
||||
Get a typeid.TypeID (stripe-style) representation of the object's ABID.
|
||||
"""
|
||||
return self.ABID.typeid
|
||||
|
||||
|
||||
|
||||
####################################################
|
||||
|
||||
# Django helpers
|
||||
def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
|
||||
"""
|
||||
Return the mapping of all ABID prefixes to their models.
|
||||
e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
|
||||
"""
|
||||
import django.apps
|
||||
prefix_map = {}
|
||||
|
||||
for model in django.apps.apps.get_models():
|
||||
abid_prefix = getattr(model, 'abid_prefix', None)
|
||||
if abid_prefix:
|
||||
prefix_map[abid_prefix] = model
|
||||
return prefix_map
|
||||
|
||||
def find_prefix_for_abid(abid: ABID) -> str:
|
||||
"""
|
||||
Find the correct prefix for a given ABID that may have be missing a prefix (slow).
|
||||
e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
|
||||
"""
|
||||
# if existing abid prefix is correct, lookup is easy
|
||||
model = find_model_from_abid(abid)
|
||||
if model:
|
||||
assert issubclass(model, ABIDModel)
|
||||
return model.abid_prefix
|
||||
|
||||
# prefix might be obj_ or missing, fuzzy-search to find any object that matches
|
||||
return find_obj_from_abid_rand(abid)[0].abid_prefix
|
||||
|
||||
def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
|
||||
"""
|
||||
Return the Django Model that corresponds to a given ABID prefix.
|
||||
e.g. 'tag_' -> core.models.Tag
|
||||
"""
|
||||
prefix = abid_part_from_prefix(prefix)
|
||||
|
||||
import django.apps
|
||||
|
||||
for model in django.apps.apps.get_models():
|
||||
if not issubclass(model, ABIDModel): continue # skip non-ABID-enabled models
|
||||
if not hasattr(model, 'objects'): continue # skip abstract models
|
||||
|
||||
if (model.abid_prefix == prefix):
|
||||
return model
|
||||
|
||||
return None
|
||||
|
||||
def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
|
||||
"""
|
||||
Shortcut for find_model_from_abid_prefix(abid.prefix)
|
||||
"""
|
||||
return find_model_from_abid_prefix(abid.prefix)
|
||||
|
||||
def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
|
||||
"""
|
||||
Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
|
||||
e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
|
||||
"""
|
||||
|
||||
# convert str to ABID if necessary
|
||||
if isinstance(rand, ABID):
|
||||
abid: ABID = rand
|
||||
else:
|
||||
rand = str(rand)
|
||||
if len(rand) < ABID_SUFFIX_LEN:
|
||||
padding_needed = ABID_SUFFIX_LEN - len(rand)
|
||||
rand = ('0'*padding_needed) + rand
|
||||
abid = ABID.parse(rand)
|
||||
|
||||
import django.apps
|
||||
|
||||
partial_matches: List[ABIDModel] = []
|
||||
|
||||
models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
|
||||
model,
|
||||
find_model_from_abid(abid),
|
||||
*django.apps.apps.get_models(),
|
||||
))))
|
||||
# print(abid, abid.rand, abid.uuid, models_to_try)
|
||||
|
||||
for model in models_to_try:
|
||||
if not issubclass(model, ABIDModel): continue # skip Models that arent ABID-enabled
|
||||
if not hasattr(model, 'objects'): continue # skip abstract Models
|
||||
assert hasattr(model, 'objects') # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
|
||||
|
||||
# continue on to try fuzzy searching by randomness portion derived from uuid field
|
||||
try:
|
||||
qs = []
|
||||
if hasattr(model, 'abid'):
|
||||
qs = model.objects.filter(abid__endswith=abid.rand)
|
||||
elif hasattr(model, 'uuid'):
|
||||
qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
|
||||
elif hasattr(model, 'id'):
|
||||
# NOTE: this only works on SQLite where every column is a string
|
||||
# other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
|
||||
|
||||
# try to search for uuid=...-2354352
|
||||
# try to search for id=...2354352
|
||||
# try to search for id=2354352
|
||||
qs = model.objects.filter(
|
||||
models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
|
||||
| models.Q(id__endswith=abid.rand)
|
||||
| models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
|
||||
)
|
||||
|
||||
for obj in qs:
|
||||
if obj.generate_abid() == abid:
|
||||
# found exact match, no need to keep iterating
|
||||
return [obj]
|
||||
partial_matches.append(obj)
|
||||
except OperationalError as err:
|
||||
print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
|
||||
|
||||
return partial_matches
|
||||
|
||||
def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
|
||||
"""
|
||||
Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
|
||||
e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
|
||||
"""
|
||||
|
||||
model = model or find_model_from_abid(abid)
|
||||
assert model, f'Could not find model that could match this ABID type: {abid}'
|
||||
|
||||
try:
|
||||
if hasattr(model, 'abid'):
|
||||
return model.objects.get(abid__endswith=abid.suffix)
|
||||
if hasattr(model, 'uuid'):
|
||||
return model.objects.get(uuid=abid.uuid)
|
||||
return model.objects.get(id=abid.uuid)
|
||||
except model.DoesNotExist:
|
||||
# if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
|
||||
if hasattr(model, 'abid') or (not fuzzy):
|
||||
raise
|
||||
|
||||
# continue on to try fuzzy searching by randomness portion derived from uuid field
|
||||
match_by_rand = find_obj_from_abid_rand(abid, model=model)
|
||||
if match_by_rand:
|
||||
if match_by_rand[0].abid_prefix != abid.prefix:
|
||||
print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
|
||||
return match_by_rand
|
||||
|
||||
raise model.DoesNotExist
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from django.test import TestCase
|
||||
|
||||
# Create your tests here.
|
||||
@@ -1 +1 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
@@ -1,120 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from django import forms
|
||||
from django.contrib import admin
|
||||
from django.http import HttpRequest
|
||||
from django.utils.text import capfirst
|
||||
from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
|
||||
from signal_webhooks.settings import webhook_settings
|
||||
from signal_webhooks.utils import get_webhook_model, model_from_reference
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
|
||||
from archivebox.api.models import APIToken
|
||||
|
||||
|
||||
def _webhook_fields(*names: str) -> tuple[str, ...]:
|
||||
model_fields = {field.name for field in get_webhook_model()._meta.fields}
|
||||
return tuple(name for name in names if name in model_fields)
|
||||
|
||||
|
||||
class APITokenAdmin(BaseModelAdmin):
|
||||
list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
|
||||
sort_fields = ("id", "created_at", "created_by", "expires")
|
||||
readonly_fields = ("created_at", "modified_at")
|
||||
search_fields = ("id", "created_by__username", "token")
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Token",
|
||||
{
|
||||
"fields": ("token", "expires"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Owner",
|
||||
{
|
||||
"fields": ("created_by",),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": ("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
list_filter = ("created_by",)
|
||||
ordering = ["-created_at"]
|
||||
list_per_page = 100
|
||||
|
||||
|
||||
class OutboundWebhookAdminForm(WebhookModelForm):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["ref"] = forms.ChoiceField(
|
||||
label=self.fields["ref"].label,
|
||||
help_text=self.fields["ref"].help_text,
|
||||
choices=[
|
||||
(ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
|
||||
for ref in sorted(webhook_settings.HOOKS)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
|
||||
form = OutboundWebhookAdminForm
|
||||
list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
|
||||
sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
|
||||
readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
|
||||
|
||||
fieldsets = (
|
||||
(
|
||||
"Webhook",
|
||||
{
|
||||
"fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
|
||||
"classes": ("card", "wide"),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Authentication",
|
||||
{
|
||||
"fields": _webhook_fields("auth_token"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Status",
|
||||
{
|
||||
"fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Owner",
|
||||
{
|
||||
"fields": _webhook_fields("created_by"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
(
|
||||
"Timestamps",
|
||||
{
|
||||
"fields": _webhook_fields("created_at", "modified_at"),
|
||||
"classes": ("card",),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
|
||||
"""Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
|
||||
return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
|
||||
|
||||
|
||||
def register_admin(admin_site: admin.AdminSite) -> None:
|
||||
admin_site.register(APIToken, APITokenAdmin)
|
||||
admin_site.register(get_webhook_model(), CustomWebhookAdmin)
|
||||
@@ -1,14 +1,11 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
|
||||
class APIConfig(AppConfig):
|
||||
name = "archivebox.api"
|
||||
label = "api"
|
||||
name = 'api'
|
||||
|
||||
|
||||
def register_admin(admin_site):
|
||||
from archivebox.api.admin import register_admin
|
||||
|
||||
register_admin(admin_site)
|
||||
def ready(self):
|
||||
pass
|
||||
+65
-92
@@ -1,130 +1,102 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
from datetime import timedelta
|
||||
from typing import Optional, cast
|
||||
|
||||
from django.utils import timezone
|
||||
from django.http import HttpRequest
|
||||
from django.contrib.auth import login
|
||||
from django.contrib.auth import authenticate
|
||||
from django.contrib.auth.models import User
|
||||
from django.contrib.auth.models import AbstractBaseUser
|
||||
|
||||
from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
|
||||
from ninja.errors import HttpError
|
||||
from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
|
||||
|
||||
|
||||
def get_or_create_api_token(user: User | None):
|
||||
from archivebox.api.models import APIToken
|
||||
|
||||
if user and user.is_superuser:
|
||||
api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
|
||||
if api_tokens.exists():
|
||||
# unexpired token exists, use it
|
||||
api_token = api_tokens.last()
|
||||
else:
|
||||
# does not exist, create a new one
|
||||
api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
|
||||
|
||||
if api_token is None:
|
||||
return None
|
||||
assert api_token.is_valid(), f"API token is not valid {api_token}"
|
||||
|
||||
return api_token
|
||||
return None
|
||||
|
||||
|
||||
def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
|
||||
def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
|
||||
"""Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
|
||||
from archivebox.api.models import APIToken # lazy import model to avoid loading it at urls.py import time
|
||||
from api.models import APIToken # lazy import model to avoid loading it at urls.py import time
|
||||
|
||||
user = None
|
||||
|
||||
user: User | None = None
|
||||
|
||||
submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
|
||||
if not submitted_empty_form:
|
||||
submitted_empty_form = token in ('string', '', None)
|
||||
if submitted_empty_form:
|
||||
assert request is not None, 'No request provided for API key authentication'
|
||||
user = request.user # see if user is authed via django session and use that as the default
|
||||
else:
|
||||
try:
|
||||
api_token = APIToken.objects.get(token=token)
|
||||
if api_token.is_valid() and isinstance(api_token.created_by, User):
|
||||
user = api_token.created_by
|
||||
if request is not None:
|
||||
setattr(request, "_api_token", api_token)
|
||||
token = APIToken.objects.get(token=token)
|
||||
if token.is_valid():
|
||||
user = token.created_by
|
||||
except APIToken.DoesNotExist:
|
||||
pass
|
||||
|
||||
return user
|
||||
if not user:
|
||||
print('[❌] Failed to authenticate API user using API Key:', request)
|
||||
|
||||
|
||||
def token_from_request(request: HttpRequest) -> str:
|
||||
token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
if not token and auth_header.lower().startswith("bearer "):
|
||||
token = auth_header.split(None, 1)[1].strip()
|
||||
return token
|
||||
|
||||
|
||||
def authenticated_user_from_request(request: HttpRequest) -> User | None:
|
||||
user = request.user
|
||||
if user.is_authenticated and user.is_active:
|
||||
return user
|
||||
|
||||
token = token_from_request(request)
|
||||
token_user = auth_using_token(token=token, request=request) if token else None
|
||||
if token_user and token_user.is_active:
|
||||
request.user = token_user
|
||||
return token_user
|
||||
return None
|
||||
|
||||
|
||||
def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
|
||||
def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
|
||||
"""Given a username and password, check if they are valid and return the corresponding user"""
|
||||
user: User | None = None
|
||||
|
||||
submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
|
||||
if not submitted_empty_form:
|
||||
authenticated_user = authenticate(
|
||||
user = None
|
||||
|
||||
submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
|
||||
if submitted_empty_form:
|
||||
assert request is not None, 'No request provided for API key authentication'
|
||||
user = request.user # see if user is authed via django session and use that as the default
|
||||
else:
|
||||
user = authenticate(
|
||||
username=username,
|
||||
password=password,
|
||||
)
|
||||
if isinstance(authenticated_user, User):
|
||||
user = authenticated_user
|
||||
return user
|
||||
|
||||
if not user:
|
||||
print('[❌] Failed to authenticate API user using API Key:', request)
|
||||
user = None
|
||||
|
||||
return cast(AbstractBaseUser | None, user)
|
||||
|
||||
|
||||
### Base Auth Types
|
||||
|
||||
class APITokenAuthCheck:
|
||||
"""The base class for authentication methods that use an api.models.APIToken"""
|
||||
def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
|
||||
user = auth_using_token(
|
||||
token=key,
|
||||
request=request,
|
||||
)
|
||||
if user is not None:
|
||||
login(request, user, backend='django.contrib.auth.backends.ModelBackend')
|
||||
return user
|
||||
|
||||
def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
|
||||
if user and user.pk:
|
||||
request.user = user
|
||||
setattr(request, "_api_auth_method", auth_method)
|
||||
if not user.is_superuser:
|
||||
raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
|
||||
return user
|
||||
class UserPassAuthCheck:
|
||||
"""The base class for authentication methods that use a username & password"""
|
||||
def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
|
||||
user = auth_using_password(
|
||||
username=username,
|
||||
password=password,
|
||||
request=request,
|
||||
)
|
||||
if user is not None:
|
||||
login(request, user, backend='django.contrib.auth.backends.ModelBackend')
|
||||
return user
|
||||
|
||||
|
||||
### Django-Ninja-Provided Auth Methods
|
||||
|
||||
|
||||
class HeaderTokenAuth(APIKeyHeader):
|
||||
class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
|
||||
"""Allow authenticating by passing X-API-Key=xyz as a request header"""
|
||||
|
||||
param_name = "X-ArchiveBox-API-Key"
|
||||
|
||||
def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
|
||||
return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
|
||||
|
||||
|
||||
class BearerTokenAuth(HttpBearer):
|
||||
class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
|
||||
"""Allow authenticating by passing Bearer=xyz as a request header"""
|
||||
pass
|
||||
|
||||
def authenticate(self, request: HttpRequest, token: str) -> User | None:
|
||||
return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
|
||||
|
||||
|
||||
class QueryParamTokenAuth(APIKeyQuery):
|
||||
class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
|
||||
"""Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
|
||||
|
||||
param_name = "api_key"
|
||||
|
||||
def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
|
||||
return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
|
||||
class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
|
||||
"""Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
|
||||
pass
|
||||
|
||||
|
||||
### Enabled Auth Methods
|
||||
@@ -132,6 +104,7 @@ class QueryParamTokenAuth(APIKeyQuery):
|
||||
API_AUTH_METHODS = [
|
||||
HeaderTokenAuth(),
|
||||
BearerTokenAuth(),
|
||||
QueryParamTokenAuth(),
|
||||
# django_auth_superuser, # django admin cookie auth, not secure to use with csrf=False
|
||||
QueryParamTokenAuth(),
|
||||
django_auth_superuser,
|
||||
UsernameAndPasswordAuth(),
|
||||
]
|
||||
@@ -1,32 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from django.http import HttpResponse
|
||||
|
||||
|
||||
class ApiCorsMiddleware:
|
||||
"""Attach permissive CORS headers for API routes (token-based auth)."""
|
||||
|
||||
def __init__(self, get_response):
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request):
|
||||
if request.path.startswith("/api/"):
|
||||
if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
|
||||
response = HttpResponse(status=204)
|
||||
return self._add_cors_headers(request, response)
|
||||
|
||||
response = self.get_response(request)
|
||||
return self._add_cors_headers(request, response)
|
||||
|
||||
return self.get_response(request)
|
||||
|
||||
def _add_cors_headers(self, request, response):
|
||||
origin = request.META.get("HTTP_ORIGIN")
|
||||
if not origin:
|
||||
return response
|
||||
|
||||
response["Access-Control-Allow-Origin"] = "*"
|
||||
response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
|
||||
response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
|
||||
response["Access-Control-Max-Age"] = "600"
|
||||
return response
|
||||
@@ -1,281 +1,29 @@
|
||||
# Generated by hand on 2025-12-29
|
||||
# Creates APIToken and OutboundWebhook tables using raw SQL
|
||||
# Generated by Django 4.2.11 on 2024-04-25 04:19
|
||||
|
||||
import api.models
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.conf import settings
|
||||
from archivebox.uuid_compat import uuid7
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
import archivebox.api.models
|
||||
import signal_webhooks.fields
|
||||
import signal_webhooks.utils
|
||||
|
||||
|
||||
# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
|
||||
# RunSQL (identical statement splitting → byte-for-byte identical behavior).
|
||||
INITIAL_SQL = """
|
||||
-- Create api_apitoken table
|
||||
CREATE TABLE IF NOT EXISTS api_apitoken (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
modified_at DATETIME NOT NULL,
|
||||
|
||||
token VARCHAR(32) NOT NULL UNIQUE,
|
||||
expires DATETIME,
|
||||
|
||||
created_by_id INTEGER NOT NULL,
|
||||
|
||||
FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
|
||||
CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
|
||||
CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
|
||||
|
||||
-- Create api_outboundwebhook table
|
||||
CREATE TABLE IF NOT EXISTS api_outboundwebhook (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
modified_at DATETIME NOT NULL,
|
||||
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
signal VARCHAR(255) NOT NULL,
|
||||
ref VARCHAR(1024) NOT NULL,
|
||||
endpoint VARCHAR(2048) NOT NULL,
|
||||
headers TEXT NOT NULL DEFAULT '{}',
|
||||
auth_token TEXT NOT NULL DEFAULT '',
|
||||
enabled BOOLEAN NOT NULL DEFAULT 1,
|
||||
keep_last_response BOOLEAN NOT NULL DEFAULT 0,
|
||||
created DATETIME NOT NULL,
|
||||
updated DATETIME NOT NULL,
|
||||
last_response TEXT NOT NULL DEFAULT '',
|
||||
last_success DATETIME,
|
||||
last_failure DATETIME,
|
||||
|
||||
created_by_id INTEGER NOT NULL,
|
||||
|
||||
FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
|
||||
CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
|
||||
"""
|
||||
|
||||
INITIAL_REVERSE_SQL = """
|
||||
DROP TABLE IF EXISTS api_outboundwebhook;
|
||||
DROP TABLE IF EXISTS api_apitoken;
|
||||
"""
|
||||
|
||||
|
||||
def _run_sqlite_only_sql(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "sqlite":
|
||||
return
|
||||
migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
|
||||
"api",
|
||||
schema_editor,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _run_sqlite_only_sql_reverse(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "sqlite":
|
||||
return
|
||||
migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
|
||||
"api",
|
||||
schema_editor,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _pg_sync_schema(apps, schema_editor):
|
||||
from archivebox.misc.db import rebuild_models_from_migration_state
|
||||
|
||||
rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
|
||||
|
||||
|
||||
def _pg_drop_schema(apps, schema_editor):
|
||||
from archivebox.misc.db import drop_models_on_postgres
|
||||
|
||||
drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
|
||||
import uuid
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.SeparateDatabaseAndState(
|
||||
database_operations=[
|
||||
migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
|
||||
],
|
||||
state_operations=[
|
||||
migrations.CreateModel(
|
||||
name="APIToken",
|
||||
fields=[
|
||||
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
||||
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
||||
("modified_at", models.DateTimeField(auto_now=True)),
|
||||
("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
|
||||
("expires", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"created_by",
|
||||
models.ForeignKey(
|
||||
default=get_or_create_system_user_pk,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "API Key",
|
||||
"verbose_name_plural": "API Keys",
|
||||
"app_label": "api",
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="OutboundWebhook",
|
||||
fields=[
|
||||
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
||||
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
||||
("modified_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"name",
|
||||
models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
|
||||
),
|
||||
(
|
||||
"signal",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("CREATE", "Create"),
|
||||
("UPDATE", "Update"),
|
||||
("DELETE", "Delete"),
|
||||
("M2M", "M2M changed"),
|
||||
("CREATE_OR_UPDATE", "Create or Update"),
|
||||
("CREATE_OR_DELETE", "Create or Delete"),
|
||||
("CREATE_OR_M2M", "Create or M2M changed"),
|
||||
("UPDATE_OR_DELETE", "Update or Delete"),
|
||||
("UPDATE_OR_M2M", "Update or M2M changed"),
|
||||
("DELETE_OR_M2M", "Delete or M2M changed"),
|
||||
("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
|
||||
("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
|
||||
("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
|
||||
("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
|
||||
("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
|
||||
],
|
||||
help_text="Signal the webhook fires to.",
|
||||
max_length=255,
|
||||
verbose_name="signal",
|
||||
),
|
||||
),
|
||||
(
|
||||
"ref",
|
||||
models.CharField(
|
||||
db_index=True,
|
||||
help_text="Dot import notation to the model the webhook is for.",
|
||||
max_length=1023,
|
||||
validators=[signal_webhooks.utils.model_from_reference],
|
||||
verbose_name="referenced model",
|
||||
),
|
||||
),
|
||||
(
|
||||
"endpoint",
|
||||
models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
|
||||
),
|
||||
(
|
||||
"headers",
|
||||
models.JSONField(
|
||||
blank=True,
|
||||
default=dict,
|
||||
help_text="Headers to send with the webhook request.",
|
||||
validators=[signal_webhooks.utils.is_dict],
|
||||
verbose_name="headers",
|
||||
),
|
||||
),
|
||||
(
|
||||
"auth_token",
|
||||
signal_webhooks.fields.TokenField(
|
||||
blank=True,
|
||||
default="",
|
||||
help_text="Authentication token to use in an Authorization header.",
|
||||
max_length=8000,
|
||||
validators=[signal_webhooks.utils.decode_cipher_key],
|
||||
verbose_name="authentication token",
|
||||
),
|
||||
),
|
||||
("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
|
||||
(
|
||||
"keep_last_response",
|
||||
models.BooleanField(
|
||||
default=False,
|
||||
help_text="Should the webhook keep a log of the latest response it got?",
|
||||
verbose_name="keep last response",
|
||||
),
|
||||
),
|
||||
(
|
||||
"created",
|
||||
models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
|
||||
),
|
||||
(
|
||||
"updated",
|
||||
models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
|
||||
),
|
||||
(
|
||||
"last_response",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
default="",
|
||||
help_text="Latest response to this webhook.",
|
||||
max_length=8000,
|
||||
verbose_name="last response",
|
||||
),
|
||||
),
|
||||
(
|
||||
"last_success",
|
||||
models.DateTimeField(
|
||||
default=None,
|
||||
help_text="When the webhook last succeeded.",
|
||||
null=True,
|
||||
verbose_name="last success",
|
||||
),
|
||||
),
|
||||
(
|
||||
"last_failure",
|
||||
models.DateTimeField(
|
||||
default=None,
|
||||
help_text="When the webhook last failed.",
|
||||
null=True,
|
||||
verbose_name="last failure",
|
||||
),
|
||||
),
|
||||
(
|
||||
"created_by",
|
||||
models.ForeignKey(
|
||||
default=get_or_create_system_user_pk,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "API Outbound Webhook",
|
||||
"app_label": "api",
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="outboundwebhook",
|
||||
constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='APIToken',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('expires', models.DateTimeField(blank=True, null=True)),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
# On non-sqlite backends the raw DDL above is skipped, so the real
|
||||
# schema diverges from Django state. Rebuild both tables (including the
|
||||
# unique constraint) from the post-migration state (no-op on sqlite).
|
||||
migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
|
||||
]
|
||||
@@ -0,0 +1,17 @@
|
||||
# Generated by Django 5.0.4 on 2024-04-26 05:28
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelOptions(
|
||||
name='apitoken',
|
||||
options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
|
||||
),
|
||||
]
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
# Generated by Django 5.0.6 on 2024-06-03 01:52
|
||||
|
||||
import abid_utils.models
|
||||
import charidfield.fields
|
||||
import django.db.models.deletion
|
||||
import signal_webhooks.fields
|
||||
import signal_webhooks.utils
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0002_alter_apitoken_options'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name='apitoken',
|
||||
old_name='user',
|
||||
new_name='created_by',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='apitoken',
|
||||
name='abid',
|
||||
field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='apitoken',
|
||||
name='modified',
|
||||
field=models.DateTimeField(auto_now=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='apitoken',
|
||||
name='uuid',
|
||||
field=models.UUIDField(blank=True, null=True, unique=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='id',
|
||||
field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='OutboundWebhook',
|
||||
fields=[
|
||||
('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
|
||||
('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
|
||||
('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
|
||||
('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
|
||||
('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
|
||||
('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
|
||||
('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
|
||||
('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
|
||||
('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
|
||||
('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
|
||||
('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
|
||||
('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('modified', models.DateTimeField(auto_now=True)),
|
||||
('id', models.UUIDField(blank=True, null=True, unique=True)),
|
||||
('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
|
||||
('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
|
||||
('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'API Outbound Webhook',
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='outboundwebhook',
|
||||
constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,24 @@
|
||||
# Generated by Django 5.1 on 2024-08-20 10:44
|
||||
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='id',
|
||||
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='uuid',
|
||||
field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
|
||||
),
|
||||
]
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# Generated by Django 5.1 on 2024-08-20 22:40
|
||||
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='apitoken',
|
||||
name='uuid',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='outboundwebhook',
|
||||
name='id',
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,29 @@
|
||||
# Generated by Django 5.1 on 2024-08-20 22:43
|
||||
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RenameField(
|
||||
model_name='outboundwebhook',
|
||||
old_name='uuid',
|
||||
new_name='id'
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='outboundwebhook',
|
||||
name='id',
|
||||
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='id',
|
||||
field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Generated by Django 5.1 on 2024-08-20 22:52
|
||||
|
||||
import abid_utils.models
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
]
|
||||
Executable → Regular
+89
-21
@@ -1,54 +1,122 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
import uuid
|
||||
import secrets
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
from datetime import timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
from signal_webhooks.models import WebhookBase
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
|
||||
|
||||
|
||||
|
||||
def generate_secret_token() -> str:
|
||||
# returns cryptographically secure string with len() == 32
|
||||
return secrets.token_hex(16)
|
||||
|
||||
|
||||
class APIToken(models.Model):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
class APIToken(ABIDModel):
|
||||
"""
|
||||
A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
|
||||
"""
|
||||
# ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
|
||||
abid_prefix = 'apt_'
|
||||
abid_ts_src = 'self.created'
|
||||
abid_uri_src = 'self.token'
|
||||
abid_subtype_src = 'self.created_by_id'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
|
||||
expires = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
app_label = "api"
|
||||
verbose_name = "API Key"
|
||||
verbose_name_plural = "API Keys"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.token
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
|
||||
|
||||
def __json__(self) -> dict:
|
||||
return {
|
||||
"TYPE": "APIToken",
|
||||
"id": str(self.pk),
|
||||
"abid": str(self.ABID),
|
||||
"created_by_id": str(self.created_by_id),
|
||||
"token": self.token,
|
||||
"created": self.created.isoformat(),
|
||||
"expires": self.expires_as_iso8601,
|
||||
}
|
||||
|
||||
@property
|
||||
def expires_as_iso8601(self):
|
||||
"""Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
|
||||
expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
|
||||
|
||||
return expiry_date.isoformat()
|
||||
|
||||
@property
|
||||
def token_redacted(self):
|
||||
return f"************{self.token[-4:]}"
|
||||
return f'************{self.token[-4:]}'
|
||||
|
||||
def is_valid(self, for_date=None):
|
||||
return not self.expires or self.expires >= (for_date or timezone.now())
|
||||
for_date = for_date or timezone.now()
|
||||
|
||||
if self.expires and self.expires < for_date:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class OutboundWebhook(WebhookBase):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
|
||||
|
||||
|
||||
# monkey patch django-signals-webhooks to change how it shows up in Admin UI
|
||||
|
||||
class OutboundWebhook(ABIDModel, WebhookBase):
|
||||
"""
|
||||
Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
|
||||
settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
|
||||
"""
|
||||
abid_prefix = 'whk_'
|
||||
abid_ts_src = 'self.created'
|
||||
abid_uri_src = 'self.endpoint'
|
||||
abid_subtype_src = 'self.ref'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
# More fields here: WebhookBase...
|
||||
|
||||
WebhookBase._meta.get_field('name').help_text = (
|
||||
'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
|
||||
WebhookBase._meta.get_field('signal').help_text = (
|
||||
'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
|
||||
WebhookBase._meta.get_field('ref').help_text = (
|
||||
'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
|
||||
WebhookBase._meta.get_field('endpoint').help_text = (
|
||||
'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
|
||||
|
||||
class Meta(WebhookBase.Meta):
|
||||
app_label = "api"
|
||||
verbose_name = "API Outbound Webhook"
|
||||
verbose_name = 'API Outbound Webhook'
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"[{self.id}] {self.ref} -> {self.endpoint}"
|
||||
@@ -0,0 +1,30 @@
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
from django.test import TestCase
|
||||
from ninja.testing import TestClient
|
||||
|
||||
from .routes_cli import router
|
||||
|
||||
class ArchiveBoxCLIAPITestCase(TestCase):
|
||||
def setUp(self):
|
||||
self.client = TestClient(router)
|
||||
|
||||
def test_add_endpoint(self):
|
||||
response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertTrue(response.json()["success"])
|
||||
|
||||
def test_remove_endpoint(self):
|
||||
response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertTrue(response.json()["success"])
|
||||
|
||||
def test_update_endpoint(self):
|
||||
response = self.client.post("/update", json={})
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertTrue(response.json()["success"])
|
||||
|
||||
def test_list_all_endpoint(self):
|
||||
response = self.client.post("/list_all", json={})
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertTrue(response.json()["success"])
|
||||
+9
-18
@@ -1,26 +1,17 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
from urllib.parse import quote
|
||||
|
||||
from django.http import HttpRequest, HttpResponseRedirect
|
||||
from django.shortcuts import redirect
|
||||
from django.urls import path
|
||||
from django.views.generic.base import RedirectView
|
||||
|
||||
from archivebox.core.routes_util import build_web_url
|
||||
from .v1_api import urls as v1_api_urls
|
||||
|
||||
|
||||
def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
|
||||
if request.META.get("QUERY_STRING"):
|
||||
url = f"{url}?{request.META['QUERY_STRING']}"
|
||||
return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
|
||||
|
||||
|
||||
urlpatterns = [
|
||||
path("", RedirectView.as_view(url="/api/v1/docs")),
|
||||
path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
|
||||
path("v1/", RedirectView.as_view(url="/api/v1/docs")),
|
||||
path("v1/", v1_api_urls),
|
||||
path("v1", RedirectView.as_view(url="/api/v1/docs")),
|
||||
path("", RedirectView.as_view(url='/api/v1')),
|
||||
|
||||
path("v1/", v1_api_urls),
|
||||
path("v1", RedirectView.as_view(url='/api/v1/docs')),
|
||||
|
||||
# ... v2 can be added here ...
|
||||
# path("v2/", v2_api_urls),
|
||||
# path("v2", RedirectView.as_view(url='/api/v2/docs')),
|
||||
]
|
||||
+39
-42
@@ -1,4 +1,4 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
|
||||
from io import StringIO
|
||||
@@ -7,22 +7,18 @@ from contextlib import redirect_stdout, redirect_stderr
|
||||
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from ninja import NinjaAPI, Swagger
|
||||
|
||||
# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
|
||||
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.version import get_COMMIT_HASH
|
||||
|
||||
from archivebox.api.auth import API_AUTH_METHODS
|
||||
from archivebox.api.models import APIToken
|
||||
from api.auth import API_AUTH_METHODS
|
||||
from ..config import VERSION, COMMIT_HASH
|
||||
|
||||
|
||||
COMMIT_HASH = get_COMMIT_HASH() or "unknown"
|
||||
COMMIT_HASH = COMMIT_HASH or 'unknown'
|
||||
|
||||
html_description = f"""
|
||||
html_description=f'''
|
||||
<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
|
||||
<br/>
|
||||
<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
|
||||
@@ -35,59 +31,42 @@ html_description = f"""
|
||||
<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
|
||||
</ul>
|
||||
<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
|
||||
"""
|
||||
'''
|
||||
|
||||
|
||||
def register_urls(api: NinjaAPI) -> NinjaAPI:
|
||||
api.add_router("/auth/", "archivebox.api.v1_auth.router")
|
||||
api.add_router("/core/", "archivebox.api.v1_core.router")
|
||||
api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
|
||||
api.add_router("/cli/", "archivebox.api.v1_cli.router")
|
||||
api.add_router("/machine/", "archivebox.api.v1_machine.router")
|
||||
api.add_router("/personas/", "archivebox.api.v1_personas.router")
|
||||
api.add_router('/auth/', 'api.v1_auth.router')
|
||||
api.add_router('/core/', 'api.v1_core.router')
|
||||
api.add_router('/cli/', 'api.v1_cli.router')
|
||||
return api
|
||||
|
||||
|
||||
class NinjaAPIWithIOCapture(NinjaAPI):
|
||||
class NinjaAPIWithIOCapture(NinjaAPI):
|
||||
def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
|
||||
stdout, stderr = StringIO(), StringIO()
|
||||
|
||||
with redirect_stderr(stderr):
|
||||
with redirect_stdout(stdout):
|
||||
setattr(request, "stdout", stdout)
|
||||
setattr(request, "stderr", stderr)
|
||||
request.stdout = stdout
|
||||
request.stderr = stderr
|
||||
|
||||
response = super().create_temporal_response(request)
|
||||
|
||||
# Disable caching of API responses entirely
|
||||
response["Cache-Control"] = "no-store"
|
||||
|
||||
# Add debug stdout and stderr headers to response
|
||||
response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
|
||||
response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
|
||||
# response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
|
||||
|
||||
# Add Auth Headers to response
|
||||
api_token_attr = request.__dict__.get("_api_token")
|
||||
api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
|
||||
token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
|
||||
|
||||
response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
|
||||
response["X-ArchiveBox-Auth-Expires"] = token_expiry
|
||||
response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
|
||||
response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
|
||||
response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
|
||||
print('RESPONDING NOW', response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
api = NinjaAPIWithIOCapture(
|
||||
title="ArchiveBox API",
|
||||
title='ArchiveBox API',
|
||||
description=html_description,
|
||||
version=VERSION,
|
||||
version='1.0.0',
|
||||
csrf=False,
|
||||
auth=API_AUTH_METHODS,
|
||||
urls_namespace="api-1",
|
||||
docs=Swagger(settings={"persistAuthorization": True}),
|
||||
# docs_decorator=login_required,
|
||||
# renderer=ORJSONRenderer(),
|
||||
)
|
||||
api = register_urls(api)
|
||||
urls = api.urls
|
||||
@@ -99,16 +78,34 @@ def generic_exception_handler(request, err):
|
||||
if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
|
||||
status = 404
|
||||
|
||||
print("".join(format_exception(err)))
|
||||
print(''.join(format_exception(err)))
|
||||
|
||||
return api.create_response(
|
||||
request,
|
||||
{
|
||||
"succeeded": False,
|
||||
"message": f"{err.__class__.__name__}: {err}",
|
||||
"message": f'{err.__class__.__name__}: {err}',
|
||||
"errors": [
|
||||
"".join(format_exception(err)),
|
||||
''.join(format_exception(err)),
|
||||
# or send simpler parent-only traceback:
|
||||
# *([str(err.__context__)] if getattr(err, '__context__', None) else []),
|
||||
],
|
||||
},
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
|
||||
# import orjson
|
||||
# from ninja.renderers import BaseRenderer
|
||||
# class ORJSONRenderer(BaseRenderer):
|
||||
# media_type = "application/json"
|
||||
# def render(self, request, data, *, response_status):
|
||||
# return {
|
||||
# "success": True,
|
||||
# "errors": [],
|
||||
# "result": data,
|
||||
# "stdout": ansi_to_html(stdout.getvalue().strip()),
|
||||
# "stderr": ansi_to_html(stderr.getvalue().strip()),
|
||||
# }
|
||||
# return orjson.dumps(data)
|
||||
+18
-72
@@ -1,106 +1,52 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
from django.http import HttpRequest
|
||||
from typing import Optional
|
||||
|
||||
from ninja import Router, Schema
|
||||
|
||||
from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
|
||||
from api.models import APIToken
|
||||
from api.auth import auth_using_token, auth_using_password
|
||||
|
||||
|
||||
router = Router(tags=["Authentication"], auth=None)
|
||||
router = Router(tags=['Authentication'])
|
||||
|
||||
|
||||
class PasswordAuthSchema(Schema):
|
||||
"""Schema for a /get_api_token request"""
|
||||
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
username: Optional[str] = None
|
||||
password: Optional[str] = None
|
||||
|
||||
|
||||
@router.post(
|
||||
"/get_api_token",
|
||||
auth=None,
|
||||
summary="Generate an API token for a given username & password (or currently logged-in user)",
|
||||
) # auth=None because they are not authed yet
|
||||
def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
|
||||
@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)') # auth=None because they are not authed yet
|
||||
def get_api_token(request, auth_data: PasswordAuthSchema):
|
||||
user = auth_using_password(
|
||||
username=auth_data.username,
|
||||
password=auth_data.password,
|
||||
request=request,
|
||||
)
|
||||
|
||||
if user and user.is_superuser:
|
||||
api_token = get_or_create_api_token(user)
|
||||
assert api_token is not None, "Failed to create API token"
|
||||
return {
|
||||
"success": True,
|
||||
"user_id": str(user.pk),
|
||||
"username": user.username,
|
||||
"token": api_token.token,
|
||||
"expires": api_token.expires.isoformat() if api_token.expires else None,
|
||||
}
|
||||
if user:
|
||||
# TODO: support multiple tokens in the future, for now we just have one per user
|
||||
api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
|
||||
|
||||
return api_token.__json__()
|
||||
|
||||
return {"success": False, "errors": ["Invalid credentials"]}
|
||||
|
||||
|
||||
|
||||
class TokenAuthSchema(Schema):
|
||||
"""Schema for a /check_api_token request"""
|
||||
|
||||
token: str
|
||||
|
||||
|
||||
@router.post(
|
||||
"/check_api_token",
|
||||
auth=None,
|
||||
summary="Validate an API token to make sure its valid and non-expired",
|
||||
) # auth=None because they are not authed yet
|
||||
def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
|
||||
@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired') # auth=None because they are not authed yet
|
||||
def check_api_token(request, token_data: TokenAuthSchema):
|
||||
user = auth_using_token(
|
||||
token=token_data.token,
|
||||
request=request,
|
||||
)
|
||||
if user:
|
||||
return {"success": True, "user_id": str(user.pk)}
|
||||
|
||||
|
||||
return {"success": False, "user_id": None}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/browser_session",
|
||||
auth=[HeaderTokenAuth(), BearerTokenAuth()],
|
||||
summary="Exchange an administrator API key for a normal browser session",
|
||||
)
|
||||
def browser_session(request: HttpRequest):
|
||||
from django.conf import settings
|
||||
from django.contrib.auth import login
|
||||
from django.http import JsonResponse
|
||||
from django.utils import timezone
|
||||
from ninja.errors import HttpError
|
||||
from archivebox.core.routes_util import get_admin_base_url
|
||||
|
||||
user = request.auth
|
||||
if not user.is_active or not user.is_superuser:
|
||||
raise HttpError(403, "An active administrator is required")
|
||||
login(request, user, backend="django.contrib.auth.backends.ModelBackend")
|
||||
# A browser session must not outlive the key used to establish it.
|
||||
token = request._api_token
|
||||
lifetime = settings.SESSION_COOKIE_AGE
|
||||
if token.expires:
|
||||
lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
|
||||
request.session.set_expiry(lifetime)
|
||||
request.session.save()
|
||||
response = JsonResponse(
|
||||
{
|
||||
"admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
|
||||
"cookie": {
|
||||
"name": settings.SESSION_COOKIE_NAME,
|
||||
"value": request.session.session_key,
|
||||
"expires": request.session.get_expiry_date().timestamp(),
|
||||
"secure": settings.SESSION_COOKIE_SECURE,
|
||||
},
|
||||
},
|
||||
)
|
||||
# The native client installs a host-only HttpOnly cookie on the admin origin;
|
||||
# API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
|
||||
response["Cache-Control"] = "no-store"
|
||||
return response
|
||||
+158
-253
@@ -1,329 +1,234 @@
|
||||
__package__ = "archivebox.api"
|
||||
__package__ = 'archivebox.api'
|
||||
|
||||
import json
|
||||
from io import StringIO
|
||||
from typing import Any
|
||||
from typing import List, Dict, Any, Optional
|
||||
from enum import Enum
|
||||
|
||||
from django.http import HttpRequest
|
||||
|
||||
from ninja import Router, Schema
|
||||
from ninja.errors import HttpError
|
||||
from pydantic import Field
|
||||
|
||||
from archivebox.misc.util import ansi_to_html
|
||||
from archivebox.core.models import SnapshotQuerySet
|
||||
from ..main import (
|
||||
add,
|
||||
remove,
|
||||
update,
|
||||
list_all,
|
||||
schedule,
|
||||
)
|
||||
from ..util import ansi_to_html
|
||||
from ..config import ONLY_NEW
|
||||
|
||||
|
||||
# router for API that exposes archivebox cli subcommands as REST endpoints
|
||||
router = Router(tags=["ArchiveBox CLI Sub-Commands"])
|
||||
router = Router(tags=['ArchiveBox CLI Sub-Commands'])
|
||||
|
||||
|
||||
# Schemas
|
||||
|
||||
JSONType = list[Any] | dict[str, Any] | bool | int | str | None
|
||||
FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
|
||||
|
||||
JSONType = List[Any] | Dict[str, Any] | bool | int | str | None
|
||||
|
||||
class CLICommandResponseSchema(Schema):
|
||||
success: bool
|
||||
errors: list[str]
|
||||
errors: List[str]
|
||||
result: JSONType
|
||||
result_format: str = "str"
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
class FilterTypeChoices(str, Enum):
|
||||
exact = 'exact'
|
||||
substring = 'substring'
|
||||
regex = 'regex'
|
||||
domain = 'domain'
|
||||
tag = 'tag'
|
||||
timestamp = 'timestamp'
|
||||
|
||||
FilterTypeChoices = Enum(
|
||||
"FilterTypeChoices",
|
||||
{filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
|
||||
type=str,
|
||||
)
|
||||
class StatusChoices(str, Enum):
|
||||
indexed = 'indexed'
|
||||
archived = 'archived'
|
||||
unarchived = 'unarchived'
|
||||
present = 'present'
|
||||
valid = 'valid'
|
||||
invalid = 'invalid'
|
||||
duplicate = 'duplicate'
|
||||
orphaned = 'orphaned'
|
||||
corrupted = 'corrupted'
|
||||
unrecognized = 'unrecognized'
|
||||
|
||||
|
||||
class AddCommandSchema(Schema):
|
||||
urls: list[str]
|
||||
snapshot_ids: list[str] | None = None
|
||||
urls: List[str]
|
||||
tag: str = ""
|
||||
depth: int = 0
|
||||
max_urls: int = 0
|
||||
crawl_max_size: int = 0
|
||||
crawl_timeout: int = 0
|
||||
snapshot_max_size: int = 0
|
||||
parser: str = "auto"
|
||||
plugins: str = ""
|
||||
persona: str = "Default"
|
||||
only_new: bool | None = None
|
||||
update: bool = False
|
||||
update: bool = not ONLY_NEW # Default to the opposite of ONLY_NEW
|
||||
update_all: bool = False
|
||||
index_only: bool = False
|
||||
overwrite: bool = False
|
||||
init: bool = False
|
||||
extractors: str = ""
|
||||
parser: str = "auto"
|
||||
|
||||
class UpdateCommandSchema(Schema):
|
||||
resume: Optional[float] = 0
|
||||
only_new: bool = ONLY_NEW
|
||||
index_only: bool = False
|
||||
|
||||
|
||||
class SnapshotFilterCommandSchema(Schema):
|
||||
after: float | None = 0
|
||||
before: float | None = None
|
||||
filter_type: str | None = FilterTypeChoices.substring
|
||||
filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
|
||||
status: str | None = None
|
||||
url__icontains: str | None = None
|
||||
url__istartswith: str | None = None
|
||||
tag: str | None = None
|
||||
crawl_id: str | None = None
|
||||
limit: int | None = None
|
||||
sort: str | None = None
|
||||
search: str | None = None
|
||||
|
||||
|
||||
class UpdateCommandSchema(SnapshotFilterCommandSchema):
|
||||
resume: str | None = None
|
||||
batch_size: int = 100
|
||||
continuous: bool = False
|
||||
index_only: bool = False
|
||||
migrate_only: bool = False
|
||||
|
||||
overwrite: bool = False
|
||||
after: Optional[float] = 0
|
||||
before: Optional[float] = 999999999999999
|
||||
status: Optional[StatusChoices] = StatusChoices.unarchived
|
||||
filter_type: Optional[str] = FilterTypeChoices.substring
|
||||
filter_patterns: Optional[List[str]] = ['https://example.com']
|
||||
extractors: Optional[str] = ""
|
||||
|
||||
class ScheduleCommandSchema(Schema):
|
||||
import_path: str | None = None
|
||||
import_path: Optional[str] = None
|
||||
add: bool = False
|
||||
show: bool = False
|
||||
foreground: bool = False
|
||||
run_all: bool = False
|
||||
quiet: bool = False
|
||||
every: str | None = None
|
||||
tag: str = ""
|
||||
every: Optional[str] = None
|
||||
tag: str = ''
|
||||
depth: int = 0
|
||||
only_new: bool | None = None
|
||||
update: bool = False
|
||||
overwrite: bool = False
|
||||
update: bool = not ONLY_NEW
|
||||
clear: bool = False
|
||||
|
||||
|
||||
class ListCommandSchema(SnapshotFilterCommandSchema):
|
||||
class ListCommandSchema(Schema):
|
||||
filter_patterns: Optional[List[str]] = ['https://example.com']
|
||||
filter_type: str = FilterTypeChoices.substring
|
||||
status: Optional[StatusChoices] = StatusChoices.indexed
|
||||
after: Optional[float] = 0
|
||||
before: Optional[float] = 999999999999999
|
||||
sort: str = 'added'
|
||||
as_json: bool = True
|
||||
as_html: bool = False
|
||||
as_csv: str | None = "timestamp,url"
|
||||
as_csv: str | bool = 'timestamp,url'
|
||||
with_headers: bool = False
|
||||
|
||||
|
||||
class RemoveCommandSchema(SnapshotFilterCommandSchema):
|
||||
class RemoveCommandSchema(Schema):
|
||||
delete: bool = True
|
||||
after: Optional[float] = 0
|
||||
before: Optional[float] = 999999999999999
|
||||
filter_type: str = FilterTypeChoices.exact
|
||||
timeout: float = 60.0
|
||||
filter_patterns: Optional[List[str]] = ['https://example.com']
|
||||
|
||||
|
||||
def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
|
||||
kwargs = args.dict()
|
||||
kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
|
||||
kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
|
||||
return kwargs
|
||||
|
||||
|
||||
@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
|
||||
def cli_add(request: HttpRequest, args: AddCommandSchema):
|
||||
from archivebox.cli.archivebox_add import add
|
||||
from archivebox.misc.util import validate_url
|
||||
|
||||
config_overrides: dict[str, object] = {}
|
||||
if args.only_new is not None:
|
||||
config_overrides["ONLY_NEW"] = bool(args.only_new)
|
||||
if args.update or args.overwrite:
|
||||
config_overrides["ONLY_NEW"] = False
|
||||
submitted_urls: str | list[str] = args.urls
|
||||
if len(args.urls) == 1:
|
||||
try:
|
||||
validate_url(args.urls[0])
|
||||
except ValueError:
|
||||
submitted_urls = args.urls[0]
|
||||
else:
|
||||
submitted_urls = []
|
||||
for url in args.urls:
|
||||
try:
|
||||
submitted_urls.append(validate_url(url))
|
||||
except ValueError:
|
||||
continue
|
||||
if not submitted_urls:
|
||||
raise HttpError(400, "No valid URLs were submitted")
|
||||
crawl, snapshots = add(
|
||||
urls=submitted_urls,
|
||||
snapshot_ids=args.snapshot_ids,
|
||||
@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
|
||||
def cli_add(request, args: AddCommandSchema):
|
||||
result = add(
|
||||
urls=args.urls,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
max_urls=args.max_urls,
|
||||
crawl_max_size=args.crawl_max_size,
|
||||
crawl_timeout=args.crawl_timeout,
|
||||
snapshot_max_size=args.snapshot_max_size,
|
||||
update=args.update,
|
||||
update_all=args.update_all,
|
||||
index_only=args.index_only,
|
||||
plugins=args.plugins,
|
||||
persona=args.persona,
|
||||
overwrite=args.overwrite,
|
||||
init=args.init,
|
||||
extractors=args.extractors,
|
||||
parser=args.parser,
|
||||
bg=True, # Always run in background for API calls
|
||||
created_by_id=request.user.pk,
|
||||
config=config_overrides or None,
|
||||
)
|
||||
|
||||
snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
|
||||
result_payload = {
|
||||
"crawl_id": str(crawl.id),
|
||||
"num_snapshots": len(snapshot_ids),
|
||||
"snapshot_ids": snapshot_ids,
|
||||
"queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
|
||||
}
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result_payload,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
|
||||
def cli_update(request: HttpRequest, args: UpdateCommandSchema):
|
||||
from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
|
||||
from archivebox.core.snapshot_status import normalize_snapshot_status
|
||||
|
||||
try:
|
||||
status = normalize_snapshot_status(args.status)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
|
||||
update_kwargs["status"] = status
|
||||
update_kwargs["stop_daemon_stack"] = False
|
||||
|
||||
is_filtered_update = any(
|
||||
(update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
|
||||
)
|
||||
matched_snapshot_ids = []
|
||||
if is_filtered_update:
|
||||
matched_snapshot_ids = [
|
||||
str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
|
||||
]
|
||||
|
||||
update(**update_kwargs)
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": {
|
||||
"matched_count": len(matched_snapshot_ids),
|
||||
"snapshot_ids": matched_snapshot_ids,
|
||||
}
|
||||
if is_filtered_update
|
||||
else None,
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
|
||||
def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
|
||||
from archivebox.cli.archivebox_schedule import schedule
|
||||
|
||||
config_overrides: dict[str, object] = {}
|
||||
if args.only_new is not None:
|
||||
config_overrides["ONLY_NEW"] = bool(args.only_new)
|
||||
if args.update or args.overwrite:
|
||||
config_overrides["ONLY_NEW"] = False
|
||||
result = schedule(
|
||||
import_path=args.import_path,
|
||||
add=args.add,
|
||||
show=args.show,
|
||||
foreground=args.foreground,
|
||||
run_all=args.run_all,
|
||||
quiet=args.quiet,
|
||||
clear=args.clear,
|
||||
every=args.every,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
config=config_overrides or None,
|
||||
)
|
||||
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
"stdout": ansi_to_html(request.stdout.getvalue().strip()),
|
||||
"stderr": ansi_to_html(request.stderr.getvalue().strip()),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
|
||||
def cli_search(request: HttpRequest, args: ListCommandSchema):
|
||||
from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
|
||||
@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
|
||||
def cli_update(request, args: UpdateCommandSchema):
|
||||
result = update(
|
||||
resume=args.resume,
|
||||
only_new=args.only_new,
|
||||
index_only=args.index_only,
|
||||
overwrite=args.overwrite,
|
||||
before=args.before,
|
||||
after=args.after,
|
||||
status=args.status,
|
||||
filter_type=args.filter_type,
|
||||
filter_patterns=args.filter_patterns,
|
||||
extractors=args.extractors,
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"stdout": ansi_to_html(request.stdout.getvalue().strip()),
|
||||
"stderr": ansi_to_html(request.stderr.getvalue().strip()),
|
||||
}
|
||||
|
||||
search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
|
||||
as_json = search_kwargs.pop("as_json")
|
||||
as_html = search_kwargs.pop("as_html")
|
||||
as_csv = search_kwargs.pop("as_csv")
|
||||
with_headers = search_kwargs.pop("with_headers")
|
||||
try:
|
||||
snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
result_format = "txt"
|
||||
if as_json:
|
||||
@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
|
||||
def cli_schedule(request, args: ScheduleCommandSchema):
|
||||
result = schedule(
|
||||
import_path=args.import_path,
|
||||
add=args.add,
|
||||
show=args.show,
|
||||
clear=args.clear,
|
||||
every=args.every,
|
||||
tag=args.tag,
|
||||
depth=args.depth,
|
||||
overwrite=args.overwrite,
|
||||
update=args.update,
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"stdout": ansi_to_html(request.stdout.getvalue().strip()),
|
||||
"stderr": ansi_to_html(request.stderr.getvalue().strip()),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
|
||||
def cli_list(request, args: ListCommandSchema):
|
||||
result = list_all(
|
||||
filter_patterns=args.filter_patterns,
|
||||
filter_type=args.filter_type,
|
||||
status=args.status,
|
||||
after=args.after,
|
||||
before=args.before,
|
||||
sort=args.sort,
|
||||
csv=args.as_csv,
|
||||
json=args.as_json,
|
||||
html=args.as_html,
|
||||
with_headers=args.with_headers,
|
||||
)
|
||||
|
||||
result_format = 'txt'
|
||||
if args.as_json:
|
||||
result_format = "json"
|
||||
result = [
|
||||
json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
|
||||
for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
|
||||
]
|
||||
elif as_html:
|
||||
elif args.as_html:
|
||||
result_format = "html"
|
||||
result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
|
||||
elif as_csv:
|
||||
elif args.as_csv:
|
||||
result_format = "csv"
|
||||
cols = [col.strip() for col in as_csv.split(",") if col.strip()]
|
||||
rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
|
||||
result = "\n".join((",".join(cols), *rows) if with_headers else rows)
|
||||
else:
|
||||
result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
|
||||
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": result_format,
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
"stdout": ansi_to_html(request.stdout.getvalue().strip()),
|
||||
"stderr": ansi_to_html(request.stderr.getvalue().strip()),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
|
||||
def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
|
||||
from archivebox.cli.archivebox_remove import remove
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
|
||||
timeout_arg = remove_kwargs.pop("timeout")
|
||||
timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
|
||||
try:
|
||||
snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
|
||||
@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
|
||||
def cli_remove(request, args: RemoveCommandSchema):
|
||||
result = remove(
|
||||
yes=True, # no way to interactively ask for confirmation via API, so we force yes
|
||||
snapshots=snapshots_to_remove,
|
||||
timeout=timeout,
|
||||
yes=True, # no way to interactively ask for confirmation via API, so we force yes
|
||||
delete=args.delete,
|
||||
before=args.before,
|
||||
after=args.after,
|
||||
filter_type=args.filter_type,
|
||||
filter_patterns=args.filter_patterns,
|
||||
)
|
||||
stdout = request.__dict__.get("stdout")
|
||||
stderr = request.__dict__.get("stderr")
|
||||
return {
|
||||
"success": bool(result["success"]),
|
||||
"errors": [str(result["error"])] if result["error"] else [],
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"result": result,
|
||||
"result_format": "json",
|
||||
"stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
|
||||
"stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
|
||||
"stdout": ansi_to_html(request.stdout.getvalue().strip()),
|
||||
"stderr": ansi_to_html(request.stderr.getvalue().strip()),
|
||||
}
|
||||
|
||||
+256
-1281
File diff suppressed because it is too large.
Load diff
@@ -1,270 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
from datetime import datetime
|
||||
from django.http import FileResponse, HttpRequest
|
||||
from django.shortcuts import redirect
|
||||
from django.utils import timezone
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from ninja import Router, Schema
|
||||
from ninja.errors import HttpError
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_PUBLIC,
|
||||
PERMISSIONS_UNLISTED,
|
||||
is_admin_user,
|
||||
normalize_permissions,
|
||||
)
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
|
||||
|
||||
from .auth import API_AUTH_METHODS, authenticated_user_from_request
|
||||
|
||||
router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
|
||||
|
||||
|
||||
class CrawlSchema(Schema):
|
||||
TYPE: str = "crawls.models.Crawl"
|
||||
|
||||
id: UUID
|
||||
|
||||
modified_at: datetime
|
||||
created_at: datetime
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
|
||||
status: str
|
||||
retry_at: datetime | None
|
||||
is_paused: bool
|
||||
|
||||
urls: str
|
||||
max_depth: int
|
||||
tags_str: str
|
||||
config: dict
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by_id)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj):
|
||||
user_model = get_user_model()
|
||||
user = user_model.objects.get(id=obj.created_by_id)
|
||||
username = user.username
|
||||
return username if isinstance(username, str) else str(user)
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses can never leak the raw
|
||||
# token/secret/api-key that the operator stored in Crawl.config.
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return redact_sensitive_config(obj.config)
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshots(obj, context):
|
||||
if bool(context["request"].__dict__.get("with_snapshots", False)):
|
||||
return obj.snapshot_set.all().distinct()
|
||||
return Snapshot.objects.none()
|
||||
|
||||
|
||||
class CrawlUpdateSchema(Schema):
|
||||
action: str | None = None
|
||||
status: str | None = None
|
||||
retry_at: datetime | None = None
|
||||
tags: list[str] | None = None
|
||||
tags_str: str | None = None
|
||||
|
||||
|
||||
class CrawlCreateSchema(Schema):
|
||||
urls: list[str]
|
||||
max_depth: int = 0
|
||||
tags: list[str] | None = None
|
||||
tags_str: str = ""
|
||||
label: str = ""
|
||||
notes: str = ""
|
||||
config: dict = {}
|
||||
|
||||
|
||||
class CrawlDeleteResponseSchema(Schema):
|
||||
success: bool
|
||||
crawl_id: str
|
||||
deleted_count: int
|
||||
deleted_snapshots: int
|
||||
|
||||
|
||||
def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
|
||||
if tags is not None:
|
||||
return [tag.strip() for tag in tags if tag and tag.strip()]
|
||||
return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
|
||||
|
||||
|
||||
@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
|
||||
def get_crawls(request: HttpRequest):
|
||||
return Crawl.objects.all().distinct()
|
||||
|
||||
|
||||
@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
|
||||
def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
|
||||
try:
|
||||
urls = [validate_url(url) for url in data.urls if url and url.strip()]
|
||||
except ValueError as err:
|
||||
raise HttpError(400, str(err)) from err
|
||||
if not urls:
|
||||
raise HttpError(400, "At least one URL is required")
|
||||
if data.max_depth not in (0, 1, 2, 3, 4):
|
||||
raise HttpError(400, "max_depth must be between 0 and 4")
|
||||
|
||||
tags = normalize_tag_list(data.tags, data.tags_str)
|
||||
config = dict(data.config or {})
|
||||
config.setdefault("PARSER", "url_list")
|
||||
config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
|
||||
crawl = Crawl.objects.create(
|
||||
urls="\n".join(urls),
|
||||
max_depth=data.max_depth,
|
||||
tags_str=",".join(tags),
|
||||
label=data.label,
|
||||
notes=data.notes,
|
||||
config=config,
|
||||
status=Crawl.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
created_by=request.user if isinstance(request.user, User) else None,
|
||||
)
|
||||
return crawl
|
||||
|
||||
|
||||
def get_crawl_by_ref(crawl_id: str):
|
||||
return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
|
||||
def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
|
||||
"""Get a specific Crawl by id."""
|
||||
setattr(request, "with_snapshots", with_snapshots)
|
||||
setattr(request, "with_archiveresults", with_archiveresults)
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
|
||||
if crawl and as_rss:
|
||||
query = request.GET.copy()
|
||||
query.pop("as_rss", None)
|
||||
query["crawl_id"] = str(crawl.id)
|
||||
return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
|
||||
|
||||
return crawl
|
||||
|
||||
|
||||
def crawl_file(request: HttpRequest, crawl_id: str, path: str):
|
||||
# Try to resolve the crawl first; if it doesn't exist, return 404.
|
||||
try:
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
except Crawl.DoesNotExist:
|
||||
raise HttpError(404, "Crawl not found")
|
||||
|
||||
user = authenticated_user_from_request(request)
|
||||
|
||||
# Gate access using the same model as SnapshotView/can_view_snapshot:
|
||||
# admins always pass; owners can see their own crawls; otherwise the crawl
|
||||
# must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
|
||||
if not is_admin_user(request):
|
||||
permissions = normalize_permissions(crawl.permissions)
|
||||
is_owner = bool(user and crawl.created_by_id == user.id)
|
||||
if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
|
||||
raise HttpError(404, "Crawl not found")
|
||||
|
||||
crawl_root = Path(crawl.output_dir).resolve()
|
||||
file_path = (crawl_root / path).resolve()
|
||||
if not file_path.is_file() or crawl_root not in file_path.parents:
|
||||
raise HttpError(404, "Crawl file not found")
|
||||
|
||||
response = FileResponse(file_path.open("rb"))
|
||||
response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
|
||||
response["Pragma"] = "no-cache"
|
||||
response["Expires"] = "0"
|
||||
response["X-Content-Type-Options"] = "nosniff"
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
|
||||
def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
|
||||
return crawl_file(request, crawl_id, filename)
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
|
||||
def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
|
||||
return crawl_file(request, crawl_id, f"{folder}/{filename}")
|
||||
|
||||
|
||||
@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
|
||||
def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
|
||||
return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
|
||||
|
||||
|
||||
@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
|
||||
def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
|
||||
"""Update a crawl (e.g., set status=sealed to cancel queued work)."""
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
payload = data.dict(exclude_unset=True)
|
||||
update_fields = ["modified_at"]
|
||||
|
||||
action = payload.pop("action", None)
|
||||
if action:
|
||||
if action == "pause":
|
||||
crawl.pause()
|
||||
return crawl
|
||||
if action in ("resume", "unpause"):
|
||||
crawl.resume()
|
||||
return crawl
|
||||
if action == "cancel":
|
||||
crawl.cancel()
|
||||
return crawl
|
||||
raise HttpError(400, f"Invalid action: {action}")
|
||||
|
||||
tags = payload.pop("tags", None)
|
||||
tags_str = payload.pop("tags_str", None)
|
||||
if tags is not None or tags_str is not None:
|
||||
crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
|
||||
update_fields.append("tags_str")
|
||||
|
||||
if "status" in payload:
|
||||
if payload["status"] not in Crawl.StatusChoices.values:
|
||||
raise HttpError(400, f"Invalid status: {payload['status']}")
|
||||
if payload["status"] == Crawl.StatusChoices.SEALED:
|
||||
crawl.cancel()
|
||||
return crawl
|
||||
crawl.status = payload["status"]
|
||||
update_fields.append("status")
|
||||
|
||||
if "retry_at" in payload:
|
||||
crawl.retry_at = payload["retry_at"]
|
||||
update_fields.append("retry_at")
|
||||
|
||||
crawl.save(update_fields=update_fields)
|
||||
return crawl
|
||||
|
||||
|
||||
@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
|
||||
def delete_crawl(request: HttpRequest, crawl_id: str):
|
||||
crawl = get_crawl_by_ref(crawl_id)
|
||||
crawl_id_str = str(crawl.id)
|
||||
crawl.cancel()
|
||||
|
||||
from archivebox.crawls.locks import crawl_lifecycle_lock
|
||||
from archivebox.services.runner import run_pending_crawls
|
||||
|
||||
with crawl_lifecycle_lock(crawl_id_str):
|
||||
run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
|
||||
crawl = get_crawl_by_ref(crawl_id_str)
|
||||
snapshot_count = crawl.snapshot_set.count()
|
||||
deleted_count, _ = crawl.delete()
|
||||
return {
|
||||
"success": True,
|
||||
"crawl_id": crawl_id_str,
|
||||
"deleted_count": deleted_count,
|
||||
"deleted_snapshots": snapshot_count,
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from uuid import UUID
|
||||
from typing import Annotated
|
||||
from datetime import datetime
|
||||
|
||||
from django.http import HttpRequest
|
||||
|
||||
from ninja import FilterLookup, FilterSchema, Query, Router, Schema
|
||||
from ninja.pagination import paginate
|
||||
|
||||
from archivebox.api.v1_core import CustomPagination
|
||||
|
||||
|
||||
router = Router(tags=["Machine and Dependencies"])
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Machine Schemas
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class MachineSchema(Schema):
|
||||
"""Schema for Machine model."""
|
||||
|
||||
TYPE: str = "machine.Machine"
|
||||
id: UUID
|
||||
created_at: datetime
|
||||
modified_at: datetime
|
||||
guid: str
|
||||
hostname: str
|
||||
hw_in_docker: bool
|
||||
hw_in_vm: bool
|
||||
hw_manufacturer: str
|
||||
hw_product: str
|
||||
hw_uuid: str
|
||||
os_arch: str
|
||||
os_family: str
|
||||
os_platform: str
|
||||
os_release: str
|
||||
os_kernel: str
|
||||
stats: dict
|
||||
num_uses_succeeded: int
|
||||
num_uses_failed: int
|
||||
|
||||
|
||||
class MachineFilterSchema(FilterSchema):
|
||||
id: Annotated[str | None, FilterLookup("id__startswith")] = None
|
||||
hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
|
||||
os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
|
||||
os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
|
||||
hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
|
||||
hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
|
||||
bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Binary Schemas
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BinarySchema(Schema):
|
||||
"""Schema for Binary model."""
|
||||
|
||||
TYPE: str = "machine.Binary"
|
||||
id: UUID
|
||||
created_at: datetime
|
||||
modified_at: datetime
|
||||
machine_id: UUID
|
||||
machine_hostname: str
|
||||
name: str
|
||||
binproviders: str
|
||||
binprovider: str
|
||||
abspath: str
|
||||
version: str
|
||||
sha256: str
|
||||
status: str
|
||||
is_valid: bool
|
||||
num_uses_succeeded: int
|
||||
num_uses_failed: int
|
||||
|
||||
@staticmethod
|
||||
def resolve_machine_hostname(obj) -> str:
|
||||
return obj.machine.hostname
|
||||
|
||||
@staticmethod
|
||||
def resolve_is_valid(obj) -> bool:
|
||||
return obj.is_valid
|
||||
|
||||
|
||||
class BinaryFilterSchema(FilterSchema):
|
||||
id: Annotated[str | None, FilterLookup("id__startswith")] = None
|
||||
name: Annotated[str | None, FilterLookup("name__icontains")] = None
|
||||
binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
|
||||
status: Annotated[str | None, FilterLookup("status")] = None
|
||||
machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
|
||||
version: Annotated[str | None, FilterLookup("version__icontains")] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Machine Endpoints
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
|
||||
@paginate(CustomPagination)
|
||||
def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
|
||||
"""List all machines."""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
return filters.filter(Machine.objects.all()).distinct()
|
||||
|
||||
|
||||
@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
|
||||
def get_current_machine(request: HttpRequest):
|
||||
"""Get the current machine."""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
return Machine.current()
|
||||
|
||||
|
||||
@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
|
||||
def get_machine(request: HttpRequest, machine_id: str):
|
||||
"""Get a specific machine by ID."""
|
||||
from archivebox.machine.models import Machine
|
||||
from django.db.models import Q
|
||||
|
||||
return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Binary Endpoints
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
|
||||
@paginate(CustomPagination)
|
||||
def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
|
||||
"""List all binaries."""
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
return filters.filter(Binary.objects.all().select_related("machine")).distinct()
|
||||
|
||||
|
||||
@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
|
||||
def get_binary(request: HttpRequest, binary_id: str):
|
||||
"""Get a specific binary by ID."""
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
return Binary.objects.select_related("machine").get(id__startswith=binary_id)
|
||||
|
||||
|
||||
@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
|
||||
def get_binaries_by_name(request: HttpRequest, name: str):
|
||||
"""Get all binaries with the given name."""
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
|
||||
@@ -1,171 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from django.db.models import Q
|
||||
from django.http import HttpRequest
|
||||
from ninja import Router, Schema
|
||||
from ninja.pagination import paginate
|
||||
from pydantic import Field
|
||||
|
||||
from archivebox.api.v1_core import CustomPagination
|
||||
from archivebox.personas.importers import validate_persona_name
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
|
||||
router = Router(tags=["Personas"])
|
||||
|
||||
|
||||
class PersonaBrowserSettingsSchema(Schema):
|
||||
user_agent: str = ""
|
||||
viewport_size: str = ""
|
||||
viewport_device_scale_factor: float | None = None
|
||||
color_scheme: Literal["", "light", "dark"] = ""
|
||||
language: str = ""
|
||||
timezone: str = ""
|
||||
geolocation: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class PersonaSyncSchema(Schema):
|
||||
extension_persona_id: str
|
||||
name: str
|
||||
settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
|
||||
cookies_txt: str = ""
|
||||
auth_json: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class PersonaSchema(Schema):
|
||||
TYPE: str = "personas.models.Persona"
|
||||
id: UUID
|
||||
name: str
|
||||
created_at: datetime
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
config: dict[str, Any] | None
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by.pk)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_username(obj) -> str:
|
||||
return obj.created_by.username
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses don't leak the raw
|
||||
# token/secret/api-key the operator stored in Persona.config.
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return redact_sensitive_config(obj.config)
|
||||
|
||||
|
||||
class PersonaSyncResponseSchema(Schema):
|
||||
success: bool
|
||||
created: bool
|
||||
persona: PersonaSchema
|
||||
cookies_file_written: bool
|
||||
auth_file_written: bool
|
||||
|
||||
|
||||
def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
|
||||
config: dict[str, Any] = {
|
||||
"BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
|
||||
"BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
|
||||
if settings.user_agent:
|
||||
config.update(
|
||||
{
|
||||
"USER_AGENT": settings.user_agent,
|
||||
"CHROME_USER_AGENT": settings.user_agent,
|
||||
"WGET_USER_AGENT": settings.user_agent,
|
||||
"CURL_USER_AGENT": settings.user_agent,
|
||||
},
|
||||
)
|
||||
if settings.viewport_size:
|
||||
config.update(
|
||||
{
|
||||
"RESOLUTION": settings.viewport_size,
|
||||
"CHROME_RESOLUTION": settings.viewport_size,
|
||||
},
|
||||
)
|
||||
if settings.viewport_device_scale_factor is not None:
|
||||
config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
|
||||
if "color_scheme" in settings.model_fields_set:
|
||||
config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
|
||||
if settings.language:
|
||||
config["BROWSER_LANGUAGE"] = settings.language
|
||||
if settings.timezone:
|
||||
config["BROWSER_TIMEZONE"] = settings.timezone
|
||||
if settings.geolocation:
|
||||
config["BROWSER_GEOLOCATION"] = settings.geolocation
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def find_persona(extension_persona_id: str, name: str) -> Persona | None:
|
||||
return (
|
||||
Persona.objects.filter(
|
||||
Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
|
||||
)
|
||||
.order_by("created_at")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
|
||||
@paginate(CustomPagination)
|
||||
def get_personas(request: HttpRequest):
|
||||
"""List personas available on this ArchiveBox server."""
|
||||
return Persona.objects.all().order_by("name")
|
||||
|
||||
|
||||
@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
|
||||
def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
|
||||
"""
|
||||
Create or update a Persona from a browser extension profile export.
|
||||
|
||||
The extension sends browser settings plus portable auth artifacts. The server
|
||||
keeps browser override settings in Persona.config and writes cookies.txt /
|
||||
auth.json into the persona directory for extractors to consume.
|
||||
"""
|
||||
name = payload.name.strip()
|
||||
is_valid, error_message = validate_persona_name(name)
|
||||
if not is_valid:
|
||||
raise ValueError(error_message)
|
||||
|
||||
persona = find_persona(payload.extension_persona_id, name)
|
||||
created = persona is None
|
||||
if persona is None:
|
||||
persona = Persona(name=name)
|
||||
if request.user.is_authenticated:
|
||||
persona.created_by = request.user
|
||||
|
||||
persona.config = {
|
||||
**(persona.config or {}),
|
||||
**browser_settings_to_config(payload.extension_persona_id, payload.settings),
|
||||
}
|
||||
persona.save()
|
||||
persona.ensure_dirs()
|
||||
|
||||
cookies_written = False
|
||||
if "cookies_txt" in payload.model_fields_set:
|
||||
(persona.path / "cookies.txt").write_text(payload.cookies_txt)
|
||||
cookies_written = True
|
||||
|
||||
auth_written = False
|
||||
if "auth_json" in payload.model_fields_set:
|
||||
(persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
|
||||
auth_written = True
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"created": created,
|
||||
"persona": persona,
|
||||
"cookies_file_written": cookies_written,
|
||||
"auth_file_written": auth_written,
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
__package__ = "archivebox.api"
|
||||
|
||||
from typing import Any
|
||||
from collections.abc import Callable
|
||||
import logging
|
||||
|
||||
from django.db import transaction
|
||||
from signal_webhooks.handlers import sync_task_handler
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def warning_error_handler(hook: Any, error: Exception | None) -> None:
|
||||
if error is not None:
|
||||
logger.warning("Outbound webhook %r failed: %s", hook.name, error)
|
||||
return
|
||||
|
||||
logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
|
||||
|
||||
|
||||
def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
|
||||
def run_webhook() -> None:
|
||||
try:
|
||||
sync_task_handler(hook, **kwargs)
|
||||
except Exception:
|
||||
logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
|
||||
|
||||
try:
|
||||
transaction.on_commit(run_webhook)
|
||||
except Exception:
|
||||
logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
|
||||
@@ -1 +0,0 @@
|
||||
__package__ = "archivebox.base_models"
|
||||
@@ -1,930 +0,0 @@
|
||||
"""Base admin classes for models using UUIDv7."""
|
||||
|
||||
__package__ = "archivebox.base_models"
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from typing import ClassVar, NotRequired, TypedDict, cast
|
||||
|
||||
from django import forms
|
||||
from django.contrib import admin
|
||||
from django.db import DatabaseError, models
|
||||
from django.forms.renderers import BaseRenderer
|
||||
from django.http import HttpRequest, QueryDict
|
||||
from django.urls import path, register_converter
|
||||
from django.utils.safestring import SafeString, mark_safe
|
||||
from django_object_actions import DjangoObjectActions
|
||||
|
||||
|
||||
class HexUUIDConverter:
|
||||
"""URL path converter that canonicalizes UUIDs to their 32-char hex form.
|
||||
|
||||
Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
|
||||
(``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
|
||||
parses either), but ``to_url`` always emits the bare-hex form. This is
|
||||
what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
|
||||
``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
|
||||
hyphenated rendering — admin links throughout the app reverse through
|
||||
this converter once ``BaseModelAdmin.get_urls`` swaps in
|
||||
``<hexuuid:object_id>`` below.
|
||||
"""
|
||||
|
||||
regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
|
||||
def to_python(self, value: str) -> str:
|
||||
# Strip hyphens but stay as a string — Django admin treats the
|
||||
# captured object_id as a string and calls ``model._meta.pk.to_python``
|
||||
# itself, so we don't want to short-circuit that.
|
||||
return value.replace("-", "")
|
||||
|
||||
def to_url(self, value) -> str:
|
||||
if isinstance(value, uuid.UUID):
|
||||
return value.hex
|
||||
return str(value).replace("-", "")
|
||||
|
||||
|
||||
register_converter(HexUUIDConverter, "hexuuid")
|
||||
|
||||
|
||||
class ConfigOption(TypedDict):
|
||||
plugin: str
|
||||
type: str | list[str]
|
||||
default: object
|
||||
description: str
|
||||
enum: NotRequired[list[object]]
|
||||
pattern: NotRequired[str]
|
||||
minimum: NotRequired[int | float]
|
||||
maximum: NotRequired[int | float]
|
||||
|
||||
|
||||
class KeyValueWidget(forms.Widget):
|
||||
"""
|
||||
A widget that renders JSON dict as editable key-value input fields
|
||||
with + and - buttons to add/remove rows.
|
||||
Includes autocomplete for available config keys from the plugin system.
|
||||
"""
|
||||
|
||||
template_name = "" # We render manually
|
||||
|
||||
class Media:
|
||||
css: ClassVar[dict[str, list[str]]] = {
|
||||
"all": [],
|
||||
}
|
||||
js: ClassVar[list[str]] = []
|
||||
|
||||
def _get_config_options(self) -> dict[str, ConfigOption]:
|
||||
"""Get available config options from plugins."""
|
||||
try:
|
||||
from archivebox.config.common import config_field_metadata
|
||||
|
||||
options: dict[str, ConfigOption] = {}
|
||||
for key, metadata in config_field_metadata().items():
|
||||
option_type = metadata.get("type", "string")
|
||||
option: ConfigOption = {
|
||||
"plugin": str(metadata.get("plugin", "archivebox")),
|
||||
"type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
|
||||
"default": metadata.get("default", ""),
|
||||
"description": str(metadata.get("description", "")),
|
||||
}
|
||||
schema = metadata.get("schema")
|
||||
if isinstance(schema, Mapping):
|
||||
for schema_key in ("enum", "pattern", "minimum", "maximum"):
|
||||
if schema_key in schema:
|
||||
option[schema_key] = schema[schema_key]
|
||||
options[key] = option
|
||||
return options
|
||||
except (ImportError, KeyError, TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
def _parse_value(self, value: object) -> dict[str, object]:
|
||||
# Parse JSON value to dict
|
||||
if value is None:
|
||||
return {}
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value) if value else {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): item for key, item in value.items()}
|
||||
return {}
|
||||
|
||||
def render(
|
||||
self,
|
||||
name: str,
|
||||
value: object,
|
||||
attrs: Mapping[str, str] | None = None,
|
||||
renderer: BaseRenderer | None = None,
|
||||
) -> SafeString:
|
||||
data = self._parse_value(value)
|
||||
|
||||
widget_id = attrs.get("id", name) if attrs else name
|
||||
config_options = self._get_config_options()
|
||||
|
||||
# Build datalist options
|
||||
datalist_options = "\n".join(
|
||||
f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
|
||||
for key, opt in sorted(config_options.items())
|
||||
)
|
||||
|
||||
# Build config metadata as JSON for JS
|
||||
config_meta_json = json.dumps(config_options)
|
||||
|
||||
html = f'''
|
||||
<div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
|
||||
<datalist id="{widget_id}_keys">
|
||||
{datalist_options}
|
||||
</datalist>
|
||||
<div id="{widget_id}_rows" class="key-value-rows">
|
||||
'''
|
||||
|
||||
# Render existing key-value pairs
|
||||
for key, val in data.items():
|
||||
val_str = json.dumps(val) if not isinstance(val, str) else val
|
||||
html += self._render_row(widget_id, key, val_str)
|
||||
|
||||
# Always add one empty row for new entries
|
||||
html += self._render_row(widget_id, "", "")
|
||||
|
||||
html += f'''
|
||||
</div>
|
||||
<div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
|
||||
<button type="button" onclick="addKeyValueRow_{widget_id}()"
|
||||
style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
|
||||
+ Add Row
|
||||
</button>
|
||||
</div>
|
||||
<input type="hidden" name="{name}" id="{widget_id}" value="">
|
||||
<script>
|
||||
(function() {{
|
||||
var configMeta_{widget_id} = {config_meta_json};
|
||||
var rowCounter_{widget_id} = 0;
|
||||
|
||||
function stringifyValue_{widget_id}(value) {{
|
||||
return typeof value === 'string' ? value : JSON.stringify(value);
|
||||
}}
|
||||
|
||||
function getTypes_{widget_id}(meta) {{
|
||||
if (!meta || meta.type === undefined || meta.type === null) {{
|
||||
return [];
|
||||
}}
|
||||
return Array.isArray(meta.type) ? meta.type : [meta.type];
|
||||
}}
|
||||
|
||||
function getMetaForKey_{widget_id}(key) {{
|
||||
if (!key) {{
|
||||
return null;
|
||||
}}
|
||||
|
||||
var explicitMeta = configMeta_{widget_id}[key];
|
||||
if (explicitMeta) {{
|
||||
return Object.assign({{ key: key }}, explicitMeta);
|
||||
}}
|
||||
|
||||
if (key.endsWith('_BINARY')) {{
|
||||
return {{
|
||||
key: key,
|
||||
plugin: 'custom',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Path to binary executable',
|
||||
}};
|
||||
}}
|
||||
|
||||
if (isRegexConfigKey_{widget_id}(key)) {{
|
||||
return {{
|
||||
key: key,
|
||||
plugin: 'custom',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'Regex pattern list',
|
||||
}};
|
||||
}}
|
||||
|
||||
return null;
|
||||
}}
|
||||
|
||||
function describeMeta_{widget_id}(meta) {{
|
||||
if (!meta) {{
|
||||
return '';
|
||||
}}
|
||||
|
||||
var details = '';
|
||||
if (Array.isArray(meta.enum) && meta.enum.length) {{
|
||||
details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
|
||||
}} else {{
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (types.length) {{
|
||||
details = 'Expected: ' + types.join(' or ');
|
||||
}}
|
||||
}}
|
||||
|
||||
if (meta.minimum !== undefined || meta.maximum !== undefined) {{
|
||||
var bounds = [];
|
||||
if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
|
||||
if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
|
||||
details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
|
||||
}}
|
||||
|
||||
return [meta.description || '', details].filter(Boolean).join(' ');
|
||||
}}
|
||||
|
||||
function getExampleInput_{widget_id}(key, meta) {{
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (key.endsWith('_BINARY')) {{
|
||||
return 'Example: wget or /usr/bin/wget';
|
||||
}}
|
||||
if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
|
||||
return 'Example: ["--extra-arg"]';
|
||||
}}
|
||||
if (types.includes('array')) {{
|
||||
return 'Example: ["value"]';
|
||||
}}
|
||||
if (types.includes('object')) {{
|
||||
return 'Example: {{"key": "value"}}';
|
||||
}}
|
||||
return '';
|
||||
}}
|
||||
|
||||
function isRegexConfigKey_{widget_id}(key) {{
|
||||
return key === 'URL_ALLOWLIST' ||
|
||||
key === 'URL_DENYLIST' ||
|
||||
key.endsWith('_PATTERN') ||
|
||||
key.includes('REGEX');
|
||||
}}
|
||||
|
||||
function isSimpleFilterPattern_{widget_id}(pattern) {{
|
||||
return /^[\\w.*:-]+$/.test(pattern);
|
||||
}}
|
||||
|
||||
function validateRegexPattern_{widget_id}(pattern) {{
|
||||
if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
|
||||
return '';
|
||||
}}
|
||||
|
||||
try {{
|
||||
new RegExp(pattern);
|
||||
}} catch (error) {{
|
||||
return error && error.message ? error.message : 'Invalid regex';
|
||||
}}
|
||||
return '';
|
||||
}}
|
||||
|
||||
function validateRegexConfig_{widget_id}(key, raw, typeName) {{
|
||||
if (typeName === 'object') {{
|
||||
var parsed;
|
||||
try {{
|
||||
parsed = JSON.parse(raw);
|
||||
}} catch (error) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
|
||||
}}
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
|
||||
return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
|
||||
}}
|
||||
for (var regexKey in parsed) {{
|
||||
var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
|
||||
if (objectRegexError) {{
|
||||
return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
|
||||
}}
|
||||
}}
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
|
||||
var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
|
||||
return pattern.trim();
|
||||
}}).filter(Boolean);
|
||||
for (var i = 0; i < patterns.length; i++) {{
|
||||
var regexError = validateRegexPattern_{widget_id}(patterns[i]);
|
||||
if (regexError) {{
|
||||
return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
|
||||
}}
|
||||
}}
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
function validateBinaryValue_{widget_id}(raw) {{
|
||||
if (!raw) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
if (/['"`]/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
|
||||
}}
|
||||
|
||||
if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
|
||||
}}
|
||||
|
||||
if (raw.startsWith('/')) {{
|
||||
if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
|
||||
}}
|
||||
|
||||
if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
|
||||
}}
|
||||
|
||||
function parseValue_{widget_id}(raw) {{
|
||||
try {{
|
||||
if (raw === 'true') return true;
|
||||
if (raw === 'false') return false;
|
||||
if (raw === 'null') return null;
|
||||
if (raw !== '' && !isNaN(raw)) return Number(raw);
|
||||
if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
|
||||
(raw.startsWith('[') && raw.endsWith(']')) ||
|
||||
(raw.startsWith('"') && raw.endsWith('"'))) {{
|
||||
return JSON.parse(raw);
|
||||
}}
|
||||
}} catch (error) {{
|
||||
return raw;
|
||||
}}
|
||||
return raw;
|
||||
}}
|
||||
|
||||
function sameValue_{widget_id}(left, right) {{
|
||||
return left === right || JSON.stringify(left) === JSON.stringify(right);
|
||||
}}
|
||||
|
||||
function parseTypedValue_{widget_id}(raw, typeName, meta) {{
|
||||
var numberValue;
|
||||
var parsed;
|
||||
|
||||
if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
|
||||
return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
|
||||
}}
|
||||
|
||||
if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
|
||||
return validateBinaryValue_{widget_id}(raw);
|
||||
}}
|
||||
|
||||
if (typeName === 'string') {{
|
||||
if (meta.pattern) {{
|
||||
try {{
|
||||
if (!(new RegExp(meta.pattern)).test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
|
||||
}}
|
||||
}} catch (error) {{}}
|
||||
}}
|
||||
return {{ ok: true, value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'integer') {{
|
||||
if (!/^-?\\d+$/.test(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be an integer' }};
|
||||
}}
|
||||
numberValue = Number(raw);
|
||||
if (meta.minimum !== undefined && numberValue < meta.minimum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
|
||||
}}
|
||||
if (meta.maximum !== undefined && numberValue > meta.maximum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
|
||||
}}
|
||||
return {{ ok: true, value: numberValue, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'number') {{
|
||||
if (raw === '' || isNaN(raw)) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be a number' }};
|
||||
}}
|
||||
numberValue = Number(raw);
|
||||
if (meta.minimum !== undefined && numberValue < meta.minimum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
|
||||
}}
|
||||
if (meta.maximum !== undefined && numberValue > meta.maximum) {{
|
||||
return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
|
||||
}}
|
||||
return {{ ok: true, value: numberValue, message: '' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'boolean') {{
|
||||
var lowered = raw.toLowerCase();
|
||||
if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
|
||||
if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
|
||||
return {{ ok: false, value: raw, message: 'Must be true or false' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'null') {{
|
||||
return raw === 'null'
|
||||
? {{ ok: true, value: null, message: '' }}
|
||||
: {{ ok: false, value: raw, message: 'Must be null' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'array' || typeName === 'object') {{
|
||||
try {{
|
||||
parsed = JSON.parse(raw);
|
||||
}} catch (error) {{
|
||||
return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
|
||||
}}
|
||||
|
||||
if (typeName === 'array' && Array.isArray(parsed)) {{
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
|
||||
return {{ ok: true, value: parsed, message: '' }};
|
||||
}}
|
||||
|
||||
return {{
|
||||
ok: false,
|
||||
value: parsed,
|
||||
message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
|
||||
}};
|
||||
}}
|
||||
|
||||
return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
|
||||
}}
|
||||
|
||||
function validateValueAgainstMeta_{widget_id}(raw, meta) {{
|
||||
if (!meta || raw === '') {{
|
||||
return {{ state: 'neutral', value: raw, message: '' }};
|
||||
}}
|
||||
|
||||
var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (!types.length) {{
|
||||
types = ['string'];
|
||||
}}
|
||||
|
||||
var error = 'Invalid value';
|
||||
for (var i = 0; i < types.length; i++) {{
|
||||
var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
|
||||
if (!candidate.ok) {{
|
||||
error = candidate.message || error;
|
||||
continue;
|
||||
}}
|
||||
if (enumValues.length && !enumValues.some(function(enumValue) {{
|
||||
return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
|
||||
}})) {{
|
||||
error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
|
||||
continue;
|
||||
}}
|
||||
return {{ state: 'valid', value: candidate.value, message: '' }};
|
||||
}}
|
||||
|
||||
return {{ state: 'invalid', value: raw, message: error }};
|
||||
}}
|
||||
|
||||
function ensureRowId_{widget_id}(row) {{
|
||||
if (!row.dataset.rowId) {{
|
||||
row.dataset.rowId = String(rowCounter_{widget_id}++);
|
||||
}}
|
||||
return row.dataset.rowId;
|
||||
}}
|
||||
|
||||
function setRowHelp_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var help = row.querySelector('.kv-help');
|
||||
if (!keyInput || !help) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var key = keyInput.value.trim();
|
||||
if (!key) {{
|
||||
help.textContent = '';
|
||||
return;
|
||||
}}
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (meta) {{
|
||||
var extra = isRegexConfigKey_{widget_id}(key)
|
||||
? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
|
||||
? ' Expected: JSON object with regex keys.'
|
||||
: ' Expected: valid regex.')
|
||||
: '';
|
||||
var example = getExampleInput_{widget_id}(key, meta);
|
||||
help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
|
||||
}} else {{
|
||||
help.textContent = 'Custom key';
|
||||
}}
|
||||
}}
|
||||
|
||||
function configureValueInput_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valueInput = row.querySelector('.kv-value');
|
||||
var datalist = row.querySelector('.kv-value-options');
|
||||
if (!keyInput || !valueInput || !datalist) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var rowId = ensureRowId_{widget_id}(row);
|
||||
datalist.id = '{widget_id}_value_options_' + rowId;
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
|
||||
var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
|
||||
var types = getTypes_{widget_id}(meta);
|
||||
if (!enumValues.length && types.includes('boolean')) {{
|
||||
enumValues = ['True', 'False'];
|
||||
}}
|
||||
if (enumValues.length) {{
|
||||
datalist.innerHTML = enumValues.map(function(enumValue) {{
|
||||
return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '"') + '"></option>';
|
||||
}}).join('');
|
||||
valueInput.setAttribute('list', datalist.id);
|
||||
}} else {{
|
||||
datalist.innerHTML = '';
|
||||
valueInput.removeAttribute('list');
|
||||
}}
|
||||
}}
|
||||
|
||||
function setValueValidationState_{widget_id}(input, state, message) {{
|
||||
if (!input) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
if (state === 'valid') {{
|
||||
input.style.borderColor = '#2da44e';
|
||||
input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
|
||||
input.style.backgroundColor = '#f6ffed';
|
||||
}} else if (state === 'invalid') {{
|
||||
input.style.borderColor = '#cf222e';
|
||||
input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
|
||||
input.style.backgroundColor = '#fff8f8';
|
||||
}} else {{
|
||||
input.style.borderColor = '#ccc';
|
||||
input.style.boxShadow = 'none';
|
||||
input.style.backgroundColor = '';
|
||||
}}
|
||||
input.title = message || '';
|
||||
}}
|
||||
|
||||
function applyValueValidation_{widget_id}(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valueInput = row.querySelector('.kv-value');
|
||||
if (!keyInput || !valueInput) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
var key = keyInput.value.trim();
|
||||
if (!key) {{
|
||||
setValueValidationState_{widget_id}(valueInput, 'neutral', '');
|
||||
return;
|
||||
}}
|
||||
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (!meta) {{
|
||||
setValueValidationState_{widget_id}(valueInput, 'neutral', '');
|
||||
return;
|
||||
}}
|
||||
|
||||
var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
|
||||
setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
|
||||
}}
|
||||
|
||||
function coerceValueForStorage_{widget_id}(key, raw) {{
|
||||
var meta = getMetaForKey_{widget_id}(key);
|
||||
if (!meta) {{
|
||||
return parseValue_{widget_id}(raw);
|
||||
}}
|
||||
|
||||
var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
|
||||
return validation.state === 'valid' ? validation.value : raw;
|
||||
}}
|
||||
|
||||
function initializeRows_{widget_id}() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
container.querySelectorAll('.key-value-row').forEach(function(row) {{
|
||||
ensureRowId_{widget_id}(row);
|
||||
configureValueInput_{widget_id}(row);
|
||||
setRowHelp_{widget_id}(row);
|
||||
applyValueValidation_{widget_id}(row);
|
||||
}});
|
||||
}}
|
||||
|
||||
function updateHiddenField_{widget_id}() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
var rows = container.querySelectorAll('.key-value-row');
|
||||
var result = {{}};
|
||||
rows.forEach(function(row) {{
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
var valInput = row.querySelector('.kv-value');
|
||||
if (keyInput && valInput && keyInput.value.trim()) {{
|
||||
var key = keyInput.value.trim();
|
||||
var val = valInput.value.trim();
|
||||
result[key] = coerceValueForStorage_{widget_id}(key, val);
|
||||
}}
|
||||
}});
|
||||
document.getElementById('{widget_id}').value = JSON.stringify(result);
|
||||
}}
|
||||
|
||||
window.addKeyValueRow_{widget_id} = function() {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
var newRow = document.createElement('div');
|
||||
newRow.className = 'key-value-row';
|
||||
newRow.style.cssText = 'margin-bottom: 6px;';
|
||||
newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
|
||||
'<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
|
||||
'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
|
||||
'<input type="text" class="kv-value" placeholder="value" ' +
|
||||
'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
|
||||
'<datalist class="kv-value-options"></datalist>' +
|
||||
'<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
|
||||
'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
|
||||
'</div>' +
|
||||
'<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
|
||||
container.appendChild(newRow);
|
||||
ensureRowId_{widget_id}(newRow);
|
||||
configureValueInput_{widget_id}(newRow);
|
||||
setRowHelp_{widget_id}(newRow);
|
||||
applyValueValidation_{widget_id}(newRow);
|
||||
updateHiddenField_{widget_id}();
|
||||
newRow.querySelector('.kv-key').focus();
|
||||
}};
|
||||
|
||||
window.removeKeyValueRow_{widget_id} = function(btn) {{
|
||||
var row = btn.closest('.key-value-row');
|
||||
row.remove();
|
||||
updateHiddenField_{widget_id}();
|
||||
}};
|
||||
|
||||
window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
|
||||
|
||||
function configRowForKey_{widget_id}(key) {{
|
||||
var container = document.getElementById('{widget_id}_rows');
|
||||
if (!container) {{
|
||||
return null;
|
||||
}}
|
||||
var match = null;
|
||||
container.querySelectorAll('.key-value-row').forEach(function(row) {{
|
||||
if (match) {{ return; }}
|
||||
var keyInput = row.querySelector('.kv-key');
|
||||
if (keyInput && keyInput.value.trim() === key) {{
|
||||
match = row;
|
||||
}}
|
||||
}});
|
||||
if (!match) {{
|
||||
window.addKeyValueRow_{widget_id}();
|
||||
var rows = container.querySelectorAll('.key-value-row');
|
||||
match = rows[rows.length - 1];
|
||||
var keyInput = match.querySelector('.kv-key');
|
||||
if (keyInput) {{
|
||||
keyInput.value = key;
|
||||
keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
}}
|
||||
}}
|
||||
return match;
|
||||
}}
|
||||
|
||||
function prefillConfigFromQuery_{widget_id}() {{
|
||||
var params = new URLSearchParams(window.location.search);
|
||||
var consumedConfigKeys = [];
|
||||
params.forEach(function(value, key) {{
|
||||
if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
|
||||
return;
|
||||
}}
|
||||
consumedConfigKeys.push(key);
|
||||
var match = configRowForKey_{widget_id}(key);
|
||||
var valueInput = match && match.querySelector('.kv-value');
|
||||
if (valueInput) {{
|
||||
valueInput.value = value;
|
||||
valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
|
||||
}}
|
||||
}});
|
||||
updateHiddenField_{widget_id}();
|
||||
if (consumedConfigKeys.length) {{
|
||||
var cleanUrl = new URL(window.location.href);
|
||||
consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
|
||||
window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
|
||||
}}
|
||||
}}
|
||||
|
||||
function focusConfigKeyFromHash_{widget_id}() {{
|
||||
// Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
|
||||
// to (or creates) the matching row in this editor. Used by
|
||||
// the setup wizard and live-config detail pages.
|
||||
var hash = (window.location.hash || '').replace(/^#/, '').trim();
|
||||
if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
|
||||
return;
|
||||
}}
|
||||
var match = configRowForKey_{widget_id}(hash);
|
||||
if (!match) {{
|
||||
return;
|
||||
}}
|
||||
match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
|
||||
var prevOutline = match.style.outline;
|
||||
match.style.outline = '2px solid #f59e0b';
|
||||
match.style.outlineOffset = '2px';
|
||||
match.style.transition = 'outline 1.2s ease-out';
|
||||
setTimeout(function() {{
|
||||
match.style.outline = prevOutline || 'none';
|
||||
}}, 1400);
|
||||
var valueInput = match.querySelector('.kv-value');
|
||||
if (valueInput) {{
|
||||
valueInput.focus();
|
||||
try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
|
||||
}}
|
||||
}}
|
||||
|
||||
// Initialize on load
|
||||
document.addEventListener('DOMContentLoaded', function() {{
|
||||
initializeRows_{widget_id}();
|
||||
prefillConfigFromQuery_{widget_id}();
|
||||
updateHiddenField_{widget_id}();
|
||||
focusConfigKeyFromHash_{widget_id}();
|
||||
}});
|
||||
// Also run immediately in case DOM is already ready
|
||||
if (document.readyState !== 'loading') {{
|
||||
initializeRows_{widget_id}();
|
||||
prefillConfigFromQuery_{widget_id}();
|
||||
updateHiddenField_{widget_id}();
|
||||
focusConfigKeyFromHash_{widget_id}();
|
||||
}}
|
||||
|
||||
window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
|
||||
|
||||
// Update on any input change
|
||||
var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
|
||||
|
||||
rowsEl_{widget_id}.addEventListener('input', function(event) {{
|
||||
var row = event.target.closest('.key-value-row');
|
||||
if (!row) {{
|
||||
return;
|
||||
}}
|
||||
|
||||
if (event.target.classList.contains('kv-key')) {{
|
||||
configureValueInput_{widget_id}(row);
|
||||
setRowHelp_{widget_id}(row);
|
||||
}}
|
||||
|
||||
if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
|
||||
applyValueValidation_{widget_id}(row);
|
||||
updateHiddenField_{widget_id}();
|
||||
}}
|
||||
}});
|
||||
}})();
|
||||
</script>
|
||||
</div>
|
||||
'''
|
||||
return mark_safe(html)
|
||||
|
||||
def _render_row(self, widget_id: str, key: str, value: str) -> str:
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
# Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
|
||||
# rendered write-only: the input is a password field with a placeholder
|
||||
# showing the value is set, but the raw value is NEVER sent to the browser.
|
||||
# When the user submits the form with the field left blank, the
|
||||
# ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
|
||||
# value so leaving it untouched is a no-op rather than a destructive clear.
|
||||
is_sensitive = is_sensitive_config_key(key)
|
||||
has_value = bool(value)
|
||||
if is_sensitive:
|
||||
input_type = "password"
|
||||
rendered_value = ""
|
||||
placeholder = (
|
||||
"•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
|
||||
)
|
||||
extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
|
||||
else:
|
||||
input_type = "text"
|
||||
rendered_value = self._escape(value)
|
||||
placeholder = "value"
|
||||
extra_attrs = ""
|
||||
return f'''
|
||||
<div class="key-value-row" style="margin-bottom: 6px;">
|
||||
<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
|
||||
<input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
|
||||
style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
|
||||
<input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
|
||||
style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
|
||||
<datalist class="kv-value-options"></datalist>
|
||||
<button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
|
||||
style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
|
||||
</div>
|
||||
<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
|
||||
</div>
|
||||
'''
|
||||
|
||||
def _escape(self, s: object) -> str:
|
||||
"""Escape HTML special chars in attribute values."""
|
||||
if not s:
|
||||
return ""
|
||||
return str(s).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)
|
||||
|
||||
def value_from_datadict(
|
||||
self,
|
||||
data: QueryDict | Mapping[str, object],
|
||||
files: object,
|
||||
name: str,
|
||||
) -> str:
|
||||
value = data.get(name, "{}")
|
||||
return value if isinstance(value, str) else "{}"
|
||||
|
||||
|
||||
class ConfigEditorMixin(admin.ModelAdmin):
|
||||
"""
|
||||
Mixin for admin classes with a config JSON field.
|
||||
|
||||
Provides a key-value editor widget with autocomplete for available config keys.
|
||||
"""
|
||||
|
||||
def formfield_for_dbfield(
|
||||
self,
|
||||
db_field: models.Field,
|
||||
request: HttpRequest,
|
||||
**kwargs: object,
|
||||
) -> forms.Field | None:
|
||||
"""Use KeyValueWidget for the config JSON field."""
|
||||
if db_field.name == "config":
|
||||
kwargs["widget"] = KeyValueWidget()
|
||||
return super().formfield_for_dbfield(db_field, request, **kwargs)
|
||||
|
||||
def save_model(self, request: HttpRequest, obj, form, change):
|
||||
"""Preserve write-only redacted credentials on save.
|
||||
|
||||
The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
|
||||
``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
|
||||
the real value never leaves the server. On submit, an empty value
|
||||
for a sensitive key that was previously set means "leave untouched",
|
||||
not "clear it." We honor that here by re-merging the stored value
|
||||
before the row is written. Explicitly removing the row in the UI
|
||||
still clears it (the key is gone from the submitted JSON, so there's
|
||||
nothing to merge over).
|
||||
"""
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
if change and obj.pk and obj.config is not None:
|
||||
try:
|
||||
stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
|
||||
except (AttributeError, DatabaseError, TypeError, ValueError):
|
||||
stored = {}
|
||||
if isinstance(stored, dict):
|
||||
new_config = dict(obj.config or {})
|
||||
for key, new_value in list(new_config.items()):
|
||||
if not is_sensitive_config_key(key):
|
||||
continue
|
||||
if new_value not in (None, "") and new_value != "********":
|
||||
continue
|
||||
if key in stored:
|
||||
new_config[key] = stored[key]
|
||||
obj.config = new_config
|
||||
super().save_model(request, obj, form, change)
|
||||
|
||||
|
||||
class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
|
||||
list_display = ("id", "created_at", "created_by")
|
||||
readonly_fields = ("id", "created_at", "modified_at")
|
||||
show_search_mode_selector = False
|
||||
change_form_template = "admin/archivebox_change_form.html"
|
||||
|
||||
def get_admin_toolbar_actions(self, request, obj):
|
||||
"""Return extra action button dicts for the shared change-form toolbar.
|
||||
|
||||
See ``templates/admin/includes/archivebox_toolbar.html`` for the
|
||||
accepted keys. Default: no extras (toolbar renders Save/History/Delete
|
||||
plus any ``django-object-actions`` change_actions).
|
||||
"""
|
||||
return []
|
||||
|
||||
def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
|
||||
context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
|
||||
return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
|
||||
|
||||
def get_default_search_mode(self) -> str:
|
||||
# The shared changelist template always asks every admin for a default
|
||||
# search mode, even when the search-mode toggle is hidden.
|
||||
return "meta"
|
||||
|
||||
def get_form(
|
||||
self,
|
||||
request: HttpRequest,
|
||||
obj: models.Model | None = None,
|
||||
change: bool = False,
|
||||
**kwargs: object,
|
||||
):
|
||||
form = super().get_form(request, obj, change=change, **kwargs)
|
||||
if "created_by" in form.base_fields:
|
||||
form.base_fields["created_by"].initial = request.user
|
||||
return form
|
||||
|
||||
def get_urls(self):
|
||||
"""Swap the per-object admin URLs from ``<path:object_id>`` to
|
||||
``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
|
||||
32-char hex form. The hyphenated form still resolves because the
|
||||
converter's regex accepts both — Django reverses through ``to_url``
|
||||
which always emits hex, so links in templates / changelists / inline
|
||||
formsets all canonicalize automatically.
|
||||
|
||||
Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
|
||||
won't match the converter's regex and fall back to the default
|
||||
``<path:object_id>`` patterns we still include after our swap.
|
||||
"""
|
||||
info = self.opts.app_label, self.opts.model_name
|
||||
object_routes = [
|
||||
path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
|
||||
path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
|
||||
path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
|
||||
]
|
||||
# Append after super().get_urls() so our patterns are the
|
||||
# *last-registered* ones with the canonical admin URL names — Django's
|
||||
# reverse() picks the later registration when names collide, which is
|
||||
# how we make ``reverse("admin:app_model_change", args=[obj.pk])``
|
||||
# emit the hex form. The original ``<path:object_id>`` routes stay in
|
||||
# place as a fallback for non-UUID PKs and for resolving inbound
|
||||
# hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
|
||||
return super().get_urls() + object_routes
|
||||
@@ -1,307 +0,0 @@
|
||||
"""Base models using UUIDv7 for all id fields."""
|
||||
|
||||
__package__ = "archivebox.base_models"
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
from archivebox.uuid_compat import CompactUUIDField, uuid7
|
||||
from pathlib import Path
|
||||
|
||||
from django.db import models
|
||||
from django.db.models import F
|
||||
from django.db import transaction
|
||||
from django.db.models.signals import pre_delete
|
||||
from django.utils import timezone
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.urls import reverse_lazy
|
||||
from django.conf import settings
|
||||
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
|
||||
|
||||
def normalize_config_json_values(config: Any) -> Any:
|
||||
if not isinstance(config, dict):
|
||||
return config
|
||||
|
||||
normalized = dict(config)
|
||||
for key, value in list(normalized.items()):
|
||||
if not isinstance(value, str) or len(value) < 2:
|
||||
continue
|
||||
if value[:1] != '"' or value[-1:] != '"':
|
||||
continue
|
||||
try:
|
||||
decoded = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(decoded, str):
|
||||
normalized[key] = decoded
|
||||
return normalized
|
||||
|
||||
|
||||
def get_or_create_system_user_pk(username="system"):
|
||||
User = get_user_model()
|
||||
# If there's exactly one superuser, use that for all system operations
|
||||
if User.objects.filter(is_superuser=True).count() == 1:
|
||||
return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
|
||||
# Otherwise get or create the system user
|
||||
user, _ = User.objects.get_or_create(
|
||||
username=username,
|
||||
defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
|
||||
)
|
||||
return user.pk
|
||||
|
||||
|
||||
class AutoDateTimeField(models.DateTimeField):
|
||||
"""DateTimeField that automatically updates on save (legacy compatibility)."""
|
||||
|
||||
def pre_save(self, model_instance, add):
|
||||
if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
|
||||
value = timezone.now()
|
||||
setattr(model_instance, self.attname, value)
|
||||
return value
|
||||
return super().pre_save(model_instance, add)
|
||||
|
||||
|
||||
class ModelWithUUID(models.Model):
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
modified_at = models.DateTimeField(auto_now=True)
|
||||
created_by = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
default=get_or_create_system_user_pk,
|
||||
null=False,
|
||||
db_index=True,
|
||||
)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"[{self.id}] {self.__class__.__name__}"
|
||||
|
||||
@property
|
||||
def admin_change_url(self) -> str:
|
||||
return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
|
||||
|
||||
@property
|
||||
def api_url(self) -> str:
|
||||
return str(reverse_lazy("api-1:get_any", args=[self.id]))
|
||||
|
||||
@property
|
||||
def api_docs_url(self) -> str:
|
||||
return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
|
||||
|
||||
|
||||
class ModelWithNotes(models.Model):
|
||||
"""Mixin for models with a notes field."""
|
||||
|
||||
notes = models.TextField(blank=True, null=False, default="")
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
|
||||
class ModelWithHealthStats(models.Model):
|
||||
"""Mixin for models with health tracking fields."""
|
||||
|
||||
num_uses_failed = models.PositiveIntegerField(default=0)
|
||||
num_uses_succeeded = models.PositiveIntegerField(default=0)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
@property
|
||||
def admin_change_url(self) -> str:
|
||||
return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
|
||||
|
||||
@property
|
||||
def health(self) -> int:
|
||||
total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
|
||||
return round((self.num_uses_succeeded / total) * 100)
|
||||
|
||||
def increment_health_stats(self, success: bool):
|
||||
"""Atomically increment success or failure counter using F() expression."""
|
||||
field = "num_uses_succeeded" if success else "num_uses_failed"
|
||||
type(self).objects.filter(pk=self.pk).update(
|
||||
**{
|
||||
field: F(field) + 1,
|
||||
"modified_at": timezone.now(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class ModelWithConfig(models.Model):
|
||||
"""Mixin for models with a JSON config field."""
|
||||
|
||||
config = models.JSONField(default=dict, null=True, blank=True, editable=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
normalized_config = normalize_config_json_values(self.config)
|
||||
if normalized_config != self.config:
|
||||
self.config = normalized_config
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if update_fields is not None:
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
|
||||
class ModelWithDeleteAfter(models.Model):
|
||||
delete_after_final_statuses: tuple[str, ...] = ()
|
||||
delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if self.delete_at is None:
|
||||
self.set_delete_at_from_config()
|
||||
if self.delete_at is not None and update_fields is not None:
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
def get_delete_after_config_value(self):
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
|
||||
|
||||
def set_delete_at_from_config(self, config_value=None) -> bool:
|
||||
if self.delete_at is not None:
|
||||
return False
|
||||
|
||||
from archivebox.config.common import parse_delete_after
|
||||
|
||||
duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
|
||||
if duration is None:
|
||||
return False
|
||||
|
||||
self.delete_at = (self.created_at or timezone.now()) + duration
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def missing_delete_at_candidates(cls):
|
||||
return cls.objects.none()
|
||||
|
||||
@classmethod
|
||||
def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
|
||||
if backfill_missing:
|
||||
missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
|
||||
for obj in missing_delete_at:
|
||||
if obj.set_delete_at_from_config():
|
||||
cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
|
||||
delete_at=obj.delete_at,
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
|
||||
# Keep the expiration sweep anchored on delete_at. Some large tables
|
||||
# have millions of final-status rows but almost no retained rows; adding
|
||||
# the status filter before SQLite has narrowed by delete_at can make the
|
||||
# runner scan the hot status index before it claims new work.
|
||||
due_pks = list(
|
||||
cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
|
||||
.order_by("delete_at", "pk")
|
||||
.values_list("pk", flat=True)[:batch_size],
|
||||
)
|
||||
if not due_pks:
|
||||
return 0
|
||||
|
||||
queryset = cls.objects.filter(pk__in=due_pks)
|
||||
if cls.delete_after_final_statuses:
|
||||
queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
|
||||
|
||||
count = 0
|
||||
expired = list(queryset.order_by("delete_at", "pk"))
|
||||
for obj in expired:
|
||||
obj.delete()
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
class ModelWithOutputDir(ModelWithUUID):
|
||||
class Meta(ModelWithUUID.Meta):
|
||||
abstract = True
|
||||
|
||||
_delete_signal_registered = False
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
super().save(*args, **kwargs)
|
||||
output_dir = Path(self.output_dir)
|
||||
# Avoid holding SQLite write transactions open across slow filesystem work.
|
||||
transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
|
||||
# Note: index.json is deprecated, models should use write_index_jsonl() for full data
|
||||
|
||||
@property
|
||||
def output_dir_parent(self) -> str:
|
||||
return f"{self._meta.model_name}s"
|
||||
|
||||
@property
|
||||
def output_dir_name(self) -> str:
|
||||
return str(self.id)
|
||||
|
||||
@property
|
||||
def output_dir_str(self) -> str:
|
||||
return f"{self.output_dir_parent}/{self.output_dir_name}"
|
||||
|
||||
@property
|
||||
def output_dir(self) -> Path:
|
||||
raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
|
||||
|
||||
def output_paths_for_delete(self) -> tuple[Path, ...]:
|
||||
return (Path(self.output_dir),)
|
||||
|
||||
@classmethod
|
||||
def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
|
||||
data_dir = CONSTANTS.DATA_DIR.resolve()
|
||||
safe_paths = []
|
||||
for raw_path in paths:
|
||||
path = Path(raw_path)
|
||||
is_safe = False
|
||||
for candidate in (path.absolute(), path.resolve()):
|
||||
try:
|
||||
candidate.relative_to(data_dir)
|
||||
is_safe = True
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
if not is_safe:
|
||||
raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
|
||||
safe_paths.append(path)
|
||||
return tuple(safe_paths)
|
||||
|
||||
@classmethod
|
||||
def delete_output_paths(cls, paths) -> None:
|
||||
for path in cls.validate_output_paths_for_delete(paths):
|
||||
if path.is_symlink() or path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
elif path.is_dir():
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
|
||||
"""Capture output paths before DB deletion and remove them after commit."""
|
||||
paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
|
||||
transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
|
||||
|
||||
@classmethod
|
||||
def register_delete_signal(cls) -> None:
|
||||
if cls._delete_signal_registered:
|
||||
return
|
||||
|
||||
def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
|
||||
if not isinstance(instance, ModelWithOutputDir):
|
||||
return
|
||||
instance.schedule_delete_cleanup(using=using)
|
||||
|
||||
pre_delete.connect(
|
||||
schedule_output_dir_cleanup,
|
||||
dispatch_uid="archivebox.output_dir_cleanup_on_delete",
|
||||
weak=False,
|
||||
)
|
||||
cls._delete_signal_registered = True
|
||||
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,3 @@
|
||||
from django.contrib import admin
|
||||
|
||||
# Register your models here.
|
||||
@@ -0,0 +1,82 @@
|
||||
import sys
|
||||
import inspect
|
||||
from typing import List, Dict, Any, Optional
|
||||
from pathlib import Path
|
||||
|
||||
import django
|
||||
from django.apps import AppConfig
|
||||
from django.core.checks import Tags, Warning, register
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3
|
||||
|
||||
from pydantic import (
|
||||
Field,
|
||||
SerializeAsAny,
|
||||
)
|
||||
|
||||
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
|
||||
|
||||
from plugantic.extractors import Extractor, ExtractorName
|
||||
from plugantic.plugins import Plugin
|
||||
from plugantic.configs import ConfigSet, ConfigSectionName
|
||||
from plugantic.replayers import Replayer
|
||||
|
||||
|
||||
class PythonBinary(Binary):
|
||||
name: BinName = 'python'
|
||||
|
||||
providers_supported: List[BinProvider] = [EnvProvider()]
|
||||
provider_overrides: Dict[str, Any] = {
|
||||
'env': {
|
||||
'subdeps': \
|
||||
lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
|
||||
'abspath': \
|
||||
lambda: sys.executable,
|
||||
'version': \
|
||||
lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
|
||||
},
|
||||
}
|
||||
|
||||
class SqliteBinary(Binary):
|
||||
name: BinName = 'sqlite'
|
||||
providers_supported: List[BinProvider] = [EnvProvider()]
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
'env': {
|
||||
'abspath': \
|
||||
lambda: inspect.getfile(sqlite3),
|
||||
'version': \
|
||||
lambda: sqlite3.version,
|
||||
},
|
||||
}
|
||||
|
||||
class DjangoBinary(Binary):
|
||||
name: BinName = 'django'
|
||||
|
||||
providers_supported: List[BinProvider] = [EnvProvider()]
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
'env': {
|
||||
'abspath': \
|
||||
lambda: inspect.getfile(django),
|
||||
'version': \
|
||||
lambda: django.VERSION[:3],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class BasicReplayer(Replayer):
|
||||
name: str = 'basic'
|
||||
|
||||
|
||||
class BasePlugin(Plugin):
|
||||
name: str = 'base'
|
||||
configs: List[SerializeAsAny[ConfigSet]] = []
|
||||
binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
|
||||
extractors: List[SerializeAsAny[Extractor]] = []
|
||||
replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
|
||||
|
||||
|
||||
PLUGINS = [BasePlugin()]
|
||||
|
||||
|
||||
class BaseConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
name = 'builtin_plugins.base'
|
||||
Loaded 100 of 1089 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user