Compare commits
26
Commits
v0.9.72rc93
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3fcda80297 | ||
|
|
e94a489daa | ||
|
|
df605452ba | ||
|
|
b472401d2f | ||
|
|
e4a1e400e8 | ||
|
|
2ce7074118 | ||
|
|
03bd48e399 | ||
|
|
edd1265b42 | ||
|
|
6bafaa99d7 | ||
|
|
53e3e9787b | ||
|
|
8e82de7f4a | ||
|
|
41dfc384aa | ||
|
|
66d88542f6 | ||
|
|
268b520394 | ||
|
|
3a3557ac72 | ||
|
|
b8ba9a64cd | ||
|
|
58a7955835 | ||
|
|
564af2fa3e | ||
|
|
ccaa630821 | ||
|
|
a3c53aca45 | ||
|
|
e0e3a7d09b | ||
|
|
c97cea180e | ||
|
|
5efc69d04b | ||
|
|
912713a732 | ||
|
|
f4c84ff6ac | ||
|
|
3ced9981f7 |
No files matched your search
@@ -25,7 +25,7 @@ def main() -> None:
|
||||
for path in archivebox_tests:
|
||||
test_path = path.relative_to(root).as_posix()
|
||||
duration = durations.get(test_path, 60)
|
||||
if test_path not in durations or duration >= 90 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
|
||||
if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
|
||||
batches.append([test_path])
|
||||
continue
|
||||
if batch and (len(batch) == 8 or batch_seconds + duration > 180):
|
||||
|
||||
@@ -451,7 +451,9 @@ jobs:
|
||||
"$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
|
||||
|
||||
"$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
|
||||
# The argument form above exercises the complete default pipeline.
|
||||
# Keep a real capture for stdin parsing without repeating every plugin.
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
|
||||
|
||||
archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
|
||||
[[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
|
||||
@@ -526,17 +526,27 @@ jobs:
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Verify published Docker images run
|
||||
- name: Verify published tags contain the exact tested images
|
||||
env:
|
||||
DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
|
||||
GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION='${{ needs.python-release.outputs.version }}'
|
||||
SHORT_SHA="${RELEASE_SHA::12}"
|
||||
for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
|
||||
"$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
|
||||
INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
|
||||
test "$INSTALLED_VERSION" = "$VERSION"
|
||||
done
|
||||
# Both architectures already passed CLI/version/capture acceptance by
|
||||
# digest. Verify promotion preserves those exact images and attestations
|
||||
# instead of downloading and running the same image again on one arch.
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
|
||||
EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
|
||||
"$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
|
||||
while IFS= read -r image; do
|
||||
[[ -n "$image" ]] || continue
|
||||
ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
|
||||
[[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
|
||||
echo "Verified exact tested images: $image"
|
||||
done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
|
||||
|
||||
- name: Record published image source
|
||||
shell: bash
|
||||
|
||||
@@ -120,6 +120,9 @@ jobs:
|
||||
if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
|
||||
workers="$(nproc)"
|
||||
cache_suffix="-workers-$workers"
|
||||
# Avoid the hosted image's cold Chromium executable page faults.
|
||||
# abxpkg owns installation and reuse in the existing runtime cache.
|
||||
echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
|
||||
fi
|
||||
{
|
||||
echo "CI_TEST_WORKERS=$workers"
|
||||
|
||||
@@ -170,6 +170,18 @@ class ResourceAdmission:
|
||||
used_bytes = 0
|
||||
for child in processes:
|
||||
try:
|
||||
if sys.platform.startswith("linux"):
|
||||
try:
|
||||
memory = child.memory_full_info()
|
||||
# Chrome's processes share substantial resident
|
||||
# memory. RSS charges those pages once per process
|
||||
# and can falsely serialize captures on an idle host.
|
||||
used_bytes += memory.pss + memory.swap
|
||||
continue
|
||||
except psutil.AccessDenied:
|
||||
# Restricted /proc access still gets conservative
|
||||
# accounting through the available RSS measurement.
|
||||
pass
|
||||
used_bytes += child.memory_info().rss
|
||||
except (psutil.Error, OSError):
|
||||
continue
|
||||
|
||||
@@ -52,6 +52,7 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from werkzeug.wrappers import Response
|
||||
from archivebox.tests.conftest import resolve_abxpkg_chrome_env
|
||||
@@ -167,7 +168,9 @@ const puppeteer = require(process.argv[1]);
|
||||
assert archived.returncode == 0, archived.stderr
|
||||
launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
|
||||
assert len(launch_commands) == 1
|
||||
assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
|
||||
launched_browser = Path(shlex.split(launch_commands[0].read_text().splitlines()[1])[0])
|
||||
# abxpkg's environment and provider projections can name the same binary.
|
||||
assert launched_browser.samefile(browser_env["CHROME_BINARY"])
|
||||
outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
|
||||
assert len(outputs) == 1
|
||||
html = outputs[0].read_text()
|
||||
|
||||
@@ -159,7 +159,7 @@ print(json.dumps({{
|
||||
assert not Path(retained["archiveresult_dir"]).exists()
|
||||
|
||||
|
||||
def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
def test_delete_after_real_add_page_and_rest_create_paths(client, recursive_test_site):
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
# Match init/server startup; visiting /add/ must not create persona records.
|
||||
@@ -172,6 +172,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
)
|
||||
|
||||
client.force_login(admin_user)
|
||||
ui_url, rest_url = recursive_test_site["child_urls"][:2]
|
||||
response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST)
|
||||
assert response.status_code == 200
|
||||
assert 'name="delete_after"' in response.content.decode()
|
||||
@@ -179,7 +180,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
response = client.post(
|
||||
reverse("add"),
|
||||
data={
|
||||
"url": "https://example.com/delete-after-ui",
|
||||
"url": ui_url,
|
||||
"tag": "retention-ui",
|
||||
"depth": "0",
|
||||
"max_urls": "1",
|
||||
@@ -203,13 +204,13 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
ui_crawl = Crawl.objects.get(urls__contains="https://example.com/delete-after-ui")
|
||||
ui_crawl = Crawl.objects.get(urls=ui_url)
|
||||
assert ui_crawl.config["DELETE_AFTER"] == "2h"
|
||||
assert ui_crawl.delete_at is not None
|
||||
from archivebox.services.runner import run_due_crawl
|
||||
|
||||
assert run_due_crawl(ui_crawl, lock_seconds=10)
|
||||
ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui")
|
||||
ui_snapshot = ui_crawl.snapshot_set.get(url=ui_url)
|
||||
assert ui_snapshot.delete_at is not None
|
||||
assert not ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").exists()
|
||||
|
||||
@@ -221,10 +222,12 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
"/api/v1/crawls/crawls",
|
||||
data=json.dumps(
|
||||
{
|
||||
"urls": ["https://example.com/delete-after-rest"],
|
||||
"urls": [rest_url],
|
||||
"max_depth": 0,
|
||||
"max_urls": 1,
|
||||
"config": {"DELETE_AFTER": "3h"},
|
||||
# Exercise retention through a real capture. The full plugin
|
||||
# pipeline has its own recursive-crawl integration coverage.
|
||||
"config": {"DELETE_AFTER": "3h", "PLUGINS": "wget,hashes"},
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
@@ -232,9 +235,16 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
rest_crawl = Crawl.objects.get(urls__contains="https://example.com/delete-after-rest")
|
||||
rest_crawl = Crawl.objects.get(urls=rest_url)
|
||||
assert rest_crawl.config["DELETE_AFTER"] == "3h"
|
||||
assert rest_crawl.delete_at is not None
|
||||
assert run_due_crawl(rest_crawl, lock_seconds=10)
|
||||
rest_snapshot = rest_crawl.snapshot_set.get(url="https://example.com/delete-after-rest")
|
||||
rest_snapshot = rest_crawl.snapshot_set.get(url=rest_url)
|
||||
assert rest_snapshot.delete_at is not None
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
for plugin in ("wget", "hashes"):
|
||||
result = rest_snapshot.archiveresult_set.get(plugin=plugin)
|
||||
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result.delete_at is not None
|
||||
assert result.output_size > 0
|
||||
@@ -215,6 +215,8 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
|
||||
step for step in docker_workflow["jobs"]["build"]["steps"] if step.get("name") == "Validate exact built image version and commit"
|
||||
)
|
||||
assert version_check["if"] == "inputs.full_tests"
|
||||
assert "archivebox version" in version_check["run"]
|
||||
assert "Missing exact commit marker" in version_check["run"]
|
||||
pip_workflow = yaml.safe_load(PIP_WORKFLOW.read_text())
|
||||
install_script = next(
|
||||
step["run"] for step in pip_workflow["jobs"]["build"]["steps"] if step.get("name") == "Release wheel import and CLI smoke"
|
||||
@@ -238,11 +240,20 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
|
||||
assert 'echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"' in tag_script
|
||||
assert 'echo "${DOCKERHUB_IMAGE}:${VERSION}"' in tag_script
|
||||
|
||||
docker_verify = next(step for step in docker_release["steps"] if step.get("name") == "Verify published Docker images run")
|
||||
docker_verify = next(
|
||||
step for step in docker_release["steps"] if step.get("name") == "Verify published tags contain the exact tested images"
|
||||
)
|
||||
verify_script = docker_verify["run"]
|
||||
assert '"${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"' in verify_script
|
||||
assert '"${GHCR_IMAGE}:sha-${SHORT_SHA}"' in verify_script
|
||||
assert '"${DOCKERHUB_IMAGE}:${VERSION}"' not in verify_script
|
||||
assert docker_verify["env"] == {
|
||||
"DOCKERHUB_TAGS": "${{ steps.docker_meta.outputs.dockerhub_tags }}",
|
||||
"GHCR_TAGS": "${{ steps.docker_meta.outputs.ghcr_tags }}",
|
||||
}
|
||||
assert '"${DOCKERHUB_IMAGE}@sha256:${digest}"' in verify_script
|
||||
assert 'imagetools create --dry-run "${REFS[@]}"' in verify_script
|
||||
assert 'imagetools inspect --raw "$image"' in verify_script
|
||||
assert 'sort == ["amd64", "arm64"]' in verify_script
|
||||
assert '[[ "$ACTUAL" == "$EXPECTED" ]]' in verify_script
|
||||
assert '"$DOCKERHUB_TAGS" "$GHCR_TAGS"' in verify_script
|
||||
|
||||
release_script = (REPO_ROOT / "bin" / "release.sh").read_text()
|
||||
assert "Never create GitHub Releases for automated rc builds" in release_script
|
||||
@@ -311,7 +322,7 @@ def test_stable_publication_requires_live_acceptance_before_upload():
|
||||
def test_staging_acceptance_uses_published_image_source_not_workflow_run_head():
|
||||
release = yaml.safe_load(RELEASE_WORKFLOW.read_text())
|
||||
docker_steps = release["jobs"]["docker-release"]["steps"]
|
||||
verified = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Verify published Docker images run")
|
||||
verified = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Verify published tags contain the exact tested images")
|
||||
source = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Record published image source")
|
||||
uploaded = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Upload published image source")
|
||||
assert verified < source < uploaded
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from archivebox.services.resource_admission import ResourceAdmission, disk_backed_swap_free_bytes
|
||||
|
||||
@@ -55,6 +56,59 @@ def test_memory_observation_tracks_a_real_child_process():
|
||||
assert child.wait(timeout=10) == 0
|
||||
|
||||
|
||||
def test_memory_observation_counts_shared_pages_once_on_linux():
|
||||
admission = ResourceAdmission()
|
||||
before = admission.memory_headroom()
|
||||
assert before is not None
|
||||
size = 128 * 1024 * 1024
|
||||
child = subprocess.Popen(
|
||||
[
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"-c",
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
allocation = bytearray(128 * 1024 * 1024)
|
||||
children = []
|
||||
for _ in range(2):
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
sys.stdin.read()
|
||||
os._exit(0)
|
||||
children.append(pid)
|
||||
print(len(allocation), flush=True)
|
||||
sys.stdin.read()
|
||||
for pid in children:
|
||||
os.waitpid(pid, 0)
|
||||
""",
|
||||
],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
assert child.stdout is not None
|
||||
assert child.stdout.readline().strip() == str(size)
|
||||
during = admission.memory_headroom()
|
||||
assert during is not None
|
||||
growth = during[0] - before[0]
|
||||
assert growth > size // 2
|
||||
finite_cgroup = any(
|
||||
(path / "memory.max").is_file() and (path / "memory.max").read_text().strip() != "max" for path in admission.cgroup_paths()
|
||||
)
|
||||
if sys.platform.startswith("linux") and not finite_cgroup:
|
||||
# These three real processes share one allocation. Linux PSS and
|
||||
# summing RSS differ by two copies. A finite cgroup instead measures
|
||||
# the entire container, including unrelated concurrent test workers.
|
||||
assert growth < size * 2, f"Shared allocation counted repeatedly: {growth}"
|
||||
finally:
|
||||
assert child.stdin is not None
|
||||
child.stdin.close()
|
||||
assert child.wait(timeout=10) == 0
|
||||
|
||||
|
||||
def test_disk_backed_swap_headroom_excludes_zram_but_keeps_backing_swap():
|
||||
swaps = """Filename Type Size Used Priority
|
||||
/dev/zram0 partition 1000 100 100
|
||||
|
||||
@@ -659,6 +659,14 @@ def test_plugin_grid_includes_all_visible_plugins_and_timestamps_last(admin_clie
|
||||
assert len(names) == len(expected)
|
||||
for name in expected:
|
||||
assert f'data-plugin-name="{name}"'.encode() in response.content
|
||||
# The same schema drives both editors, including newly installed plugins.
|
||||
# Check every plugin's category and rendered controls, not a fixed allowlist.
|
||||
catalog = get_plugin_catalog()
|
||||
for group in form.plugin_groups:
|
||||
for card in group["plugins"]:
|
||||
assert group["field_name"] == f"{catalog[card['name']].config.category}_plugins"
|
||||
for field in card["config_fields"]:
|
||||
assert f'name="{field["input_name"]}"'.encode() in response.content
|
||||
postprocessing = next(group for group in form.plugin_groups if group["field_name"] == "postprocessing_plugins")
|
||||
assert postprocessing["plugins"][-1]["name"] == "opentimestamps"
|
||||
assert b'name="plugin_config__opentimestamps__OPENTIMESTAMPS_TIMEOUT"' in response.content
|
||||
|
||||
@@ -156,7 +156,6 @@ if [[ "$HAS_USEFUL_SNAPSHOT" == "0" ]]; then
|
||||
--depth=0 \
|
||||
--overwrite \
|
||||
--tag=documentation,reference \
|
||||
--plugins=title,headers,wget,screenshot,pdf,dom,readability,htmltotext,hashes,dns \
|
||||
https://docs.sweeting.me/s/cookie-dilemma https://archivebox.io
|
||||
)
|
||||
fi
|
||||
@@ -267,11 +266,11 @@ VIEWS=(
|
||||
add_supplementary_template_capture() {
|
||||
local plugin_names="$1"
|
||||
local source_url="$2"
|
||||
local prepare_plugins="$3"
|
||||
local capture_plugins="${prepare_plugins:-$plugin_names}"
|
||||
local capture_log="$CAPTURE_ROOT/${plugin_names}-supplementary-capture.log"
|
||||
local supplementary_plugins=()
|
||||
IFS=',' read -r -a supplementary_plugins <<<"$plugin_names"
|
||||
# These are real all-plugin captures. The recipe only selects which output
|
||||
# to photograph, never which extractors run or which dependencies get saved.
|
||||
echo "[*] Capturing real $plugin_names source for template coverage"
|
||||
if ! (
|
||||
cd "$DATA_DIR"
|
||||
@@ -279,7 +278,6 @@ add_supplementary_template_capture() {
|
||||
--depth=0 \
|
||||
--overwrite \
|
||||
--tag=screenshot-gallery \
|
||||
--plugins="$capture_plugins" \
|
||||
"$source_url"
|
||||
) >"$capture_log" 2>&1; then
|
||||
echo "[!] Supplementary $plugin_names capture failed" >&2
|
||||
@@ -302,20 +300,6 @@ add_supplementary_template_capture() {
|
||||
|
||||
local snapshot_id snapshot_view_url snapshot_view_path
|
||||
IFS=$'\t' read -r snapshot_id snapshot_view_url snapshot_view_path <<<"$snapshot_record"
|
||||
if [[ -n "$prepare_plugins" ]]; then
|
||||
if ! (
|
||||
cd "$DATA_DIR"
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox extract \
|
||||
--plugins="$plugin_names" \
|
||||
"$snapshot_id"
|
||||
) >>"$capture_log" 2>&1; then
|
||||
echo "[!] $plugin_names extraction failed for the prepared snapshot" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
stop_background_runner
|
||||
fi
|
||||
|
||||
local discovery_report="$CAPTURE_ROOT/${plugin_names}-output-discovery.json"
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
|
||||
@@ -360,6 +344,12 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
|
||||
[[ "$capture_mode" != wait-text:* ]] || wait_for_text="${capture_mode#wait-text:}"
|
||||
preview_wait_for_text=""
|
||||
[[ "$capture_mode" != wait-preview-text:* ]] || preview_wait_for_text="${capture_mode#wait-preview-text:}"
|
||||
config_selector=""
|
||||
config_click_selector=""
|
||||
if [[ "$capture_mode" == plugin-config:* ]]; then
|
||||
config_selector="[data-plugin-name='${capture_mode#plugin-config:}']"
|
||||
config_click_selector="$config_selector .plugin-config-marker"
|
||||
fi
|
||||
view_timing_report=""
|
||||
while IFS='|' read -r profile _viewport_width _viewport_height; do
|
||||
filename="$slug-$profile.png"
|
||||
@@ -483,6 +473,8 @@ PY
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
SCREENSHOT_VARIANTS_JSON="$output_variants" \
|
||||
SCREENSHOT_COLLAPSE_FILTERS=1 \
|
||||
SCREENSHOT_CLICK_SELECTOR="$config_click_selector" \
|
||||
SCREENSHOT_SCROLL_SELECTOR="$config_selector" \
|
||||
SCREENSHOT_SNAPSHOT_HEADER="$snapshot_header" \
|
||||
SCREENSHOT_WAIT_FOR_TEXT="$wait_for_text" \
|
||||
SCREENSHOT_PREVIEW_WAIT_FOR_TEXT="$preview_wait_for_text" \
|
||||
@@ -655,11 +647,18 @@ PY
|
||||
[[ -z "$plugin_name" ]] && continue
|
||||
VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$output_path|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|$output_capture_mode")
|
||||
done <<<"$SNAPSHOT_OUTPUT_PLUGINS"
|
||||
while IFS='|' read -r example_plugins example_url example_prepare; do
|
||||
while IFS='|' read -r example_plugins example_url; do
|
||||
[[ -z "$example_plugins" ]] && continue
|
||||
add_supplementary_template_capture "$example_plugins" "$example_url" "$example_prepare"
|
||||
add_supplementary_template_capture "$example_plugins" "$example_url"
|
||||
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; plan=json.load(open(sys.argv[1])); print("\n".join("|".join((",".join(example["plugins"]), example["url"], ",".join(example["prepare_plugins"]))) for example in plan["examples"]))' \
|
||||
'import json,sys; plan=json.load(open(sys.argv[1])); print("\n".join("|".join((",".join(example["plugins"]), example["url"])) for example in plan["examples"]))' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN")
|
||||
|
||||
while IFS='|' read -r config_name config_plugin; do
|
||||
[[ -z "$config_plugin" ]] && continue
|
||||
VIEWS+=("$config_name|$ADMIN_BASE_URL/add/|/add/|archivebox/templates/plugins/plugin_config_grid.html|plugin-config:$config_plugin")
|
||||
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; print("\n".join(view["name"] + "|" + view["plugin"] for view in json.load(open(sys.argv[1]))["config_views"]))' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN")
|
||||
|
||||
VIEWS+=("Snapshot View (header collapsed)|$LIVE_SNAPSHOT_VIEW_URL|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|snapshot-collapsed")
|
||||
|
||||
@@ -67,6 +67,7 @@ def plugin_screenshot_plan() -> dict:
|
||||
required = []
|
||||
examples = {}
|
||||
overrides = {}
|
||||
config_views = []
|
||||
enabled_plugins = []
|
||||
for plugin_dir in sorted(get_plugins_dir().iterdir()):
|
||||
config_path = plugin_dir / "config.json"
|
||||
@@ -81,6 +82,8 @@ def plugin_screenshot_plan() -> dict:
|
||||
enabled = True
|
||||
if recipe.get("view"):
|
||||
required.append(recipe["view"])
|
||||
if recipe.get("config"):
|
||||
config_views.append({"name": recipe["view"], "plugin": plugin})
|
||||
elif enabled and (
|
||||
recipe.get("url")
|
||||
or (plugin_dir / "templates/full.html").is_file()
|
||||
@@ -89,10 +92,16 @@ def plugin_screenshot_plan() -> dict:
|
||||
required.append(f"Snapshot View ({plugin})")
|
||||
if recipe.get("url"):
|
||||
overrides[plugin] = recipe
|
||||
key = (recipe["url"], tuple(recipe.get("prepare_plugins", [])))
|
||||
example = examples.setdefault(key, {"url": key[0], "prepare_plugins": list(key[1]), "plugins": []})
|
||||
key = recipe["url"]
|
||||
example = examples.setdefault(key, {"url": key, "plugins": []})
|
||||
example["plugins"].append(plugin)
|
||||
return {"required": required, "examples": list(examples.values()), "overrides": overrides, "enabled": enabled_plugins}
|
||||
return {
|
||||
"required": required,
|
||||
"examples": list(examples.values()),
|
||||
"overrides": overrides,
|
||||
"enabled": enabled_plugins,
|
||||
"config_views": config_views,
|
||||
}
|
||||
|
||||
|
||||
def build_provenance() -> dict[str, str]:
|
||||
|
||||
@@ -374,12 +374,7 @@ async function main() {
|
||||
if (process.env.SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR, { timeout: 45000 });
|
||||
}
|
||||
if (process.env.SCREENSHOT_SCROLL_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_SCROLL_SELECTOR, { timeout: 45000 }).catch(() => {});
|
||||
await page.evaluate((selector) => {
|
||||
document.querySelector(selector)?.scrollIntoView({ block: 'start', inline: 'nearest' });
|
||||
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
|
||||
}
|
||||
|
||||
|
||||
if (requireLiveProgress) {
|
||||
await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
|
||||
@@ -446,6 +441,14 @@ async function main() {
|
||||
const screenshotPaths = [];
|
||||
if (requireLiveProgress) checks.liveProgressCaptures = [];
|
||||
const capture = async (screenshotPath) => {
|
||||
// Resizing can move the target to a different row/column. Re-anchor each
|
||||
// gallery variant so it photographs the requested controls at every width.
|
||||
if (process.env.SCREENSHOT_SCROLL_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_SCROLL_SELECTOR, { timeout: 45000 }).catch(() => {});
|
||||
await page.evaluate((selector) => {
|
||||
document.querySelector(selector)?.scrollIntoView({ block: 'start', inline: 'nearest' });
|
||||
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
|
||||
}
|
||||
await waitForVisibleCardPreviews(page, requireLiveProgress ? Math.max(1, liveProgressDeadline - Date.now()) : 45000);
|
||||
if (requireLiveProgress) {
|
||||
const liveProgress = await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
|
||||
|
||||
+1
-1
@@ -56,7 +56,7 @@ View: [`/admin/`](http://admin.archivebox.localhost:5797/admin/) · [View code](
|
||||
|
||||
## AI agent
|
||||
|
||||
View: [`/admin/agent/`](http://admin.archivebox.localhost:5797/admin/agent/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/abx_plugins/plugins/opencode/views.py)
|
||||
View: [`/admin/agent/`](http://admin.archivebox.localhost:5797/admin/agent/) · [View code](https://github.com/ArchiveBox/abx-plugins/blob/main/abx_plugins/plugins/opencode/archivebox/views.py)
|
||||
|
||||
<table><thead><tr>
|
||||
<th>Desktop</th><th>Tablet</th><th>Mobile</th>
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.72rc93",
|
||||
"version": "0.9.72rc105",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.72rc93"
|
||||
version = "0.9.72rc105"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@@ -79,8 +79,8 @@ dependencies = [
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.74", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abxpkg==1.13.33", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.13.146", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.13.121", # shared ArchiveBox downloader package
|
||||
"abx-plugins[opencode]==1.13.161", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.13.126", # shared ArchiveBox downloader package
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
"browser-cookie3>=0.20.1",
|
||||
@@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
|
||||
|
||||
|
||||
[tool.bumpver]
|
||||
current_version = "v0.9.72rc93"
|
||||
current_version = "v0.9.72rc105"
|
||||
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
|
||||
commit_message = "bump version {old_version} -> {new_version}"
|
||||
tag_message = "{new_version}"
|
||||
|
||||
@@ -13,18 +13,18 @@ supported-markers = [
|
||||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-09-30T01:37:52.816536614Z"
|
||||
exclude-newer = "2026-10-01T07:11:37.418401316Z"
|
||||
exclude-newer-span = "P5D"
|
||||
|
||||
[options.exclude-newer-package]
|
||||
abxbus = { timestamp = "2026-10-05T01:37:51.816553339Z", span = "PT1S" }
|
||||
abxbus = { timestamp = "2026-10-06T07:11:36.418414Z", span = "PT1S" }
|
||||
abx-plugins = "2100-01-01T00:00:00Z"
|
||||
abx-dl = "2100-01-01T00:00:00Z"
|
||||
abxpkg = "2100-01-01T00:00:00Z"
|
||||
|
||||
[[package]]
|
||||
name = "abx-dl"
|
||||
version = "1.13.121"
|
||||
version = "1.13.126"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -38,14 +38,14 @@ dependencies = [
|
||||
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0c/c6/001320cf547b6680168dc71502ea059b689f4039db93b56d78d299317953/abx_dl-1.13.121.tar.gz", hash = "sha256:2d180daa473051b32b313eba16ff58aeff0feaf5ca3f5a5c694a59cdc6dcdd87", size = 98182, upload-time = "2026-10-05T01:36:38.625Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b7/80/f30434e2e081d33c1b46233e9b284c98cff9eb8895b5fd546204953849b7/abx_dl-1.13.126.tar.gz", hash = "sha256:bdd29b626fba7a35f39865e5769b3b6ea1145a9953f2cf3aa14379eae192c248", size = 98185, upload-time = "2026-10-06T07:10:19.966Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/25/e9/036a9cd48fb9a8ac3f306d2edbadb15e71907d9211d17070ee4eef4e09c4/abx_dl-1.13.121-py3-none-any.whl", hash = "sha256:1cc3acb9c25ee6ad7614d342490ab96b9b874de7f9326b98b782dfa3a9a99f16", size = 103547, upload-time = "2026-10-05T01:36:37.099Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/3e/f666f8294a31a8acd9189d790585f453e0ccecbb0a9ed9c9163c28ba6a10/abx_dl-1.13.126-py3-none-any.whl", hash = "sha256:7eb1e7295b4d11c5225d6fcd100e51069dc1022818e0c8f50abdf7d24ba4434d", size = 103547, upload-time = "2026-10-06T07:10:21.456Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "abx-plugins"
|
||||
version = "1.13.146"
|
||||
version = "1.13.161"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -54,9 +54,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4d/14/ffaf2c2912a0130123ae498e6cb8e0ad2670767d072c783a3c9628ae54ac/abx_plugins-1.13.146.tar.gz", hash = "sha256:a04cf3b2ea0a8338f4d01e3cbc056c4b249657bcdf6e2f2a22d1d8de66817863", size = 504056, upload-time = "2026-10-05T01:29:08.662Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3a/9d/9e442f90b30eb34fe4be44444d399685f453d3e452961018f7f91eaca173/abx_plugins-1.13.161.tar.gz", hash = "sha256:eb80af30d259716d4c713c708197328ca7fe0ad0b7fe05c27ef3bcb8112913e3", size = 509069, upload-time = "2026-10-06T07:02:15.305Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/fc/050aec5f6ffdb6ed19c651611b76975e5df3f9e43cab0c67ad0260b5e031/abx_plugins-1.13.146-py3-none-any.whl", hash = "sha256:c43a20dad8940d46e212ac6095ed52c30750cae82ec8216ffbafe01d96195743", size = 733172, upload-time = "2026-10-05T01:29:06.678Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/2b/da4c917bd49e92ba059b5a93b21f49499b5d385c15db23e2bc4ced834eb9/abx_plugins-1.13.161-py3-none-any.whl", hash = "sha256:1a88ec1580fa522718cf77049fd841441bd041cea63d8e38d4d1947e8b5b7b6a", size = 739583, upload-time = "2026-10-06T07:02:12.819Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -127,7 +127,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "archivebox"
|
||||
version = "0.9.72rc93"
|
||||
version = "0.9.72rc105"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -228,8 +228,8 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "abx-dl", specifier = "==1.13.121" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.146" },
|
||||
{ name = "abx-dl", specifier = "==1.13.126" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.161" },
|
||||
{ name = "abxbus", specifier = "==2.5.74" },
|
||||
{ name = "abxpkg", specifier = "==1.13.33" },
|
||||
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
|
||||
|
||||
Reference in new issue
Block a user