Merge branch 'dev' into plugins-browsertrix

Signed-off-by: longzai <437172242@qq.com>

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/"
+    target-branch: "dev"
+    schedule:
+      interval: "weekly"

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+  schedule:
+    - cron: '33 17 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker.yml

@@ -11,7 +11,7 @@ on:
 
 env:
   DOCKER_IMAGE: archivebox-ci
-      
+
 jobs:
   buildx:
     runs-on: ubuntu-latest
@@ -24,21 +24,21 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-      
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
-      
+          platforms: linux/amd64,linux/arm64
+
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
-      
+
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
-      
+
       - name: Cache Docker layers
         uses: actions/cache@v3
         with:
@@ -51,21 +51,27 @@ jobs:
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
-           username: ${{ secrets.DOCKER_USERNAME }}
-           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Collect Docker tags
+        # https://github.com/docker/metadata-action
         id: docker_meta
         uses: docker/metadata-action@v5
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
           tags: |
+              # :stable
               type=ref,event=branch
+              # :0.7.3
               type=semver,pattern={{version}}
+              # :0.7
               type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
               type=sha
-              type=raw,value=latest,enable={{is_default_branch}}
-      
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+
       - name: Build and push
         id: docker_build
         uses: docker/build-push-action@v5
@@ -77,7 +83,7 @@ jobs:
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
@@ -88,7 +94,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           repository: archivebox/archivebox
-       
+
       # This ugly bit is necessary if you don't want your cache to grow forever
       # until it hits GitHub's limit of 5GB.
       # Temp fix

.gitignore

@@ -12,6 +12,10 @@ venv/
 .docker-venv/
 node_modules/
 
+# Ignore dev lockfiles (should always be built fresh)
+requirements-dev.txt
+pdm.dev.lock
+
 # Packaging artifacts
 .pdm-python
 .pdm-build
@@ -22,11 +26,9 @@ dist/
 
 # Data folders
 data/
-data1/
-data2/
-data3/
 data*/
 output/
 
 # vim
 *.sw?
+.vscode/

Dockerfile

@@ -15,15 +15,29 @@
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
 
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
 FROM python:3.11-slim-bookworm
+# Uses Debian 12 w/ faster-updating apt-lists added below: https://packages.debian.org/bookworm-backports/
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one personal internet archiving container" \
+    description="All-in-one self-hosted internet archiving solution" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
-
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
+    org.opencontainers.image.title="ArchiveBox" \
+    org.opencontainers.image.vendor="ArchiveBox" \
+    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
+    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'
+    
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
@@ -113,9 +127,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep \
+        zlib1g-dev dumb-init gosu cron unzip grep ncat \
         # 3. frivolous CLI helpers to make debugging failed archiving easier
-        # nano iputils-ping dnsutils htop procps jq yq
+        # nano iputils-ping dnsutils htop procps jq yq \
     && rm -rf /var/lib/apt/lists/*
 
 ######### Language Environments ####################################

README.md

@@ -124,8 +124,8 @@ curl -fsSL 'https://get.archivebox.io' | sh
 
 ## Key Features
 
-- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), doesn't require signing up online, stores all data locally
-- [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies) 
+- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
+- [**Powerful command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
@@ -669,7 +669,7 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add < ~/Downloads/firefox_bookmarks_export.html
+archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
@@ -865,6 +865,7 @@ Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <
 
 <h4>Learn More</h4>
 <ul>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>

archivebox/api/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class APIConfig(AppConfig):
+    name = 'api'

archivebox/api/archive.py

@@ -0,0 +1,184 @@
+# archivebox_api.py
+from typing import List, Optional
+from enum import Enum
+from pydantic import BaseModel
+from ninja import Router
+from main import (
+    add,
+    remove,
+    update,
+    list_all,
+    ONLY_NEW,
+)  # Assuming these functions are defined in main.py
+
+
+# Schemas
+
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'
+
+
+class AddURLSchema(BaseModel):
+    urls: List[str]
+    tag: str = ""
+    depth: int = 0
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
+    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+
+class RemoveURLSchema(BaseModel):
+    yes: bool = False
+    delete: bool = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    filter_type: str = "exact"
+    filter_patterns: Optional[List[str]] = None
+
+
+class UpdateSchema(BaseModel):
+    resume: Optional[float] = None
+    only_new: Optional[bool] = None
+    index_only: Optional[bool] = False
+    overwrite: Optional[bool] = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    status: Optional[StatusChoices] = None
+    filter_type: Optional[str] = 'exact'
+    filter_patterns: Optional[List[str]] = None
+    extractors: Optional[str] = ""
+
+
+class ListAllSchema(BaseModel):
+    filter_patterns: Optional[List[str]] = None
+    filter_type: str = 'exact'
+    status: Optional[StatusChoices] = None
+    after: Optional[float] = None
+    before: Optional[float] = None
+    sort: Optional[str] = None
+    csv: Optional[str] = None
+    json: bool = False
+    html: bool = False
+    with_headers: bool = False
+
+
+# API Router
+router = Router()
+
+
+@router.post("/add", response={200: dict})
+def api_add(request, payload: AddURLSchema):
+    try:
+        result = add(
+            urls=payload.urls,
+            tag=payload.tag,
+            depth=payload.depth,
+            update=payload.update,
+            update_all=payload.update_all,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            init=payload.init,
+            extractors=payload.extractors,
+            parser=payload.parser,
+        )
+        # Currently the add function returns a list of ALL items in the DB, ideally only return new items
+        return {
+            "status": "success",
+            "message": "URLs added successfully.",
+            "result": str(result),
+        }
+    except Exception as e:
+        # Handle exceptions raised by the add function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/remove", response={200: dict})
+def api_remove(request, payload: RemoveURLSchema):
+    try:
+        result = remove(
+            yes=payload.yes,
+            delete=payload.delete,
+            before=payload.before,
+            after=payload.after,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+        )
+        return {
+            "status": "success",
+            "message": "URLs removed successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the remove function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/update", response={200: dict})
+def api_update(request, payload: UpdateSchema):
+    try:
+        result = update(
+            resume=payload.resume,
+            only_new=payload.only_new,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            before=payload.before,
+            after=payload.after,
+            status=payload.status,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+            extractors=payload.extractors,
+        )
+        return {
+            "status": "success",
+            "message": "Archive updated successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the update function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/list_all", response={200: dict})
+def api_list_all(request, payload: ListAllSchema):
+    try:
+        result = list_all(
+            filter_patterns=payload.filter_patterns,
+            filter_type=payload.filter_type,
+            status=payload.status,
+            after=payload.after,
+            before=payload.before,
+            sort=payload.sort,
+            csv=payload.csv,
+            json=payload.json,
+            html=payload.html,
+            with_headers=payload.with_headers,
+        )
+        # TODO: This is kind of bad, make the format a choice field
+        if payload.json:
+            return {"status": "success", "format": "json", "data": result}
+        elif payload.html:
+            return {"status": "success", "format": "html", "data": result}
+        elif payload.csv:
+            return {"status": "success", "format": "csv", "data": result}
+        else:
+            return {
+                "status": "success",
+                "message": "List generated successfully.",
+                "data": result,
+            }
+    except Exception as e:
+        # Handle exceptions raised by the list_all function or during processing
+        return {"status": "error", "message": str(e)}

archivebox/api/auth.py

@@ -0,0 +1,48 @@
+from django.contrib.auth import authenticate
+from ninja import Form, Router, Schema
+from ninja.security import HttpBearer
+
+from api.models import Token
+
+router = Router()
+
+
+class GlobalAuth(HttpBearer):
+    def authenticate(self, request, token):
+        try:
+            return Token.objects.get(token=token).user
+        except Token.DoesNotExist:
+            pass
+
+
+class AuthSchema(Schema):
+    email: str
+    password: str
+
+
+@router.post("/authenticate", auth=None)  # overriding global auth
+def get_token(request, auth_data: AuthSchema):
+    user = authenticate(username=auth_data.email, password=auth_data.password)
+    if user:
+        # Assuming a user can have multiple tokens and you want to create a new one every time
+        new_token = Token.objects.create(user=user)
+        return {"token": new_token.token, "expires": new_token.expiry_as_iso8601}
+    else:
+        return {"error": "Invalid credentials"}
+
+
+class TokenValidationSchema(Schema):
+    token: str
+
+
+@router.post("/validate_token", auth=None) # No authentication required for this endpoint
+def validate_token(request, token_data: TokenValidationSchema):
+    try:
+        # Attempt to authenticate using the provided token
+        user = GlobalAuth().authenticate(request, token_data.token)
+        if user:
+            return {"status": "valid"}
+        else:
+            return {"status": "invalid"}
+    except Token.DoesNotExist:
+        return {"status": "invalid"}

archivebox/api/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.14 on 2024-04-09 18:52
+
+import api.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Token',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('token', models.CharField(default=auth.models.hex_uuid, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expiry', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tokens', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

archivebox/api/models.py

@@ -0,0 +1,30 @@
+import uuid
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+def hex_uuid():
+    return uuid.uuid4().hex
+
+
+class Token(models.Model):
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tokens"
+    )
+    token = models.CharField(max_length=32, default=hex_uuid, unique=True)
+    created = models.DateTimeField(auto_now_add=True)
+    expiry = models.DateTimeField(null=True, blank=True)
+
+    @property
+    def expiry_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = (
+            self.expiry if self.expiry else timezone.now() + timedelta(days=365 * 100)
+        )
+        return expiry_date.isoformat()
+
+    def __str__(self):
+        return self.token

archivebox/api/tests.py

@@ -0,0 +1,27 @@
+from django.test import TestCase
+from ninja.testing import TestClient
+from archivebox.api.archive import router as archive_router
+
+class ArchiveBoxAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(archive_router)
+
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "test"})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue("success" in response.json()["status"])

archivebox/config.py

@@ -38,7 +38,7 @@ from hashlib import md5
 from pathlib import Path
 from datetime import datetime, timezone
 from typing import Optional, Type, Tuple, Dict, Union, List
-from subprocess import run, PIPE, DEVNULL
+from subprocess import run, PIPE, STDOUT, DEVNULL
 from configparser import ConfigParser
 from collections import defaultdict
 import importlib.metadata
@@ -854,7 +854,7 @@ def hint(text: Union[Tuple[str, ...], List[str], str], prefix='    ', config: Op
 
 
 # Dependency Metadata Helpers
-def bin_version(binary: Optional[str]) -> Optional[str]:
+def bin_version(binary: Optional[str], cmd=None) -> Optional[str]:
     """check the presence and return valid version line of a specified binary"""
 
     abspath = bin_path(binary)
@@ -863,11 +863,21 @@ def bin_version(binary: Optional[str]) -> Optional[str]:
 
     try:
         bin_env = os.environ | {'LANG': 'C'}
-        version_str = run([abspath, "--version"], stdout=PIPE, env=bin_env).stdout.strip().decode()
+        is_cmd_str = cmd and isinstance(cmd, str)
+        version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT, env=bin_env).stdout.strip().decode()
         if not version_str:
-            version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
-        # take first 3 columns of first line of version info
-        return ' '.join(version_str.split('\n')[0].strip().split()[:3])
+            version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT).stdout.strip().decode()
+        
+        version_ptn = re.compile(r"\d+?\.\d+?\.?\d*", re.MULTILINE)
+        try:
+            version_nums = version_ptn.findall(version_str.split('\n')[0])[0]
+            if version_nums:
+                return version_nums
+            else:
+                raise IndexError
+        except IndexError:
+            # take first 3 columns of first line of version info
+            return ' '.join(version_str.split('\n')[0].strip().split()[:3])
     except OSError:
         pass
         # stderr(f'[X] Unable to find working version of dependency: {binary}', color='red')

archivebox/config_stubs.py

@@ -9,6 +9,8 @@ SimpleConfigValueDict = Dict[str, SimpleConfigValue]
 SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
 ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
 
+SHArgs = List[str]   # shell command args list e.g. ["--something=1", "--someotherarg"]
+
 
 class BaseConfig(TypedDict):
     pass
@@ -16,10 +18,10 @@ class BaseConfig(TypedDict):
 class ConfigDict(BaseConfig, total=False):
     """
     # Regenerate by pasting this quine into `archivebox shell` 🥚
-    from archivebox.config import ConfigDict, CONFIG_DEFAULTS
+    from archivebox.config import ConfigDict, CONFIG_SCHEMA
     print('class ConfigDict(BaseConfig, total=False):')
     print('    ' + '"'*3 + ConfigDict.__doc__ + '"'*3)
-    for section, configs in CONFIG_DEFAULTS.items():
+    for section, configs in CONFIG_SCHEMA.items():
         for key, attrs in configs.items():
             Type, default = attrs['type'], attrs['default']
             if default is None:
@@ -32,16 +34,23 @@ class ConfigDict(BaseConfig, total=False):
     USE_COLOR: bool
     SHOW_PROGRESS: bool
     IN_DOCKER: bool
+    IN_QEMU: bool
+    PUID: int
+    PGID: int
 
-    PACKAGE_DIR: Path
-    OUTPUT_DIR: Path
-    CONFIG_FILE: Path
+    OUTPUT_DIR: Optional[str]
+    CONFIG_FILE: Optional[str]
     ONLY_NEW: bool
     TIMEOUT: int
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
     RESTRICT_FILE_NAMES: str
     URL_DENYLIST: str
+    URL_ALLOWLIST: Optional[str]
+    ADMIN_USERNAME: Optional[str]
+    ADMIN_PASSWORD: Optional[str]
+    ENFORCE_ATOMIC_WRITES: bool
+    TAG_SEPARATOR_PATTERN: str
 
     SECRET_KEY: Optional[str]
     BIND_ADDR: str
@@ -49,7 +58,27 @@ class ConfigDict(BaseConfig, total=False):
     DEBUG: bool
     PUBLIC_INDEX: bool
     PUBLIC_SNAPSHOTS: bool
+    PUBLIC_ADD_VIEW: bool
     FOOTER_INFO: str
+    SNAPSHOTS_PER_PAGE: int
+    CUSTOM_TEMPLATES_DIR: Optional[str]
+    TIME_ZONE: str
+    TIMEZONE: str
+    REVERSE_PROXY_USER_HEADER: str
+    REVERSE_PROXY_WHITELIST: str
+    LOGOUT_REDIRECT_URL: str
+    PREVIEW_ORIGINALS: bool
+    LDAP: bool
+    LDAP_SERVER_URI: Optional[str]
+    LDAP_BIND_DN: Optional[str]
+    LDAP_BIND_PASSWORD: Optional[str]
+    LDAP_USER_BASE: Optional[str]
+    LDAP_USER_FILTER: Optional[str]
+    LDAP_USERNAME_ATTR: Optional[str]
+    LDAP_FIRSTNAME_ATTR: Optional[str]
+    LDAP_LASTNAME_ATTR: Optional[str]
+    LDAP_EMAIL_ATTR: Optional[str]
+    LDAP_CREATE_SUPERUSER: bool
 
     SAVE_TITLE: bool
     SAVE_FAVICON: bool
@@ -58,25 +87,50 @@ class ConfigDict(BaseConfig, total=False):
     SAVE_SINGLEFILE: bool
     SAVE_READABILITY: bool
     SAVE_MERCURY: bool
+    SAVE_HTMLTOTEXT: bool
     SAVE_PDF: bool
     SAVE_SCREENSHOT: bool
     SAVE_DOM: bool
+    SAVE_HEADERS: bool
     SAVE_WARC: bool
     SAVE_GIT: bool
     SAVE_MEDIA: bool
     SAVE_ARCHIVE_DOT_ORG: bool
+    SAVE_ALLOWLIST: dict
+    SAVE_DENYLIST: dict
 
     RESOLUTION: str
     GIT_DOMAINS: str
     CHECK_SSL_VALIDITY: bool
+    MEDIA_MAX_SIZE: str
     CURL_USER_AGENT: str
     WGET_USER_AGENT: str
     CHROME_USER_AGENT: str
-    COOKIES_FILE: Union[str, Path, None]
-    CHROME_USER_DATA_DIR: Union[str, Path, None]
+    COOKIES_FILE: Optional[str]
+    CHROME_USER_DATA_DIR: Optional[str]
     CHROME_TIMEOUT: int
     CHROME_HEADLESS: bool
     CHROME_SANDBOX: bool
+    YOUTUBEDL_ARGS: list
+    WGET_ARGS: list
+    CURL_ARGS: list
+    GIT_ARGS: list
+    SINGLEFILE_ARGS: Optional[list]
+    FAVICON_PROVIDER: str
+
+    USE_INDEXING_BACKEND: bool
+    USE_SEARCHING_BACKEND: bool
+    SEARCH_BACKEND_ENGINE: str
+    SEARCH_BACKEND_HOST_NAME: str
+    SEARCH_BACKEND_PORT: int
+    SEARCH_BACKEND_PASSWORD: str
+    SEARCH_PROCESS_HTML: bool
+    SONIC_COLLECTION: str
+    SONIC_BUCKET: str
+    SEARCH_BACKEND_TIMEOUT: int
+    FTS_SEPARATE_DATABASE: bool
+    FTS_TOKENIZERS: str
+    FTS_SQLITE_MAX_LENGTH: int
 
     USE_CURL: bool
     USE_WGET: bool
@@ -85,7 +139,9 @@ class ConfigDict(BaseConfig, total=False):
     USE_MERCURY: bool
     USE_GIT: bool
     USE_CHROME: bool
+    USE_NODE: bool
     USE_YOUTUBEDL: bool
+    USE_RIPGREP: bool
     CURL_BINARY: str
     GIT_BINARY: str
     WGET_BINARY: str
@@ -93,13 +149,12 @@ class ConfigDict(BaseConfig, total=False):
     READABILITY_BINARY: str
     MERCURY_BINARY: str
     YOUTUBEDL_BINARY: str
+    NODE_BINARY: str
+    RIPGREP_BINARY: str
     CHROME_BINARY: Optional[str]
-
-    YOUTUBEDL_ARGS: List[str]
-    WGET_ARGS: List[str]
-    CURL_ARGS: List[str]
-    GIT_ARGS: List[str]
-    TAG_SEPARATOR_PATTERN: str
+    POCKET_CONSUMER_KEY: Optional[str]
+    POCKET_ACCESS_TOKENS: dict
+    READWISE_READER_TOKENS: dict
 
 
 ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

archivebox/core/__init__.py

@@ -1,2 +1,3 @@
 __package__ = 'archivebox.core'
 
+# default_app_config = 'core.apps.CoreAppConfig'

archivebox/core/admin.py

@@ -12,6 +12,7 @@ from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group, Permission
 from django import forms
 
 from ..util import htmldecode, urldecode, ansi_to_html
@@ -159,6 +160,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
     action_form = SnapshotActionForm
 
+
     def changelist_view(self, request, extra_context=None):
         extra_context = extra_context or {}
         return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)

archivebox/core/apps.py

@@ -1,9 +1,16 @@
 from django.apps import AppConfig
 
 
-class CoreConfig(AppConfig):
+class CoreAppConfig(AppConfig):
     name = 'core'
 
+    # label = 'Archive Data'
+    verbose_name = "Archive Data"
+
+    # WIP: broken by Django 3.1.2 -> 4.0 migration
+    # default_auto_field = 'django.db.models.UUIDField'
+
+
     def ready(self):
         from .auth import register_signals
 

archivebox/core/auth.py

@@ -1,5 +1,7 @@
-import os
-from django.conf import settings
+__package__ = 'archivebox.core'
+
+
+
 from ..config import (
     LDAP
 )

archivebox/core/models.py

@@ -50,7 +50,7 @@ class Tag(models.Model):
 
     class Meta:
         verbose_name = "Tag"
-        verbose_name_plural = "Tags"
+        verbose_name_plural = "🏷️ Tags"
 
     def __str__(self):
         return self.name
@@ -98,6 +98,10 @@ class Snapshot(models.Model):
 
     keys = ('url', 'timestamp', 'title', 'tags', 'updated')
 
+    class Meta:
+        verbose_name = "Snapshot"
+        verbose_name_plural = "⭐️ Archived Webpages (Snapshots)"
+
     def __repr__(self) -> str:
         title = self.title or '-'
         return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
@@ -282,5 +286,9 @@ class ArchiveResult(models.Model):
 
     objects = ArchiveResultManager()
 
+    class Meta:
+        verbose_name = "ArchiveResult"
+        verbose_name_plural = "📑 Logs (ArchiveResults)"
+
     def __str__(self):
         return self.extractor

archivebox/core/settings.py

@@ -1,5 +1,10 @@
 __package__ = 'archivebox.core'
 
+# TODO: add this after we upgrade to Django >=3.2
+# https://github.com/typeddjango/django-stubs
+# import django_stubs_ext
+# django_stubs_ext.monkeypatch()
+
 import os
 import sys
 import re
@@ -59,12 +64,88 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
+    'solo',
+
 
     'core',
+    'api',
+    
+    # Plugins
+    
+    'plugins.defaults',
+    'plugins.system',
+    # 'plugins.replaywebpage',           # provides UI to view WARC files
+    # 'plugins.gallerydl',               # provides gallerydl dependency + extractor
+    # 'plugins.browsertrix',             # provides browsertrix dependency + extractor
+    # 'plugins.playwright',              # provides playwright dependency
+    # ...
+    # someday we may have enough plugins to justify dynamic loading:
+    # *(path.parent.name for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/apps.py')),,
 
     'django_extensions',
 ]
 
+################################################################################
+### Staticfile and Template Settings
+################################################################################
+
+STATIC_URL = '/static/'
+
+STATIC_ROOT = Path(PACKAGE_DIR) / 'collected_static'
+
+STATICFILES_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
+
+    # Plugins
+    # str(Path(PACKAGE_DIR) / 'plugins/defaults/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/gallerydl/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/browsertrix/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/playwright/static'),
+    # ...
+    # someday if there are many more plugins / user-addable plugins:
+    # *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/static')),
+]
+
+MEDIA_URL = '/archive/'
+MEDIA_ROOT = OUTPUT_DIR / 'archive'
+
+
+TEMPLATE_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
+
+    # Plugins
+    # added by plugins.<PluginName>.apps.<AppName>.ready -> .settings.register_plugin_settings
+    # str(Path(PACKAGE_DIR) / 'plugins/defaults/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/gallerydl/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/browsertrix/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/playwright/templates'),
+    # ...
+    #
+    # someday if there are many more plugins / user-addable plugins:
+    # *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/templates')),
+]
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': TEMPLATE_DIRS,
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
 
 # For usage with https://www.jetadmin.io/integrations/django
 # INSTALLED_APPS += ['jet_django']
@@ -162,7 +243,7 @@ if DEBUG_TOOLBAR:
         'debug_toolbar.panels.request.RequestPanel',
         'debug_toolbar.panels.sql.SQLPanel',
         'debug_toolbar.panels.staticfiles.StaticFilesPanel',
-        # 'debug_toolbar.panels.templates.TemplatesPanel',
+        # 'debug_toolbar.panels.templates.TemplatesPanel',    # buggy/slow
         'debug_toolbar.panels.cache.CachePanel',
         'debug_toolbar.panels.signals.SignalsPanel',
         'debug_toolbar.panels.logging.LoggingPanel',
@@ -172,39 +253,6 @@ if DEBUG_TOOLBAR:
     ]
     MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
 
-################################################################################
-### Staticfile and Template Settings
-################################################################################
-
-STATIC_URL = '/static/'
-
-STATICFILES_DIRS = [
-    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
-]
-
-TEMPLATE_DIRS = [
-    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
-]
-
-TEMPLATES = [
-    {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': TEMPLATE_DIRS,
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
-            ],
-        },
-    },
-]
 
 
 ################################################################################
@@ -311,21 +359,21 @@ IGNORABLE_404_URLS = [
 ]
 
 class NoisyRequestsFilter(logging.Filter):
-    def filter(self, record):
+    def filter(self, record) -> bool:
         logline = record.getMessage()
 
         # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
         for ignorable_url_pattern in IGNORABLE_404_URLS:
             ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
             if ignorable_log_pattern.match(logline):
-                return 0
+                return False
 
         # ignore staticfile requests that 200 or 30*
         ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M)
         if ignoreable_200_log_pattern.match(logline):
-            return 0
+            return False
 
-        return 1
+        return True
 
 if LOGS_DIR.exists():
     ERROR_LOG = (LOGS_DIR / 'errors.log')

archivebox/core/urls.py

@@ -8,6 +8,18 @@ from django.views.generic.base import RedirectView
 
 from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
 
+from ninja import NinjaAPI
+from api.auth import GlobalAuth
+
+api = NinjaAPI(auth=GlobalAuth())
+api.add_router("/auth/", "api.auth.router")
+api.add_router("/archive/", "api.archive.router")
+
+# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
+# from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
+# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
+
+
 # print('DEBUG', settings.DEBUG)
 
 urlpatterns = [
@@ -20,6 +32,10 @@ urlpatterns = [
 
     path('archive/', RedirectView.as_view(url='/')),
     path('archive/<path:path>', SnapshotView.as_view(), name='Snapshot'),
+    path('web/<path:path>', SnapshotView.as_view()),   # support archive.org-style URLs
+
+    path('plugins/replaywebpage/', include('plugins.replaywebpage.urls')),
+    # ... dynamic load these someday if there are more of them
 
     path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
     path('add/', AddView.as_view(), name='add'),
@@ -31,6 +47,8 @@ urlpatterns = [
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', archivebox_admin.urls),
     
+    path("api/", api.urls),
+
     path('health/', HealthCheckView.as_view(), name='healthcheck'),
     path('error/', lambda _: 1/0),
 

archivebox/core/views.py

@@ -56,12 +56,18 @@ class SnapshotView(View):
             slug, archivefile = path.split('/', 1)[0], 'index.html'
 
         # slug is a timestamp
-        if slug.replace('.','').isdigit():
+        if slug.replace('.', '').isdigit():
 
             # missing trailing slash -> redirect to index
             if '/' not in path:
                 return redirect(f'{path}/index.html')
 
+            # TODO: add support for archive.org-style URLs where timestamp may be a human-readable date
+            # https://web.archivebox.io / web / 2022-01 / https://example.com
+            # https://web.archivebox.io / web / 20220505103616 / https://example.com
+            # https://web.archivebox.io / web / 2022-05-05__0:36:16 / https://example.com
+            # use archivebox.util.parse_date (supports unix timestamps, iso date strings, and lots more etc.)
+
             try:
                 try:
                     snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))

archivebox/logging_util.py

@@ -494,12 +494,12 @@ def log_removal_started(links: List["Link"], yes: bool, delete: bool):
     if delete:
         file_counts = [link.num_outputs for link in links if Path(link.link_dir).exists()]
         print(
-            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
+            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n' +
             f'    ({len(file_counts)} data folders with {sum(file_counts)} archived files will be deleted!)'
         )
     else:
         print(
-            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
+            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n' +
             '    (Pass --delete if you also want to permanently delete the data folders)'
         )
 
@@ -638,17 +638,15 @@ def printable_folder_status(name: str, folder: Dict) -> str:
 
 @enforce_types
 def printable_dependency_version(name: str, dependency: Dict) -> str:
-    version = None
+    color, symbol, note, version = 'red', 'X', 'invalid', '?'
+
     if dependency['enabled']:
         if dependency['is_valid']:
-            color, symbol, note, version = 'green', '√', 'valid', ''
+            color, symbol, note = 'green', '√', 'valid'
 
             parsed_version_num = re.search(r'[\d\.]+', dependency['version'])
             if parsed_version_num:
                 version = f'v{parsed_version_num[0]}'
-
-        if not version:
-            color, symbol, note, version = 'red', 'X', 'invalid', '?'
     else:
         color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
 

archivebox/manage.py

@@ -7,7 +7,7 @@ if __name__ == '__main__':
     # versions of ./manage.py commands whenever possible. When that's not possible
     # (e.g. makemigrations), you can comment out this check temporarily
 
-    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv):
+    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'collectstatic' in sys.argv):
         print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
         print()
         print('    Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')

archivebox/mypy.ini

@@ -1,3 +0,0 @@
-[mypy]
-plugins =
-    mypy_django_plugin.main

archivebox/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "description": "ArchiveBox: The self-hosted internet archive",
   "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
   "repository": "github:ArchiveBox/ArchiveBox",
@@ -8,6 +8,6 @@
   "dependencies": {
     "@postlight/parser": "^2.2.3",
     "readability-extractor": "github:ArchiveBox/readability-extractor",
-    "single-file-cli": "^1.1.46"
+    "single-file-cli": "^1.1.54"
   }
 }

archivebox/plugins/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins'
+
+

archivebox/plugins/defaults/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins.defaults'
+
+default_app_config = 'plugins.defaults.apps.DefaultsPluginAppConfig'

archivebox/plugins/defaults/admin.py

@@ -0,0 +1,20 @@
+from django.contrib import admin
+from solo.admin import SingletonModelAdmin
+
+from .models import (
+	ArchiveBoxDefaultDependency,
+	ArchiveBoxDefaultExtractor,
+)
+
+
+class DependencyAdmin(SingletonModelAdmin):
+	readonly_fields = ('id', 'NAME', 'LABEL', 'REQUIRED', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
+
+class ExtractorAdmin(SingletonModelAdmin):
+	readonly_fields = ('id', 'NAME', 'LABEL', 'DEFAULT_ENABLED', 'DEFAULT_CMD', 'CMD', 'ARGS', 'TIMEOUT', 'dependency', 'is_valid', 'is_enabled')
+
+print('DefaultsPluginConfig.admin')
+
+
+admin.site.register(ArchiveBoxDefaultDependency, DependencyAdmin)
+admin.site.register(ArchiveBoxDefaultExtractor, ExtractorAdmin)

archivebox/plugins/defaults/apps.py

@@ -0,0 +1,24 @@
+# __package__ = 'archivebox.plugins.defaults'
+
+
+
+from django.apps import AppConfig
+
+
+class DefaultsPluginAppConfig(AppConfig):
+    name = "plugins.defaults"
+
+    # label = "ArchiveBox Defaults"
+    verbose_name = "Plugin Configuration Defaults"
+    
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        print('plugins.defaults.apps.DefaultsPluginConfig.ready')
+
+        from django.conf import settings
+
+        from .settings import register_plugin_settings
+
+        register_plugin_settings(settings, name=self.name)
+        

archivebox/plugins/defaults/migrations/0001_initial.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.1.14 on 2024-01-24 08:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ArchiveBoxDefaultDependency',
+            fields=[
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='/bin/false', max_length=255)),
+                ('ARGS', models.CharField(default='', max_length=255)),
+                ('id', models.AutoField(default=1, primary_key=True, serialize=False)),
+            ],
+            options={
+                'verbose_name': 'Dependency Configuration Defaults',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ArchiveBoxDefaultExtractor',
+            fields=[
+                ('ENABLED', models.BooleanField(default=True)),
+                ('CMD', models.CharField(default=['{DEPENDENCY.BINARY}', '{ARGS}', '{url}'], max_length=255)),
+                ('ARGS', models.CharField(default=['--timeout={TIMEOUT}'], max_length=255)),
+                ('TIMEOUT', models.CharField(default='{TIMEOUT}', max_length=255)),
+                ('id', models.AutoField(default=1, primary_key=True, serialize=False)),
+            ],
+            options={
+                'verbose_name': 'Extractor Configuration Defaults',
+                'abstract': False,
+            },
+        ),
+    ]

archivebox/plugins/defaults/migrations/0002_auto_20240124_0943.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.1.14 on 2024-01-24 09:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('defaults', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='archiveboxdefaultdependency',
+            options={'verbose_name': 'Default Configuration: Dependencies'},
+        ),
+        migrations.AlterModelOptions(
+            name='archiveboxdefaultextractor',
+            options={'verbose_name': 'Default Configuration: Extractors'},
+        ),
+        migrations.AlterField(
+            model_name='archiveboxdefaultdependency',
+            name='BINARY',
+            field=models.CharField(default='/bin/bash', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='archiveboxdefaultdependency',
+            name='ENABLED',
+            field=models.BooleanField(default=True),
+        ),
+    ]

archivebox/plugins/defaults/models.py

@@ -0,0 +1,385 @@
+__package__ = 'archivebox.plugins.defaults'
+
+# import shutil
+
+import re
+
+from typing import List, Dict, Any
+from pathlib import Path
+
+from django.db import models, transaction
+from django.utils.functional import cached_property
+
+from solo.models import SingletonModel                        # type: ignore[import-untyped]
+
+
+from config import bin_path, bin_version
+
+ConfigDict = Dict[str, Any]
+
+
+# def bin_path(binary: str) -> str | None:
+#     return shutil.which(str(Path(binary).expanduser())) or shutil.which(str(binary)) or binary
+
+# def bin_version(bin_path: str, cmd: str | None=None) -> str | None:
+#     return '0.0.0'
+
+# def pretty_path(path: Path) -> str:
+#     """take a Path object and return the path as a string relative to the current directory"""
+
+#     if not path:
+#         return ''
+
+#     return str(path.expanduser().resolve().relative_to(Path.cwd().resolve()))
+
+
+class ArchiveBoxBaseDependency(models.Model):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    NAME = 'DEFAULT'
+    LABEL = "Default"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES: List[str] = []
+
+    BIN_DEPENDENCIES: List[str] = []
+    APT_DEPENDENCIES: List[str] = []
+    BREW_DEPENDENCIES: List[str] = []
+    PIP_DEPENDENCIES: List[str] = []
+    NPM_DEPENDENCIES: List[str] = []
+
+    DEFAULT_BINARY: str | None              = '/bin/bash'
+    DEFAULT_START_CMD: str | None           = '/bin/bash -c "while true; do sleep 1; done"'
+    DEFAULT_PID_FILE: str | None            = 'logs/{NAME}_WORKER.pid'
+    DEFAULT_STOP_CMD: str | None            = 'kill "$(<{PID_FILE})"'
+    DEFAULT_VERSION_COMMAND: str | None     = '{BINARY} --version'
+    DEFAULT_ARGS: str | None                = ''
+
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=False)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+    
+    # START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
+    # WORKERS = models.IntegerField(default=1)
+
+    class Meta:
+        abstract = True
+        app_label = 'defaults'
+
+    def __str__(self):
+        return f"{self.LABEL} Dependency Configuration"
+
+    def __json__(self):
+        return {
+            'type': 'ArchiveBoxDependency',
+            '__class__': self.__class__.__name__,
+            'NAME': self.NAME,
+            'LABEL': self.LABEL,
+            'ENABLED': self.ENABLED,
+            'BINARY': self.BINARY,
+            'ARGS': self.ARGS,
+            # 'START_CMD': self.START_CMD,
+            # 'WORKERS': self.WORKERS,
+        }
+
+    @cached_property
+    def bin_path(self) -> str:
+        return bin_path(self.BINARY or self.DEFAULT_BINARY)
+
+    @cached_property
+    def bin_version(self) -> str | None:
+        print(f'ArchiveBoxBaseDependency.bin_version({self.bin_path}, cmd={self.VERSION_CMD.format(BINARY=self.BINARY)})')
+        return bin_version(self.bin_path, cmd=self.VERSION_CMD.format(BINARY=self.BINARY))
+        # return bin_version(self.bin_path, cmd=self.VERSION_CMD)
+
+    @cached_property
+    def is_valid(self) -> bool:
+        return bool(self.bin_path and self.bin_version)
+
+    @cached_property
+    def is_enabled(self) -> bool:
+        return bool(self.ENABLED and self.is_valid)
+
+    @cached_property
+    def pretty_version(self) -> str:
+        if self.is_enabled:
+            if self.is_valid:
+                color, symbol, note, version = 'green', '√', 'valid', ''
+
+                parsed_version_num = re.search(r'[\d\.]+', self.bin_version)
+                if parsed_version_num:
+                    version = f'v{parsed_version_num[0]}'
+
+            if not self.bin_version:
+                color, symbol, note, version = 'red', 'X', 'invalid', '?'
+        else:
+            color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
+
+        path = pretty_path(self.bin_path)
+
+        return ' '.join((
+            ANSI[color],
+            symbol,
+            ANSI['reset'],
+            name.ljust(21),
+            version.ljust(14),
+            ANSI[color],
+            note.ljust(8),
+            ANSI['reset'],
+            path.ljust(76),
+        ))
+
+    # @helper
+    def install_parents(self, config):
+        return {
+            # parent_dependency.NAME: parent_dependency.get_solo().install_self()
+            parent_dependency: parent_dependency
+            for parent_dependency in self.PARENT_DEPENDENCIES
+        }
+
+    # @helper
+    def install_self(self, config):
+        assert all(self.install_parents(config=config).values())
+
+        BashEnvironmentDependency.get_solo().install_pkgs(self.BIN_DEPENDENCIES)
+        AptEnvironmentDependency.get_solo().install_pkgs(self.APT_DEPENDENCIES)
+        BrewEnvironmentDependency.get_solo().install_pkgs(self.BREW_DEPENDENCIES)
+        PipEnvironmentDependency.get_solo().install_pkgs(self.PIP_DEPENDENCIES)
+        NPMEnvironmentDependency.get_solo().install_pkgs(self.NPM_DEPENDENCIES)
+
+        assert self.is_valid
+        return self.bin_version
+
+    # @task
+    def run(args, pwd, timeout):
+        errors = None
+        timer = TimedProgress(timeout, prefix='      ')
+        try:
+            proc = run(cmd=[self.bin_path, *args], pwd=pwd, timeout=timeout)
+
+        except Exception as err:
+            errors = err
+        finally:
+            timer.end()
+
+        return proc, timer, errors
+
+class ArchiveBoxDefaultDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+
+    class Meta:                 # pyright: ignore [reportIncompatibleVariableOverride]
+        abstract = False
+        app_label = 'defaults'
+        verbose_name = 'Default Configuration: Dependencies'
+
+
+class ArchiveBoxBaseExtractor(models.Model):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    NAME = 'DEFAULT'
+    LABEL = 'Default'
+
+    DEFAULT_DEPENDENCY = ArchiveBoxDefaultDependency
+    DEPENDENCY = DEFAULT_DEPENDENCY
+
+
+    DEFAULT_ENABLED = True
+    DEFAULT_CMD = ['{DEPENDENCY.BINARY}', '{ARGS}', '{url}']
+    DEFAULT_ARGS = ['--timeout={TIMEOUT}']
+    DEFAULT_TIMEOUT = '{TIMEOUT}'
+    # DEFAULT_USER_AGENT = '{USER_AGENT}'
+    # DEFAULT_COOKIES_TXT = '{COOKIES_TXT}'
+
+    ENABLED = models.BooleanField(default=DEFAULT_ENABLED, editable=True)
+
+    CMD = models.CharField(max_length=255, default=DEFAULT_CMD)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+    TIMEOUT = models.CharField(max_length=255, default=DEFAULT_TIMEOUT)
+    
+    ALIASES = {
+        'ENABLED': (f'SAVE_{NAME}', f'USE_{NAME}', f'FETCH_{NAME}'),
+    }
+
+    def __str__(self):
+        return f"{self.LABEL} Extractor Configuration"
+
+    class Meta:             # pyright: ignore [reportIncompatibleVariableOverride]
+        abstract = True
+        verbose_name = "Default Extractor Configuration"
+        app_label = 'defaults'
+
+    @cached_property
+    def dependency(self):
+        return self.DEPENDENCY.get_solo()
+
+    def __json__(self):
+        return {
+            'type': 'ArchiveBoxExtractor',
+            '__class__': self.__class__.__name__,
+            'NAME': self.NAME,
+            'LABEL': self.LABEL,
+            'ENABLED': self.ENABLED,
+            'DEPENDENCY': self.dependency.__json__(),
+            'ARGS': self.ARGS,
+            'CMD': self.CMD,
+            'TIMEOUT': self.TIMEOUT,
+            'is_valid': self.is_valid,
+            'is_enabled': self.is_enabled,
+        }
+
+
+    def format_args(self, csv: List[str], **config):
+        un_prefixed_config = {**self.__json__()}          # e.g. ENABLED=True
+        prefixed_config = {                               # e.g. GALLERYDL_ENABLED=True
+            f'{self.NAME}_{key}': value
+            for key, value in un_prefixed_config.items()
+        }
+
+        merged_config = {
+            **config,                  # e.g. TIMEOUT=60
+            **un_prefixed_config,      # e.g. ENABLED=True
+            **prefixed_config,         # e.g. GALLERYDL_ENABLED=True
+        }
+        formatted_config = [
+            arg.format(**merged_config)
+            for arg in csv
+        ]
+
+        return formatted_config
+
+    @cached_property
+    def is_valid(self):
+        if not self.dependency.is_valid:
+            return False
+
+        # TIMEOUT must be at least 5 seconds
+        # if self.TIMEOUT < 5:
+        #     return False
+
+        # assert Path(self.COOKIES_TXT).exists()
+        # TODO: validate user agent with uaparser
+        # TODO: validate args, cookies.txt?
+        return True
+
+    @cached_property
+    def is_enabled(self):
+        return self.ENABLED and self.is_valid and self.dependency.is_enabled
+
+
+    def save(self, *args, **kwargs):
+        # assert self.is_valid
+
+        with transaction.atomic():
+            result = super().save(*args, **kwargs)
+            # post to message bus:
+            print({
+                'type': f'{self.__class__.__name__}.save',
+                'diff': self.__json__(),
+                'kwargs': kwargs,
+            })
+            # potential consumers of this event:
+            #    - event logger: write to events.log
+            #    - config file updater: writes to ArchiveBox.conf
+            #    - supervisor: restarts relevant dependencies/extractors
+            #    - etc...
+
+        return result
+
+    def out_dir(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        return (snapshot_dir / self.NAME)
+
+    def create_out_dir(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        out_dir = self.out_dir(url=url, snapshot_dir=snapshot_dir, config=config)
+        return out_dir.mkdir(exist_ok=True)
+
+    def should_extract(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        # return False if extractor is disabled
+        if not self.is_enabled:
+            return False
+
+        out_dir = self.out_dir(url=url, snapshot_dir=snapshot_dir, config=config)
+        
+        if has_existing_output := out_dir.glob('*'):
+            return False
+
+        if not (has_write_access := os.access(out_dir, os.W_OK | os.X_OK)):
+            return False
+
+        return True
+
+
+    def get_dependency_cmd(self, url: str, extractor_dir: Path, config: ConfigDict):
+        return [
+            self.format_args(self.CMD, **config),
+            url,
+            *self.format_args(self.ARGS, **config),   # TODO: split and requote this properly
+        ]
+
+    # @requires_config('HOSTNAME', 'TIMEOUT', 'USER_AGENT', 'CHECK_SSL_VALIDITY')
+    def extract(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        if not self.ENABLED:
+            return
+
+        extractor_dir = self.create_extractor_directory(snapshot_dir)
+
+        cmd = self.get_dependency_cmd(url=url, extractor_dir=extractor_dir, config=config)
+
+        status, stdout, stderr, output_path = 'failed', '', '', None
+        try:
+            proc, timer, errors = self.dependency.run(cmd, cwd=extractor_dir, timeout=self.TIMEOUT)
+            stdout, stderr = proc.stdout, proc.stderr
+            
+            if 'ERROR: Unsupported URL' in stderr:
+                hints = ('gallery-dl doesnt support this type of url yet',)
+                raise ArchiveError('Failed to save gallerydl', hints)
+
+            if proc.returncode == 0 and 'finished' in stdout:
+                output_path = extractor_dir / 'index.html'
+                status = 'succeeded'
+        except Exception as err:
+            stderr += err
+
+        num_bytes, num_dirs, num_files = get_dir_size(extractor_dir)
+
+        return ArchiveResult(
+            cmd=cmd,
+            pwd=str(out_dir),
+            cmd_version=self.dependency.bin_version,
+            cmd_path=self.dependency.bin_path,
+            cmd_hostname=config.HOSTNAME,
+
+            output_path=output_path,
+            stdout=stdout,
+            stderr=stderr,
+            status=status,
+
+            num_bytes=num_bytes,
+            num_files=num_files,
+            num_dirs=num_dirs,
+            **timer.stats,
+        )
+
+
+class ArchiveBoxDefaultExtractor(ArchiveBoxBaseExtractor, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    DEPENDENCY = ArchiveBoxDefaultDependency
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+
+    class Meta:
+        abstract = False
+        app_label = 'defaults'
+        verbose_name = 'Default Configuration: Extractors'

archivebox/plugins/defaults/settings.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from django.conf import settings
+
+def register_plugin_settings(settings=settings, name='defaults'):
+
+	settings.STATICFILES_DIRS += [
+		str(Path(settings.PACKAGE_DIR) / f'plugins/{name}/static'),
+	]
+
+	settings.TEMPLATE_DIRS += [
+		str(Path(settings.PACKAGE_DIR) / f'plugins/{name}/templates'),
+	]
+
+	print('REGISTERED PLUGIN SETTINGS', name)

archivebox/plugins/gallerydl/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.plugins.replaywebpage'

archivebox/plugins/gallerydl/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+from solo.admin import SingletonModelAdmin
+
+from .models import GalleryDLDependency, GalleryDLExtractor
+
+
+admin.site.register(GalleryDLDependency, SingletonModelAdmin)
+admin.site.register(GalleryDLExtractor, SingletonModelAdmin)

archivebox/plugins/gallerydl/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+
+
+class GalleryDLAppConfig(AppConfig):
+    label = "Gallery-DL"
+    name = "plugin_gallerydl"
+    
+    default_auto_field = "django.db.models.BigAutoField"
+
+    def ready(self):
+        # querying models is ok, but don't fetch rows from DB or perform stateful actions here
+
+        print('√ Loaded GalleryDL Plugin')

archivebox/plugins/gallerydl/extractors.py

@@ -0,0 +1,50 @@
+# browsertrix extractor
+
+def save_browsertrix(link, out_dir, timeout, config):
+
+
+	browsertrix_dir = out_dir / 'browsertrix'
+	browsertrix_dir.mkdir(exist_ok=True)
+
+	crawl_id = link.timestamp
+
+	browsertrix_crawler_cmd = [
+		'crawl',
+		f'--url', link.url,
+		f'--collection={crawl_id}',
+		'--scopeType=page',
+		'--generateWACZ',
+		'--text=final-to-warc',
+		'--timeLimit=60',
+	]
+
+	remote_cmd = """
+	rm /tmp/dump.rdb;
+	rm -rf /crawls/collections;
+	mkdir /crawls/collections;
+	env CRAWL_ID={crawl_id} 
+	"""
+
+	local_cmd = ['nc', 'browsertrix', '2222']
+
+	status = 'succeeded'
+    timer = TimedProgress(timeout, prefix='      ')
+    try:
+        result = run(local_cmd, cwd=str(out_dir), input=remote_cmd, timeout=timeout)
+		
+		cmd_output = result.stdout.decode()
+
+		wacz_output_file = Path('/browsertrix/crawls') / crawl_id / f'{crawl_id}'.wacz
+
+		copy_and_overwrite(wacz_output_file, browsertrix_dir / wacz_output_file.name)
+
+
+
+TEMPLATE = """
+
+"""
+
+# rm /tmp/dump.rdb;
+# rm -rf /crawls/collections;
+# mkdir /crawls/collections;
+# env CRAWL_ID=tec2342 crawl --url 'https://example.com' --scopeType page --generateWACZ --collection tec2342 --text final-to-warc --timeLimit 60

archivebox/plugins/gallerydl/models.py

@@ -0,0 +1,121 @@
+from django.db import models
+from django.utils.functional import cached_property
+
+from solo.models import SingletonModel
+
+from archivebox.plugins.defaults.models import (
+    ArchiveBoxDefaultDependency,
+    ArchiveBoxDefaultExtractor,
+    BashEnvironmentDependency,
+    PipEnvironmentDependency,
+)
+
+
+class GalleryDLDependency(ArchiveBoxDefaultDependency, SingletonModel):
+    NAME = 'GALLERYDL'
+    LABEL = "GalleryDL"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES = [
+        BashEnvironmentDependency,
+        PipEnvironmentDependency,
+    ]
+
+    BIN_DEPENDENCIES = ['gallery-dl']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = ['gallery-dl']
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'gallery-dl'
+    DEFAULT_START_CMD = None
+    DEFAULT_ARGS = []
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True)
+    BINARY = models.CharField(max_length=255, default='gallery-dl')
+
+    WORKERS = models.IntegerField(default='1')
+
+
+class GalleryDLExtractor(ArchiveBoxDefaultExtractor, SingletonModel):
+    NAME = 'GALLERYDL'
+    LABEL = 'gallery-dl'
+
+    DEPENDENCY = GalleryDLDependency.get_solo()
+
+    # https://github.com/mikf/gallery-dl
+    DEFAULT_CMD = [
+        '{DEPENDENCY.BINARY}',
+        '{ARGS}'
+        '{url}',
+    ]
+    DEFAULT_ARGS = [
+        '--timeout', self.TIMEOUT.format(**config),
+        '--cookies', self.COOKIES_TXT.format(**config),
+        '--user-agent', self.COOKIES_TXT.format(**config),
+        '--verify', self.CHECK_SSL_VALIDITY.format(**config),
+    ]
+
+    ENABLED = models.BooleanField(default=True)
+
+    CMD = models.CharField(max_length=255, default=DEFAULT_CMD)
+    ARGS = models.CSVField(max_length=255, default=DEFAULT_ARGS)
+    
+    TIMEOUT = models.CharField(max_length=255, default='{TIMEOUT}')
+    USER_AGENT = models.CharField(max_length=255, default='{USER_AGENT}')
+    COOKIES_TXT = models.CharField(max_length=255, default='{COOKIES_TXT}')
+    CHECK_SSL_VALIDITY = models.CharField(default='{CHECK_SSL_VALIDITY}')
+
+    # @task
+    # @requires_config('HOSTNAME', 'TIMEOUT', 'USER_AGENT', 'CHECK_SSL_VALIDITY')
+    def extract(self, url: str, out_dir: Path, config: ConfigDict):
+        if not self.ENABLED:
+            return
+
+        extractor_dir = self.create_extractor_directory(out_dir)
+
+        cmd = [
+            self.CMD,
+            url,
+            '--timeout', self.TIMEOUT.format(**config),
+            '--cookies', self.COOKIES_TXT.format(**config),
+            '--user-agent', self.COOKIES_TXT.format(**config),
+            '--verify', self.CHECK_SSL_VALIDITY.format(**config),
+            *split_args(self.ARGS.format(**config)),
+        ]
+
+        status, stdout, stderr, output_path = 'failed', '', '', None
+        try:
+            proc, timer, errors = self.DEPENDENCY.run(cmd, cwd=extractor_dir, timeout=self.GALLERYDL_TIMEOUT)
+            stdout, stderr = proc.stdout, proc.stderr
+            
+            if 'ERROR: Unsupported URL' in stderr:
+                hints = ('gallery-dl doesnt support this type of url yet',)
+                raise ArchiveError('Failed to save gallerydl', hints)
+
+            if proc.returncode == 0 and 'finished' in stdout:
+                output_path = extractor_dir / 'index.html'
+                status = 'succeeded'
+        except Exception as err:
+            stderr += err
+
+        num_bytes, num_dirs, num_files = get_dir_size(extractor_dir)
+
+        return ArchiveResult(
+            cmd=cmd,
+            pwd=str(out_dir),
+            cmd_version=self.DEPENDENCY.bin_version,
+            cmd_path=self.DEPENDENCY.bin_path,
+            cmd_hostname=config.HOSTNAME,
+
+            output_path=output_path,
+            stdout=stdout,
+            stderr=stderr,
+            status=status,
+
+            num_bytes=num_bytes,
+            num_files=num_files,
+            num_dirs=num_dirs,
+            **timer.stats,
+        )

archivebox/plugins/gallerydl/plugin.yaml

@@ -0,0 +1,59 @@
+dependencies:
+	GalleryDLDependency:
+		ID: gallerydl
+		LABEL: GalleryDL
+		REQUIRED: false
+
+		PARENT_DEPENDENCIES:
+			- BashEnvironmentDependency
+			- PipEnvironmentDependency
+
+		PIP_DEPENDENCIES:
+			- gallery-dl
+
+		USER_CONFIG:
+			ENABLED: models.BooleanField(max_length=255, default={DEFAULT_CONFIG.ENABLED})
+			BINARY: models.CharField(max_length=255, default={DEFAULT_CONFIG.BINARY})
+
+		DEFAULT_CONFIG:
+			ENABLED: true
+			BINARY: 'gallery-dl'
+
+		CONFIG_ALIASES:
+			- SAVE_GALLERYDL: ENABLED
+			- USE_GALLERYDL: ENABLED
+			- GALLERYDL_ENABLED: ENABLED
+			- GALLERYDL_BINARY: BINARY
+
+		TASKS:
+			# plugins.GalleryDLDependency
+			run_dependency: plugins.gallerydl.models.GalleryDLDependency.run_dependency
+
+
+extractors:
+	GalleryDLExtractor:
+		ID: GALLERYDL
+		LABEL: GalleryDL
+		ENABLED: true
+
+		DEPENDENCY: GalleryDLDependency
+
+		CONFIG:
+			ENABLED: models.BooleanField(default={DEFAULT_CONFIG.ENABLED})
+			CMD: models.CharField(max_length=255, default={DEFAULT_CONFIG.CMD})
+			ARGS: models.CharField(max_length=255, default={DEFAULT_CONFIG.ARGS})
+			USER_AGENT: models.CharField(max_length=255, default={DEFAULT_CONFIG.USER_AGENT})
+			CHECK_SSL_VALIDITY: models.CharField(max_length=255, default={DEFAULT_CONFIG.CHECK_SSL_VALIDITY})
+
+		DEFAULT_CONFIG:
+			ENABLED: true
+			CMD: gallery-dl {args} {url}
+			ARGS: --user-agent={USER_AGENT} --check-ssl={CHECK_SSL_VALIDITY}
+      CHECK_SSL_VALIDITY: {CHECK_SSL_VALIDITY}
+      USER_AGENT: {USER_AGENT}
+
+
+		TASKS:
+			CREATE_OUT_DIR: plugins.gallerydl.tasks.create_out_dir
+			SHOULD_EXTRACT: plugins.gallerydl.tasks.should_extract
+			EXTRACT: plugins.gallerydl.tasks.extract

archivebox/plugins/gallerydl/static/sw.js

@@ -0,0 +1,124 @@
+/**
+ * Skipped minification because the original files appears to be already minified.
+ * Original file: /npm/replaywebpage@1.8.14/sw.js
+ *
+ * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files
+ */
+/*! sw.js (wabac.js 2.16.13) is part of Webrecorder project. Copyright (C) 2020-2023, Webrecorder Software. Licensed under the Affero General Public License v3. */(()=>{var t={9742:(t,e)=>{"use strict";e.byteLength=function(t){var e=c(t),r=e[0],i=e[1];return 3*(r+i)/4-i},e.toByteArray=function(t){var e,r,s=c(t),o=s[0],a=s[1],l=new n(function(t,e,r){return 3*(e+r)/4-r}(0,o,a)),h=0,u=a>0?o-4:o;for(r=0;r<u;r+=4)e=i[t.charCodeAt(r)]<<18|i[t.charCodeAt(r+1)]<<12|i[t.charCodeAt(r+2)]<<6|i[t.charCodeAt(r+3)],l[h++]=e>>16&255,l[h++]=e>>8&255,l[h++]=255&e;return 2===a&&(e=i[t.charCodeAt(r)]<<2|i[t.charCodeAt(r+1)]>>4,l[h++]=255&e),1===a&&(e=i[t.charCodeAt(r)]<<10|i[t.charCodeAt(r+1)]<<4|i[t.charCodeAt(r+2)]>>2,l[h++]=e>>8&255,l[h++]=255&e),l},e.fromByteArray=function(t){for(var e,i=t.length,n=i%3,s=[],o=16383,a=0,c=i-n;a<c;a+=o)s.push(l(t,a,a+o>c?c:a+o));return 1===n?(e=t[i-1],s.push(r[e>>2]+r[e<<4&63]+"==")):2===n&&(e=(t[i-2]<<8)+t[i-1],s.push(r[e>>10]+r[e>>4&63]+r[e<<2&63]+"=")),s.join("")};for(var r=[],i=[],n="undefined"!=typeof Uint8Array?Uint8Array:Array,s="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",o=0,a=s.length;o<a;++o)r[o]=s[o],i[s.charCodeAt(o)]=o;function c(t){var e=t.length;if(e%4>0)throw new Error("Invalid string. Length must be a multiple of 4");var r=t.indexOf("=");return-1===r&&(r=e),[r,r===e?0:4-r%4]}function l(t,e,i){for(var n,s,o=[],a=e;a<i;a+=3)n=(t[a]<<16&16711680)+(t[a+1]<<8&65280)+(255&t[a+2]),o.push(r[(s=n)>>18&63]+r[s>>12&63]+r[s>>6&63]+r[63&s]);return o.join("")}i["-".charCodeAt(0)]=62,i["_".charCodeAt(0)]=63},4181:t=>{var e=4096,r=new Uint32Array([0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535,131071,262143,524287,1048575,2097151,4194303,8388607,16777215]);function i(t){this.buf_=new Uint8Array(8224),this.input_=t,this.reset()}i.READ_SIZE=e,i.IBUF_MASK=8191,i.prototype.reset=function(){this.buf_ptr_=0,this.val_=0,this.pos_=0,this.bit_pos_=0,this.bit_end_pos_=0,this.eos_=0,this.readMoreInput();for(var t=0;t<4;t++)this.val_|=this.buf_[this.pos_]<<8*t,++this.pos_;return this.bit_end_pos_>0},i.prototype.readMoreInput=function(){if(!(this.bit_end_pos_>256))if(this.eos_){if(this.bit_pos_>this.bit_end_pos_)throw new Error("Unexpected end of input "+this.bit_pos_+" "+this.bit_end_pos_)}else{var t=this.buf_ptr_,r=this.input_.read(this.buf_,t,e);if(r<0)throw new Error("Unexpected end of input");if(r<e){this.eos_=1;for(var i=0;i<32;i++)this.buf_[t+r+i]=0}if(0===t){for(i=0;i<32;i++)this.buf_[8192+i]=this.buf_[i];this.buf_ptr_=e}else this.buf_ptr_=0;this.bit_end_pos_+=r<<3}},i.prototype.fillBitWindow=function(){for(;this.bit_pos_>=8;)this.val_>>>=8,this.val_|=this.buf_[8191&this.pos_]<<24,++this.pos_,this.bit_pos_=this.bit_pos_-8>>>0,this.bit_end_pos_=this.bit_end_pos_-8>>>0},i.prototype.readBits=function(t){32-this.bit_pos_<t&&this.fillBitWindow();var e=this.val_>>>this.bit_pos_&r[t];return this.bit_pos_+=t,e},t.exports=i},7080:(t,e)=>{e.lookup=new Uint8Array([0,0,0,0,0,0,0,0,0,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,12,16,12,12,20,12,16,24,28,12,12,32,12,36,12,44,44,44,44,44,44,44,44,44,44,32,32,24,40,28,12,12,48,52,52,52,48,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,24,12,28,12,12,12,56,60,60,60,56,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,24,12,28,12,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,0,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,32,32,32,32,32,32,32,32,32,32,32,32,32,Line truncated
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */const i=r(9742),n=r(645),s="function"==typeof Symbol&&"function"==typeof Symbol.for?Symbol.for("nodejs.util.inspect.custom"):null;e.Buffer=c,e.SlowBuffer=function(t){return+t!=t&&(t=0),c.alloc(+t)},e.INSPECT_MAX_BYTES=50;const o=2147483647;function a(t){if(t>o)throw new RangeError('The value "'+t+'" is invalid for option "size"');const e=new Uint8Array(t);return Object.setPrototypeOf(e,c.prototype),e}function c(t,e,r){if("number"==typeof t){if("string"==typeof e)throw new TypeError('The "string" argument must be of type string. Received type number');return u(t)}return l(t,e,r)}function l(t,e,r){if("string"==typeof t)return function(t,e){if("string"==typeof e&&""!==e||(e="utf8"),!c.isEncoding(e))throw new TypeError("Unknown encoding: "+e);const r=0|g(t,e);let i=a(r);const n=i.write(t,e);return n!==r&&(i=i.slice(0,n)),i}(t,e);if(ArrayBuffer.isView(t))return function(t){if(Y(t,Uint8Array)){const e=new Uint8Array(t);return d(e.buffer,e.byteOffset,e.byteLength)}return p(t)}(t);if(null==t)throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t);if(Y(t,ArrayBuffer)||t&&Y(t.buffer,ArrayBuffer))return d(t,e,r);if("undefined"!=typeof SharedArrayBuffer&&(Y(t,SharedArrayBuffer)||t&&Y(t.buffer,SharedArrayBuffer)))return d(t,e,r);if("number"==typeof t)throw new TypeError('The "value" argument must not be of type number. Received type number');const i=t.valueOf&&t.valueOf();if(null!=i&&i!==t)return c.from(i,e,r);const n=function(t){if(c.isBuffer(t)){const e=0|f(t.length),r=a(e);return 0===r.length||t.copy(r,0,0,e),r}return void 0!==t.length?"number"!=typeof t.length||X(t.length)?a(0):p(t):"Buffer"===t.type&&Array.isArray(t.data)?p(t.data):void 0}(t);if(n)return n;if("undefined"!=typeof Symbol&&null!=Symbol.toPrimitive&&"function"==typeof t[Symbol.toPrimitive])return c.from(t[Symbol.toPrimitive]("string"),e,r);throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t)}function h(t){if("number"!=typeof t)throw new TypeError('"size" argument must be of type number');if(t<0)throw new RangeError('The value "'+t+'" is invalid for option "size"')}function u(t){return h(t),a(t<0?0:0|f(t))}function p(t){const e=t.length<0?0:0|f(t.length),r=a(e);for(let i=0;i<e;i+=1)r[i]=255&t[i];return r}function d(t,e,r){if(e<0||t.byteLength<e)throw new RangeError('"offset" is outside of buffer bounds');if(t.byteLength<e+(r||0))throw new RangeError('"length" is outside of buffer bounds');let i;return i=void 0===e&&void 0===r?new Uint8Array(t):void 0===r?new Uint8Array(t,e):new Uint8Array(t,e,r),Object.setPrototypeOf(i,c.prototype),i}function f(t){if(t>=o)throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+o.toString(16)+" bytes");return 0|t}function g(t,e){if(c.isBuffer(t))return t.length;if(ArrayBuffer.isView(t)||Y(t,ArrayBuffer))return t.byteLength;if("string"!=typeof t)throw new TypeError('The "string" argument must be one of type string, Buffer, or ArrayBuffer. Received type '+typeof t);const r=t.length,i=arguments.length>2&&!0===arguments[2];if(!i&&0===r)return 0;let n=!1;for(;;)switch(e){case"ascii":case"latin1":case"binary":return r;case"utf8":case"utf-8":return q(t).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*r;case"hex":return r>>>1;case"base64":return Q(t).length;default:if(n)return i?-1:q(t).length;e=(""+e).toLowerCase(),n=!0}}function m(t,e,r){let i=!1;if((void 0===e||e<0)&&(e=0),e>this.length)return"";if((void 0===r||r>this.length)&&(r=this.length),r<=0)return"";if((r>>>=0)<=(e>>>=0))return"";for(t||(t="utf8");;)switch(t){case"hex":return k(this,e,r);case"utf8":case"utf-8":return x(this,e,r);case"ascii":return I(this,e,r);case"latin1":case"binary":return R(this,e,r);case"base64":return T(this,e,r);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return N(this,e,r);default:if(i)throw new TypeError("Unknown encoding: "+t);t=(t+"").toLowerCase(),i=!0}}function w(t,e,r){const i=t[e];t[e]=t[r],t[r]=i}function y(t,e,r,i,n){if(0===t.length)return-1;if("string"==typeof r?(i=r,r=0):r>2147483647?r=2147483647:r<-2147483648&&(r=-2147483648),X(r=+r)&&(r=n?0:t.length-1),r<0&&(r=t.length+r),r>=t.length){if(n)return-1;r=t.length-1}else if(r<0){if(!n)return-1;r=0}if("string"==typeof e&&(e=c.from(e,i)),c.isBuffer(e))return 0===e.length?-1:b(t,e,r,i,n);if("number"==typeof e)return e&=255,"function"==typeof Uint8Array.prototype.indexOf?n?Uint8Array.prototype.indexOf.call(t,e,r):Uint8Array.prototype.lastIndexOf.call(t,e,r):b(t,[e],r,i,n);throw new TypeError("val must be string, number or Buffer")}function b(t,e,r,i,n){let s,o=1,a=t.length,c=e.length;if(void 0!==i&&("ucs2"===(i=String(i).toLowerCase())||"ucs-2"===i||"utf16le"===i||"utf-16le"===i)){if(t.length<2||e.length<2)return-1;o=2,a/=2,c/=2,r/=2}function l(t,e){return 1===o?t[e]:t.readUInt16BE(e*o)}if(n){let i=-1;for(s=r;s<a;s++)if(l(t,s)===l(e,-1===i?0:s-i)){if(-1===i&&(i=Line truncated
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+e.read=function(t,e,r,i,n){var s,o,a=8*n-i-1,c=(1<<a)-1,l=c>>1,h=-7,u=r?n-1:0,p=r?-1:1,d=t[e+u];for(u+=p,s=d&(1<<-h)-1,d>>=-h,h+=a;h>0;s=256*s+t[e+u],u+=p,h-=8);for(o=s&(1<<-h)-1,s>>=-h,h+=i;h>0;o=256*o+t[e+u],u+=p,h-=8);if(0===s)s=1-l;else{if(s===c)return o?NaN:1/0*(d?-1:1);o+=Math.pow(2,i),s-=l}return(d?-1:1)*o*Math.pow(2,s-i)},e.write=function(t,e,r,i,n,s){var o,a,c,l=8*s-n-1,h=(1<<l)-1,u=h>>1,p=23===n?Math.pow(2,-24)-Math.pow(2,-77):0,d=i?0:s-1,f=i?1:-1,g=e<0||0===e&&1/e<0?1:0;for(e=Math.abs(e),isNaN(e)||e===1/0?(a=isNaN(e)?1:0,o=h):(o=Math.floor(Math.log(e)/Math.LN2),e*(c=Math.pow(2,-o))<1&&(o--,c*=2),(e+=o+u>=1?p/c:p*Math.pow(2,1-u))*c>=2&&(o++,c/=2),o+u>=h?(a=0,o=h):o+u>=1?(a=(e*c-1)*Math.pow(2,n),o+=u):(a=e*Math.pow(2,u-1)*Math.pow(2,n),o=0));n>=8;t[r+d]=255&a,d+=f,a/=256,n-=8);for(o=o<<n|a,l+=n;l>0;t[r+d]=255&o,d+=f,o/=256,l-=8);t[r+d-f]|=128*g}},5717:t=>{"function"==typeof Object.create?t.exports=function(t,e){e&&(t.super_=e,t.prototype=Object.create(e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}))}:t.exports=function(t,e){if(e){t.super_=e;var r=function(){};r.prototype=e.prototype,t.prototype=new r,t.prototype.constructor=t}}},6512:function(t){!function(e){"use strict";const r="(0?\\d+|0x[a-f0-9]+)",i={fourOctet:new RegExp(`^${r}\\.${r}\\.${r}\\.${r}$`,"i"),threeOctet:new RegExp(`^${r}\\.${r}\\.${r}$`,"i"),twoOctet:new RegExp(`^${r}\\.${r}$`,"i"),longValue:new RegExp(`^${r}$`,"i")},n=new RegExp("^0[0-7]+$","i"),s=new RegExp("^0x[a-f0-9]+$","i"),o="%[0-9a-z]{1,}",a="(?:[0-9a-f]+::?)+",c={zoneIndex:new RegExp(o,"i"),native:new RegExp(`^(::)?(${a})?([0-9a-f]+)?(::)?(${o})?$`,"i"),deprecatedTransitional:new RegExp(`^(?:::)(${r}\\.${r}\\.${r}\\.${r}(${o})?)$`,"i"),transitional:new RegExp(`^((?:${a})|(?:::)(?:${a})?)${r}\\.${r}\\.${r}\\.${r}(${o})?$`,"i")};function l(t,e){if(t.indexOf("::")!==t.lastIndexOf("::"))return null;let r,i,n=0,s=-1,o=(t.match(c.zoneIndex)||[])[0];for(o&&(o=o.substring(1),t=t.replace(/%.+$/,""));(s=t.indexOf(":",s+1))>=0;)n++;if("::"===t.substr(0,2)&&n--,"::"===t.substr(-2,2)&&n--,n>e)return null;for(i=e-n,r=":";i--;)r+="0:";return":"===(t=t.replace("::",r))[0]&&(t=t.slice(1)),":"===t[t.length-1]&&(t=t.slice(0,-1)),{parts:e=function(){const e=t.split(":"),r=[];for(let t=0;t<e.length;t++)r.push(parseInt(e[t],16));return r}(),zoneId:o}}function h(t,e,r,i){if(t.length!==e.length)throw new Error("ipaddr: cannot match CIDR for objects with different lengths");let n,s=0;for(;i>0;){if(n=r-i,n<0&&(n=0),t[s]>>n!=e[s]>>n)return!1;i-=r,s+=1}return!0}function u(t){if(s.test(t))return parseInt(t,16);if("0"===t[0]&&!isNaN(parseInt(t[1],10))){if(n.test(t))return parseInt(t,8);throw new Error(`ipaddr: cannot parse ${t} as octal`)}return parseInt(t,10)}function p(t,e){for(;t.length<e;)t=`0${t}`;return t}const d={};d.IPv4=function(){function t(t){if(4!==t.length)throw new Error("ipaddr: ipv4 octet count should be 4");let e,r;for(e=0;e<t.length;e++)if(r=t[e],!(0<=r&&r<=255))throw new Error("ipaddr: ipv4 octet should fit in 8 bits");this.octets=t}return t.prototype.SpecialRanges={unspecified:[[new t([0,0,0,0]),8]],broadcast:[[new t([255,255,255,255]),32]],multicast:[[new t([224,0,0,0]),4]],linkLocal:[[new t([169,254,0,0]),16]],loopback:[[new t([127,0,0,0]),8]],carrierGradeNat:[[new t([100,64,0,0]),10]],private:[[new t([10,0,0,0]),8],[new t([172,16,0,0]),12],[new t([192,168,0,0]),16]],reserved:[[new t([192,0,0,0]),24],[new t([192,0,2,0]),24],[new t([192,88,99,0]),24],[new t([198,51,100,0]),24],[new t([203,0,113,0]),24],[new t([240,0,0,0]),4]]},t.prototype.kind=function(){return"ipv4"},t.prototype.match=function(t,e){let r;if(void 0===e&&(r=t,t=r[0],e=r[1]),"ipv4"!==t.kind())throw new Error("ipaddr: cannot match ipv4 address with non-ipv4 one");return h(this.octets,t.octets,8,e)},t.prototype.prefixLengthFromSubnetMask=function(){let t=0,e=!1;const r={0:8,128:7,192:6,224:5,240:4,248:3,252:2,254:1,255:0};let i,n,s;for(i=3;i>=0;i-=1){if(n=this.octets[i],!(n in r))return null;if(s=r[n],e&&0!==s)return null;8!==s&&(e=!0),t+=s}return 32-t},t.prototype.range=function(){return d.subnetMatch(this,this.SpecialRanges)},t.prototype.toByteArray=function(){return this.octets.slice(0)},t.prototype.toIPv4MappedAddress=function(){return d.IPv6.parse(`::ffff:${this.toString()}`)},t.prototype.toNormalizedString=function(){return this.toString()},t.prototype.toString=function(){return this.octets.join(".")},t}(),d.IPv4.broadcastAddressFromCIDR=function(t){try{const e=this.parseCIDR(t),r=e[0].toByteArray(),i=this.subnetMaskFromPrefixLength(e[1]).toByteArray(),n=[];let s=0;for(;s<4;)n.push(parseInt(r[s],10)|255^parseInt(i[s],10)),s++;return new this(n)}catch(t){throw new Error("ipaddr: the address does not have IPv4 CIDR format")}},d.IPv4.isIPv4=function(t){return null!==this.parser(t)},d.IPv4.isValid=function(t){try{return new this(this.parser(t)),!0}catch(t){return!1}},d.IPv4.isValidFourPartDecimal=function(t){return!(!d.IPv4.isValid(t)||!t.match(/^(0|[1-9]\d*)(\.(0|[1-9]\d*)){3}$/))},d.Line truncated
+/*! *****************************************************************************
+Copyright (C) Microsoft. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */!function(t){!function(e){var i="object"==typeof r.g?r.g:"object"==typeof self?self:"object"==typeof this?this:Function("return this;")(),s=o(t);function o(t,e){return function(r,i){"function"!=typeof t[r]&&Object.defineProperty(t,r,{configurable:!0,writable:!0,value:i}),e&&e(r,i)}}void 0===i.Reflect?i.Reflect=t:s=o(i.Reflect,s),function(t){var e=Object.prototype.hasOwnProperty,r="function"==typeof Symbol,i=r&&void 0!==Symbol.toPrimitive?Symbol.toPrimitive:"@@toPrimitive",s=r&&void 0!==Symbol.iterator?Symbol.iterator:"@@iterator",o="function"==typeof Object.create,a={__proto__:[]}instanceof Array,c=!o&&!a,l={create:o?function(){return M(Object.create(null))}:a?function(){return M({__proto__:null})}:function(){return M({})},has:c?function(t,r){return e.call(t,r)}:function(t,e){return e in t},get:c?function(t,r){return e.call(t,r)?t[r]:void 0}:function(t,e){return t[e]}},h=Object.getPrototypeOf(Function),u="object"==typeof n&&n.env&&"true"===n.env.REFLECT_METADATA_USE_MAP_POLYFILL,p=u||"function"!=typeof Map||"function"!=typeof Map.prototype.entries?function(){var t={},e=[],r=function(){function t(t,e,r){this._index=0,this._keys=t,this._values=e,this._selector=r}return t.prototype["@@iterator"]=function(){return this},t.prototype[s]=function(){return this},t.prototype.next=function(){var t=this._index;if(t>=0&&t<this._keys.length){var r=this._selector(this._keys[t],this._values[t]);return t+1>=this._keys.length?(this._index=-1,this._keys=e,this._values=e):this._index++,{value:r,done:!1}}return{value:void 0,done:!0}},t.prototype.throw=function(t){throw this._index>=0&&(this._index=-1,this._keys=e,this._values=e),t},t.prototype.return=function(t){return this._index>=0&&(this._index=-1,this._keys=e,this._values=e),{value:t,done:!0}},t}();return function(){function e(){this._keys=[],this._values=[],this._cacheKey=t,this._cacheIndex=-2}return Object.defineProperty(e.prototype,"size",{get:function(){return this._keys.length},enumerable:!0,configurable:!0}),e.prototype.has=function(t){return this._find(t,!1)>=0},e.prototype.get=function(t){var e=this._find(t,!1);return e>=0?this._values[e]:void 0},e.prototype.set=function(t,e){var r=this._find(t,!0);return this._values[r]=e,this},e.prototype.delete=function(e){var r=this._find(e,!1);if(r>=0){for(var i=this._keys.length,n=r+1;n<i;n++)this._keys[n-1]=this._keys[n],this._values[n-1]=this._values[n];return this._keys.length--,this._values.length--,e===this._cacheKey&&(this._cacheKey=t,this._cacheIndex=-2),!0}return!1},e.prototype.clear=function(){this._keys.length=0,this._values.length=0,this._cacheKey=t,this._cacheIndex=-2},e.prototype.keys=function(){return new r(this._keys,this._values,i)},e.prototype.values=function(){return new r(this._keys,this._values,n)},e.prototype.entries=function(){return new r(this._keys,this._values,o)},e.prototype["@@iterator"]=function(){return this.entries()},e.prototype[s]=function(){return this.entries()},e.prototype._find=function(t,e){return this._cacheKey!==t&&(this._cacheIndex=this._keys.indexOf(this._cacheKey=t)),this._cacheIndex<0&&e&&(this._cacheIndex=this._keys.length,this._keys.push(t),this._values.push(void 0)),this._cacheIndex},e}();function i(t,e){return t}function n(t,e){return e}function o(t,e){return[t,e]}}():Map,d=u||"function"!=typeof Set||"function"!=typeof Set.prototype.entries?function(){function t(){this._map=new p}return Object.defineProperty(t.prototype,"size",{get:function(){return this._map.size},enumerable:!0,configurable:!0}),t.prototype.has=function(t){return this._map.has(t)},t.prototype.add=function(t){return this._map.set(t,t),this},t.prototype.delete=function(t){return this._map.delete(t)},t.prototype.clear=function(){this._map.clear()},t.prototype.keys=function(){return this._map.keys()},t.prototype.values=function(){return this._map.values()},t.prototype.entries=function(){return this._map.entries()},t.prototype["@@iterator"]=function(){return this.keys()},t.prototype[s]=function(){return this.keys()},t}():Set,f=new(u||"function"!=typeof WeakMap?function(){var t=l.create(),r=i();return function(){function t(){this._key=i()}return t.prototype.has=function(t){var e=n(t,!1);return void 0!==e&&l.has(e,this._key)},t.prototype.get=function(t){var e=n(t,!1);return void 0!==e?l.get(e,this._key):void 0},t.prototype.set=function(t,e){return n(t,!0)[this._key]=e,this},t.prototype.delete=function(t){var e=n(t,!1);return void 0!==e&&delete e[this._key]},t.prototype.clear=function(){this._key=i()},t}();function i(){var e;do{e="@@WeakMap@@"+o()}while(l.has(t,e));return t[e]=!0,e}function n(t,i){if(!e.call(t,r)){if(!i)return;Object.defineProperty(t,r,{value:l.create()})}return t[r]}function s(t,e){for(var r=0;r<e;++r)t[r]=255*Math.random()|0;return t}function o(){var t=function(t){return"function"==typeof Uint8Array?"undefined"!=typeof crypto?crypto.getRandomValues(new Uint8Array(t)):"undefined"!=typeof msCrypto?msCrypto.getRandomValues(nLine truncated
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+var i=r(8764),n=i.Buffer;function s(t,e){for(var r in t)e[r]=t[r]}function o(t,e,r){return n(t,e,r)}n.from&&n.alloc&&n.allocUnsafe&&n.allocUnsafeSlow?t.exports=i:(s(i,e),e.Buffer=o),o.prototype=Object.create(n.prototype),s(n,o),o.from=function(t,e,r){if("number"==typeof t)throw new TypeError("Argument must not be a number");return n(t,e,r)},o.alloc=function(t,e,r){if("number"!=typeof t)throw new TypeError("Argument must be a number");var i=n(t);return void 0!==e?"string"==typeof r?i.fill(e,r):i.fill(e):i.fill(0),i},o.allocUnsafe=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return n(t)},o.allocUnsafeSlow=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return i.SlowBuffer(t)}},2830:(t,e,r)=>{t.exports=n;var i=r(7187).EventEmitter;function n(){i.call(this)}r(5717)(n,i),n.Readable=r(9481),n.Writable=r(4229),n.Duplex=r(6753),n.Transform=r(4605),n.PassThrough=r(2725),n.finished=r(8610),n.pipeline=r(9946),n.Stream=n,n.prototype.pipe=function(t,e){var r=this;function n(e){t.writable&&!1===t.write(e)&&r.pause&&r.pause()}function s(){r.readable&&r.resume&&r.resume()}r.on("data",n),t.on("drain",s),t._isStdio||e&&!1===e.end||(r.on("end",a),r.on("close",c));var o=!1;function a(){o||(o=!0,t.end())}function c(){o||(o=!0,"function"==typeof t.destroy&&t.destroy())}function l(t){if(h(),0===i.listenerCount(this,"error"))throw t}function h(){r.removeListener("data",n),t.removeListener("drain",s),r.removeListener("end",a),r.removeListener("close",c),r.removeListener("error",l),t.removeListener("error",l),r.removeListener("end",h),r.removeListener("close",h),t.removeListener("close",h)}return r.on("error",l),t.on("error",l),r.on("end",h),r.on("close",h),t.on("close",h),t.emit("pipe",r),t}},2553:(t,e,r)=>{"use strict";var i=r(9509).Buffer,n=i.isEncoding||function(t){switch((t=""+t)&&t.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function s(t){var e;switch(this.encoding=function(t){var e=function(t){if(!t)return"utf8";for(var e;;)switch(t){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return t;default:if(e)return;t=(""+t).toLowerCase(),e=!0}}(t);if("string"!=typeof e&&(i.isEncoding===n||!n(t)))throw new Error("Unknown encoding: "+t);return e||t}(t),this.encoding){case"utf16le":this.text=c,this.end=l,e=4;break;case"utf8":this.fillLast=a,e=4;break;case"base64":this.text=h,this.end=u,e=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=i.allocUnsafe(e)}function o(t){return t<=127?0:t>>5==6?2:t>>4==14?3:t>>3==30?4:t>>6==2?-1:-2}function a(t){var e=this.lastTotal-this.lastNeed,r=function(t,e,r){if(128!=(192&e[0]))return t.lastNeed=0,"�";if(t.lastNeed>1&&e.length>1){if(128!=(192&e[1]))return t.lastNeed=1,"�";if(t.lastNeed>2&&e.length>2&&128!=(192&e[2]))return t.lastNeed=2,"�"}}(this,t);return void 0!==r?r:this.lastNeed<=t.length?(t.copy(this.lastChar,e,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(t.copy(this.lastChar,e,0,t.length),void(this.lastNeed-=t.length))}function c(t,e){if((t.length-e)%2==0){var r=t.toString("utf16le",e);if(r){var i=r.charCodeAt(r.length-1);if(i>=55296&&i<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1],r.slice(0,-1)}return r}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=t[t.length-1],t.toString("utf16le",e,t.length-1)}function l(t){var e=t&&t.length?this.write(t):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return e+this.lastChar.toString("utf16le",0,r)}return e}function h(t,e){var r=(t.length-e)%3;return 0===r?t.toString("base64",e):(this.lastNeed=3-r,this.lastTotal=3,1===r?this.lastChar[0]=t[t.length-1]:(this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1]),t.toString("base64",e,t.length-r))}function u(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+this.lastChar.toString("base64",0,3-this.lastNeed):e}function p(t){return t.toString(this.encoding)}function d(t){return t&&t.length?this.write(t):""}e.s=s,s.prototype.write=function(t){if(0===t.length)return"";var e,r;if(this.lastNeed){if(void 0===(e=this.fillLast(t)))return"";r=this.lastNeed,this.lastNeed=0}else r=0;return r<t.length?e?e+this.text(t,r):this.text(t,r):e||""},s.prototype.end=function(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+"�":e},s.prototype.text=function(t,e){var r=function(t,e,r){var i=e.length-1;if(i<r)return 0;var n=o(e[i]);return n>=0?(n>0&&(t.lastNeed=n-1),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(t.lastNeed=n-2),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(2===n?n=0:t.lastNeed=n-3),n):0}(this,t,e);if(!this.lastNeed)return t.toString("utf8",e);this.lastTotal=r;var i=t.length-(r-this.lastNeed);return t.copy(tLine truncated
+/*!
+ * hash-wasm (https://www.npmjs.com/package/hash-wasm)
+ * (c) Dani Biro
+ * @license MIT
+ */
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */function B(t,e,r,i){return new(r||(r=Promise))((function(n,s){function o(t){try{c(i.next(t))}catch(t){s(t)}}function a(t){try{c(i.throw(t))}catch(t){s(t)}}function c(t){var e;t.done?n(t.value):(e=t.value,e instanceof r?e:new r((function(t){t(e)}))).then(o,a)}c((i=i.apply(t,e||[])).next())}))}class P{constructor(){this.mutex=Promise.resolve()}lock(){let t=()=>{};return this.mutex=this.mutex.then((()=>new Promise(t))),new Promise((e=>{t=e}))}dispatch(t){return B(this,void 0,void 0,(function*(){const e=yield this.lock();try{return yield Promise.resolve(t())}finally{e()}}))}}const L="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:r.g,D=null!==(I=L.Buffer)&&void 0!==I?I:null,U=L.TextEncoder?new L.TextEncoder:null;function M(t,e){return(15&t)+(t>>6|t>>3&8)<<4|(15&e)+(e>>6|e>>3&8)}const F="a".charCodeAt(0)-10,H="0".charCodeAt(0);function W(t,e,r){let i=0;for(let n=0;n<r;n++){let r=e[n]>>>4;t[i++]=r>9?r+F:r+H,r=15&e[n],t[i++]=r>9?r+F:r+H}return String.fromCharCode.apply(null,t)}const j=null!==D?t=>{if("string"==typeof t){const e=D.from(t,"utf8");return new Uint8Array(e.buffer,e.byteOffset,e.length)}if(D.isBuffer(t))return new Uint8Array(t.buffer,t.byteOffset,t.length);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")}:t=>{if("string"==typeof t)return U.encode(t);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")},V="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",G=new Uint8Array(256);for(let t=0;t<V.length;t++)G[V.charCodeAt(t)]=t;function z(t){const e=function(t){let e=Math.floor(.75*t.length);const r=t.length;return"="===t[r-1]&&(e-=1,"="===t[r-2]&&(e-=1)),e}(t),r=t.length,i=new Uint8Array(e);let n=0;for(let e=0;e<r;e+=4){const r=G[t.charCodeAt(e)],s=G[t.charCodeAt(e+1)],o=G[t.charCodeAt(e+2)],a=G[t.charCodeAt(e+3)];i[n]=r<<2|s>>4,n+=1,i[n]=(15&s)<<4|o>>2,n+=1,i[n]=(3&o)<<6|63&a,n+=1}return i}const q=16384,Q=new P,K=new Map;function Y(t,e){return B(this,void 0,void 0,(function*(){let r=null,i=null,n=!1;if("undefined"==typeof WebAssembly)throw new Error("WebAssembly is not supported in this environment!");const s=()=>new DataView(r.exports.memory.buffer).getUint32(r.exports.STATE_SIZE,!0),o=Q.dispatch((()=>B(this,void 0,void 0,(function*(){if(!K.has(t.name)){const e=z(t.data),r=WebAssembly.compile(e);K.set(t.name,r)}const e=yield K.get(t.name);r=yield WebAssembly.instantiate(e,{})})))),a=(t=null)=>{n=!0,r.exports.Hash_Init(t)},c=t=>{if(!n)throw new Error("update() called before init()");(t=>{let e=0;for(;e<t.length;){const n=t.subarray(e,e+q);e+=n.length,i.set(n),r.exports.Hash_Update(n.length)}})(j(t))},l=new Uint8Array(2*e),h=(t,s=null)=>{if(!n)throw new Error("digest() called before init()");return n=!1,r.exports.Hash_Final(s),"binary"===t?i.slice(0,e):W(l,i,e)},u=t=>"string"==typeof t?t.length<4096:t.byteLength<q;let p=u;switch(t.name){case"argon2":case"scrypt":p=()=>!0;break;case"blake2b":case"blake2s":p=(t,e)=>e<=512&&u(t);break;case"blake3":p=(t,e)=>0===e&&u(t);break;case"xxhash64":case"xxhash3":case"xxhash128":p=()=>!1}return yield(()=>B(this,void 0,void 0,(function*(){r||(yield o);const t=r.exports.Hash_GetBuffer(),e=r.exports.memory.buffer;i=new Uint8Array(e,t,q)})))(),{getMemory:()=>i,writeMemory:(t,e=0)=>{i.set(t,e)},getExports:()=>r.exports,setMemorySize:t=>{r.exports.Hash_SetMemorySize(t);const e=r.exports.Hash_GetBuffer(),n=r.exports.memory.buffer;i=new Uint8Array(n,e,t)},init:a,update:c,digest:h,save:()=>{if(!n)throw new Error("save() can only be called after init() and before digest()");const e=r.exports.Hash_GetState(),i=s(),o=r.exports.memory.buffer,a=new Uint8Array(o,e,i),c=new Uint8Array(4+i);return function(t,e){const r=e.length>>1;for(let i=0;i<r;i++){const r=i<<1;t[i]=M(e.charCodeAt(r),e.charCodeAt(r+1))}}(c,t.hash),c.set(a,4),c},load:e=>{if(!(e instanceof Uint8Array))throw new Error("load() expects an Uint8Array generated by save()");const i=r.exports.Hash_GetState(),o=s(),a=4+o,c=r.exports.memory.buffer;if(e.length!==a)throw new Error(`Bad state length (expected ${a} bytes, got ${e.length})`);if(!function(t,e){if(t.length!==2*e.length)return!1;for(let r=0;r<e.length;r++){const i=r<<1;if(e[r]!==M(t.charCodeAt(i),t.charCodeAt(i+1)))return!1}return!0}(t.hash,e.subarray(0,4)))throw new Error("This state was written by an incompatible hash implementation");const l=e.subarray(4);new Uint8Array(c,i,o).set(l),n=!0},calculate:(t,n=null,s=null)=>{if(!p(t,n))return a(n),c(t),h("hex",s);const o=j(t);return i.set(o),r.exports.Hash_Calculate(o.length,n,s),W(l,i,e)},hashLength:e}}))}new P,new P,new DataView(new ArrayBuffer(4)),new P,new P,new P,new P,new P,new P,new P,new P,new P;var X={name:"sha256",data:"AGFzbQEAAAABEQRgAAF/YAF/AGACf38AYAAAAwgHAAEBAgMAAgQFAXABAQEFBAEBAgIGDgJ/AUHwiQULfwBBgAgLB3AIBm1lbW9yeQIADLine truncated
+/*! js-yaml 4.1.0 https://github.com/nodeca/js-yaml @license MIT */function so(t){return null==t}var oo={isNothing:so,isObject:function(t){return"object"==typeof t&&null!==t},toArray:function(t){return Array.isArray(t)?t:so(t)?[]:[t]},repeat:function(t,e){var r,i="";for(r=0;r<e;r+=1)i+=t;return i},isNegativeZero:function(t){return 0===t&&Number.NEGATIVE_INFINITY===1/t},extend:function(t,e){var r,i,n,s;if(e)for(r=0,i=(s=Object.keys(e)).length;r<i;r+=1)t[n=s[r]]=e[n];return t}};function ao(t,e){var r="",i=t.reason||"(unknown reason)";return t.mark?(t.mark.name&&(r+='in "'+t.mark.name+'" '),r+="("+(t.mark.line+1)+":"+(t.mark.column+1)+")",!e&&t.mark.snippet&&(r+="\n\n"+t.mark.snippet),i+" "+r):i}function co(t,e){Error.call(this),this.name="YAMLException",this.reason=t,this.mark=e,this.message=ao(this,!1),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack||""}co.prototype=Object.create(Error.prototype),co.prototype.constructor=co,co.prototype.toString=function(t){return this.name+": "+ao(this,t)};var lo=co;function ho(t,e,r,i,n){var s="",o="",a=Math.floor(n/2)-1;return i-e>a&&(e=i-a+(s=" ... ").length),r-i>a&&(r=i+a-(o=" ...").length),{str:s+t.slice(e,r).replace(/\t/g,"→")+o,pos:i-e+s.length}}function uo(t,e){return oo.repeat(" ",e-t.length)+t}var po=["kind","multi","resolve","construct","instanceOf","predicate","represent","representName","defaultStyle","styleAliases"],fo=["scalar","sequence","mapping"],go=function(t,e){if(e=e||{},Object.keys(e).forEach((function(e){if(-1===po.indexOf(e))throw new lo('Unknown option "'+e+'" is met in definition of "'+t+'" YAML type.')})),this.options=e,this.tag=t,this.kind=e.kind||null,this.resolve=e.resolve||function(){return!0},this.construct=e.construct||function(t){return t},this.instanceOf=e.instanceOf||null,this.predicate=e.predicate||null,this.represent=e.represent||null,this.representName=e.representName||null,this.defaultStyle=e.defaultStyle||null,this.multi=e.multi||!1,this.styleAliases=function(t){var e={};return null!==t&&Object.keys(t).forEach((function(r){t[r].forEach((function(t){e[String(t)]=r}))})),e}(e.styleAliases||null),-1===fo.indexOf(this.kind))throw new lo('Unknown kind "'+this.kind+'" is specified for "'+t+'" YAML type.')};function mo(t,e){var r=[];return t[e].forEach((function(t){var e=r.length;r.forEach((function(r,i){r.tag===t.tag&&r.kind===t.kind&&r.multi===t.multi&&(e=i)})),r[e]=t})),r}function wo(t){return this.extend(t)}wo.prototype.extend=function(t){var e=[],r=[];if(t instanceof go)r.push(t);else if(Array.isArray(t))r=r.concat(t);else{if(!t||!Array.isArray(t.implicit)&&!Array.isArray(t.explicit))throw new lo("Schema.extend argument should be a Type, [ Type ], or a schema definition ({ implicit: [...], explicit: [...] })");t.implicit&&(e=e.concat(t.implicit)),t.explicit&&(r=r.concat(t.explicit))}e.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.");if(t.loadKind&&"scalar"!==t.loadKind)throw new lo("There is a non-scalar type in the implicit list of a schema. Implicit resolving of such types is not supported.");if(t.multi)throw new lo("There is a multi type in the implicit list of a schema. Multi tags can only be listed as explicit.")})),r.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.")}));var i=Object.create(wo.prototype);return i.implicit=(this.implicit||[]).concat(e),i.explicit=(this.explicit||[]).concat(r),i.compiledImplicit=mo(i,"implicit"),i.compiledExplicit=mo(i,"explicit"),i.compiledTypeMap=function(){var t,e,r={scalar:{},sequence:{},mapping:{},fallback:{},multi:{scalar:[],sequence:[],mapping:[],fallback:[]}};function i(t){t.multi?(r.multi[t.kind].push(t),r.multi.fallback.push(t)):r[t.kind][t.tag]=r.fallback[t.tag]=t}for(t=0,e=arguments.length;t<e;t+=1)arguments[t].forEach(i);return r}(i.compiledImplicit,i.compiledExplicit),i};var yo=wo,bo=new go("tag:yaml.org,2002:str",{kind:"scalar",construct:function(t){return null!==t?t:""}}),Ao=new go("tag:yaml.org,2002:seq",{kind:"sequence",construct:function(t){return null!==t?t:[]}}),_o=new go("tag:yaml.org,2002:map",{kind:"mapping",construct:function(t){return null!==t?t:{}}}),vo=new yo({explicit:[bo,Ao,_o]}),Eo=new go("tag:yaml.org,2002:null",{kind:"scalar",resolve:function(t){if(null===t)return!0;var e=t.length;return 1===e&&"~"===t||4===e&&("null"===t||"Null"===t||"NULL"===t)},construct:function(){return null},predicate:function(t){return null===t},represent:{canonical:function(){return"~"},lowercase:function(){return"null"},uppercase:function(){return"NULL"},camelcase:function(){return"Null"},empty:function(){return""}},defaultStyle:"lowercase"}),So=new go("tag:yaml.org,2002:bool",{kind:"scalar",resolve:function(t){if(null===t)return!1;var e=t.length;return 4===e&&("true"===t||"True"===t||"TRUE"===t)||5===e&&("false"===t||"False"===t||"FALSE"===t)},construct:fuLine truncated
+/*!
+ * Copyright (c) 2014, GMO GlobalSign
+ * Copyright (c) 2015-2022, Peculiar Ventures
+ * All rights reserved.
+ * 
+ * Author 2014-2019, Yury Strozhevsky
+ * 
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ * 
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * 
+ */
+function lc(){if("undefined"==typeof BigInt)throw new Error("BigInt is not defined. Your environment doesn't implement BigInt.")}function hc(t){let e=0,r=0;for(let r=0;r<t.length;r++)e+=t[r].byteLength;const i=new Uint8Array(e);for(let e=0;e<t.length;e++){const n=t[e];i.set(new Uint8Array(n),r),r+=n.byteLength}return i.buffer}function uc(t,e,r,i){return e instanceof Uint8Array?e.byteLength?r<0?(t.error="Wrong parameter: inputOffset less than zero",!1):i<0?(t.error="Wrong parameter: inputLength less than zero",!1):!(e.byteLength-r-i<0&&(t.error="End of input reached before message was fully decoded (inconsistent offset and length values)",1)):(t.error="Wrong parameter: inputBuffer has zero length",!1):(t.error="Wrong parameter: inputBuffer must be 'Uint8Array'",!1)}Math.log(2);class pc{constructor(){this.items=[]}write(t){this.items.push(t)}final(){return hc(this.items)}}const dc=[new Uint8Array([1])],fc="0123456789",gc="name",mc="valueHexView",wc="",yc=new ArrayBuffer(0),bc=new Uint8Array(0),Ac="EndOfContent",_c="OCTET STRING",vc="BIT STRING";function Ec(t){var e;return(e=class extends t{constructor(...t){var e;super(...t);const r=t[0]||{};this.isHexOnly=null!==(e=r.isHexOnly)&&void 0!==e&&e,this.valueHexView=r.valueHex?ic.vJ.toUint8Array(r.valueHex):bc}get valueHex(){return this.valueHexView.slice().buffer}set valueHex(t){this.valueHexView=new Uint8Array(t)}fromBER(t,e,r){const i=t instanceof ArrayBuffer?new Uint8Array(t):t;if(!uc(this,i,e,r))return-1;const n=e+r;return this.valueHexView=i.subarray(e,n),this.valueHexView.length?(this.blockLength=r,n):(this.warnings.push("Zero buffer length"),e)}toBER(t=!1){return this.isHexOnly?t?new ArrayBuffer(this.valueHexView.byteLength):this.valueHexView.byteLength===this.valueHexView.buffer.byteLength?this.valueHexView.buffer:this.valueHexView.slice().buffer:(this.error="Flag 'isHexOnly' is not set, abort",yc)}toJSON(){return{...super.toJSON(),isHexOnly:this.isHexOnly,valueHex:ic.ep.ToHex(this.valueHexView)}}}).NAME="hexBlock",e}class Sc{constructor({blockLength:t=0,error:e="",warnings:r=[],valueBeforeDecode:i=bc}={}){this.blockLength=t,this.error=e,this.warnings=r,this.valueBeforeDecodeView=ic.vJ.toUint8Array(i)}static blockName(){return this.NAME}get valueBeforeDecode(){return this.valueBeforeDecodeView.slice().buffer}set valueBeforeDecode(t){this.valueBeforeDecodeView=new Uint8Array(t)}toJSON(){return{blockName:this.constructor.NAME,blockLength:this.blockLength,error:this.error,warnings:this.warnings,valueBeforeDecode:ic.ep.ToHex(this.valueBeforeDecodeView)}}}Sc.NAME="baseBlock";class Tc extends Sc{fromBER(t,e,r){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}toBER(t,e){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}}Tc.NAME="valueBlock";class xc extends(Ec(Sc)){constructor({idBlock:t={}}={}){var e,r,i,n;super(),t?(this.isHexOnly=null!==(e=t.isHexOnly)&&void 0!==e&&e,this.valueHexView=t.valueHex?ic.vJ.toUint8Array(t.valueHex):bc,this.tagClass=null!==(r=t.tagClass)&&void 0!==r?r:-1,this.tagNumber=null!==(i=t.tagNumber)&&void 0!==i?i:-1,this.isConstructed=null!==(n=t.isConstructed)&&void 0!==n&&n):(this.tagClass=-1,this.tagNumber=-1,this.isConstructed=!1)}toBER(t=!1){let e=0;switch(this.tagClass){case 1:e|=0;break;case 2:e|=64;break;case 3:e|=128;break;case 4:e|=192;break;default:return this.error="Unknown tag class",yc}if(this.isConstructed&&(e|=32),this.tagNumber<31&&!this.isHexOnly){const r=new Uint8Array(1);if(!t){let t=this.tagNumber;t&=31,e|=t,r[0]=e}return r.buffer}if(!this.isHexOnly){const r=sc(this.tagNumber,7),i=new Uint8Array(r),n=r.byteLength,s=new Uint8Array(n+1);if(s[0]=31|e,!t){for(let t=0;t<n-1;t++)s[t+1]=128|i[t];s[n]=i[n-1]}return s.buffer}const r=new Uint8Array(this.valueHexView.byteLength+1);if(r[0]=31|e,!t){const t=this.valueHexView;for(let e=0;e<t.length-1;e++)r[e+1]=128|t[e];r[this.valueHexView.byteLength]=t[t.length-1]}return r.buffer}fromBER(t,e,r){const i=ic.vJ.toUint8Array(t);if(!uc(this,i,e,r))return-1;const n=i.subarray(e,e+r);if(0===n.length)return this.error="Zero buffer length",-1;switch(192&n[0]){case 0:this.tagClass=1;break;case 64:this.tagClass=2;break;case 128:this.tagClass=3;break;case 192:this.tagClass=4;break;default:return this.error="Unknown tag class",-1}this.isConstructed=32==(32&n[0]),this.isHexOnly=!1;const s=31&n[0];if(31!==s)this.tagNumber=s,this.blockLength=1;else{let t=1,e=this.valueHexView=new Uint8Array(255),r=255;for(;128&n[t];){if(e[t-1]=127&n[t],t++,t>=n.length)return this.error="End of input reached before message was fully decoded",-1;if(t===r){r+=255;const t=new Uint8Array(r);for(let r=0;r<e.length;r++)t[r]=e[r];e=this.valueHexView=new Uint8Array(r)}}this.blockLength=t+1,e[t-1]=127&n[t];const i=new Uint8Array(t);for(let r=0;r<t;r++)i[r]=e[r];e=this.valueHexView=new Uint8Array(t),e.set(i),this.blockLength<=9?this.tagNumber=nc(e,7):(this.isHexOnly=!0,this.warnings.push("Tag too long, represented as hex-coded"))}if(1=Line truncated
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */var vg=function(t,e){return vg=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])},vg(t,e)};function Eg(t,e){function r(){this.constructor=t}vg(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}function Sg(t){var e="function"==typeof Symbol&&Symbol.iterator,r=e&&t[e],i=0;if(r)return r.call(t);if(t&&"number"==typeof t.length)return{next:function(){return t&&i>=t.length&&(t=void 0),{value:t&&t[i++],done:!t}}};throw new TypeError(e?"Object is not iterable.":"Symbol.iterator is not defined.")}function Tg(t,e){var r="function"==typeof Symbol&&t[Symbol.iterator];if(!r)return t;var i,n,s=r.call(t),o=[];try{for(;(void 0===e||e-- >0)&&!(i=s.next()).done;)o.push(i.value)}catch(t){n={error:t}}finally{try{i&&!i.done&&(r=s.return)&&r.call(s)}finally{if(n)throw n.error}}return o}function xg(){for(var t=[],e=0;e<arguments.length;e++)t=t.concat(Tg(arguments[e]));return t}function Cg(t){return!!t.useClass}function Ig(t){return!!t.useFactory}var Rg=function(){function t(t){this.wrap=t,this.reflectMethods=["get","getPrototypeOf","setPrototypeOf","getOwnPropertyDescriptor","defineProperty","has","set","deleteProperty","apply","construct","ownKeys"]}return t.prototype.createProxy=function(t){var e,r=this,i=!1;return new Proxy({},this.createHandler((function(){return i||(e=t(r.wrap()),i=!0),e})))},t.prototype.createHandler=function(t){var e={};return this.reflectMethods.forEach((function(r){e[r]=function(){for(var e=[],i=0;i<arguments.length;i++)e[i]=arguments[i];e[0]=t();var n=Reflect[r];return n.apply(void 0,xg(e))}})),e},t}();function kg(t){return"string"==typeof t||"symbol"==typeof t}function Ng(t){return"object"==typeof t&&"token"in t&&"transform"in t}function Og(t){return!!t.useToken}function Bg(t){return null!=t.useValue}const Pg=function(){function t(){this._registryMap=new Map}return t.prototype.entries=function(){return this._registryMap.entries()},t.prototype.getAll=function(t){return this.ensure(t),this._registryMap.get(t)},t.prototype.get=function(t){this.ensure(t);var e=this._registryMap.get(t);return e[e.length-1]||null},t.prototype.set=function(t,e){this.ensure(t),this._registryMap.get(t).push(e)},t.prototype.setAll=function(t,e){this._registryMap.set(t,e)},t.prototype.has=function(t){return this.ensure(t),this._registryMap.get(t).length>0},t.prototype.clear=function(){this._registryMap.clear()},t.prototype.ensure=function(t){this._registryMap.has(t)||this._registryMap.set(t,[])},t}(),Lg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Dg=function(){this.scopedResolutions=new Map};var Ug=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Mg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg);const Fg=function(){this.preResolution=new Ug,this.postResolution=new Mg};var Hg=new Map,Wg=function(){function t(t){this.parent=t,this._registry=new Lg,this.interceptors=new Fg,this.disposed=!1,this.disposables=new Set}return t.prototype.register=function(t,e,r){var i;if(void 0===r&&(r={lifecycle:_g.Transient}),this.ensureNotDisposed(),i=function(t){return Cg(t)||Bg(t)||Og(t)||Ig(t)}(e)?e:{useClass:e},Og(i))for(var n=[t],s=i;null!=s;){var o=s.useToken;if(n.includes(o))throw new Error("Token registration cycle detected! "+xg(n,[o]).join(" -> "));n.push(o);var a=this._registry.get(o);s=a&&Og(a.provider)?a.provider:null}if((r.lifecycle===_g.Singleton||r.lifecycle==_g.ContainerScoped||r.lifecycle==_g.ResolutionScoped)&&(Bg(i)||Ig(i)))throw new Error('Cannot use lifecycle "'+_g[r.lifecycle]+'" with ValueProviders or FactoryProviders');return this._registry.set(t,{provider:i,options:r}),this},t.prototype.registerType=function(t,e){return this.ensureNotDisposed(),kg(e)?this.register(t,{useToken:e}):this.register(t,{useClass:e})},t.prototype.registerInstance=function(t,e){return this.ensureNotDisposed(),this.register(t,{useValue:e})},t.prototype.registerSingleton=function(t,e){if(this.ensureNotDisposed(),kg(t)){if(kg(e))return this.register(t,{useToken:e},{lifecycle:_g.Singleton});if(e)return this.register(t,{useClass:e},{lifecycle:_g.Singleton});throw new Error('Cannot register a type name as a singleton without a "to" token')}var r=t;return e&&!kg(e)&&(r=e),this.register(t,{useClass:r},{lifecycle:_g.Singleton})},t.prototype.resolve=function(t,e){void 0===e&&(e=new Dg),this.ensureNotDisposed();var r=this.getRegistration(t);if(!r&&kg(t))throw new Error('Attempted to resolve unregistered dependency token: "'+t.toString()+'"');if(this.executePreResolutionInterceptor(t,"Single"),r){var i=this.resolveRegistration(r,e);return this.executePostResolutionInterceptor(t,i,"Single"),i}if(function(t){return"function"==typeof t||t instanceof Rg}(t))return i=this.construct(t,e),this.executePostResolutionInterceptor(t,i,"Line truncated
+/*!
+ * MIT License
+ * 
+ * Copyright (c) Peculiar Ventures. All rights reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ * 
+ */
+const zm="crypto.algorithm",qm="crypto.algorithmProvider";var Qm;jg.registerSingleton(qm,class{getAlgorithms(){return jg.resolveAll(zm)}toAsnAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toAsnAlgorithm(t);if(r)return r}if(/[0-9.]+/.test(t.name)){const e=new Fp({algorithm:t.name});if("parameters"in t){const r=t;e.parameters=r.parameters}return e}throw new Error("Cannot convert WebCrypto algorithm to ASN.1 algorithm")}toWebAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toWebAlgorithm(t);if(r)return r}return{name:t.algorithm,parameters:t.parameters}}});const Km="1.3.36.3.3.2.8.1.1",Ym=`${Km}.1`,Xm=`${Km}.2`,Jm=`${Km}.3`,Zm=`${Km}.4`,$m=`${Km}.5`,tw=`${Km}.6`,ew=`${Km}.7`,rw=`${Km}.8`,iw=`${Km}.9`,nw=`${Km}.10`,sw=`${Km}.11`,ow=`${Km}.12`,aw=`${Km}.13`,cw=`${Km}.14`,lw="brainpoolP160r1",hw="brainpoolP160t1",uw="brainpoolP192r1",pw="brainpoolP192t1",dw="brainpoolP224r1",fw="brainpoolP224t1",gw="brainpoolP256r1",mw="brainpoolP256t1",ww="brainpoolP320r1",yw="brainpoolP320t1",bw="brainpoolP384r1",Aw="brainpoolP384t1",_w="brainpoolP512r1",vw="brainpoolP512t1",Ew="ECDSA";let Sw=Qm=class{toAsnAlgorithm(t){if(t.name.toLowerCase()===Ew.toLowerCase())if("hash"in t)switch(("string"==typeof t.hash?t.hash:t.hash.name).toLowerCase()){case"sha-1":return Pf;case"sha-256":return Lf;case"sha-384":return Df;case"sha-512":return Uf}else if("namedCurve"in t){let e="";switch(t.namedCurve){case"P-256":e=kf;break;case"K-256":e=Qm.SECP256K1;break;case"P-384":e=Nf;break;case"P-521":e=Of;break;case lw:e=Ym;break;case hw:e=Xm;break;case uw:e=Jm;break;case pw:e=Zm;break;case dw:e=$m;break;case fw:e=tw;break;case gw:e=ew;break;case mw:e=rw;break;case ww:e=iw;break;case yw:e=nw;break;case bw:e=sw;break;case Aw:e=ow;break;case _w:e=aw;break;case vw:e=cw}if(e)return new Fp({algorithm:Sf,parameters:iu.serialize(new Mf({namedCurve:e}))})}return null}toWebAlgorithm(t){switch(t.algorithm){case Tf:return{name:Ew,hash:{name:"SHA-1"}};case Cf:return{name:Ew,hash:{name:"SHA-256"}};case If:return{name:Ew,hash:{name:"SHA-384"}};case Rf:return{name:Ew,hash:{name:"SHA-512"}};case Sf:if(!t.parameters)throw new TypeError("Cannot get required parameters from EC algorithm");switch(iu.parse(t.parameters,Mf).namedCurve){case kf:return{name:Ew,namedCurve:"P-256"};case Qm.SECP256K1:return{name:Ew,namedCurve:"K-256"};case Nf:return{name:Ew,namedCurve:"P-384"};case Of:return{name:Ew,namedCurve:"P-521"};case Ym:return{name:Ew,namedCurve:lw};case Xm:return{name:Ew,namedCurve:hw};case Jm:return{name:Ew,namedCurve:uw};case Zm:return{name:Ew,namedCurve:pw};case $m:return{name:Ew,namedCurve:dw};case tw:return{name:Ew,namedCurve:fw};case ew:return{name:Ew,namedCurve:gw};case rw:return{name:Ew,namedCurve:mw};case iw:return{name:Ew,namedCurve:ww};case nw:return{name:Ew,namedCurve:yw};case sw:return{name:Ew,namedCurve:bw};case ow:return{name:Ew,namedCurve:Aw};case aw:return{name:Ew,namedCurve:_w};case cw:return{name:Ew,namedCurve:vw}}}return null}};Sw.SECP256K1="1.3.132.0.10",Sw=Qm=bg([Vg()],Sw),jg.registerSingleton(zm,Sw);const Tw=Symbol("name"),xw=Symbol("value");class Cw{constructor(t,e={},r=""){this[Tw]=t,this[xw]=r;for(const t in e)this[t]=e[t]}}Cw.NAME=Tw,Cw.VALUE=xw;class Iw{static toString(t){return this.items[t]||t}}Iw.items={[eg]:"sha1",[rg]:"sha224",[ig]:"sha256",[ng]:"sha384",[sg]:"sha512",[jf]:"rsaEncryption",[Kf]:"sha1WithRSAEncryption",[Yf]:"sha224WithRSAEncryption",[Xf]:"sha256WithRSAEncryption",[Jf]:"sha384WithRSAEncryption",[Zf]:"sha512WithRSAEncryption",[Sf]:"ecPublicKey",[Tf]:"ecdsaWithSHA1",[xf]:"ecdsaWithSHA224",[Cf]:"ecdsaWithSHA256",[If]:"ecdsaWithSHA384",[Rf]:"ecdsaWithSHA512",[np]:"TLS WWW server authentication",[sp]:"TLS WWW client authentication",[op]:"Code Signing",[ap]:"E-mail Protection",[cp]:"Time Stamping",[lp]:"OCSP Signing","1.2.840.113549.1.7.2":"Signed Data"};class Rw{static serialize(t){return this.serializeObj(t).join("\n")}static pad(t=0){return"".padStart(2*t," ")}static serializeObj(t,e=0){const r=[];let i=this.pad(e++),n="";const s=t[Cw.VALUE];s&&(n=` ${s}`),r.push(`${i}${t[Cw.NAME]}:${n}`),i=this.pad(e);for(const n in t){if("symbol"==typeof n)continue;const s=t[n],o=n?`${n}: `:"";if("string"==typeof s||"number"==typeof s||"boolean"==typeof s)r.push(`${i}${o}${s}`);else if(s instanceof Date)r.push(`${i}${o}${s.toUTCString()}`);else if(Array.isArray(s))for(const t of s)t[Cw.NAME]=n,r.push(...this.serializeObj(t,e));else if(s instanceof Cw)s[Cw.NAME]=n,r.push(...this.serializeObj(s,e));else if(ic.vJ.isBufferSource(s))n?(r.push(`${i}${o}`),r.push(...this.serializeBufferSource(s,e+1))):r.push(...this.serializeBufferSource(s,e));else{if(!("toTextObject"in s))throw new TypeError("Cannot serialize data in text format. Unsupported type.");{const t=s.toTextObject();t[Cw.NAME]=n,r.push(...this.serializeObj(t,e))}}}return r}static serializeBufferSource(t,e=0){const r=this.pad(e),i=ic.vJ.toUint8Array(t),n=[];for(let t=0;t<i.length;){const e=[];for(let r=0;r<16&&t<i.length;r++){8===r&&e.push("");const n=i[t++].toStrinLine truncated

archivebox/plugins/gallerydl/static/test.txt

@@ -0,0 +1 @@
+test content this should be visible

archivebox/plugins/gallerydl/templates/plugin_replaywebpage__viewer.html

@@ -0,0 +1,40 @@
+{% load tz core_tags static %}
+
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>{{title}}</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+      
+        </style>
+        <style>
+            html, body {
+                width: 100%;
+                height: 100%;
+                background-color: #ddd;
+            }
+        </style>
+    </head>
+    <body>
+        ReplayWeb.page for: {{snapshot.url}} ({{timestamp}}) /{{warc_filename}}
+
+        {{snapshot}}
+
+        <script>
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/sw.min.js
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/ui.min.js
+        </script>
+
+        <style>
+        </style>
+        <script src="/static/ui.js"></script>
+
+        <replay-web-page
+            style="height: 600px"
+            embed="replay"
+            replayBase="/static/"
+            source="/static/test.wacz"
+            url="https://example.com/">
+        </replay-web-page>
+    </body>
+</html>

archivebox/plugins/gallerydl/urls.py

@@ -0,0 +1,12 @@
+from django.urls import path
+
+from .views import GalleryDLIconView, GalleryDLEmbedView, GalleryDLOutputView, GalleryDLDependencyView, GalleryDLExtractorView
+
+urlpatterns = [
+	path('/plugins/gallerydl/icon/<path:path>', GalleryDLIconView(.as_view), name='gallerydl_icon'),
+	path('/plugins/gallerydl/embed/<path:path>', GalleryDLEmbedView.as_view(), name='gallerydl_embed'),
+	path('/plugins/gallerydl/output/<path:path>', GalleryDLOutputView.as_view(), name='gallerydl_output'),
+
+	path('/plugins/gallerydl/dependency/', GalleryDLDependencyView.as_view(), name='gallerydl_dependency'),
+	path('/plugins/gallerydl/extractor/', GalleryDLExtractorView.as_view(), name='gallerydl_extractor'),
+]

archivebox/plugins/gallerydl/views.py

@@ -0,0 +1,78 @@
+import os
+import sys
+from pathlib import Path
+
+from django.views import View
+from django.shortcuts import render
+from django.db.models import Q
+
+from core.models import Snapshot
+
+# from archivebox.config import PUBLIC_SNAPSHOTS
+PUBLIC_SNAPSHOTS = True
+
+
+class GalleryDLIconView(View):
+    template_name = 'plugin_gallerydl__icon.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        # ...
+        return render(template_name=self.template_name, request=self.request, context=context)
+
+
+class GalleryDLEmbedView(View):
+    template_name = 'plugin_gallerydl__embed.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        # ...
+        return render(template_name=self.template_name, request=self.request, context=context)
+
+
+class GalleryDLOutputView(View):
+    template_name = 'plugin_gallerydl__output.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        # ...
+        return render(template_name=self.template_name, request=self.request, context=context)

archivebox/plugins/replaywebpage/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.plugins.replaywebpage'

archivebox/plugins/replaywebpage/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class ReplayWebPageConfig(AppConfig):
+    label = "ReplayWeb.Page"
+    name = "plugin_replaywebpage"
+    
+    default_auto_field = "django.db.models.BigAutoField"

archivebox/plugins/replaywebpage/extractors.py

@@ -0,0 +1,50 @@
+# browsertrix extractor
+
+def save_browsertrix(link, out_dir, timeout, config):
+
+
+	browsertrix_dir = out_dir / 'browsertrix'
+	browsertrix_dir.mkdir(exist_ok=True)
+
+	crawl_id = link.timestamp
+
+	browsertrix_crawler_cmd = [
+		'crawl',
+		f'--url', link.url,
+		f'--collection={crawl_id}',
+		'--scopeType=page',
+		'--generateWACZ',
+		'--text=final-to-warc',
+		'--timeLimit=60',
+	]
+
+	remote_cmd = """
+	rm /tmp/dump.rdb;
+	rm -rf /crawls/collections;
+	mkdir /crawls/collections;
+	env CRAWL_ID={crawl_id} 
+	"""
+
+	local_cmd = ['nc', 'browsertrix', '2222']
+
+	status = 'succeeded'
+    timer = TimedProgress(timeout, prefix='      ')
+    try:
+        result = run(local_cmd, cwd=str(out_dir), input=remote_cmd, timeout=timeout)
+		
+		cmd_output = result.stdout.decode()
+
+		wacz_output_file = Path('/browsertrix/crawls') / crawl_id / f'{crawl_id}'.wacz
+
+		copy_and_overwrite(wacz_output_file, browsertrix_dir / wacz_output_file.name)
+
+
+
+TEMPLATE = """
+
+"""
+
+# rm /tmp/dump.rdb;
+# rm -rf /crawls/collections;
+# mkdir /crawls/collections;
+# env CRAWL_ID=tec2342 crawl --url 'https://example.com' --scopeType page --generateWACZ --collection tec2342 --text final-to-warc --timeLimit 60

archivebox/plugins/replaywebpage/models.py

@@ -0,0 +1,12 @@
+# from solo.models import SingletonModel
+
+
+# class ReplayWebPageConfiguration(SingletonModel):
+#     site_name = models.CharField(max_length=255, default='Site Name')
+#     maintenance_mode = models.BooleanField(default=False)
+
+#     def __str__(self):
+#         return "Site Configuration"
+
+#     class Meta:
+#         verbose_name = "Site Configuration"

archivebox/plugins/replaywebpage/static/sw.js

@@ -0,0 +1,124 @@
+/**
+ * Skipped minification because the original files appears to be already minified.
+ * Original file: /npm/replaywebpage@1.8.14/sw.js
+ *
+ * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files
+ */
+/*! sw.js (wabac.js 2.16.13) is part of Webrecorder project. Copyright (C) 2020-2023, Webrecorder Software. Licensed under the Affero General Public License v3. */(()=>{var t={9742:(t,e)=>{"use strict";e.byteLength=function(t){var e=c(t),r=e[0],i=e[1];return 3*(r+i)/4-i},e.toByteArray=function(t){var e,r,s=c(t),o=s[0],a=s[1],l=new n(function(t,e,r){return 3*(e+r)/4-r}(0,o,a)),h=0,u=a>0?o-4:o;for(r=0;r<u;r+=4)e=i[t.charCodeAt(r)]<<18|i[t.charCodeAt(r+1)]<<12|i[t.charCodeAt(r+2)]<<6|i[t.charCodeAt(r+3)],l[h++]=e>>16&255,l[h++]=e>>8&255,l[h++]=255&e;return 2===a&&(e=i[t.charCodeAt(r)]<<2|i[t.charCodeAt(r+1)]>>4,l[h++]=255&e),1===a&&(e=i[t.charCodeAt(r)]<<10|i[t.charCodeAt(r+1)]<<4|i[t.charCodeAt(r+2)]>>2,l[h++]=e>>8&255,l[h++]=255&e),l},e.fromByteArray=function(t){for(var e,i=t.length,n=i%3,s=[],o=16383,a=0,c=i-n;a<c;a+=o)s.push(l(t,a,a+o>c?c:a+o));return 1===n?(e=t[i-1],s.push(r[e>>2]+r[e<<4&63]+"==")):2===n&&(e=(t[i-2]<<8)+t[i-1],s.push(r[e>>10]+r[e>>4&63]+r[e<<2&63]+"=")),s.join("")};for(var r=[],i=[],n="undefined"!=typeof Uint8Array?Uint8Array:Array,s="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",o=0,a=s.length;o<a;++o)r[o]=s[o],i[s.charCodeAt(o)]=o;function c(t){var e=t.length;if(e%4>0)throw new Error("Invalid string. Length must be a multiple of 4");var r=t.indexOf("=");return-1===r&&(r=e),[r,r===e?0:4-r%4]}function l(t,e,i){for(var n,s,o=[],a=e;a<i;a+=3)n=(t[a]<<16&16711680)+(t[a+1]<<8&65280)+(255&t[a+2]),o.push(r[(s=n)>>18&63]+r[s>>12&63]+r[s>>6&63]+r[63&s]);return o.join("")}i["-".charCodeAt(0)]=62,i["_".charCodeAt(0)]=63},4181:t=>{var e=4096,r=new Uint32Array([0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535,131071,262143,524287,1048575,2097151,4194303,8388607,16777215]);function i(t){this.buf_=new Uint8Array(8224),this.input_=t,this.reset()}i.READ_SIZE=e,i.IBUF_MASK=8191,i.prototype.reset=function(){this.buf_ptr_=0,this.val_=0,this.pos_=0,this.bit_pos_=0,this.bit_end_pos_=0,this.eos_=0,this.readMoreInput();for(var t=0;t<4;t++)this.val_|=this.buf_[this.pos_]<<8*t,++this.pos_;return this.bit_end_pos_>0},i.prototype.readMoreInput=function(){if(!(this.bit_end_pos_>256))if(this.eos_){if(this.bit_pos_>this.bit_end_pos_)throw new Error("Unexpected end of input "+this.bit_pos_+" "+this.bit_end_pos_)}else{var t=this.buf_ptr_,r=this.input_.read(this.buf_,t,e);if(r<0)throw new Error("Unexpected end of input");if(r<e){this.eos_=1;for(var i=0;i<32;i++)this.buf_[t+r+i]=0}if(0===t){for(i=0;i<32;i++)this.buf_[8192+i]=this.buf_[i];this.buf_ptr_=e}else this.buf_ptr_=0;this.bit_end_pos_+=r<<3}},i.prototype.fillBitWindow=function(){for(;this.bit_pos_>=8;)this.val_>>>=8,this.val_|=this.buf_[8191&this.pos_]<<24,++this.pos_,this.bit_pos_=this.bit_pos_-8>>>0,this.bit_end_pos_=this.bit_end_pos_-8>>>0},i.prototype.readBits=function(t){32-this.bit_pos_<t&&this.fillBitWindow();var e=this.val_>>>this.bit_pos_&r[t];return this.bit_pos_+=t,e},t.exports=i},7080:(t,e)=>{e.lookup=new Uint8Array([0,0,0,0,0,0,0,0,0,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,12,16,12,12,20,12,16,24,28,12,12,32,12,36,12,44,44,44,44,44,44,44,44,44,44,32,32,24,40,28,12,12,48,52,52,52,48,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,24,12,28,12,12,12,56,60,60,60,56,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,24,12,28,12,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,0,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,32,32,32,32,32,32,32,32,32,32,32,32,32,Line truncated
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */const i=r(9742),n=r(645),s="function"==typeof Symbol&&"function"==typeof Symbol.for?Symbol.for("nodejs.util.inspect.custom"):null;e.Buffer=c,e.SlowBuffer=function(t){return+t!=t&&(t=0),c.alloc(+t)},e.INSPECT_MAX_BYTES=50;const o=2147483647;function a(t){if(t>o)throw new RangeError('The value "'+t+'" is invalid for option "size"');const e=new Uint8Array(t);return Object.setPrototypeOf(e,c.prototype),e}function c(t,e,r){if("number"==typeof t){if("string"==typeof e)throw new TypeError('The "string" argument must be of type string. Received type number');return u(t)}return l(t,e,r)}function l(t,e,r){if("string"==typeof t)return function(t,e){if("string"==typeof e&&""!==e||(e="utf8"),!c.isEncoding(e))throw new TypeError("Unknown encoding: "+e);const r=0|g(t,e);let i=a(r);const n=i.write(t,e);return n!==r&&(i=i.slice(0,n)),i}(t,e);if(ArrayBuffer.isView(t))return function(t){if(Y(t,Uint8Array)){const e=new Uint8Array(t);return d(e.buffer,e.byteOffset,e.byteLength)}return p(t)}(t);if(null==t)throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t);if(Y(t,ArrayBuffer)||t&&Y(t.buffer,ArrayBuffer))return d(t,e,r);if("undefined"!=typeof SharedArrayBuffer&&(Y(t,SharedArrayBuffer)||t&&Y(t.buffer,SharedArrayBuffer)))return d(t,e,r);if("number"==typeof t)throw new TypeError('The "value" argument must not be of type number. Received type number');const i=t.valueOf&&t.valueOf();if(null!=i&&i!==t)return c.from(i,e,r);const n=function(t){if(c.isBuffer(t)){const e=0|f(t.length),r=a(e);return 0===r.length||t.copy(r,0,0,e),r}return void 0!==t.length?"number"!=typeof t.length||X(t.length)?a(0):p(t):"Buffer"===t.type&&Array.isArray(t.data)?p(t.data):void 0}(t);if(n)return n;if("undefined"!=typeof Symbol&&null!=Symbol.toPrimitive&&"function"==typeof t[Symbol.toPrimitive])return c.from(t[Symbol.toPrimitive]("string"),e,r);throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t)}function h(t){if("number"!=typeof t)throw new TypeError('"size" argument must be of type number');if(t<0)throw new RangeError('The value "'+t+'" is invalid for option "size"')}function u(t){return h(t),a(t<0?0:0|f(t))}function p(t){const e=t.length<0?0:0|f(t.length),r=a(e);for(let i=0;i<e;i+=1)r[i]=255&t[i];return r}function d(t,e,r){if(e<0||t.byteLength<e)throw new RangeError('"offset" is outside of buffer bounds');if(t.byteLength<e+(r||0))throw new RangeError('"length" is outside of buffer bounds');let i;return i=void 0===e&&void 0===r?new Uint8Array(t):void 0===r?new Uint8Array(t,e):new Uint8Array(t,e,r),Object.setPrototypeOf(i,c.prototype),i}function f(t){if(t>=o)throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+o.toString(16)+" bytes");return 0|t}function g(t,e){if(c.isBuffer(t))return t.length;if(ArrayBuffer.isView(t)||Y(t,ArrayBuffer))return t.byteLength;if("string"!=typeof t)throw new TypeError('The "string" argument must be one of type string, Buffer, or ArrayBuffer. Received type '+typeof t);const r=t.length,i=arguments.length>2&&!0===arguments[2];if(!i&&0===r)return 0;let n=!1;for(;;)switch(e){case"ascii":case"latin1":case"binary":return r;case"utf8":case"utf-8":return q(t).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*r;case"hex":return r>>>1;case"base64":return Q(t).length;default:if(n)return i?-1:q(t).length;e=(""+e).toLowerCase(),n=!0}}function m(t,e,r){let i=!1;if((void 0===e||e<0)&&(e=0),e>this.length)return"";if((void 0===r||r>this.length)&&(r=this.length),r<=0)return"";if((r>>>=0)<=(e>>>=0))return"";for(t||(t="utf8");;)switch(t){case"hex":return k(this,e,r);case"utf8":case"utf-8":return x(this,e,r);case"ascii":return I(this,e,r);case"latin1":case"binary":return R(this,e,r);case"base64":return T(this,e,r);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return N(this,e,r);default:if(i)throw new TypeError("Unknown encoding: "+t);t=(t+"").toLowerCase(),i=!0}}function w(t,e,r){const i=t[e];t[e]=t[r],t[r]=i}function y(t,e,r,i,n){if(0===t.length)return-1;if("string"==typeof r?(i=r,r=0):r>2147483647?r=2147483647:r<-2147483648&&(r=-2147483648),X(r=+r)&&(r=n?0:t.length-1),r<0&&(r=t.length+r),r>=t.length){if(n)return-1;r=t.length-1}else if(r<0){if(!n)return-1;r=0}if("string"==typeof e&&(e=c.from(e,i)),c.isBuffer(e))return 0===e.length?-1:b(t,e,r,i,n);if("number"==typeof e)return e&=255,"function"==typeof Uint8Array.prototype.indexOf?n?Uint8Array.prototype.indexOf.call(t,e,r):Uint8Array.prototype.lastIndexOf.call(t,e,r):b(t,[e],r,i,n);throw new TypeError("val must be string, number or Buffer")}function b(t,e,r,i,n){let s,o=1,a=t.length,c=e.length;if(void 0!==i&&("ucs2"===(i=String(i).toLowerCase())||"ucs-2"===i||"utf16le"===i||"utf-16le"===i)){if(t.length<2||e.length<2)return-1;o=2,a/=2,c/=2,r/=2}function l(t,e){return 1===o?t[e]:t.readUInt16BE(e*o)}if(n){let i=-1;for(s=r;s<a;s++)if(l(t,s)===l(e,-1===i?0:s-i)){if(-1===i&&(i=Line truncated
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+e.read=function(t,e,r,i,n){var s,o,a=8*n-i-1,c=(1<<a)-1,l=c>>1,h=-7,u=r?n-1:0,p=r?-1:1,d=t[e+u];for(u+=p,s=d&(1<<-h)-1,d>>=-h,h+=a;h>0;s=256*s+t[e+u],u+=p,h-=8);for(o=s&(1<<-h)-1,s>>=-h,h+=i;h>0;o=256*o+t[e+u],u+=p,h-=8);if(0===s)s=1-l;else{if(s===c)return o?NaN:1/0*(d?-1:1);o+=Math.pow(2,i),s-=l}return(d?-1:1)*o*Math.pow(2,s-i)},e.write=function(t,e,r,i,n,s){var o,a,c,l=8*s-n-1,h=(1<<l)-1,u=h>>1,p=23===n?Math.pow(2,-24)-Math.pow(2,-77):0,d=i?0:s-1,f=i?1:-1,g=e<0||0===e&&1/e<0?1:0;for(e=Math.abs(e),isNaN(e)||e===1/0?(a=isNaN(e)?1:0,o=h):(o=Math.floor(Math.log(e)/Math.LN2),e*(c=Math.pow(2,-o))<1&&(o--,c*=2),(e+=o+u>=1?p/c:p*Math.pow(2,1-u))*c>=2&&(o++,c/=2),o+u>=h?(a=0,o=h):o+u>=1?(a=(e*c-1)*Math.pow(2,n),o+=u):(a=e*Math.pow(2,u-1)*Math.pow(2,n),o=0));n>=8;t[r+d]=255&a,d+=f,a/=256,n-=8);for(o=o<<n|a,l+=n;l>0;t[r+d]=255&o,d+=f,o/=256,l-=8);t[r+d-f]|=128*g}},5717:t=>{"function"==typeof Object.create?t.exports=function(t,e){e&&(t.super_=e,t.prototype=Object.create(e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}))}:t.exports=function(t,e){if(e){t.super_=e;var r=function(){};r.prototype=e.prototype,t.prototype=new r,t.prototype.constructor=t}}},6512:function(t){!function(e){"use strict";const r="(0?\\d+|0x[a-f0-9]+)",i={fourOctet:new RegExp(`^${r}\\.${r}\\.${r}\\.${r}$`,"i"),threeOctet:new RegExp(`^${r}\\.${r}\\.${r}$`,"i"),twoOctet:new RegExp(`^${r}\\.${r}$`,"i"),longValue:new RegExp(`^${r}$`,"i")},n=new RegExp("^0[0-7]+$","i"),s=new RegExp("^0x[a-f0-9]+$","i"),o="%[0-9a-z]{1,}",a="(?:[0-9a-f]+::?)+",c={zoneIndex:new RegExp(o,"i"),native:new RegExp(`^(::)?(${a})?([0-9a-f]+)?(::)?(${o})?$`,"i"),deprecatedTransitional:new RegExp(`^(?:::)(${r}\\.${r}\\.${r}\\.${r}(${o})?)$`,"i"),transitional:new RegExp(`^((?:${a})|(?:::)(?:${a})?)${r}\\.${r}\\.${r}\\.${r}(${o})?$`,"i")};function l(t,e){if(t.indexOf("::")!==t.lastIndexOf("::"))return null;let r,i,n=0,s=-1,o=(t.match(c.zoneIndex)||[])[0];for(o&&(o=o.substring(1),t=t.replace(/%.+$/,""));(s=t.indexOf(":",s+1))>=0;)n++;if("::"===t.substr(0,2)&&n--,"::"===t.substr(-2,2)&&n--,n>e)return null;for(i=e-n,r=":";i--;)r+="0:";return":"===(t=t.replace("::",r))[0]&&(t=t.slice(1)),":"===t[t.length-1]&&(t=t.slice(0,-1)),{parts:e=function(){const e=t.split(":"),r=[];for(let t=0;t<e.length;t++)r.push(parseInt(e[t],16));return r}(),zoneId:o}}function h(t,e,r,i){if(t.length!==e.length)throw new Error("ipaddr: cannot match CIDR for objects with different lengths");let n,s=0;for(;i>0;){if(n=r-i,n<0&&(n=0),t[s]>>n!=e[s]>>n)return!1;i-=r,s+=1}return!0}function u(t){if(s.test(t))return parseInt(t,16);if("0"===t[0]&&!isNaN(parseInt(t[1],10))){if(n.test(t))return parseInt(t,8);throw new Error(`ipaddr: cannot parse ${t} as octal`)}return parseInt(t,10)}function p(t,e){for(;t.length<e;)t=`0${t}`;return t}const d={};d.IPv4=function(){function t(t){if(4!==t.length)throw new Error("ipaddr: ipv4 octet count should be 4");let e,r;for(e=0;e<t.length;e++)if(r=t[e],!(0<=r&&r<=255))throw new Error("ipaddr: ipv4 octet should fit in 8 bits");this.octets=t}return t.prototype.SpecialRanges={unspecified:[[new t([0,0,0,0]),8]],broadcast:[[new t([255,255,255,255]),32]],multicast:[[new t([224,0,0,0]),4]],linkLocal:[[new t([169,254,0,0]),16]],loopback:[[new t([127,0,0,0]),8]],carrierGradeNat:[[new t([100,64,0,0]),10]],private:[[new t([10,0,0,0]),8],[new t([172,16,0,0]),12],[new t([192,168,0,0]),16]],reserved:[[new t([192,0,0,0]),24],[new t([192,0,2,0]),24],[new t([192,88,99,0]),24],[new t([198,51,100,0]),24],[new t([203,0,113,0]),24],[new t([240,0,0,0]),4]]},t.prototype.kind=function(){return"ipv4"},t.prototype.match=function(t,e){let r;if(void 0===e&&(r=t,t=r[0],e=r[1]),"ipv4"!==t.kind())throw new Error("ipaddr: cannot match ipv4 address with non-ipv4 one");return h(this.octets,t.octets,8,e)},t.prototype.prefixLengthFromSubnetMask=function(){let t=0,e=!1;const r={0:8,128:7,192:6,224:5,240:4,248:3,252:2,254:1,255:0};let i,n,s;for(i=3;i>=0;i-=1){if(n=this.octets[i],!(n in r))return null;if(s=r[n],e&&0!==s)return null;8!==s&&(e=!0),t+=s}return 32-t},t.prototype.range=function(){return d.subnetMatch(this,this.SpecialRanges)},t.prototype.toByteArray=function(){return this.octets.slice(0)},t.prototype.toIPv4MappedAddress=function(){return d.IPv6.parse(`::ffff:${this.toString()}`)},t.prototype.toNormalizedString=function(){return this.toString()},t.prototype.toString=function(){return this.octets.join(".")},t}(),d.IPv4.broadcastAddressFromCIDR=function(t){try{const e=this.parseCIDR(t),r=e[0].toByteArray(),i=this.subnetMaskFromPrefixLength(e[1]).toByteArray(),n=[];let s=0;for(;s<4;)n.push(parseInt(r[s],10)|255^parseInt(i[s],10)),s++;return new this(n)}catch(t){throw new Error("ipaddr: the address does not have IPv4 CIDR format")}},d.IPv4.isIPv4=function(t){return null!==this.parser(t)},d.IPv4.isValid=function(t){try{return new this(this.parser(t)),!0}catch(t){return!1}},d.IPv4.isValidFourPartDecimal=function(t){return!(!d.IPv4.isValid(t)||!t.match(/^(0|[1-9]\d*)(\.(0|[1-9]\d*)){3}$/))},d.Line truncated
+/*! *****************************************************************************
+Copyright (C) Microsoft. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */!function(t){!function(e){var i="object"==typeof r.g?r.g:"object"==typeof self?self:"object"==typeof this?this:Function("return this;")(),s=o(t);function o(t,e){return function(r,i){"function"!=typeof t[r]&&Object.defineProperty(t,r,{configurable:!0,writable:!0,value:i}),e&&e(r,i)}}void 0===i.Reflect?i.Reflect=t:s=o(i.Reflect,s),function(t){var e=Object.prototype.hasOwnProperty,r="function"==typeof Symbol,i=r&&void 0!==Symbol.toPrimitive?Symbol.toPrimitive:"@@toPrimitive",s=r&&void 0!==Symbol.iterator?Symbol.iterator:"@@iterator",o="function"==typeof Object.create,a={__proto__:[]}instanceof Array,c=!o&&!a,l={create:o?function(){return M(Object.create(null))}:a?function(){return M({__proto__:null})}:function(){return M({})},has:c?function(t,r){return e.call(t,r)}:function(t,e){return e in t},get:c?function(t,r){return e.call(t,r)?t[r]:void 0}:function(t,e){return t[e]}},h=Object.getPrototypeOf(Function),u="object"==typeof n&&n.env&&"true"===n.env.REFLECT_METADATA_USE_MAP_POLYFILL,p=u||"function"!=typeof Map||"function"!=typeof Map.prototype.entries?function(){var t={},e=[],r=function(){function t(t,e,r){this._index=0,this._keys=t,this._values=e,this._selector=r}return t.prototype["@@iterator"]=function(){return this},t.prototype[s]=function(){return this},t.prototype.next=function(){var t=this._index;if(t>=0&&t<this._keys.length){var r=this._selector(this._keys[t],this._values[t]);return t+1>=this._keys.length?(this._index=-1,this._keys=e,this._values=e):this._index++,{value:r,done:!1}}return{value:void 0,done:!0}},t.prototype.throw=function(t){throw this._index>=0&&(this._index=-1,this._keys=e,this._values=e),t},t.prototype.return=function(t){return this._index>=0&&(this._index=-1,this._keys=e,this._values=e),{value:t,done:!0}},t}();return function(){function e(){this._keys=[],this._values=[],this._cacheKey=t,this._cacheIndex=-2}return Object.defineProperty(e.prototype,"size",{get:function(){return this._keys.length},enumerable:!0,configurable:!0}),e.prototype.has=function(t){return this._find(t,!1)>=0},e.prototype.get=function(t){var e=this._find(t,!1);return e>=0?this._values[e]:void 0},e.prototype.set=function(t,e){var r=this._find(t,!0);return this._values[r]=e,this},e.prototype.delete=function(e){var r=this._find(e,!1);if(r>=0){for(var i=this._keys.length,n=r+1;n<i;n++)this._keys[n-1]=this._keys[n],this._values[n-1]=this._values[n];return this._keys.length--,this._values.length--,e===this._cacheKey&&(this._cacheKey=t,this._cacheIndex=-2),!0}return!1},e.prototype.clear=function(){this._keys.length=0,this._values.length=0,this._cacheKey=t,this._cacheIndex=-2},e.prototype.keys=function(){return new r(this._keys,this._values,i)},e.prototype.values=function(){return new r(this._keys,this._values,n)},e.prototype.entries=function(){return new r(this._keys,this._values,o)},e.prototype["@@iterator"]=function(){return this.entries()},e.prototype[s]=function(){return this.entries()},e.prototype._find=function(t,e){return this._cacheKey!==t&&(this._cacheIndex=this._keys.indexOf(this._cacheKey=t)),this._cacheIndex<0&&e&&(this._cacheIndex=this._keys.length,this._keys.push(t),this._values.push(void 0)),this._cacheIndex},e}();function i(t,e){return t}function n(t,e){return e}function o(t,e){return[t,e]}}():Map,d=u||"function"!=typeof Set||"function"!=typeof Set.prototype.entries?function(){function t(){this._map=new p}return Object.defineProperty(t.prototype,"size",{get:function(){return this._map.size},enumerable:!0,configurable:!0}),t.prototype.has=function(t){return this._map.has(t)},t.prototype.add=function(t){return this._map.set(t,t),this},t.prototype.delete=function(t){return this._map.delete(t)},t.prototype.clear=function(){this._map.clear()},t.prototype.keys=function(){return this._map.keys()},t.prototype.values=function(){return this._map.values()},t.prototype.entries=function(){return this._map.entries()},t.prototype["@@iterator"]=function(){return this.keys()},t.prototype[s]=function(){return this.keys()},t}():Set,f=new(u||"function"!=typeof WeakMap?function(){var t=l.create(),r=i();return function(){function t(){this._key=i()}return t.prototype.has=function(t){var e=n(t,!1);return void 0!==e&&l.has(e,this._key)},t.prototype.get=function(t){var e=n(t,!1);return void 0!==e?l.get(e,this._key):void 0},t.prototype.set=function(t,e){return n(t,!0)[this._key]=e,this},t.prototype.delete=function(t){var e=n(t,!1);return void 0!==e&&delete e[this._key]},t.prototype.clear=function(){this._key=i()},t}();function i(){var e;do{e="@@WeakMap@@"+o()}while(l.has(t,e));return t[e]=!0,e}function n(t,i){if(!e.call(t,r)){if(!i)return;Object.defineProperty(t,r,{value:l.create()})}return t[r]}function s(t,e){for(var r=0;r<e;++r)t[r]=255*Math.random()|0;return t}function o(){var t=function(t){return"function"==typeof Uint8Array?"undefined"!=typeof crypto?crypto.getRandomValues(new Uint8Array(t)):"undefined"!=typeof msCrypto?msCrypto.getRandomValues(nLine truncated
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+var i=r(8764),n=i.Buffer;function s(t,e){for(var r in t)e[r]=t[r]}function o(t,e,r){return n(t,e,r)}n.from&&n.alloc&&n.allocUnsafe&&n.allocUnsafeSlow?t.exports=i:(s(i,e),e.Buffer=o),o.prototype=Object.create(n.prototype),s(n,o),o.from=function(t,e,r){if("number"==typeof t)throw new TypeError("Argument must not be a number");return n(t,e,r)},o.alloc=function(t,e,r){if("number"!=typeof t)throw new TypeError("Argument must be a number");var i=n(t);return void 0!==e?"string"==typeof r?i.fill(e,r):i.fill(e):i.fill(0),i},o.allocUnsafe=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return n(t)},o.allocUnsafeSlow=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return i.SlowBuffer(t)}},2830:(t,e,r)=>{t.exports=n;var i=r(7187).EventEmitter;function n(){i.call(this)}r(5717)(n,i),n.Readable=r(9481),n.Writable=r(4229),n.Duplex=r(6753),n.Transform=r(4605),n.PassThrough=r(2725),n.finished=r(8610),n.pipeline=r(9946),n.Stream=n,n.prototype.pipe=function(t,e){var r=this;function n(e){t.writable&&!1===t.write(e)&&r.pause&&r.pause()}function s(){r.readable&&r.resume&&r.resume()}r.on("data",n),t.on("drain",s),t._isStdio||e&&!1===e.end||(r.on("end",a),r.on("close",c));var o=!1;function a(){o||(o=!0,t.end())}function c(){o||(o=!0,"function"==typeof t.destroy&&t.destroy())}function l(t){if(h(),0===i.listenerCount(this,"error"))throw t}function h(){r.removeListener("data",n),t.removeListener("drain",s),r.removeListener("end",a),r.removeListener("close",c),r.removeListener("error",l),t.removeListener("error",l),r.removeListener("end",h),r.removeListener("close",h),t.removeListener("close",h)}return r.on("error",l),t.on("error",l),r.on("end",h),r.on("close",h),t.on("close",h),t.emit("pipe",r),t}},2553:(t,e,r)=>{"use strict";var i=r(9509).Buffer,n=i.isEncoding||function(t){switch((t=""+t)&&t.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function s(t){var e;switch(this.encoding=function(t){var e=function(t){if(!t)return"utf8";for(var e;;)switch(t){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return t;default:if(e)return;t=(""+t).toLowerCase(),e=!0}}(t);if("string"!=typeof e&&(i.isEncoding===n||!n(t)))throw new Error("Unknown encoding: "+t);return e||t}(t),this.encoding){case"utf16le":this.text=c,this.end=l,e=4;break;case"utf8":this.fillLast=a,e=4;break;case"base64":this.text=h,this.end=u,e=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=i.allocUnsafe(e)}function o(t){return t<=127?0:t>>5==6?2:t>>4==14?3:t>>3==30?4:t>>6==2?-1:-2}function a(t){var e=this.lastTotal-this.lastNeed,r=function(t,e,r){if(128!=(192&e[0]))return t.lastNeed=0,"�";if(t.lastNeed>1&&e.length>1){if(128!=(192&e[1]))return t.lastNeed=1,"�";if(t.lastNeed>2&&e.length>2&&128!=(192&e[2]))return t.lastNeed=2,"�"}}(this,t);return void 0!==r?r:this.lastNeed<=t.length?(t.copy(this.lastChar,e,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(t.copy(this.lastChar,e,0,t.length),void(this.lastNeed-=t.length))}function c(t,e){if((t.length-e)%2==0){var r=t.toString("utf16le",e);if(r){var i=r.charCodeAt(r.length-1);if(i>=55296&&i<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1],r.slice(0,-1)}return r}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=t[t.length-1],t.toString("utf16le",e,t.length-1)}function l(t){var e=t&&t.length?this.write(t):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return e+this.lastChar.toString("utf16le",0,r)}return e}function h(t,e){var r=(t.length-e)%3;return 0===r?t.toString("base64",e):(this.lastNeed=3-r,this.lastTotal=3,1===r?this.lastChar[0]=t[t.length-1]:(this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1]),t.toString("base64",e,t.length-r))}function u(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+this.lastChar.toString("base64",0,3-this.lastNeed):e}function p(t){return t.toString(this.encoding)}function d(t){return t&&t.length?this.write(t):""}e.s=s,s.prototype.write=function(t){if(0===t.length)return"";var e,r;if(this.lastNeed){if(void 0===(e=this.fillLast(t)))return"";r=this.lastNeed,this.lastNeed=0}else r=0;return r<t.length?e?e+this.text(t,r):this.text(t,r):e||""},s.prototype.end=function(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+"�":e},s.prototype.text=function(t,e){var r=function(t,e,r){var i=e.length-1;if(i<r)return 0;var n=o(e[i]);return n>=0?(n>0&&(t.lastNeed=n-1),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(t.lastNeed=n-2),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(2===n?n=0:t.lastNeed=n-3),n):0}(this,t,e);if(!this.lastNeed)return t.toString("utf8",e);this.lastTotal=r;var i=t.length-(r-this.lastNeed);return t.copy(tLine truncated
+/*!
+ * hash-wasm (https://www.npmjs.com/package/hash-wasm)
+ * (c) Dani Biro
+ * @license MIT
+ */
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */function B(t,e,r,i){return new(r||(r=Promise))((function(n,s){function o(t){try{c(i.next(t))}catch(t){s(t)}}function a(t){try{c(i.throw(t))}catch(t){s(t)}}function c(t){var e;t.done?n(t.value):(e=t.value,e instanceof r?e:new r((function(t){t(e)}))).then(o,a)}c((i=i.apply(t,e||[])).next())}))}class P{constructor(){this.mutex=Promise.resolve()}lock(){let t=()=>{};return this.mutex=this.mutex.then((()=>new Promise(t))),new Promise((e=>{t=e}))}dispatch(t){return B(this,void 0,void 0,(function*(){const e=yield this.lock();try{return yield Promise.resolve(t())}finally{e()}}))}}const L="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:r.g,D=null!==(I=L.Buffer)&&void 0!==I?I:null,U=L.TextEncoder?new L.TextEncoder:null;function M(t,e){return(15&t)+(t>>6|t>>3&8)<<4|(15&e)+(e>>6|e>>3&8)}const F="a".charCodeAt(0)-10,H="0".charCodeAt(0);function W(t,e,r){let i=0;for(let n=0;n<r;n++){let r=e[n]>>>4;t[i++]=r>9?r+F:r+H,r=15&e[n],t[i++]=r>9?r+F:r+H}return String.fromCharCode.apply(null,t)}const j=null!==D?t=>{if("string"==typeof t){const e=D.from(t,"utf8");return new Uint8Array(e.buffer,e.byteOffset,e.length)}if(D.isBuffer(t))return new Uint8Array(t.buffer,t.byteOffset,t.length);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")}:t=>{if("string"==typeof t)return U.encode(t);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")},V="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",G=new Uint8Array(256);for(let t=0;t<V.length;t++)G[V.charCodeAt(t)]=t;function z(t){const e=function(t){let e=Math.floor(.75*t.length);const r=t.length;return"="===t[r-1]&&(e-=1,"="===t[r-2]&&(e-=1)),e}(t),r=t.length,i=new Uint8Array(e);let n=0;for(let e=0;e<r;e+=4){const r=G[t.charCodeAt(e)],s=G[t.charCodeAt(e+1)],o=G[t.charCodeAt(e+2)],a=G[t.charCodeAt(e+3)];i[n]=r<<2|s>>4,n+=1,i[n]=(15&s)<<4|o>>2,n+=1,i[n]=(3&o)<<6|63&a,n+=1}return i}const q=16384,Q=new P,K=new Map;function Y(t,e){return B(this,void 0,void 0,(function*(){let r=null,i=null,n=!1;if("undefined"==typeof WebAssembly)throw new Error("WebAssembly is not supported in this environment!");const s=()=>new DataView(r.exports.memory.buffer).getUint32(r.exports.STATE_SIZE,!0),o=Q.dispatch((()=>B(this,void 0,void 0,(function*(){if(!K.has(t.name)){const e=z(t.data),r=WebAssembly.compile(e);K.set(t.name,r)}const e=yield K.get(t.name);r=yield WebAssembly.instantiate(e,{})})))),a=(t=null)=>{n=!0,r.exports.Hash_Init(t)},c=t=>{if(!n)throw new Error("update() called before init()");(t=>{let e=0;for(;e<t.length;){const n=t.subarray(e,e+q);e+=n.length,i.set(n),r.exports.Hash_Update(n.length)}})(j(t))},l=new Uint8Array(2*e),h=(t,s=null)=>{if(!n)throw new Error("digest() called before init()");return n=!1,r.exports.Hash_Final(s),"binary"===t?i.slice(0,e):W(l,i,e)},u=t=>"string"==typeof t?t.length<4096:t.byteLength<q;let p=u;switch(t.name){case"argon2":case"scrypt":p=()=>!0;break;case"blake2b":case"blake2s":p=(t,e)=>e<=512&&u(t);break;case"blake3":p=(t,e)=>0===e&&u(t);break;case"xxhash64":case"xxhash3":case"xxhash128":p=()=>!1}return yield(()=>B(this,void 0,void 0,(function*(){r||(yield o);const t=r.exports.Hash_GetBuffer(),e=r.exports.memory.buffer;i=new Uint8Array(e,t,q)})))(),{getMemory:()=>i,writeMemory:(t,e=0)=>{i.set(t,e)},getExports:()=>r.exports,setMemorySize:t=>{r.exports.Hash_SetMemorySize(t);const e=r.exports.Hash_GetBuffer(),n=r.exports.memory.buffer;i=new Uint8Array(n,e,t)},init:a,update:c,digest:h,save:()=>{if(!n)throw new Error("save() can only be called after init() and before digest()");const e=r.exports.Hash_GetState(),i=s(),o=r.exports.memory.buffer,a=new Uint8Array(o,e,i),c=new Uint8Array(4+i);return function(t,e){const r=e.length>>1;for(let i=0;i<r;i++){const r=i<<1;t[i]=M(e.charCodeAt(r),e.charCodeAt(r+1))}}(c,t.hash),c.set(a,4),c},load:e=>{if(!(e instanceof Uint8Array))throw new Error("load() expects an Uint8Array generated by save()");const i=r.exports.Hash_GetState(),o=s(),a=4+o,c=r.exports.memory.buffer;if(e.length!==a)throw new Error(`Bad state length (expected ${a} bytes, got ${e.length})`);if(!function(t,e){if(t.length!==2*e.length)return!1;for(let r=0;r<e.length;r++){const i=r<<1;if(e[r]!==M(t.charCodeAt(i),t.charCodeAt(i+1)))return!1}return!0}(t.hash,e.subarray(0,4)))throw new Error("This state was written by an incompatible hash implementation");const l=e.subarray(4);new Uint8Array(c,i,o).set(l),n=!0},calculate:(t,n=null,s=null)=>{if(!p(t,n))return a(n),c(t),h("hex",s);const o=j(t);return i.set(o),r.exports.Hash_Calculate(o.length,n,s),W(l,i,e)},hashLength:e}}))}new P,new P,new DataView(new ArrayBuffer(4)),new P,new P,new P,new P,new P,new P,new P,new P,new P;var X={name:"sha256",data:"AGFzbQEAAAABEQRgAAF/YAF/AGACf38AYAAAAwgHAAEBAgMAAgQFAXABAQEFBAEBAgIGDgJ/AUHwiQULfwBBgAgLB3AIBm1lbW9yeQIADLine truncated
+/*! js-yaml 4.1.0 https://github.com/nodeca/js-yaml @license MIT */function so(t){return null==t}var oo={isNothing:so,isObject:function(t){return"object"==typeof t&&null!==t},toArray:function(t){return Array.isArray(t)?t:so(t)?[]:[t]},repeat:function(t,e){var r,i="";for(r=0;r<e;r+=1)i+=t;return i},isNegativeZero:function(t){return 0===t&&Number.NEGATIVE_INFINITY===1/t},extend:function(t,e){var r,i,n,s;if(e)for(r=0,i=(s=Object.keys(e)).length;r<i;r+=1)t[n=s[r]]=e[n];return t}};function ao(t,e){var r="",i=t.reason||"(unknown reason)";return t.mark?(t.mark.name&&(r+='in "'+t.mark.name+'" '),r+="("+(t.mark.line+1)+":"+(t.mark.column+1)+")",!e&&t.mark.snippet&&(r+="\n\n"+t.mark.snippet),i+" "+r):i}function co(t,e){Error.call(this),this.name="YAMLException",this.reason=t,this.mark=e,this.message=ao(this,!1),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack||""}co.prototype=Object.create(Error.prototype),co.prototype.constructor=co,co.prototype.toString=function(t){return this.name+": "+ao(this,t)};var lo=co;function ho(t,e,r,i,n){var s="",o="",a=Math.floor(n/2)-1;return i-e>a&&(e=i-a+(s=" ... ").length),r-i>a&&(r=i+a-(o=" ...").length),{str:s+t.slice(e,r).replace(/\t/g,"→")+o,pos:i-e+s.length}}function uo(t,e){return oo.repeat(" ",e-t.length)+t}var po=["kind","multi","resolve","construct","instanceOf","predicate","represent","representName","defaultStyle","styleAliases"],fo=["scalar","sequence","mapping"],go=function(t,e){if(e=e||{},Object.keys(e).forEach((function(e){if(-1===po.indexOf(e))throw new lo('Unknown option "'+e+'" is met in definition of "'+t+'" YAML type.')})),this.options=e,this.tag=t,this.kind=e.kind||null,this.resolve=e.resolve||function(){return!0},this.construct=e.construct||function(t){return t},this.instanceOf=e.instanceOf||null,this.predicate=e.predicate||null,this.represent=e.represent||null,this.representName=e.representName||null,this.defaultStyle=e.defaultStyle||null,this.multi=e.multi||!1,this.styleAliases=function(t){var e={};return null!==t&&Object.keys(t).forEach((function(r){t[r].forEach((function(t){e[String(t)]=r}))})),e}(e.styleAliases||null),-1===fo.indexOf(this.kind))throw new lo('Unknown kind "'+this.kind+'" is specified for "'+t+'" YAML type.')};function mo(t,e){var r=[];return t[e].forEach((function(t){var e=r.length;r.forEach((function(r,i){r.tag===t.tag&&r.kind===t.kind&&r.multi===t.multi&&(e=i)})),r[e]=t})),r}function wo(t){return this.extend(t)}wo.prototype.extend=function(t){var e=[],r=[];if(t instanceof go)r.push(t);else if(Array.isArray(t))r=r.concat(t);else{if(!t||!Array.isArray(t.implicit)&&!Array.isArray(t.explicit))throw new lo("Schema.extend argument should be a Type, [ Type ], or a schema definition ({ implicit: [...], explicit: [...] })");t.implicit&&(e=e.concat(t.implicit)),t.explicit&&(r=r.concat(t.explicit))}e.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.");if(t.loadKind&&"scalar"!==t.loadKind)throw new lo("There is a non-scalar type in the implicit list of a schema. Implicit resolving of such types is not supported.");if(t.multi)throw new lo("There is a multi type in the implicit list of a schema. Multi tags can only be listed as explicit.")})),r.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.")}));var i=Object.create(wo.prototype);return i.implicit=(this.implicit||[]).concat(e),i.explicit=(this.explicit||[]).concat(r),i.compiledImplicit=mo(i,"implicit"),i.compiledExplicit=mo(i,"explicit"),i.compiledTypeMap=function(){var t,e,r={scalar:{},sequence:{},mapping:{},fallback:{},multi:{scalar:[],sequence:[],mapping:[],fallback:[]}};function i(t){t.multi?(r.multi[t.kind].push(t),r.multi.fallback.push(t)):r[t.kind][t.tag]=r.fallback[t.tag]=t}for(t=0,e=arguments.length;t<e;t+=1)arguments[t].forEach(i);return r}(i.compiledImplicit,i.compiledExplicit),i};var yo=wo,bo=new go("tag:yaml.org,2002:str",{kind:"scalar",construct:function(t){return null!==t?t:""}}),Ao=new go("tag:yaml.org,2002:seq",{kind:"sequence",construct:function(t){return null!==t?t:[]}}),_o=new go("tag:yaml.org,2002:map",{kind:"mapping",construct:function(t){return null!==t?t:{}}}),vo=new yo({explicit:[bo,Ao,_o]}),Eo=new go("tag:yaml.org,2002:null",{kind:"scalar",resolve:function(t){if(null===t)return!0;var e=t.length;return 1===e&&"~"===t||4===e&&("null"===t||"Null"===t||"NULL"===t)},construct:function(){return null},predicate:function(t){return null===t},represent:{canonical:function(){return"~"},lowercase:function(){return"null"},uppercase:function(){return"NULL"},camelcase:function(){return"Null"},empty:function(){return""}},defaultStyle:"lowercase"}),So=new go("tag:yaml.org,2002:bool",{kind:"scalar",resolve:function(t){if(null===t)return!1;var e=t.length;return 4===e&&("true"===t||"True"===t||"TRUE"===t)||5===e&&("false"===t||"False"===t||"FALSE"===t)},construct:fuLine truncated
+/*!
+ * Copyright (c) 2014, GMO GlobalSign
+ * Copyright (c) 2015-2022, Peculiar Ventures
+ * All rights reserved.
+ * 
+ * Author 2014-2019, Yury Strozhevsky
+ * 
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ * 
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * 
+ */
+function lc(){if("undefined"==typeof BigInt)throw new Error("BigInt is not defined. Your environment doesn't implement BigInt.")}function hc(t){let e=0,r=0;for(let r=0;r<t.length;r++)e+=t[r].byteLength;const i=new Uint8Array(e);for(let e=0;e<t.length;e++){const n=t[e];i.set(new Uint8Array(n),r),r+=n.byteLength}return i.buffer}function uc(t,e,r,i){return e instanceof Uint8Array?e.byteLength?r<0?(t.error="Wrong parameter: inputOffset less than zero",!1):i<0?(t.error="Wrong parameter: inputLength less than zero",!1):!(e.byteLength-r-i<0&&(t.error="End of input reached before message was fully decoded (inconsistent offset and length values)",1)):(t.error="Wrong parameter: inputBuffer has zero length",!1):(t.error="Wrong parameter: inputBuffer must be 'Uint8Array'",!1)}Math.log(2);class pc{constructor(){this.items=[]}write(t){this.items.push(t)}final(){return hc(this.items)}}const dc=[new Uint8Array([1])],fc="0123456789",gc="name",mc="valueHexView",wc="",yc=new ArrayBuffer(0),bc=new Uint8Array(0),Ac="EndOfContent",_c="OCTET STRING",vc="BIT STRING";function Ec(t){var e;return(e=class extends t{constructor(...t){var e;super(...t);const r=t[0]||{};this.isHexOnly=null!==(e=r.isHexOnly)&&void 0!==e&&e,this.valueHexView=r.valueHex?ic.vJ.toUint8Array(r.valueHex):bc}get valueHex(){return this.valueHexView.slice().buffer}set valueHex(t){this.valueHexView=new Uint8Array(t)}fromBER(t,e,r){const i=t instanceof ArrayBuffer?new Uint8Array(t):t;if(!uc(this,i,e,r))return-1;const n=e+r;return this.valueHexView=i.subarray(e,n),this.valueHexView.length?(this.blockLength=r,n):(this.warnings.push("Zero buffer length"),e)}toBER(t=!1){return this.isHexOnly?t?new ArrayBuffer(this.valueHexView.byteLength):this.valueHexView.byteLength===this.valueHexView.buffer.byteLength?this.valueHexView.buffer:this.valueHexView.slice().buffer:(this.error="Flag 'isHexOnly' is not set, abort",yc)}toJSON(){return{...super.toJSON(),isHexOnly:this.isHexOnly,valueHex:ic.ep.ToHex(this.valueHexView)}}}).NAME="hexBlock",e}class Sc{constructor({blockLength:t=0,error:e="",warnings:r=[],valueBeforeDecode:i=bc}={}){this.blockLength=t,this.error=e,this.warnings=r,this.valueBeforeDecodeView=ic.vJ.toUint8Array(i)}static blockName(){return this.NAME}get valueBeforeDecode(){return this.valueBeforeDecodeView.slice().buffer}set valueBeforeDecode(t){this.valueBeforeDecodeView=new Uint8Array(t)}toJSON(){return{blockName:this.constructor.NAME,blockLength:this.blockLength,error:this.error,warnings:this.warnings,valueBeforeDecode:ic.ep.ToHex(this.valueBeforeDecodeView)}}}Sc.NAME="baseBlock";class Tc extends Sc{fromBER(t,e,r){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}toBER(t,e){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}}Tc.NAME="valueBlock";class xc extends(Ec(Sc)){constructor({idBlock:t={}}={}){var e,r,i,n;super(),t?(this.isHexOnly=null!==(e=t.isHexOnly)&&void 0!==e&&e,this.valueHexView=t.valueHex?ic.vJ.toUint8Array(t.valueHex):bc,this.tagClass=null!==(r=t.tagClass)&&void 0!==r?r:-1,this.tagNumber=null!==(i=t.tagNumber)&&void 0!==i?i:-1,this.isConstructed=null!==(n=t.isConstructed)&&void 0!==n&&n):(this.tagClass=-1,this.tagNumber=-1,this.isConstructed=!1)}toBER(t=!1){let e=0;switch(this.tagClass){case 1:e|=0;break;case 2:e|=64;break;case 3:e|=128;break;case 4:e|=192;break;default:return this.error="Unknown tag class",yc}if(this.isConstructed&&(e|=32),this.tagNumber<31&&!this.isHexOnly){const r=new Uint8Array(1);if(!t){let t=this.tagNumber;t&=31,e|=t,r[0]=e}return r.buffer}if(!this.isHexOnly){const r=sc(this.tagNumber,7),i=new Uint8Array(r),n=r.byteLength,s=new Uint8Array(n+1);if(s[0]=31|e,!t){for(let t=0;t<n-1;t++)s[t+1]=128|i[t];s[n]=i[n-1]}return s.buffer}const r=new Uint8Array(this.valueHexView.byteLength+1);if(r[0]=31|e,!t){const t=this.valueHexView;for(let e=0;e<t.length-1;e++)r[e+1]=128|t[e];r[this.valueHexView.byteLength]=t[t.length-1]}return r.buffer}fromBER(t,e,r){const i=ic.vJ.toUint8Array(t);if(!uc(this,i,e,r))return-1;const n=i.subarray(e,e+r);if(0===n.length)return this.error="Zero buffer length",-1;switch(192&n[0]){case 0:this.tagClass=1;break;case 64:this.tagClass=2;break;case 128:this.tagClass=3;break;case 192:this.tagClass=4;break;default:return this.error="Unknown tag class",-1}this.isConstructed=32==(32&n[0]),this.isHexOnly=!1;const s=31&n[0];if(31!==s)this.tagNumber=s,this.blockLength=1;else{let t=1,e=this.valueHexView=new Uint8Array(255),r=255;for(;128&n[t];){if(e[t-1]=127&n[t],t++,t>=n.length)return this.error="End of input reached before message was fully decoded",-1;if(t===r){r+=255;const t=new Uint8Array(r);for(let r=0;r<e.length;r++)t[r]=e[r];e=this.valueHexView=new Uint8Array(r)}}this.blockLength=t+1,e[t-1]=127&n[t];const i=new Uint8Array(t);for(let r=0;r<t;r++)i[r]=e[r];e=this.valueHexView=new Uint8Array(t),e.set(i),this.blockLength<=9?this.tagNumber=nc(e,7):(this.isHexOnly=!0,this.warnings.push("Tag too long, represented as hex-coded"))}if(1=Line truncated
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */var vg=function(t,e){return vg=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])},vg(t,e)};function Eg(t,e){function r(){this.constructor=t}vg(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}function Sg(t){var e="function"==typeof Symbol&&Symbol.iterator,r=e&&t[e],i=0;if(r)return r.call(t);if(t&&"number"==typeof t.length)return{next:function(){return t&&i>=t.length&&(t=void 0),{value:t&&t[i++],done:!t}}};throw new TypeError(e?"Object is not iterable.":"Symbol.iterator is not defined.")}function Tg(t,e){var r="function"==typeof Symbol&&t[Symbol.iterator];if(!r)return t;var i,n,s=r.call(t),o=[];try{for(;(void 0===e||e-- >0)&&!(i=s.next()).done;)o.push(i.value)}catch(t){n={error:t}}finally{try{i&&!i.done&&(r=s.return)&&r.call(s)}finally{if(n)throw n.error}}return o}function xg(){for(var t=[],e=0;e<arguments.length;e++)t=t.concat(Tg(arguments[e]));return t}function Cg(t){return!!t.useClass}function Ig(t){return!!t.useFactory}var Rg=function(){function t(t){this.wrap=t,this.reflectMethods=["get","getPrototypeOf","setPrototypeOf","getOwnPropertyDescriptor","defineProperty","has","set","deleteProperty","apply","construct","ownKeys"]}return t.prototype.createProxy=function(t){var e,r=this,i=!1;return new Proxy({},this.createHandler((function(){return i||(e=t(r.wrap()),i=!0),e})))},t.prototype.createHandler=function(t){var e={};return this.reflectMethods.forEach((function(r){e[r]=function(){for(var e=[],i=0;i<arguments.length;i++)e[i]=arguments[i];e[0]=t();var n=Reflect[r];return n.apply(void 0,xg(e))}})),e},t}();function kg(t){return"string"==typeof t||"symbol"==typeof t}function Ng(t){return"object"==typeof t&&"token"in t&&"transform"in t}function Og(t){return!!t.useToken}function Bg(t){return null!=t.useValue}const Pg=function(){function t(){this._registryMap=new Map}return t.prototype.entries=function(){return this._registryMap.entries()},t.prototype.getAll=function(t){return this.ensure(t),this._registryMap.get(t)},t.prototype.get=function(t){this.ensure(t);var e=this._registryMap.get(t);return e[e.length-1]||null},t.prototype.set=function(t,e){this.ensure(t),this._registryMap.get(t).push(e)},t.prototype.setAll=function(t,e){this._registryMap.set(t,e)},t.prototype.has=function(t){return this.ensure(t),this._registryMap.get(t).length>0},t.prototype.clear=function(){this._registryMap.clear()},t.prototype.ensure=function(t){this._registryMap.has(t)||this._registryMap.set(t,[])},t}(),Lg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Dg=function(){this.scopedResolutions=new Map};var Ug=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Mg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg);const Fg=function(){this.preResolution=new Ug,this.postResolution=new Mg};var Hg=new Map,Wg=function(){function t(t){this.parent=t,this._registry=new Lg,this.interceptors=new Fg,this.disposed=!1,this.disposables=new Set}return t.prototype.register=function(t,e,r){var i;if(void 0===r&&(r={lifecycle:_g.Transient}),this.ensureNotDisposed(),i=function(t){return Cg(t)||Bg(t)||Og(t)||Ig(t)}(e)?e:{useClass:e},Og(i))for(var n=[t],s=i;null!=s;){var o=s.useToken;if(n.includes(o))throw new Error("Token registration cycle detected! "+xg(n,[o]).join(" -> "));n.push(o);var a=this._registry.get(o);s=a&&Og(a.provider)?a.provider:null}if((r.lifecycle===_g.Singleton||r.lifecycle==_g.ContainerScoped||r.lifecycle==_g.ResolutionScoped)&&(Bg(i)||Ig(i)))throw new Error('Cannot use lifecycle "'+_g[r.lifecycle]+'" with ValueProviders or FactoryProviders');return this._registry.set(t,{provider:i,options:r}),this},t.prototype.registerType=function(t,e){return this.ensureNotDisposed(),kg(e)?this.register(t,{useToken:e}):this.register(t,{useClass:e})},t.prototype.registerInstance=function(t,e){return this.ensureNotDisposed(),this.register(t,{useValue:e})},t.prototype.registerSingleton=function(t,e){if(this.ensureNotDisposed(),kg(t)){if(kg(e))return this.register(t,{useToken:e},{lifecycle:_g.Singleton});if(e)return this.register(t,{useClass:e},{lifecycle:_g.Singleton});throw new Error('Cannot register a type name as a singleton without a "to" token')}var r=t;return e&&!kg(e)&&(r=e),this.register(t,{useClass:r},{lifecycle:_g.Singleton})},t.prototype.resolve=function(t,e){void 0===e&&(e=new Dg),this.ensureNotDisposed();var r=this.getRegistration(t);if(!r&&kg(t))throw new Error('Attempted to resolve unregistered dependency token: "'+t.toString()+'"');if(this.executePreResolutionInterceptor(t,"Single"),r){var i=this.resolveRegistration(r,e);return this.executePostResolutionInterceptor(t,i,"Single"),i}if(function(t){return"function"==typeof t||t instanceof Rg}(t))return i=this.construct(t,e),this.executePostResolutionInterceptor(t,i,"Line truncated
+/*!
+ * MIT License
+ * 
+ * Copyright (c) Peculiar Ventures. All rights reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ * 
+ */
+const zm="crypto.algorithm",qm="crypto.algorithmProvider";var Qm;jg.registerSingleton(qm,class{getAlgorithms(){return jg.resolveAll(zm)}toAsnAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toAsnAlgorithm(t);if(r)return r}if(/[0-9.]+/.test(t.name)){const e=new Fp({algorithm:t.name});if("parameters"in t){const r=t;e.parameters=r.parameters}return e}throw new Error("Cannot convert WebCrypto algorithm to ASN.1 algorithm")}toWebAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toWebAlgorithm(t);if(r)return r}return{name:t.algorithm,parameters:t.parameters}}});const Km="1.3.36.3.3.2.8.1.1",Ym=`${Km}.1`,Xm=`${Km}.2`,Jm=`${Km}.3`,Zm=`${Km}.4`,$m=`${Km}.5`,tw=`${Km}.6`,ew=`${Km}.7`,rw=`${Km}.8`,iw=`${Km}.9`,nw=`${Km}.10`,sw=`${Km}.11`,ow=`${Km}.12`,aw=`${Km}.13`,cw=`${Km}.14`,lw="brainpoolP160r1",hw="brainpoolP160t1",uw="brainpoolP192r1",pw="brainpoolP192t1",dw="brainpoolP224r1",fw="brainpoolP224t1",gw="brainpoolP256r1",mw="brainpoolP256t1",ww="brainpoolP320r1",yw="brainpoolP320t1",bw="brainpoolP384r1",Aw="brainpoolP384t1",_w="brainpoolP512r1",vw="brainpoolP512t1",Ew="ECDSA";let Sw=Qm=class{toAsnAlgorithm(t){if(t.name.toLowerCase()===Ew.toLowerCase())if("hash"in t)switch(("string"==typeof t.hash?t.hash:t.hash.name).toLowerCase()){case"sha-1":return Pf;case"sha-256":return Lf;case"sha-384":return Df;case"sha-512":return Uf}else if("namedCurve"in t){let e="";switch(t.namedCurve){case"P-256":e=kf;break;case"K-256":e=Qm.SECP256K1;break;case"P-384":e=Nf;break;case"P-521":e=Of;break;case lw:e=Ym;break;case hw:e=Xm;break;case uw:e=Jm;break;case pw:e=Zm;break;case dw:e=$m;break;case fw:e=tw;break;case gw:e=ew;break;case mw:e=rw;break;case ww:e=iw;break;case yw:e=nw;break;case bw:e=sw;break;case Aw:e=ow;break;case _w:e=aw;break;case vw:e=cw}if(e)return new Fp({algorithm:Sf,parameters:iu.serialize(new Mf({namedCurve:e}))})}return null}toWebAlgorithm(t){switch(t.algorithm){case Tf:return{name:Ew,hash:{name:"SHA-1"}};case Cf:return{name:Ew,hash:{name:"SHA-256"}};case If:return{name:Ew,hash:{name:"SHA-384"}};case Rf:return{name:Ew,hash:{name:"SHA-512"}};case Sf:if(!t.parameters)throw new TypeError("Cannot get required parameters from EC algorithm");switch(iu.parse(t.parameters,Mf).namedCurve){case kf:return{name:Ew,namedCurve:"P-256"};case Qm.SECP256K1:return{name:Ew,namedCurve:"K-256"};case Nf:return{name:Ew,namedCurve:"P-384"};case Of:return{name:Ew,namedCurve:"P-521"};case Ym:return{name:Ew,namedCurve:lw};case Xm:return{name:Ew,namedCurve:hw};case Jm:return{name:Ew,namedCurve:uw};case Zm:return{name:Ew,namedCurve:pw};case $m:return{name:Ew,namedCurve:dw};case tw:return{name:Ew,namedCurve:fw};case ew:return{name:Ew,namedCurve:gw};case rw:return{name:Ew,namedCurve:mw};case iw:return{name:Ew,namedCurve:ww};case nw:return{name:Ew,namedCurve:yw};case sw:return{name:Ew,namedCurve:bw};case ow:return{name:Ew,namedCurve:Aw};case aw:return{name:Ew,namedCurve:_w};case cw:return{name:Ew,namedCurve:vw}}}return null}};Sw.SECP256K1="1.3.132.0.10",Sw=Qm=bg([Vg()],Sw),jg.registerSingleton(zm,Sw);const Tw=Symbol("name"),xw=Symbol("value");class Cw{constructor(t,e={},r=""){this[Tw]=t,this[xw]=r;for(const t in e)this[t]=e[t]}}Cw.NAME=Tw,Cw.VALUE=xw;class Iw{static toString(t){return this.items[t]||t}}Iw.items={[eg]:"sha1",[rg]:"sha224",[ig]:"sha256",[ng]:"sha384",[sg]:"sha512",[jf]:"rsaEncryption",[Kf]:"sha1WithRSAEncryption",[Yf]:"sha224WithRSAEncryption",[Xf]:"sha256WithRSAEncryption",[Jf]:"sha384WithRSAEncryption",[Zf]:"sha512WithRSAEncryption",[Sf]:"ecPublicKey",[Tf]:"ecdsaWithSHA1",[xf]:"ecdsaWithSHA224",[Cf]:"ecdsaWithSHA256",[If]:"ecdsaWithSHA384",[Rf]:"ecdsaWithSHA512",[np]:"TLS WWW server authentication",[sp]:"TLS WWW client authentication",[op]:"Code Signing",[ap]:"E-mail Protection",[cp]:"Time Stamping",[lp]:"OCSP Signing","1.2.840.113549.1.7.2":"Signed Data"};class Rw{static serialize(t){return this.serializeObj(t).join("\n")}static pad(t=0){return"".padStart(2*t," ")}static serializeObj(t,e=0){const r=[];let i=this.pad(e++),n="";const s=t[Cw.VALUE];s&&(n=` ${s}`),r.push(`${i}${t[Cw.NAME]}:${n}`),i=this.pad(e);for(const n in t){if("symbol"==typeof n)continue;const s=t[n],o=n?`${n}: `:"";if("string"==typeof s||"number"==typeof s||"boolean"==typeof s)r.push(`${i}${o}${s}`);else if(s instanceof Date)r.push(`${i}${o}${s.toUTCString()}`);else if(Array.isArray(s))for(const t of s)t[Cw.NAME]=n,r.push(...this.serializeObj(t,e));else if(s instanceof Cw)s[Cw.NAME]=n,r.push(...this.serializeObj(s,e));else if(ic.vJ.isBufferSource(s))n?(r.push(`${i}${o}`),r.push(...this.serializeBufferSource(s,e+1))):r.push(...this.serializeBufferSource(s,e));else{if(!("toTextObject"in s))throw new TypeError("Cannot serialize data in text format. Unsupported type.");{const t=s.toTextObject();t[Cw.NAME]=n,r.push(...this.serializeObj(t,e))}}}return r}static serializeBufferSource(t,e=0){const r=this.pad(e),i=ic.vJ.toUint8Array(t),n=[];for(let t=0;t<i.length;){const e=[];for(let r=0;r<16&&t<i.length;r++){8===r&&e.push("");const n=i[t++].toStrinLine truncated

archivebox/plugins/replaywebpage/static/test.txt

@@ -0,0 +1 @@
+test content this should be visible

archivebox/plugins/replaywebpage/templates/plugin_replaywebpage__viewer.html

@@ -0,0 +1,40 @@
+{% load tz core_tags static %}
+
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>{{title}}</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+      
+        </style>
+        <style>
+            html, body {
+                width: 100%;
+                height: 100%;
+                background-color: #ddd;
+            }
+        </style>
+    </head>
+    <body>
+        ReplayWeb.page for: {{snapshot.url}} ({{timestamp}}) /{{warc_filename}}
+
+        {{snapshot}}
+
+        <script>
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/sw.min.js
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/ui.min.js
+        </script>
+
+        <style>
+        </style>
+        <script src="/static/ui.js"></script>
+
+        <replay-web-page
+            style="height: 600px"
+            embed="replay"
+            replayBase="/static/"
+            source="/static/test.wacz"
+            url="https://example.com/">
+        </replay-web-page>
+    </body>
+</html>

archivebox/plugins/replaywebpage/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from .views import ReplayWebPageViewer
+
+urlpatterns = [
+	path('<path:path>', ReplayWebPageViewer.as_view(), name='plugin_replaywebpage__viewer'),
+]

archivebox/plugins/replaywebpage/views.py

@@ -0,0 +1,47 @@
+import os
+import sys
+from pathlib import Path
+
+from django.views import View
+from django.shortcuts import render
+from django.db.models import Q
+
+from core.models import Snapshot
+
+# from archivebox.config import PUBLIC_SNAPSHOTS
+PUBLIC_SNAPSHOTS = True
+
+
+class ReplayWebPageViewer(View):
+    template_name = 'plugin_replaywebpage__viewer.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        try:
+            timestamp, warc_filename = path.split('/', 1)
+        except (IndexError, ValueError):
+            timestamp, warc_filename = path.split('/', 1)[0], ''
+
+        snapshot = Snapshot.objects.get(Q(timestamp=timestamp) | Q(id__startswith=timestamp))
+
+        context = self.get_context_data()
+        context.update({
+            "snapshot": snapshot,
+            "timestamp": timestamp,
+            "warc_filename": warc_filename,
+        })
+        return render(template_name=self.template_name, request=self.request, context=context)
+

archivebox/plugins/system/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins.system'
+
+default_app_config = 'plugins.system.apps.SystemPluginAppConfig'

archivebox/plugins/system/admin.py

@@ -0,0 +1,49 @@
+from django.contrib import admin
+from solo.admin import SingletonModelAdmin
+
+from plugins.defaults.admin import DependencyAdmin, ExtractorAdmin
+
+from .models import (
+	BashEnvironmentDependency,
+	PythonEnvironmentDependency,
+	NodeJSEnvironmentDependency,
+
+	AptEnvironmentDependency,
+	BrewEnvironmentDependency,
+	PipEnvironmentDependency,
+	NPMEnvironmentDependency,
+
+	SQLiteDependency,
+	DjangoDependency,
+	ArchiveBoxDependency,
+
+	# ArchiveBoxDefaultExtractor,
+)
+
+
+print('DefaultsPluginConfig.admin')
+
+class MultiDependencyAdmin(admin.ModelAdmin):
+	readonly_fields = DependencyAdmin.readonly_fields
+	list_display = ('id', 'NAME', 'ENABLED', 'BINARY', 'ARGS', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
+
+class MultiExtractorAdmin(admin.ModelAdmin):
+	readonly_fields = DependencyAdmin.readonly_fields
+	list_display = ('id', 'NAME', 'CMD', 'ARGS', 'is_valid', 'is_enabled')
+
+
+# admin.site.register(BashEnvironmentDependency, DependencyAdmin)
+admin.site.register(BashEnvironmentDependency, MultiDependencyAdmin)
+admin.site.register(PythonEnvironmentDependency, DependencyAdmin)
+admin.site.register(NodeJSEnvironmentDependency, DependencyAdmin)
+
+admin.site.register(AptEnvironmentDependency, DependencyAdmin)
+admin.site.register(BrewEnvironmentDependency, DependencyAdmin)
+admin.site.register(PipEnvironmentDependency, DependencyAdmin)
+admin.site.register(NPMEnvironmentDependency, DependencyAdmin)
+
+admin.site.register(SQLiteDependency, DependencyAdmin)
+admin.site.register(DjangoDependency, DependencyAdmin)
+admin.site.register(ArchiveBoxDependency, DependencyAdmin)
+
+# admin.site.register(ArchiveBoxDefaultExtractor, ExtractorAdmin)

archivebox/plugins/system/apps.py

@@ -0,0 +1,21 @@
+__package__ = 'archivebox.plugins.system'
+
+
+from django.apps import AppConfig
+
+
+class SystemPluginAppConfig(AppConfig):
+    name = "plugins.system"
+    verbose_name = "Host System Configuration"
+    
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        print('plugins.system.apps.SystemPluginConfig.ready')
+
+        from django.conf import settings
+
+        from plugins.defaults.settings import register_plugin_settings
+
+        register_plugin_settings(settings, name=self.name)
+        

archivebox/plugins/system/migrations/0001_initial.py

@@ -0,0 +1,144 @@
+# Generated by Django 3.1.14 on 2024-01-24 08:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AptEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True)),
+                ('BINARY', models.CharField(default='apt-get', max_length=255)),
+                ('ARGS', models.CharField(default='-qq', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Package Manager: apt',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ArchiveBoxDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='archivebox', editable=False, max_length=255)),
+                ('ARGS', models.CharField(default=[], editable=False, max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Internal Dependency: ArchiveBox Package',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='BashEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='bash', max_length=255)),
+                ('ARGS', models.CharField(default='-c', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Shell Environment: bash',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='BrewEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True)),
+                ('BINARY', models.CharField(default='brew', max_length=255)),
+                ('ARGS', models.CharField(default='', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Package Manager: brew',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='DjangoDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='django-admin.py', editable=False, max_length=255)),
+                ('ARGS', models.CharField(default=[], editable=False, max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Internal Dependency: Django Package',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='NodeJSEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True)),
+                ('BINARY', models.CharField(default='node', max_length=255)),
+                ('ARGS', models.CharField(default='-c', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Shell Environment: NodeJS',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='NPMEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True)),
+                ('BINARY', models.CharField(default='node', max_length=255)),
+                ('ARGS', models.CharField(default='', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Package Manager: npm',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='PipEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True)),
+                ('BINARY', models.CharField(default='pip3', max_length=255)),
+                ('ARGS', models.CharField(default='', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Package Manager: pip',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='PythonEnvironmentDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='python3', max_length=255)),
+                ('ARGS', models.CharField(default='-c', max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Shell Environment: Python3',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='SQLiteDependency',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='sqlite3', editable=False, max_length=255)),
+                ('ARGS', models.CharField(default=[], editable=False, max_length=255)),
+            ],
+            options={
+                'verbose_name': 'Internal Dependency: SQLite3 Package',
+                'abstract': False,
+            },
+        ),
+    ]

archivebox/plugins/system/migrations/0002_auto_20240124_0943.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.1.14 on 2024-01-24 09:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('system', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='archiveboxdependency',
+            options={'verbose_name': 'Internal Dependency: archivebox'},
+        ),
+        migrations.AlterModelOptions(
+            name='djangodependency',
+            options={'verbose_name': 'Internal Dependency: django'},
+        ),
+        migrations.AlterModelOptions(
+            name='nodejsenvironmentdependency',
+            options={'verbose_name': 'Shell Environment: node'},
+        ),
+        migrations.AlterModelOptions(
+            name='pythonenvironmentdependency',
+            options={'verbose_name': 'Shell Environment: python3'},
+        ),
+        migrations.AlterModelOptions(
+            name='sqlitedependency',
+            options={'verbose_name': 'Internal Dependency: sqlite3'},
+        ),
+    ]

archivebox/plugins/system/migrations/0003_auto_20240124_0956.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.1.14 on 2024-01-24 09:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('system', '0002_auto_20240124_0943'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='bashenvironmentdependency',
+            options={'verbose_name': 'Shell Environment: bash', 'verbose_name_plural': 'Shell Environments: bash'},
+        ),
+        migrations.AddField(
+            model_name='bashenvironmentdependency',
+            name='VERSION_CMD',
+            field=models.CharField(default='{BINARY} --version', max_length=255),
+        ),
+    ]

archivebox/plugins/system/models.py

@@ -0,0 +1,448 @@
+# __package__ = 'archivebox.plugins.system'
+
+
+import os
+import shutil
+import sys
+import inspect
+import django
+from sqlite3 import dbapi2 as sqlite3
+
+from pathlib import Path
+from typing import List, Dict, Any
+
+from django.db import models
+from django.utils.functional import cached_property
+
+from solo.models import SingletonModel
+
+from config import bin_path, bin_version, VERSION
+
+from plugins.defaults.models import ArchiveBoxBaseDependency
+
+ConfigDict = Dict[str, Any]
+
+
+class BashEnvironmentDependency(ArchiveBoxBaseDependency):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'BASH'
+    LABEL = "Bash"
+    REQUIRED = True
+
+    PARENT_DEPENDENCIES = []
+
+    BIN_DEPENDENCIES: List[str] = ['bash']
+    APT_DEPENDENCIES: List[str] = []
+    BREW_DEPENDENCIES: List[str] = []    
+    PIP_DEPENDENCIES: List[str] = []
+    NPM_DEPENDENCIES: List[str] = []
+
+    DEFAULT_BINARY = 'bash'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = '-c'
+    
+    ENABLED = models.BooleanField(default=True, editable=not REQUIRED)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+
+    VERSION_CMD = models.CharField(max_length=255, default='{BINARY} --version')
+    
+    # START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
+    # WORKERS = models.IntegerField(default=1)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Shell Environment: bash"
+        verbose_name_plural = "Shell Environments: bash"
+
+    # @task
+    def install_pkgs(self, os_pkgs=()):
+        assert self.is_valid, 'Bash environment is not available on this host'
+
+        for os_dependency in os_pkgs:
+            assert bin_path(os_dependency)
+
+        return True
+
+class PythonEnvironmentDependency(ArchiveBoxBaseDependency):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'PYTHON'
+    LABEL = "Python"
+    REQUIRED = True
+
+    PARENT_DEPENDENCIES = []
+
+    BIN_DEPENDENCIES = ['python3']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []    
+    PIP_DEPENDENCIES = []
+    NPM_DEPENDENCIES = []
+
+    DEFAULT_BINARY = 'python3'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = '-c'
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=not REQUIRED)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+    
+    # START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
+    # WORKERS = models.IntegerField(default=1)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Shell Environment: python3"
+
+class NodeJSEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'NODEJS'
+    LABEL = "NodeJS"
+    REQUIRED = True
+
+    PARENT_DEPENDENCIES = []
+
+    BIN_DEPENDENCIES = ['node']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []    
+    PIP_DEPENDENCIES = []
+    NPM_DEPENDENCIES = []
+
+    DEFAULT_BINARY = 'node'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = '-c'
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+    
+    # START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
+    # WORKERS = models.IntegerField(default=1)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Shell Environment: node"
+
+
+class AptEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'APT'
+    LABEL = "apt"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES = ['BashEnvironmentDependency']
+
+    BIN_DEPENDENCIES = ['apt-get']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = []
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'apt-get'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = '-qq'
+
+    ENABLED = models.BooleanField(default=True, editable=not REQUIRED)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Package Manager: apt"
+
+    # @task
+    def install_pkgs(self, apt_pkgs=()):        
+        assert self.is_valid, 'Apt environment is not available on this host'
+
+        # with huey.lock_task('apt-install'):
+
+        run(cmd=[self.DEFAULT_BINARY, '-qq', 'update'])
+        for apt_package in apt_pkgs:
+            run(cmd=[self.DEFAULT_BINARY, 'install', '-y', apt_package])
+
+        return True
+
+class BrewEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'BREW'
+    LABEL = "homebrew"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES = ['BashEnvironmentDependency']
+
+    BIN_DEPENDENCIES = ['brew']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = []
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'brew'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = ''
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Package Manager: brew"
+
+    # @task
+    def install_pkgs(self, brew_pkgs=()):
+        assert self.is_valid, 'Brw environment is not available on this host'
+        
+        run(cmd=[self.DEFAULT_BINARY, 'update'])
+
+        for brew_pkg in brew_pkgs:
+            run(cmd=[self.DEFAULT_BINARY, 'install', brew_pkg])
+
+        return True
+
+
+
+
+class PipEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'PIP'
+    LABEL = "pip"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES = ['BashEnvironmentDependency']
+
+    BIN_DEPENDENCIES = ['python3', 'pip3']
+    APT_DEPENDENCIES = ['python3.11', 'pip3', 'pipx']
+    BREW_DEPENDENCIES = ['python@3.11', 'pipx']
+    PIP_PACKAGES = ['setuptools', 'pipx']
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'pip3'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = ''
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Package Manager: pip"
+
+    # @task
+    def install_pkgs(self, pip_pkgs=()):
+        assert self.is_valid, 'Pip environment is not available on this host'
+        
+        for pip_pkg in pip_pkgs:
+            run(cmd=[self.DEFAULT_BINARY, 'install', '--update', '--ignore-installed', pip_pkg])
+
+        return True
+
+
+class NPMEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'NODEJS'
+    LABEL = "NodeJS"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES = ['BashEnvironmentDependency']
+
+    BIN_DEPENDENCIES = ['node', 'npm']
+    APT_DEPENDENCIES = ['node', 'npm']
+    BREW_DEPENDENCIES = ['node', 'npm']
+    PIP_PACKAGES = []
+    NPM_PACKAGES = ['npm']
+
+    DEFAULT_BINARY = 'node'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = ''
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Package Manager: npm"
+
+    # @task
+    def install_pkgs(self, npm_pkgs=()):
+        assert self.is_valid, 'NPM environment is not available on this host'
+        
+        for npm_pkg in npm_pkgs:
+            run(cmd=[self.DEFAULT_BINARY, 'install', npm_pkg])
+
+        return True
+
+
+class DjangoDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'DJANGO'
+    LABEL = "Django"
+    REQUIRED = True
+
+    PARENT_DEPENDENCIES = []
+
+    BIN_DEPENDENCIES = ['django-admin.py']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = ['django==3.1.14']
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'django-admin.py'
+    DEFAULT_START_CMD = 'archivebox server 0.0.0.0:8000'
+    DEFAULT_PID_FILE = 'logs/{NAME}_WORKER.pid'
+    DEFAULT_STOP_CMD = 'kill "$(<{PID_FILE})"'
+    DEFAULT_ARGS = []
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=False)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY, editable=False)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS, editable=False)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Internal Dependency: django"
+
+    @cached_property
+    def bin_path(self):
+        return inspect.getfile(django)
+
+    @cached_property
+    def bin_version(self):
+        return '.'.join(str(v) for v in django.VERSION[:3])
+
+
+class SQLiteDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+
+    NAME = 'SQLITE'
+    LABEL = "SQLite"
+    REQUIRED = True
+
+    PARENT_DEPENDENCIES = []
+
+    BIN_DEPENDENCIES = []
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = []
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'sqlite3'
+    DEFAULT_START_CMD = None
+    DEFAULT_STOP_CMD = None
+    DEFAULT_PID_FILE = None
+    DEFAULT_ARGS = []
+    VERSION_CMD = 'python3 -c ""'
+
+    ENABLED = models.BooleanField(default=True, editable=False)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY, editable=False)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS, editable=False)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Internal Dependency: sqlite3"
+
+    @cached_property
+    def bin_path(self):
+        return inspect.getfile(sqlite3)
+
+    @cached_property
+    def bin_version(self):
+        return sqlite3.version
+
+class ArchiveBoxDependency(ArchiveBoxBaseDependency):
+    singleton_instance_id = 1
+
+    id = models.AutoField(primary_key=True)
+    
+    NAME = 'ARCHIVEBOX'
+    LABEL = "ArchiveBox"
+    REQUIRED = True
+
+    PARENT_DEPENDENCIES = [
+        'PipEnvironmentDependency',
+        'DjangoDependency',
+        'SQLiteDependency',
+    ]
+
+    BIN_DEPENDENCIES = ['archivebox']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = ['archivebox']
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'archivebox'
+    DEFAULT_START_CMD = '{BINARY} server 0.0.0.0:8000'
+    DEFAULT_ARGS = []
+    VERSION_CMD = 'archivebox --version'
+
+    ENABLED = models.BooleanField(default=True, editable=False)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY, editable=False)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS, editable=False)
+
+    class Meta:
+        abstract = False
+        app_label = 'system'
+        verbose_name = "Internal Dependency: archivebox"
+
+    @cached_property
+    def bin_path(self):
+        return sys.argv[0] or bin_path('archivebox')
+
+    @cached_property
+    def bin_version(self):
+        # return config['VERSION']
+        return VERSION
+

archivebox/plugins/system/settings.py

@@ -0,0 +1,3 @@
+from django.conf import settings
+
+from plugins.defaults import register_plugin_settings

archivebox/templates/static/sw.js

@@ -0,0 +1,118 @@
+/*! sw.js (wabac.js 2.16.13) is part of Webrecorder project. Copyright (C) 2020-2023, Webrecorder Software. Licensed under the Affero General Public License v3. */(()=>{var t={9742:(t,e)=>{"use strict";e.byteLength=function(t){var e=c(t),r=e[0],i=e[1];return 3*(r+i)/4-i},e.toByteArray=function(t){var e,r,s=c(t),o=s[0],a=s[1],l=new n(function(t,e,r){return 3*(e+r)/4-r}(0,o,a)),h=0,u=a>0?o-4:o;for(r=0;r<u;r+=4)e=i[t.charCodeAt(r)]<<18|i[t.charCodeAt(r+1)]<<12|i[t.charCodeAt(r+2)]<<6|i[t.charCodeAt(r+3)],l[h++]=e>>16&255,l[h++]=e>>8&255,l[h++]=255&e;return 2===a&&(e=i[t.charCodeAt(r)]<<2|i[t.charCodeAt(r+1)]>>4,l[h++]=255&e),1===a&&(e=i[t.charCodeAt(r)]<<10|i[t.charCodeAt(r+1)]<<4|i[t.charCodeAt(r+2)]>>2,l[h++]=e>>8&255,l[h++]=255&e),l},e.fromByteArray=function(t){for(var e,i=t.length,n=i%3,s=[],o=16383,a=0,c=i-n;a<c;a+=o)s.push(l(t,a,a+o>c?c:a+o));return 1===n?(e=t[i-1],s.push(r[e>>2]+r[e<<4&63]+"==")):2===n&&(e=(t[i-2]<<8)+t[i-1],s.push(r[e>>10]+r[e>>4&63]+r[e<<2&63]+"=")),s.join("")};for(var r=[],i=[],n="undefined"!=typeof Uint8Array?Uint8Array:Array,s="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",o=0,a=s.length;o<a;++o)r[o]=s[o],i[s.charCodeAt(o)]=o;function c(t){var e=t.length;if(e%4>0)throw new Error("Invalid string. Length must be a multiple of 4");var r=t.indexOf("=");return-1===r&&(r=e),[r,r===e?0:4-r%4]}function l(t,e,i){for(var n,s,o=[],a=e;a<i;a+=3)n=(t[a]<<16&16711680)+(t[a+1]<<8&65280)+(255&t[a+2]),o.push(r[(s=n)>>18&63]+r[s>>12&63]+r[s>>6&63]+r[63&s]);return o.join("")}i["-".charCodeAt(0)]=62,i["_".charCodeAt(0)]=63},4181:t=>{var e=4096,r=new Uint32Array([0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535,131071,262143,524287,1048575,2097151,4194303,8388607,16777215]);function i(t){this.buf_=new Uint8Array(8224),this.input_=t,this.reset()}i.READ_SIZE=e,i.IBUF_MASK=8191,i.prototype.reset=function(){this.buf_ptr_=0,this.val_=0,this.pos_=0,this.bit_pos_=0,this.bit_end_pos_=0,this.eos_=0,this.readMoreInput();for(var t=0;t<4;t++)this.val_|=this.buf_[this.pos_]<<8*t,++this.pos_;return this.bit_end_pos_>0},i.prototype.readMoreInput=function(){if(!(this.bit_end_pos_>256))if(this.eos_){if(this.bit_pos_>this.bit_end_pos_)throw new Error("Unexpected end of input "+this.bit_pos_+" "+this.bit_end_pos_)}else{var t=this.buf_ptr_,r=this.input_.read(this.buf_,t,e);if(r<0)throw new Error("Unexpected end of input");if(r<e){this.eos_=1;for(var i=0;i<32;i++)this.buf_[t+r+i]=0}if(0===t){for(i=0;i<32;i++)this.buf_[8192+i]=this.buf_[i];this.buf_ptr_=e}else this.buf_ptr_=0;this.bit_end_pos_+=r<<3}},i.prototype.fillBitWindow=function(){for(;this.bit_pos_>=8;)this.val_>>>=8,this.val_|=this.buf_[8191&this.pos_]<<24,++this.pos_,this.bit_pos_=this.bit_pos_-8>>>0,this.bit_end_pos_=this.bit_end_pos_-8>>>0},i.prototype.readBits=function(t){32-this.bit_pos_<t&&this.fillBitWindow();var e=this.val_>>>this.bit_pos_&r[t];return this.bit_pos_+=t,e},t.exports=i},7080:(t,e)=>{e.lookup=new Uint8Array([0,0,0,0,0,0,0,0,0,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,12,16,12,12,20,12,16,24,28,12,12,32,12,36,12,44,44,44,44,44,44,44,44,44,44,32,32,24,40,28,12,12,48,52,52,52,48,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,24,12,28,12,12,12,56,60,60,60,56,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,24,12,28,12,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,0,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,32,32,32,32,32,32,32,32,32,32,32,32,32,Line truncated
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */const i=r(9742),n=r(645),s="function"==typeof Symbol&&"function"==typeof Symbol.for?Symbol.for("nodejs.util.inspect.custom"):null;e.Buffer=c,e.SlowBuffer=function(t){return+t!=t&&(t=0),c.alloc(+t)},e.INSPECT_MAX_BYTES=50;const o=2147483647;function a(t){if(t>o)throw new RangeError('The value "'+t+'" is invalid for option "size"');const e=new Uint8Array(t);return Object.setPrototypeOf(e,c.prototype),e}function c(t,e,r){if("number"==typeof t){if("string"==typeof e)throw new TypeError('The "string" argument must be of type string. Received type number');return u(t)}return l(t,e,r)}function l(t,e,r){if("string"==typeof t)return function(t,e){if("string"==typeof e&&""!==e||(e="utf8"),!c.isEncoding(e))throw new TypeError("Unknown encoding: "+e);const r=0|g(t,e);let i=a(r);const n=i.write(t,e);return n!==r&&(i=i.slice(0,n)),i}(t,e);if(ArrayBuffer.isView(t))return function(t){if(Y(t,Uint8Array)){const e=new Uint8Array(t);return d(e.buffer,e.byteOffset,e.byteLength)}return p(t)}(t);if(null==t)throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t);if(Y(t,ArrayBuffer)||t&&Y(t.buffer,ArrayBuffer))return d(t,e,r);if("undefined"!=typeof SharedArrayBuffer&&(Y(t,SharedArrayBuffer)||t&&Y(t.buffer,SharedArrayBuffer)))return d(t,e,r);if("number"==typeof t)throw new TypeError('The "value" argument must not be of type number. Received type number');const i=t.valueOf&&t.valueOf();if(null!=i&&i!==t)return c.from(i,e,r);const n=function(t){if(c.isBuffer(t)){const e=0|f(t.length),r=a(e);return 0===r.length||t.copy(r,0,0,e),r}return void 0!==t.length?"number"!=typeof t.length||X(t.length)?a(0):p(t):"Buffer"===t.type&&Array.isArray(t.data)?p(t.data):void 0}(t);if(n)return n;if("undefined"!=typeof Symbol&&null!=Symbol.toPrimitive&&"function"==typeof t[Symbol.toPrimitive])return c.from(t[Symbol.toPrimitive]("string"),e,r);throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t)}function h(t){if("number"!=typeof t)throw new TypeError('"size" argument must be of type number');if(t<0)throw new RangeError('The value "'+t+'" is invalid for option "size"')}function u(t){return h(t),a(t<0?0:0|f(t))}function p(t){const e=t.length<0?0:0|f(t.length),r=a(e);for(let i=0;i<e;i+=1)r[i]=255&t[i];return r}function d(t,e,r){if(e<0||t.byteLength<e)throw new RangeError('"offset" is outside of buffer bounds');if(t.byteLength<e+(r||0))throw new RangeError('"length" is outside of buffer bounds');let i;return i=void 0===e&&void 0===r?new Uint8Array(t):void 0===r?new Uint8Array(t,e):new Uint8Array(t,e,r),Object.setPrototypeOf(i,c.prototype),i}function f(t){if(t>=o)throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+o.toString(16)+" bytes");return 0|t}function g(t,e){if(c.isBuffer(t))return t.length;if(ArrayBuffer.isView(t)||Y(t,ArrayBuffer))return t.byteLength;if("string"!=typeof t)throw new TypeError('The "string" argument must be one of type string, Buffer, or ArrayBuffer. Received type '+typeof t);const r=t.length,i=arguments.length>2&&!0===arguments[2];if(!i&&0===r)return 0;let n=!1;for(;;)switch(e){case"ascii":case"latin1":case"binary":return r;case"utf8":case"utf-8":return q(t).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*r;case"hex":return r>>>1;case"base64":return Q(t).length;default:if(n)return i?-1:q(t).length;e=(""+e).toLowerCase(),n=!0}}function m(t,e,r){let i=!1;if((void 0===e||e<0)&&(e=0),e>this.length)return"";if((void 0===r||r>this.length)&&(r=this.length),r<=0)return"";if((r>>>=0)<=(e>>>=0))return"";for(t||(t="utf8");;)switch(t){case"hex":return k(this,e,r);case"utf8":case"utf-8":return x(this,e,r);case"ascii":return I(this,e,r);case"latin1":case"binary":return R(this,e,r);case"base64":return T(this,e,r);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return N(this,e,r);default:if(i)throw new TypeError("Unknown encoding: "+t);t=(t+"").toLowerCase(),i=!0}}function w(t,e,r){const i=t[e];t[e]=t[r],t[r]=i}function y(t,e,r,i,n){if(0===t.length)return-1;if("string"==typeof r?(i=r,r=0):r>2147483647?r=2147483647:r<-2147483648&&(r=-2147483648),X(r=+r)&&(r=n?0:t.length-1),r<0&&(r=t.length+r),r>=t.length){if(n)return-1;r=t.length-1}else if(r<0){if(!n)return-1;r=0}if("string"==typeof e&&(e=c.from(e,i)),c.isBuffer(e))return 0===e.length?-1:b(t,e,r,i,n);if("number"==typeof e)return e&=255,"function"==typeof Uint8Array.prototype.indexOf?n?Uint8Array.prototype.indexOf.call(t,e,r):Uint8Array.prototype.lastIndexOf.call(t,e,r):b(t,[e],r,i,n);throw new TypeError("val must be string, number or Buffer")}function b(t,e,r,i,n){let s,o=1,a=t.length,c=e.length;if(void 0!==i&&("ucs2"===(i=String(i).toLowerCase())||"ucs-2"===i||"utf16le"===i||"utf-16le"===i)){if(t.length<2||e.length<2)return-1;o=2,a/=2,c/=2,r/=2}function l(t,e){return 1===o?t[e]:t.readUInt16BE(e*o)}if(n){let i=-1;for(s=r;s<a;s++)if(l(t,s)===l(e,-1===i?0:s-i)){if(-1===i&&(i=Line truncated
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+e.read=function(t,e,r,i,n){var s,o,a=8*n-i-1,c=(1<<a)-1,l=c>>1,h=-7,u=r?n-1:0,p=r?-1:1,d=t[e+u];for(u+=p,s=d&(1<<-h)-1,d>>=-h,h+=a;h>0;s=256*s+t[e+u],u+=p,h-=8);for(o=s&(1<<-h)-1,s>>=-h,h+=i;h>0;o=256*o+t[e+u],u+=p,h-=8);if(0===s)s=1-l;else{if(s===c)return o?NaN:1/0*(d?-1:1);o+=Math.pow(2,i),s-=l}return(d?-1:1)*o*Math.pow(2,s-i)},e.write=function(t,e,r,i,n,s){var o,a,c,l=8*s-n-1,h=(1<<l)-1,u=h>>1,p=23===n?Math.pow(2,-24)-Math.pow(2,-77):0,d=i?0:s-1,f=i?1:-1,g=e<0||0===e&&1/e<0?1:0;for(e=Math.abs(e),isNaN(e)||e===1/0?(a=isNaN(e)?1:0,o=h):(o=Math.floor(Math.log(e)/Math.LN2),e*(c=Math.pow(2,-o))<1&&(o--,c*=2),(e+=o+u>=1?p/c:p*Math.pow(2,1-u))*c>=2&&(o++,c/=2),o+u>=h?(a=0,o=h):o+u>=1?(a=(e*c-1)*Math.pow(2,n),o+=u):(a=e*Math.pow(2,u-1)*Math.pow(2,n),o=0));n>=8;t[r+d]=255&a,d+=f,a/=256,n-=8);for(o=o<<n|a,l+=n;l>0;t[r+d]=255&o,d+=f,o/=256,l-=8);t[r+d-f]|=128*g}},5717:t=>{"function"==typeof Object.create?t.exports=function(t,e){e&&(t.super_=e,t.prototype=Object.create(e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}))}:t.exports=function(t,e){if(e){t.super_=e;var r=function(){};r.prototype=e.prototype,t.prototype=new r,t.prototype.constructor=t}}},6512:function(t){!function(e){"use strict";const r="(0?\\d+|0x[a-f0-9]+)",i={fourOctet:new RegExp(`^${r}\\.${r}\\.${r}\\.${r}$`,"i"),threeOctet:new RegExp(`^${r}\\.${r}\\.${r}$`,"i"),twoOctet:new RegExp(`^${r}\\.${r}$`,"i"),longValue:new RegExp(`^${r}$`,"i")},n=new RegExp("^0[0-7]+$","i"),s=new RegExp("^0x[a-f0-9]+$","i"),o="%[0-9a-z]{1,}",a="(?:[0-9a-f]+::?)+",c={zoneIndex:new RegExp(o,"i"),native:new RegExp(`^(::)?(${a})?([0-9a-f]+)?(::)?(${o})?$`,"i"),deprecatedTransitional:new RegExp(`^(?:::)(${r}\\.${r}\\.${r}\\.${r}(${o})?)$`,"i"),transitional:new RegExp(`^((?:${a})|(?:::)(?:${a})?)${r}\\.${r}\\.${r}\\.${r}(${o})?$`,"i")};function l(t,e){if(t.indexOf("::")!==t.lastIndexOf("::"))return null;let r,i,n=0,s=-1,o=(t.match(c.zoneIndex)||[])[0];for(o&&(o=o.substring(1),t=t.replace(/%.+$/,""));(s=t.indexOf(":",s+1))>=0;)n++;if("::"===t.substr(0,2)&&n--,"::"===t.substr(-2,2)&&n--,n>e)return null;for(i=e-n,r=":";i--;)r+="0:";return":"===(t=t.replace("::",r))[0]&&(t=t.slice(1)),":"===t[t.length-1]&&(t=t.slice(0,-1)),{parts:e=function(){const e=t.split(":"),r=[];for(let t=0;t<e.length;t++)r.push(parseInt(e[t],16));return r}(),zoneId:o}}function h(t,e,r,i){if(t.length!==e.length)throw new Error("ipaddr: cannot match CIDR for objects with different lengths");let n,s=0;for(;i>0;){if(n=r-i,n<0&&(n=0),t[s]>>n!=e[s]>>n)return!1;i-=r,s+=1}return!0}function u(t){if(s.test(t))return parseInt(t,16);if("0"===t[0]&&!isNaN(parseInt(t[1],10))){if(n.test(t))return parseInt(t,8);throw new Error(`ipaddr: cannot parse ${t} as octal`)}return parseInt(t,10)}function p(t,e){for(;t.length<e;)t=`0${t}`;return t}const d={};d.IPv4=function(){function t(t){if(4!==t.length)throw new Error("ipaddr: ipv4 octet count should be 4");let e,r;for(e=0;e<t.length;e++)if(r=t[e],!(0<=r&&r<=255))throw new Error("ipaddr: ipv4 octet should fit in 8 bits");this.octets=t}return t.prototype.SpecialRanges={unspecified:[[new t([0,0,0,0]),8]],broadcast:[[new t([255,255,255,255]),32]],multicast:[[new t([224,0,0,0]),4]],linkLocal:[[new t([169,254,0,0]),16]],loopback:[[new t([127,0,0,0]),8]],carrierGradeNat:[[new t([100,64,0,0]),10]],private:[[new t([10,0,0,0]),8],[new t([172,16,0,0]),12],[new t([192,168,0,0]),16]],reserved:[[new t([192,0,0,0]),24],[new t([192,0,2,0]),24],[new t([192,88,99,0]),24],[new t([198,51,100,0]),24],[new t([203,0,113,0]),24],[new t([240,0,0,0]),4]]},t.prototype.kind=function(){return"ipv4"},t.prototype.match=function(t,e){let r;if(void 0===e&&(r=t,t=r[0],e=r[1]),"ipv4"!==t.kind())throw new Error("ipaddr: cannot match ipv4 address with non-ipv4 one");return h(this.octets,t.octets,8,e)},t.prototype.prefixLengthFromSubnetMask=function(){let t=0,e=!1;const r={0:8,128:7,192:6,224:5,240:4,248:3,252:2,254:1,255:0};let i,n,s;for(i=3;i>=0;i-=1){if(n=this.octets[i],!(n in r))return null;if(s=r[n],e&&0!==s)return null;8!==s&&(e=!0),t+=s}return 32-t},t.prototype.range=function(){return d.subnetMatch(this,this.SpecialRanges)},t.prototype.toByteArray=function(){return this.octets.slice(0)},t.prototype.toIPv4MappedAddress=function(){return d.IPv6.parse(`::ffff:${this.toString()}`)},t.prototype.toNormalizedString=function(){return this.toString()},t.prototype.toString=function(){return this.octets.join(".")},t}(),d.IPv4.broadcastAddressFromCIDR=function(t){try{const e=this.parseCIDR(t),r=e[0].toByteArray(),i=this.subnetMaskFromPrefixLength(e[1]).toByteArray(),n=[];let s=0;for(;s<4;)n.push(parseInt(r[s],10)|255^parseInt(i[s],10)),s++;return new this(n)}catch(t){throw new Error("ipaddr: the address does not have IPv4 CIDR format")}},d.IPv4.isIPv4=function(t){return null!==this.parser(t)},d.IPv4.isValid=function(t){try{return new this(this.parser(t)),!0}catch(t){return!1}},d.IPv4.isValidFourPartDecimal=function(t){return!(!d.IPv4.isValid(t)||!t.match(/^(0|[1-9]\d*)(\.(0|[1-9]\d*)){3}$/))},d.Line truncated
+/*! *****************************************************************************
+Copyright (C) Microsoft. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */!function(t){!function(e){var i="object"==typeof r.g?r.g:"object"==typeof self?self:"object"==typeof this?this:Function("return this;")(),s=o(t);function o(t,e){return function(r,i){"function"!=typeof t[r]&&Object.defineProperty(t,r,{configurable:!0,writable:!0,value:i}),e&&e(r,i)}}void 0===i.Reflect?i.Reflect=t:s=o(i.Reflect,s),function(t){var e=Object.prototype.hasOwnProperty,r="function"==typeof Symbol,i=r&&void 0!==Symbol.toPrimitive?Symbol.toPrimitive:"@@toPrimitive",s=r&&void 0!==Symbol.iterator?Symbol.iterator:"@@iterator",o="function"==typeof Object.create,a={__proto__:[]}instanceof Array,c=!o&&!a,l={create:o?function(){return M(Object.create(null))}:a?function(){return M({__proto__:null})}:function(){return M({})},has:c?function(t,r){return e.call(t,r)}:function(t,e){return e in t},get:c?function(t,r){return e.call(t,r)?t[r]:void 0}:function(t,e){return t[e]}},h=Object.getPrototypeOf(Function),u="object"==typeof n&&n.env&&"true"===n.env.REFLECT_METADATA_USE_MAP_POLYFILL,p=u||"function"!=typeof Map||"function"!=typeof Map.prototype.entries?function(){var t={},e=[],r=function(){function t(t,e,r){this._index=0,this._keys=t,this._values=e,this._selector=r}return t.prototype["@@iterator"]=function(){return this},t.prototype[s]=function(){return this},t.prototype.next=function(){var t=this._index;if(t>=0&&t<this._keys.length){var r=this._selector(this._keys[t],this._values[t]);return t+1>=this._keys.length?(this._index=-1,this._keys=e,this._values=e):this._index++,{value:r,done:!1}}return{value:void 0,done:!0}},t.prototype.throw=function(t){throw this._index>=0&&(this._index=-1,this._keys=e,this._values=e),t},t.prototype.return=function(t){return this._index>=0&&(this._index=-1,this._keys=e,this._values=e),{value:t,done:!0}},t}();return function(){function e(){this._keys=[],this._values=[],this._cacheKey=t,this._cacheIndex=-2}return Object.defineProperty(e.prototype,"size",{get:function(){return this._keys.length},enumerable:!0,configurable:!0}),e.prototype.has=function(t){return this._find(t,!1)>=0},e.prototype.get=function(t){var e=this._find(t,!1);return e>=0?this._values[e]:void 0},e.prototype.set=function(t,e){var r=this._find(t,!0);return this._values[r]=e,this},e.prototype.delete=function(e){var r=this._find(e,!1);if(r>=0){for(var i=this._keys.length,n=r+1;n<i;n++)this._keys[n-1]=this._keys[n],this._values[n-1]=this._values[n];return this._keys.length--,this._values.length--,e===this._cacheKey&&(this._cacheKey=t,this._cacheIndex=-2),!0}return!1},e.prototype.clear=function(){this._keys.length=0,this._values.length=0,this._cacheKey=t,this._cacheIndex=-2},e.prototype.keys=function(){return new r(this._keys,this._values,i)},e.prototype.values=function(){return new r(this._keys,this._values,n)},e.prototype.entries=function(){return new r(this._keys,this._values,o)},e.prototype["@@iterator"]=function(){return this.entries()},e.prototype[s]=function(){return this.entries()},e.prototype._find=function(t,e){return this._cacheKey!==t&&(this._cacheIndex=this._keys.indexOf(this._cacheKey=t)),this._cacheIndex<0&&e&&(this._cacheIndex=this._keys.length,this._keys.push(t),this._values.push(void 0)),this._cacheIndex},e}();function i(t,e){return t}function n(t,e){return e}function o(t,e){return[t,e]}}():Map,d=u||"function"!=typeof Set||"function"!=typeof Set.prototype.entries?function(){function t(){this._map=new p}return Object.defineProperty(t.prototype,"size",{get:function(){return this._map.size},enumerable:!0,configurable:!0}),t.prototype.has=function(t){return this._map.has(t)},t.prototype.add=function(t){return this._map.set(t,t),this},t.prototype.delete=function(t){return this._map.delete(t)},t.prototype.clear=function(){this._map.clear()},t.prototype.keys=function(){return this._map.keys()},t.prototype.values=function(){return this._map.values()},t.prototype.entries=function(){return this._map.entries()},t.prototype["@@iterator"]=function(){return this.keys()},t.prototype[s]=function(){return this.keys()},t}():Set,f=new(u||"function"!=typeof WeakMap?function(){var t=l.create(),r=i();return function(){function t(){this._key=i()}return t.prototype.has=function(t){var e=n(t,!1);return void 0!==e&&l.has(e,this._key)},t.prototype.get=function(t){var e=n(t,!1);return void 0!==e?l.get(e,this._key):void 0},t.prototype.set=function(t,e){return n(t,!0)[this._key]=e,this},t.prototype.delete=function(t){var e=n(t,!1);return void 0!==e&&delete e[this._key]},t.prototype.clear=function(){this._key=i()},t}();function i(){var e;do{e="@@WeakMap@@"+o()}while(l.has(t,e));return t[e]=!0,e}function n(t,i){if(!e.call(t,r)){if(!i)return;Object.defineProperty(t,r,{value:l.create()})}return t[r]}function s(t,e){for(var r=0;r<e;++r)t[r]=255*Math.random()|0;return t}function o(){var t=function(t){return"function"==typeof Uint8Array?"undefined"!=typeof crypto?crypto.getRandomValues(new Uint8Array(t)):"undefined"!=typeof msCrypto?msCrypto.getRandomValues(nLine truncated
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+var i=r(8764),n=i.Buffer;function s(t,e){for(var r in t)e[r]=t[r]}function o(t,e,r){return n(t,e,r)}n.from&&n.alloc&&n.allocUnsafe&&n.allocUnsafeSlow?t.exports=i:(s(i,e),e.Buffer=o),o.prototype=Object.create(n.prototype),s(n,o),o.from=function(t,e,r){if("number"==typeof t)throw new TypeError("Argument must not be a number");return n(t,e,r)},o.alloc=function(t,e,r){if("number"!=typeof t)throw new TypeError("Argument must be a number");var i=n(t);return void 0!==e?"string"==typeof r?i.fill(e,r):i.fill(e):i.fill(0),i},o.allocUnsafe=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return n(t)},o.allocUnsafeSlow=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return i.SlowBuffer(t)}},2830:(t,e,r)=>{t.exports=n;var i=r(7187).EventEmitter;function n(){i.call(this)}r(5717)(n,i),n.Readable=r(9481),n.Writable=r(4229),n.Duplex=r(6753),n.Transform=r(4605),n.PassThrough=r(2725),n.finished=r(8610),n.pipeline=r(9946),n.Stream=n,n.prototype.pipe=function(t,e){var r=this;function n(e){t.writable&&!1===t.write(e)&&r.pause&&r.pause()}function s(){r.readable&&r.resume&&r.resume()}r.on("data",n),t.on("drain",s),t._isStdio||e&&!1===e.end||(r.on("end",a),r.on("close",c));var o=!1;function a(){o||(o=!0,t.end())}function c(){o||(o=!0,"function"==typeof t.destroy&&t.destroy())}function l(t){if(h(),0===i.listenerCount(this,"error"))throw t}function h(){r.removeListener("data",n),t.removeListener("drain",s),r.removeListener("end",a),r.removeListener("close",c),r.removeListener("error",l),t.removeListener("error",l),r.removeListener("end",h),r.removeListener("close",h),t.removeListener("close",h)}return r.on("error",l),t.on("error",l),r.on("end",h),r.on("close",h),t.on("close",h),t.emit("pipe",r),t}},2553:(t,e,r)=>{"use strict";var i=r(9509).Buffer,n=i.isEncoding||function(t){switch((t=""+t)&&t.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function s(t){var e;switch(this.encoding=function(t){var e=function(t){if(!t)return"utf8";for(var e;;)switch(t){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return t;default:if(e)return;t=(""+t).toLowerCase(),e=!0}}(t);if("string"!=typeof e&&(i.isEncoding===n||!n(t)))throw new Error("Unknown encoding: "+t);return e||t}(t),this.encoding){case"utf16le":this.text=c,this.end=l,e=4;break;case"utf8":this.fillLast=a,e=4;break;case"base64":this.text=h,this.end=u,e=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=i.allocUnsafe(e)}function o(t){return t<=127?0:t>>5==6?2:t>>4==14?3:t>>3==30?4:t>>6==2?-1:-2}function a(t){var e=this.lastTotal-this.lastNeed,r=function(t,e,r){if(128!=(192&e[0]))return t.lastNeed=0,"�";if(t.lastNeed>1&&e.length>1){if(128!=(192&e[1]))return t.lastNeed=1,"�";if(t.lastNeed>2&&e.length>2&&128!=(192&e[2]))return t.lastNeed=2,"�"}}(this,t);return void 0!==r?r:this.lastNeed<=t.length?(t.copy(this.lastChar,e,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(t.copy(this.lastChar,e,0,t.length),void(this.lastNeed-=t.length))}function c(t,e){if((t.length-e)%2==0){var r=t.toString("utf16le",e);if(r){var i=r.charCodeAt(r.length-1);if(i>=55296&&i<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1],r.slice(0,-1)}return r}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=t[t.length-1],t.toString("utf16le",e,t.length-1)}function l(t){var e=t&&t.length?this.write(t):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return e+this.lastChar.toString("utf16le",0,r)}return e}function h(t,e){var r=(t.length-e)%3;return 0===r?t.toString("base64",e):(this.lastNeed=3-r,this.lastTotal=3,1===r?this.lastChar[0]=t[t.length-1]:(this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1]),t.toString("base64",e,t.length-r))}function u(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+this.lastChar.toString("base64",0,3-this.lastNeed):e}function p(t){return t.toString(this.encoding)}function d(t){return t&&t.length?this.write(t):""}e.s=s,s.prototype.write=function(t){if(0===t.length)return"";var e,r;if(this.lastNeed){if(void 0===(e=this.fillLast(t)))return"";r=this.lastNeed,this.lastNeed=0}else r=0;return r<t.length?e?e+this.text(t,r):this.text(t,r):e||""},s.prototype.end=function(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+"�":e},s.prototype.text=function(t,e){var r=function(t,e,r){var i=e.length-1;if(i<r)return 0;var n=o(e[i]);return n>=0?(n>0&&(t.lastNeed=n-1),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(t.lastNeed=n-2),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(2===n?n=0:t.lastNeed=n-3),n):0}(this,t,e);if(!this.lastNeed)return t.toString("utf8",e);this.lastTotal=r;var i=t.length-(r-this.lastNeed);return t.copy(tLine truncated
+/*!
+ * hash-wasm (https://www.npmjs.com/package/hash-wasm)
+ * (c) Dani Biro
+ * @license MIT
+ */
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */function B(t,e,r,i){return new(r||(r=Promise))((function(n,s){function o(t){try{c(i.next(t))}catch(t){s(t)}}function a(t){try{c(i.throw(t))}catch(t){s(t)}}function c(t){var e;t.done?n(t.value):(e=t.value,e instanceof r?e:new r((function(t){t(e)}))).then(o,a)}c((i=i.apply(t,e||[])).next())}))}class P{constructor(){this.mutex=Promise.resolve()}lock(){let t=()=>{};return this.mutex=this.mutex.then((()=>new Promise(t))),new Promise((e=>{t=e}))}dispatch(t){return B(this,void 0,void 0,(function*(){const e=yield this.lock();try{return yield Promise.resolve(t())}finally{e()}}))}}const L="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:r.g,D=null!==(I=L.Buffer)&&void 0!==I?I:null,U=L.TextEncoder?new L.TextEncoder:null;function M(t,e){return(15&t)+(t>>6|t>>3&8)<<4|(15&e)+(e>>6|e>>3&8)}const F="a".charCodeAt(0)-10,H="0".charCodeAt(0);function W(t,e,r){let i=0;for(let n=0;n<r;n++){let r=e[n]>>>4;t[i++]=r>9?r+F:r+H,r=15&e[n],t[i++]=r>9?r+F:r+H}return String.fromCharCode.apply(null,t)}const j=null!==D?t=>{if("string"==typeof t){const e=D.from(t,"utf8");return new Uint8Array(e.buffer,e.byteOffset,e.length)}if(D.isBuffer(t))return new Uint8Array(t.buffer,t.byteOffset,t.length);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")}:t=>{if("string"==typeof t)return U.encode(t);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")},V="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",G=new Uint8Array(256);for(let t=0;t<V.length;t++)G[V.charCodeAt(t)]=t;function z(t){const e=function(t){let e=Math.floor(.75*t.length);const r=t.length;return"="===t[r-1]&&(e-=1,"="===t[r-2]&&(e-=1)),e}(t),r=t.length,i=new Uint8Array(e);let n=0;for(let e=0;e<r;e+=4){const r=G[t.charCodeAt(e)],s=G[t.charCodeAt(e+1)],o=G[t.charCodeAt(e+2)],a=G[t.charCodeAt(e+3)];i[n]=r<<2|s>>4,n+=1,i[n]=(15&s)<<4|o>>2,n+=1,i[n]=(3&o)<<6|63&a,n+=1}return i}const q=16384,Q=new P,K=new Map;function Y(t,e){return B(this,void 0,void 0,(function*(){let r=null,i=null,n=!1;if("undefined"==typeof WebAssembly)throw new Error("WebAssembly is not supported in this environment!");const s=()=>new DataView(r.exports.memory.buffer).getUint32(r.exports.STATE_SIZE,!0),o=Q.dispatch((()=>B(this,void 0,void 0,(function*(){if(!K.has(t.name)){const e=z(t.data),r=WebAssembly.compile(e);K.set(t.name,r)}const e=yield K.get(t.name);r=yield WebAssembly.instantiate(e,{})})))),a=(t=null)=>{n=!0,r.exports.Hash_Init(t)},c=t=>{if(!n)throw new Error("update() called before init()");(t=>{let e=0;for(;e<t.length;){const n=t.subarray(e,e+q);e+=n.length,i.set(n),r.exports.Hash_Update(n.length)}})(j(t))},l=new Uint8Array(2*e),h=(t,s=null)=>{if(!n)throw new Error("digest() called before init()");return n=!1,r.exports.Hash_Final(s),"binary"===t?i.slice(0,e):W(l,i,e)},u=t=>"string"==typeof t?t.length<4096:t.byteLength<q;let p=u;switch(t.name){case"argon2":case"scrypt":p=()=>!0;break;case"blake2b":case"blake2s":p=(t,e)=>e<=512&&u(t);break;case"blake3":p=(t,e)=>0===e&&u(t);break;case"xxhash64":case"xxhash3":case"xxhash128":p=()=>!1}return yield(()=>B(this,void 0,void 0,(function*(){r||(yield o);const t=r.exports.Hash_GetBuffer(),e=r.exports.memory.buffer;i=new Uint8Array(e,t,q)})))(),{getMemory:()=>i,writeMemory:(t,e=0)=>{i.set(t,e)},getExports:()=>r.exports,setMemorySize:t=>{r.exports.Hash_SetMemorySize(t);const e=r.exports.Hash_GetBuffer(),n=r.exports.memory.buffer;i=new Uint8Array(n,e,t)},init:a,update:c,digest:h,save:()=>{if(!n)throw new Error("save() can only be called after init() and before digest()");const e=r.exports.Hash_GetState(),i=s(),o=r.exports.memory.buffer,a=new Uint8Array(o,e,i),c=new Uint8Array(4+i);return function(t,e){const r=e.length>>1;for(let i=0;i<r;i++){const r=i<<1;t[i]=M(e.charCodeAt(r),e.charCodeAt(r+1))}}(c,t.hash),c.set(a,4),c},load:e=>{if(!(e instanceof Uint8Array))throw new Error("load() expects an Uint8Array generated by save()");const i=r.exports.Hash_GetState(),o=s(),a=4+o,c=r.exports.memory.buffer;if(e.length!==a)throw new Error(`Bad state length (expected ${a} bytes, got ${e.length})`);if(!function(t,e){if(t.length!==2*e.length)return!1;for(let r=0;r<e.length;r++){const i=r<<1;if(e[r]!==M(t.charCodeAt(i),t.charCodeAt(i+1)))return!1}return!0}(t.hash,e.subarray(0,4)))throw new Error("This state was written by an incompatible hash implementation");const l=e.subarray(4);new Uint8Array(c,i,o).set(l),n=!0},calculate:(t,n=null,s=null)=>{if(!p(t,n))return a(n),c(t),h("hex",s);const o=j(t);return i.set(o),r.exports.Hash_Calculate(o.length,n,s),W(l,i,e)},hashLength:e}}))}new P,new P,new DataView(new ArrayBuffer(4)),new P,new P,new P,new P,new P,new P,new P,new P,new P;var X={name:"sha256",data:"AGFzbQEAAAABEQRgAAF/YAF/AGACf38AYAAAAwgHAAEBAgMAAgQFAXABAQEFBAEBAgIGDgJ/AUHwiQULfwBBgAgLB3AIBm1lbW9yeQIADLine truncated
+/*! js-yaml 4.1.0 https://github.com/nodeca/js-yaml @license MIT */function so(t){return null==t}var oo={isNothing:so,isObject:function(t){return"object"==typeof t&&null!==t},toArray:function(t){return Array.isArray(t)?t:so(t)?[]:[t]},repeat:function(t,e){var r,i="";for(r=0;r<e;r+=1)i+=t;return i},isNegativeZero:function(t){return 0===t&&Number.NEGATIVE_INFINITY===1/t},extend:function(t,e){var r,i,n,s;if(e)for(r=0,i=(s=Object.keys(e)).length;r<i;r+=1)t[n=s[r]]=e[n];return t}};function ao(t,e){var r="",i=t.reason||"(unknown reason)";return t.mark?(t.mark.name&&(r+='in "'+t.mark.name+'" '),r+="("+(t.mark.line+1)+":"+(t.mark.column+1)+")",!e&&t.mark.snippet&&(r+="\n\n"+t.mark.snippet),i+" "+r):i}function co(t,e){Error.call(this),this.name="YAMLException",this.reason=t,this.mark=e,this.message=ao(this,!1),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack||""}co.prototype=Object.create(Error.prototype),co.prototype.constructor=co,co.prototype.toString=function(t){return this.name+": "+ao(this,t)};var lo=co;function ho(t,e,r,i,n){var s="",o="",a=Math.floor(n/2)-1;return i-e>a&&(e=i-a+(s=" ... ").length),r-i>a&&(r=i+a-(o=" ...").length),{str:s+t.slice(e,r).replace(/\t/g,"→")+o,pos:i-e+s.length}}function uo(t,e){return oo.repeat(" ",e-t.length)+t}var po=["kind","multi","resolve","construct","instanceOf","predicate","represent","representName","defaultStyle","styleAliases"],fo=["scalar","sequence","mapping"],go=function(t,e){if(e=e||{},Object.keys(e).forEach((function(e){if(-1===po.indexOf(e))throw new lo('Unknown option "'+e+'" is met in definition of "'+t+'" YAML type.')})),this.options=e,this.tag=t,this.kind=e.kind||null,this.resolve=e.resolve||function(){return!0},this.construct=e.construct||function(t){return t},this.instanceOf=e.instanceOf||null,this.predicate=e.predicate||null,this.represent=e.represent||null,this.representName=e.representName||null,this.defaultStyle=e.defaultStyle||null,this.multi=e.multi||!1,this.styleAliases=function(t){var e={};return null!==t&&Object.keys(t).forEach((function(r){t[r].forEach((function(t){e[String(t)]=r}))})),e}(e.styleAliases||null),-1===fo.indexOf(this.kind))throw new lo('Unknown kind "'+this.kind+'" is specified for "'+t+'" YAML type.')};function mo(t,e){var r=[];return t[e].forEach((function(t){var e=r.length;r.forEach((function(r,i){r.tag===t.tag&&r.kind===t.kind&&r.multi===t.multi&&(e=i)})),r[e]=t})),r}function wo(t){return this.extend(t)}wo.prototype.extend=function(t){var e=[],r=[];if(t instanceof go)r.push(t);else if(Array.isArray(t))r=r.concat(t);else{if(!t||!Array.isArray(t.implicit)&&!Array.isArray(t.explicit))throw new lo("Schema.extend argument should be a Type, [ Type ], or a schema definition ({ implicit: [...], explicit: [...] })");t.implicit&&(e=e.concat(t.implicit)),t.explicit&&(r=r.concat(t.explicit))}e.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.");if(t.loadKind&&"scalar"!==t.loadKind)throw new lo("There is a non-scalar type in the implicit list of a schema. Implicit resolving of such types is not supported.");if(t.multi)throw new lo("There is a multi type in the implicit list of a schema. Multi tags can only be listed as explicit.")})),r.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.")}));var i=Object.create(wo.prototype);return i.implicit=(this.implicit||[]).concat(e),i.explicit=(this.explicit||[]).concat(r),i.compiledImplicit=mo(i,"implicit"),i.compiledExplicit=mo(i,"explicit"),i.compiledTypeMap=function(){var t,e,r={scalar:{},sequence:{},mapping:{},fallback:{},multi:{scalar:[],sequence:[],mapping:[],fallback:[]}};function i(t){t.multi?(r.multi[t.kind].push(t),r.multi.fallback.push(t)):r[t.kind][t.tag]=r.fallback[t.tag]=t}for(t=0,e=arguments.length;t<e;t+=1)arguments[t].forEach(i);return r}(i.compiledImplicit,i.compiledExplicit),i};var yo=wo,bo=new go("tag:yaml.org,2002:str",{kind:"scalar",construct:function(t){return null!==t?t:""}}),Ao=new go("tag:yaml.org,2002:seq",{kind:"sequence",construct:function(t){return null!==t?t:[]}}),_o=new go("tag:yaml.org,2002:map",{kind:"mapping",construct:function(t){return null!==t?t:{}}}),vo=new yo({explicit:[bo,Ao,_o]}),Eo=new go("tag:yaml.org,2002:null",{kind:"scalar",resolve:function(t){if(null===t)return!0;var e=t.length;return 1===e&&"~"===t||4===e&&("null"===t||"Null"===t||"NULL"===t)},construct:function(){return null},predicate:function(t){return null===t},represent:{canonical:function(){return"~"},lowercase:function(){return"null"},uppercase:function(){return"NULL"},camelcase:function(){return"Null"},empty:function(){return""}},defaultStyle:"lowercase"}),So=new go("tag:yaml.org,2002:bool",{kind:"scalar",resolve:function(t){if(null===t)return!1;var e=t.length;return 4===e&&("true"===t||"True"===t||"TRUE"===t)||5===e&&("false"===t||"False"===t||"FALSE"===t)},construct:fuLine truncated
+/*!
+ * Copyright (c) 2014, GMO GlobalSign
+ * Copyright (c) 2015-2022, Peculiar Ventures
+ * All rights reserved.
+ * 
+ * Author 2014-2019, Yury Strozhevsky
+ * 
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ * 
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * 
+ */
+function lc(){if("undefined"==typeof BigInt)throw new Error("BigInt is not defined. Your environment doesn't implement BigInt.")}function hc(t){let e=0,r=0;for(let r=0;r<t.length;r++)e+=t[r].byteLength;const i=new Uint8Array(e);for(let e=0;e<t.length;e++){const n=t[e];i.set(new Uint8Array(n),r),r+=n.byteLength}return i.buffer}function uc(t,e,r,i){return e instanceof Uint8Array?e.byteLength?r<0?(t.error="Wrong parameter: inputOffset less than zero",!1):i<0?(t.error="Wrong parameter: inputLength less than zero",!1):!(e.byteLength-r-i<0&&(t.error="End of input reached before message was fully decoded (inconsistent offset and length values)",1)):(t.error="Wrong parameter: inputBuffer has zero length",!1):(t.error="Wrong parameter: inputBuffer must be 'Uint8Array'",!1)}Math.log(2);class pc{constructor(){this.items=[]}write(t){this.items.push(t)}final(){return hc(this.items)}}const dc=[new Uint8Array([1])],fc="0123456789",gc="name",mc="valueHexView",wc="",yc=new ArrayBuffer(0),bc=new Uint8Array(0),Ac="EndOfContent",_c="OCTET STRING",vc="BIT STRING";function Ec(t){var e;return(e=class extends t{constructor(...t){var e;super(...t);const r=t[0]||{};this.isHexOnly=null!==(e=r.isHexOnly)&&void 0!==e&&e,this.valueHexView=r.valueHex?ic.vJ.toUint8Array(r.valueHex):bc}get valueHex(){return this.valueHexView.slice().buffer}set valueHex(t){this.valueHexView=new Uint8Array(t)}fromBER(t,e,r){const i=t instanceof ArrayBuffer?new Uint8Array(t):t;if(!uc(this,i,e,r))return-1;const n=e+r;return this.valueHexView=i.subarray(e,n),this.valueHexView.length?(this.blockLength=r,n):(this.warnings.push("Zero buffer length"),e)}toBER(t=!1){return this.isHexOnly?t?new ArrayBuffer(this.valueHexView.byteLength):this.valueHexView.byteLength===this.valueHexView.buffer.byteLength?this.valueHexView.buffer:this.valueHexView.slice().buffer:(this.error="Flag 'isHexOnly' is not set, abort",yc)}toJSON(){return{...super.toJSON(),isHexOnly:this.isHexOnly,valueHex:ic.ep.ToHex(this.valueHexView)}}}).NAME="hexBlock",e}class Sc{constructor({blockLength:t=0,error:e="",warnings:r=[],valueBeforeDecode:i=bc}={}){this.blockLength=t,this.error=e,this.warnings=r,this.valueBeforeDecodeView=ic.vJ.toUint8Array(i)}static blockName(){return this.NAME}get valueBeforeDecode(){return this.valueBeforeDecodeView.slice().buffer}set valueBeforeDecode(t){this.valueBeforeDecodeView=new Uint8Array(t)}toJSON(){return{blockName:this.constructor.NAME,blockLength:this.blockLength,error:this.error,warnings:this.warnings,valueBeforeDecode:ic.ep.ToHex(this.valueBeforeDecodeView)}}}Sc.NAME="baseBlock";class Tc extends Sc{fromBER(t,e,r){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}toBER(t,e){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}}Tc.NAME="valueBlock";class xc extends(Ec(Sc)){constructor({idBlock:t={}}={}){var e,r,i,n;super(),t?(this.isHexOnly=null!==(e=t.isHexOnly)&&void 0!==e&&e,this.valueHexView=t.valueHex?ic.vJ.toUint8Array(t.valueHex):bc,this.tagClass=null!==(r=t.tagClass)&&void 0!==r?r:-1,this.tagNumber=null!==(i=t.tagNumber)&&void 0!==i?i:-1,this.isConstructed=null!==(n=t.isConstructed)&&void 0!==n&&n):(this.tagClass=-1,this.tagNumber=-1,this.isConstructed=!1)}toBER(t=!1){let e=0;switch(this.tagClass){case 1:e|=0;break;case 2:e|=64;break;case 3:e|=128;break;case 4:e|=192;break;default:return this.error="Unknown tag class",yc}if(this.isConstructed&&(e|=32),this.tagNumber<31&&!this.isHexOnly){const r=new Uint8Array(1);if(!t){let t=this.tagNumber;t&=31,e|=t,r[0]=e}return r.buffer}if(!this.isHexOnly){const r=sc(this.tagNumber,7),i=new Uint8Array(r),n=r.byteLength,s=new Uint8Array(n+1);if(s[0]=31|e,!t){for(let t=0;t<n-1;t++)s[t+1]=128|i[t];s[n]=i[n-1]}return s.buffer}const r=new Uint8Array(this.valueHexView.byteLength+1);if(r[0]=31|e,!t){const t=this.valueHexView;for(let e=0;e<t.length-1;e++)r[e+1]=128|t[e];r[this.valueHexView.byteLength]=t[t.length-1]}return r.buffer}fromBER(t,e,r){const i=ic.vJ.toUint8Array(t);if(!uc(this,i,e,r))return-1;const n=i.subarray(e,e+r);if(0===n.length)return this.error="Zero buffer length",-1;switch(192&n[0]){case 0:this.tagClass=1;break;case 64:this.tagClass=2;break;case 128:this.tagClass=3;break;case 192:this.tagClass=4;break;default:return this.error="Unknown tag class",-1}this.isConstructed=32==(32&n[0]),this.isHexOnly=!1;const s=31&n[0];if(31!==s)this.tagNumber=s,this.blockLength=1;else{let t=1,e=this.valueHexView=new Uint8Array(255),r=255;for(;128&n[t];){if(e[t-1]=127&n[t],t++,t>=n.length)return this.error="End of input reached before message was fully decoded",-1;if(t===r){r+=255;const t=new Uint8Array(r);for(let r=0;r<e.length;r++)t[r]=e[r];e=this.valueHexView=new Uint8Array(r)}}this.blockLength=t+1,e[t-1]=127&n[t];const i=new Uint8Array(t);for(let r=0;r<t;r++)i[r]=e[r];e=this.valueHexView=new Uint8Array(t),e.set(i),this.blockLength<=9?this.tagNumber=nc(e,7):(this.isHexOnly=!0,this.warnings.push("Tag too long, represented as hex-coded"))}if(1=Line truncated
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */var vg=function(t,e){return vg=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])},vg(t,e)};function Eg(t,e){function r(){this.constructor=t}vg(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}function Sg(t){var e="function"==typeof Symbol&&Symbol.iterator,r=e&&t[e],i=0;if(r)return r.call(t);if(t&&"number"==typeof t.length)return{next:function(){return t&&i>=t.length&&(t=void 0),{value:t&&t[i++],done:!t}}};throw new TypeError(e?"Object is not iterable.":"Symbol.iterator is not defined.")}function Tg(t,e){var r="function"==typeof Symbol&&t[Symbol.iterator];if(!r)return t;var i,n,s=r.call(t),o=[];try{for(;(void 0===e||e-- >0)&&!(i=s.next()).done;)o.push(i.value)}catch(t){n={error:t}}finally{try{i&&!i.done&&(r=s.return)&&r.call(s)}finally{if(n)throw n.error}}return o}function xg(){for(var t=[],e=0;e<arguments.length;e++)t=t.concat(Tg(arguments[e]));return t}function Cg(t){return!!t.useClass}function Ig(t){return!!t.useFactory}var Rg=function(){function t(t){this.wrap=t,this.reflectMethods=["get","getPrototypeOf","setPrototypeOf","getOwnPropertyDescriptor","defineProperty","has","set","deleteProperty","apply","construct","ownKeys"]}return t.prototype.createProxy=function(t){var e,r=this,i=!1;return new Proxy({},this.createHandler((function(){return i||(e=t(r.wrap()),i=!0),e})))},t.prototype.createHandler=function(t){var e={};return this.reflectMethods.forEach((function(r){e[r]=function(){for(var e=[],i=0;i<arguments.length;i++)e[i]=arguments[i];e[0]=t();var n=Reflect[r];return n.apply(void 0,xg(e))}})),e},t}();function kg(t){return"string"==typeof t||"symbol"==typeof t}function Ng(t){return"object"==typeof t&&"token"in t&&"transform"in t}function Og(t){return!!t.useToken}function Bg(t){return null!=t.useValue}const Pg=function(){function t(){this._registryMap=new Map}return t.prototype.entries=function(){return this._registryMap.entries()},t.prototype.getAll=function(t){return this.ensure(t),this._registryMap.get(t)},t.prototype.get=function(t){this.ensure(t);var e=this._registryMap.get(t);return e[e.length-1]||null},t.prototype.set=function(t,e){this.ensure(t),this._registryMap.get(t).push(e)},t.prototype.setAll=function(t,e){this._registryMap.set(t,e)},t.prototype.has=function(t){return this.ensure(t),this._registryMap.get(t).length>0},t.prototype.clear=function(){this._registryMap.clear()},t.prototype.ensure=function(t){this._registryMap.has(t)||this._registryMap.set(t,[])},t}(),Lg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Dg=function(){this.scopedResolutions=new Map};var Ug=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Mg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg);const Fg=function(){this.preResolution=new Ug,this.postResolution=new Mg};var Hg=new Map,Wg=function(){function t(t){this.parent=t,this._registry=new Lg,this.interceptors=new Fg,this.disposed=!1,this.disposables=new Set}return t.prototype.register=function(t,e,r){var i;if(void 0===r&&(r={lifecycle:_g.Transient}),this.ensureNotDisposed(),i=function(t){return Cg(t)||Bg(t)||Og(t)||Ig(t)}(e)?e:{useClass:e},Og(i))for(var n=[t],s=i;null!=s;){var o=s.useToken;if(n.includes(o))throw new Error("Token registration cycle detected! "+xg(n,[o]).join(" -> "));n.push(o);var a=this._registry.get(o);s=a&&Og(a.provider)?a.provider:null}if((r.lifecycle===_g.Singleton||r.lifecycle==_g.ContainerScoped||r.lifecycle==_g.ResolutionScoped)&&(Bg(i)||Ig(i)))throw new Error('Cannot use lifecycle "'+_g[r.lifecycle]+'" with ValueProviders or FactoryProviders');return this._registry.set(t,{provider:i,options:r}),this},t.prototype.registerType=function(t,e){return this.ensureNotDisposed(),kg(e)?this.register(t,{useToken:e}):this.register(t,{useClass:e})},t.prototype.registerInstance=function(t,e){return this.ensureNotDisposed(),this.register(t,{useValue:e})},t.prototype.registerSingleton=function(t,e){if(this.ensureNotDisposed(),kg(t)){if(kg(e))return this.register(t,{useToken:e},{lifecycle:_g.Singleton});if(e)return this.register(t,{useClass:e},{lifecycle:_g.Singleton});throw new Error('Cannot register a type name as a singleton without a "to" token')}var r=t;return e&&!kg(e)&&(r=e),this.register(t,{useClass:r},{lifecycle:_g.Singleton})},t.prototype.resolve=function(t,e){void 0===e&&(e=new Dg),this.ensureNotDisposed();var r=this.getRegistration(t);if(!r&&kg(t))throw new Error('Attempted to resolve unregistered dependency token: "'+t.toString()+'"');if(this.executePreResolutionInterceptor(t,"Single"),r){var i=this.resolveRegistration(r,e);return this.executePostResolutionInterceptor(t,i,"Single"),i}if(function(t){return"function"==typeof t||t instanceof Rg}(t))return i=this.construct(t,e),this.executePostResolutionInterceptor(t,i,"Line truncated
+/*!
+ * MIT License
+ * 
+ * Copyright (c) Peculiar Ventures. All rights reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ * 
+ */
+const zm="crypto.algorithm",qm="crypto.algorithmProvider";var Qm;jg.registerSingleton(qm,class{getAlgorithms(){return jg.resolveAll(zm)}toAsnAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toAsnAlgorithm(t);if(r)return r}if(/[0-9.]+/.test(t.name)){const e=new Fp({algorithm:t.name});if("parameters"in t){const r=t;e.parameters=r.parameters}return e}throw new Error("Cannot convert WebCrypto algorithm to ASN.1 algorithm")}toWebAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toWebAlgorithm(t);if(r)return r}return{name:t.algorithm,parameters:t.parameters}}});const Km="1.3.36.3.3.2.8.1.1",Ym=`${Km}.1`,Xm=`${Km}.2`,Jm=`${Km}.3`,Zm=`${Km}.4`,$m=`${Km}.5`,tw=`${Km}.6`,ew=`${Km}.7`,rw=`${Km}.8`,iw=`${Km}.9`,nw=`${Km}.10`,sw=`${Km}.11`,ow=`${Km}.12`,aw=`${Km}.13`,cw=`${Km}.14`,lw="brainpoolP160r1",hw="brainpoolP160t1",uw="brainpoolP192r1",pw="brainpoolP192t1",dw="brainpoolP224r1",fw="brainpoolP224t1",gw="brainpoolP256r1",mw="brainpoolP256t1",ww="brainpoolP320r1",yw="brainpoolP320t1",bw="brainpoolP384r1",Aw="brainpoolP384t1",_w="brainpoolP512r1",vw="brainpoolP512t1",Ew="ECDSA";let Sw=Qm=class{toAsnAlgorithm(t){if(t.name.toLowerCase()===Ew.toLowerCase())if("hash"in t)switch(("string"==typeof t.hash?t.hash:t.hash.name).toLowerCase()){case"sha-1":return Pf;case"sha-256":return Lf;case"sha-384":return Df;case"sha-512":return Uf}else if("namedCurve"in t){let e="";switch(t.namedCurve){case"P-256":e=kf;break;case"K-256":e=Qm.SECP256K1;break;case"P-384":e=Nf;break;case"P-521":e=Of;break;case lw:e=Ym;break;case hw:e=Xm;break;case uw:e=Jm;break;case pw:e=Zm;break;case dw:e=$m;break;case fw:e=tw;break;case gw:e=ew;break;case mw:e=rw;break;case ww:e=iw;break;case yw:e=nw;break;case bw:e=sw;break;case Aw:e=ow;break;case _w:e=aw;break;case vw:e=cw}if(e)return new Fp({algorithm:Sf,parameters:iu.serialize(new Mf({namedCurve:e}))})}return null}toWebAlgorithm(t){switch(t.algorithm){case Tf:return{name:Ew,hash:{name:"SHA-1"}};case Cf:return{name:Ew,hash:{name:"SHA-256"}};case If:return{name:Ew,hash:{name:"SHA-384"}};case Rf:return{name:Ew,hash:{name:"SHA-512"}};case Sf:if(!t.parameters)throw new TypeError("Cannot get required parameters from EC algorithm");switch(iu.parse(t.parameters,Mf).namedCurve){case kf:return{name:Ew,namedCurve:"P-256"};case Qm.SECP256K1:return{name:Ew,namedCurve:"K-256"};case Nf:return{name:Ew,namedCurve:"P-384"};case Of:return{name:Ew,namedCurve:"P-521"};case Ym:return{name:Ew,namedCurve:lw};case Xm:return{name:Ew,namedCurve:hw};case Jm:return{name:Ew,namedCurve:uw};case Zm:return{name:Ew,namedCurve:pw};case $m:return{name:Ew,namedCurve:dw};case tw:return{name:Ew,namedCurve:fw};case ew:return{name:Ew,namedCurve:gw};case rw:return{name:Ew,namedCurve:mw};case iw:return{name:Ew,namedCurve:ww};case nw:return{name:Ew,namedCurve:yw};case sw:return{name:Ew,namedCurve:bw};case ow:return{name:Ew,namedCurve:Aw};case aw:return{name:Ew,namedCurve:_w};case cw:return{name:Ew,namedCurve:vw}}}return null}};Sw.SECP256K1="1.3.132.0.10",Sw=Qm=bg([Vg()],Sw),jg.registerSingleton(zm,Sw);const Tw=Symbol("name"),xw=Symbol("value");class Cw{constructor(t,e={},r=""){this[Tw]=t,this[xw]=r;for(const t in e)this[t]=e[t]}}Cw.NAME=Tw,Cw.VALUE=xw;class Iw{static toString(t){return this.items[t]||t}}Iw.items={[eg]:"sha1",[rg]:"sha224",[ig]:"sha256",[ng]:"sha384",[sg]:"sha512",[jf]:"rsaEncryption",[Kf]:"sha1WithRSAEncryption",[Yf]:"sha224WithRSAEncryption",[Xf]:"sha256WithRSAEncryption",[Jf]:"sha384WithRSAEncryption",[Zf]:"sha512WithRSAEncryption",[Sf]:"ecPublicKey",[Tf]:"ecdsaWithSHA1",[xf]:"ecdsaWithSHA224",[Cf]:"ecdsaWithSHA256",[If]:"ecdsaWithSHA384",[Rf]:"ecdsaWithSHA512",[np]:"TLS WWW server authentication",[sp]:"TLS WWW client authentication",[op]:"Code Signing",[ap]:"E-mail Protection",[cp]:"Time Stamping",[lp]:"OCSP Signing","1.2.840.113549.1.7.2":"Signed Data"};class Rw{static serialize(t){return this.serializeObj(t).join("\n")}static pad(t=0){return"".padStart(2*t," ")}static serializeObj(t,e=0){const r=[];let i=this.pad(e++),n="";const s=t[Cw.VALUE];s&&(n=` ${s}`),r.push(`${i}${t[Cw.NAME]}:${n}`),i=this.pad(e);for(const n in t){if("symbol"==typeof n)continue;const s=t[n],o=n?`${n}: `:"";if("string"==typeof s||"number"==typeof s||"boolean"==typeof s)r.push(`${i}${o}${s}`);else if(s instanceof Date)r.push(`${i}${o}${s.toUTCString()}`);else if(Array.isArray(s))for(const t of s)t[Cw.NAME]=n,r.push(...this.serializeObj(t,e));else if(s instanceof Cw)s[Cw.NAME]=n,r.push(...this.serializeObj(s,e));else if(ic.vJ.isBufferSource(s))n?(r.push(`${i}${o}`),r.push(...this.serializeBufferSource(s,e+1))):r.push(...this.serializeBufferSource(s,e));else{if(!("toTextObject"in s))throw new TypeError("Cannot serialize data in text format. Unsupported type.");{const t=s.toTextObject();t[Cw.NAME]=n,r.push(...this.serializeObj(t,e))}}}return r}static serializeBufferSource(t,e=0){const r=this.pad(e),i=ic.vJ.toUint8Array(t),n=[];for(let t=0;t<i.length;){const e=[];for(let r=0;r<16&&t<i.length;r++){8===r&&e.push("");const n=i[t++].toStrinLine truncated

archivebox/util.py

@@ -57,19 +57,57 @@ short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0]
 ts_to_date_str = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
 ts_to_iso = lambda ts: ts and parse_date(ts).isoformat()
 
+COLOR_REGEX = re.compile(r'\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m')
 
+
+# https://mathiasbynens.be/demo/url-regex
 URL_REGEX = re.compile(
-    r'(?=('
-    r'http[s]?://'                    # start matching from allowed schemes
-    r'(?:[a-zA-Z]|[0-9]'              # followed by allowed alphanum characters
-    r'|[-_$@.&+!*\(\),]'           #    or allowed symbols (keep hyphen first to match literal hyphen)
-    r'|(?:%[0-9a-fA-F][0-9a-fA-F]))'  #    or allowed unicode bytes
-    r'[^\]\[\(\)<>"\'\s]+'          # stop parsing at these symbols
+    r'(?=('                           +
+    r'http[s]?://'                    +  # start matching from allowed schemes
+    r'(?:[a-zA-Z]|[0-9]'              +  # followed by allowed alphanum characters
+    r'|[-_$@.&+!*\(\),]'              +  #   or allowed symbols (keep hyphen first to match literal hyphen)
+    r'|[^\u0000-\u007F])+'            +  #   or allowed unicode bytes
+    r'[^\]\[<>"\'\s]+'                +  # stop parsing at these symbols
     r'))',
-    re.IGNORECASE,
+    re.IGNORECASE | re.UNICODE,
 )
 
-COLOR_REGEX = re.compile(r'\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m')
+def parens_are_matched(string: str, open_char='(', close_char=')'):
+    """check that all parentheses in a string are balanced and nested properly"""
+    count = 0
+    for c in string:
+        if c == open_char:
+            count += 1
+        elif c == close_char:
+            count -= 1
+        if count < 0:
+            return False
+    return count == 0
+
+def fix_url_from_markdown(url_str: str) -> str:
+    """
+    cleanup a regex-parsed url that may contain dangling trailing parens from markdown link syntax
+    helpful to fix URLs parsed from markdown e.g.
+      input:  https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def).somemoretext
+      result: https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def
+    """
+    trimmed_url = url_str
+
+    # cut off one trailing character at a time
+    # until parens are balanced e.g. /a(b)c).x(y)z -> /a(b)c
+    while not parens_are_matched(trimmed_url):
+        trimmed_url = trimmed_url[:-1]
+    
+    # make sure trimmed url is still valid
+    if re.findall(URL_REGEX, trimmed_url):
+        return trimmed_url
+    
+    return url_str
+
+def find_all_urls(urls_str: str):
+    for url in re.findall(URL_REGEX, urls_str):
+        yield fix_url_from_markdown(url)
+
 
 def is_static_file(url: str):
     # TODO: the proper way is with MIME type detection + ext, not only extension
@@ -228,7 +266,11 @@ def get_headers(url: str, timeout: int=None) -> str:
     
     return pyjson.dumps(
         {
+            'URL': url,
             'Status-Code': response.status_code,
+            'Elapsed': response.elapsed,
+            'Encoding': response.encoding,
+            'Apparent-Encoding': response.apparent_encoding,
             **dict(response.headers),
         },
         indent=4,
@@ -399,3 +441,48 @@ class ExtendedEncoder(pyjson.JSONEncoder):
 
         return pyjson.JSONEncoder.default(self, obj)
 
+
+### URL PARSING TESTS / ASSERTIONS
+# they run at runtime because I like having them inline in this file,
+# I like the peace of mind knowing it's enforced at runtime across all OS's (in case the regex engine ever has any weird locale-specific quirks),
+# and these assertions are basically instant, so not a big performance cost to do it on startup
+
+assert fix_url_from_markdown('/a(b)c).x(y)z') == '/a(b)c'
+assert fix_url_from_markdown('https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def).link(with)_trailingtext') == 'https://wikipedia.org/en/some_article_(Disambiguation).html?abc=def'
+
+URL_REGEX_TESTS = [
+    ('https://example.com', ['https://example.com']),
+    ('http://abc-file234example.com/abc?def=abc&23423=sdfsdf#abc=234&234=a234', ['http://abc-file234example.com/abc?def=abc&23423=sdfsdf#abc=234&234=a234']),
+
+    ('https://twitter.com/share?url=https://akaao.success-corp.co.jp&text=ア@サ!ト&hashtags=ア%オ,元+ア.ア-オ_イ*シ$ロ abc', ['https://twitter.com/share?url=https://akaao.success-corp.co.jp&text=ア@サ!ト&hashtags=ア%オ,元+ア.ア-オ_イ*シ$ロ', 'https://akaao.success-corp.co.jp&text=ア@サ!ト&hashtags=ア%オ,元+ア.ア-オ_イ*シ$ロ']),
+    ('<a href="https://twitter.com/share#url=https://akaao.success-corp.co.jp&text=ア@サ!ト?hashtags=ア%オ,元+ア&abc=.ア-オ_イ*シ$ロ"> abc', ['https://twitter.com/share#url=https://akaao.success-corp.co.jp&text=ア@サ!ト?hashtags=ア%オ,元+ア&abc=.ア-オ_イ*シ$ロ', 'https://akaao.success-corp.co.jp&text=ア@サ!ト?hashtags=ア%オ,元+ア&abc=.ア-オ_イ*シ$ロ']),
+
+    ('///a',                                                []),
+    ('http://',                                             []),
+    ('http://../',                                          ['http://../']),
+    ('http://-error-.invalid/',                             ['http://-error-.invalid/']),
+    ('https://a(b)c+1#2?3&4/',                              ['https://a(b)c+1#2?3&4/']),
+    ('http://उदाहरण.परीक्षा',                                   ['http://उदाहरण.परीक्षा']),
+    ('http://例子.测试',                                     ['http://例子.测试']),
+    ('http://➡.ws/䨹 htps://abc.1243?234',                  ['http://➡.ws/䨹']),
+    ('http://⌘.ws">https://exa+mple.com//:abc ',            ['http://⌘.ws', 'https://exa+mple.com//:abc']),
+    ('http://مثال.إختبار/abc?def=ت&ب=abc#abc=234',          ['http://مثال.إختبار/abc?def=ت&ب=abc#abc=234']),
+    ('http://-.~_!$&()*+,;=:%40:80%2f::::::@example.c\'om', ['http://-.~_!$&()*+,;=:%40:80%2f::::::@example.c']),
+    
+    ('http://us:pa@ex.co:42/http://ex.co:19/a?_d=4#-a=2.3', ['http://us:pa@ex.co:42/http://ex.co:19/a?_d=4#-a=2.3', 'http://ex.co:19/a?_d=4#-a=2.3']),
+    ('http://code.google.com/events/#&product=browser',     ['http://code.google.com/events/#&product=browser']),
+    ('http://foo.bar?q=Spaces should be encoded',           ['http://foo.bar?q=Spaces']),
+    ('http://foo.com/blah_(wikipedia)#c(i)t[e]-1',          ['http://foo.com/blah_(wikipedia)#c(i)t']),
+    ('http://foo.com/(something)?after=parens',             ['http://foo.com/(something)?after=parens']),
+    ('http://foo.com/unicode_(✪)_in_parens) abc',           ['http://foo.com/unicode_(✪)_in_parens']),
+    ('http://foo.bar/?q=Test%20URL-encoded%20stuff',        ['http://foo.bar/?q=Test%20URL-encoded%20stuff']),
+
+    ('[xyz](http://a.b/?q=(Test)%20U)RL-encoded%20stuff',   ['http://a.b/?q=(Test)%20U']),
+    ('[xyz](http://a.b/?q=(Test)%20U)-ab https://abc+123',  ['http://a.b/?q=(Test)%20U', 'https://abc+123']),
+    ('[xyz](http://a.b/?q=(Test)%20U) https://a(b)c+12)3',  ['http://a.b/?q=(Test)%20U', 'https://a(b)c+12']),
+    ('[xyz](http://a.b/?q=(Test)a\nabchttps://a(b)c+12)3',  ['http://a.b/?q=(Test)a', 'https://a(b)c+12']),
+    ('http://foo.bar/?q=Test%20URL-encoded%20stuff',        ['http://foo.bar/?q=Test%20URL-encoded%20stuff']),
+]
+for urls_str, expected_url_matches in URL_REGEX_TESTS:
+    url_matches = list(find_all_urls(urls_str))
+    assert url_matches == expected_url_matches, 'FAILED URL_REGEX CHECK!'

archivebox/vendor/requirements.txt

@@ -0,0 +1,6 @@
+# this folder contains vendored versions of these packages
+
+atomicwrites==1.4.0
+pocket==0.3.7
+django-taggit==1.3.0
+base32-crockford==0.3.0

bin/build_deb.sh

@@ -31,6 +31,20 @@ else
     echo "[!] Warning: No virtualenv presesnt in $REPO_DIR.venv"
 fi
 
+
+# Build python package lists
+# https://pdm-project.org/latest/usage/lockfile/
+echo "[+] Generating requirements.txt and pdm.lock from pyproject.toml..."
+pdm lock --group=':all' --production --lockfile pdm.lock --strategy="cross_platform"
+pdm sync --group=':all' --production --lockfile pdm.lock --clean || pdm sync --group=':all' --production --lockfile pdm.lock --clean
+pdm export --group=':all' --production --lockfile pdm.lock --without-hashes -o requirements.txt
+
+pdm lock --group=':all' --dev --lockfile pdm.dev.lock --strategy="cross_platform" 
+pdm sync --group=':all' --dev --lockfile pdm.dev.lock --clean || pdm sync --group=':all' --dev --lockfile pdm.dev.lock --clean
+pdm export --group=':all' --dev --lockfile pdm.dev.lock --without-hashes -o requirements-dev.txt
+
+
+
 # cleanup build artifacts
 rm -Rf build deb_dist dist archivebox-*.tar.gz
 

bin/build_dev.sh

@@ -21,6 +21,20 @@ VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
 SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
 REQUIRED_PLATFORMS="${2:-"linux/arm64,linux/amd64,linux/arm/v7"}"
 
+
+# Build python package lists
+# https://pdm-project.org/latest/usage/lockfile/
+echo "[+] Generating requirements.txt and pdm.lock from pyproject.toml..."
+pdm lock --group=':all' --production --lockfile pdm.lock --strategy="cross_platform"
+pdm sync --group=':all' --production --lockfile pdm.lock --clean || pdm sync --group=':all' --production --lockfile pdm.lock --clean
+pdm export --group=':all' --production --lockfile pdm.lock --without-hashes -o requirements.txt
+
+pdm lock --group=':all' --dev --lockfile pdm.dev.lock --strategy="cross_platform" 
+pdm sync --group=':all' --dev --lockfile pdm.dev.lock --clean || pdm sync --group=':all' --dev --lockfile pdm.dev.lock --clean
+pdm export --group=':all' --dev --lockfile pdm.dev.lock --without-hashes -o requirements-dev.txt
+
+
+
 echo "[+] Building Docker image: tag=$TAG_NAME version=$SHORT_VERSION arch=$REQUIRED_PLATFORMS"
 
 
@@ -32,4 +46,4 @@ docker build . --no-cache -t archivebox-dev --load
 #                -t archivebox \
 #                -t archivebox:$TAG_NAME \
 #                -t archivebox:$VERSION \
-#                -t archivebox:$SHORT_VERSION
+#                -t archivebox:$SHORT_VERSION

bin/build_docker.sh

@@ -71,10 +71,8 @@ docker buildx use xbuilder 2>&1 >/dev/null || create_builder
 check_platforms || (recreate_builder && check_platforms) || exit 1
 
 
-# Build python package lists
-echo "[+] Generating requirements.txt and pdm.lock from pyproject.toml..."
-pdm lock --group=':all' --strategy="cross_platform" --production
-pdm export --group=':all' --production --without-hashes -o requirements.txt
+# Make sure pyproject.toml, pdm{.dev}.lock, requirements{-dev}.txt, package{-lock}.json are all up-to-date
+bash ./bin/lock_pkgs.sh
 
 
 echo "[+] Building archivebox:$VERSION docker image..."

bin/build_pip.sh

@@ -20,20 +20,13 @@ else
 fi
 cd "$REPO_DIR"
 
-echo "[*] Cleaning up build dirs"
-cd "$REPO_DIR"
-rm -Rf build dist
+# Generate pdm.lock, requirements.txt, and package-lock.json
+bash ./bin/lock_pkgs.sh
 
 echo "[+] Building sdist, bdist_wheel, and egg_info"
-rm -f archivebox/package.json
-cp package.json archivebox/package.json
-
-pdm self update
-pdm install
+rm -Rf build dist
 pdm build
-pdm export --without-hashes -o ./pip_dist/requirements.txt
-
 cp dist/* ./pip_dist/
 
 echo
-echo "[√] Finished. Don't forget to commit the new sdist and wheel files in ./pip_dist/"
+echo "[√] Finished. Don't forget to commit the new sdist and wheel files in ./pip_dist/"

bin/docker_entrypoint.sh

@@ -64,7 +64,7 @@ if [[ -d "$DATA_DIR/archive" ]]; then
         rm -f "$DATA_DIR/archive/.permissions_test_safe_to_delete"
         # echo "[√] Permissions are correct"
     else
-     # the only time this fails is if the host filesystem doesn't allow us to write as root (e.g. some NFS mapall/maproot problems, connection issues, drive dissapeared, etc.)
+        # the only time this fails is if the host filesystem doesn't allow us to write as root (e.g. some NFS mapall/maproot problems, connection issues, drive dissapeared, etc.)
         echo -e "\n[X] Error: archivebox user (PUID=$PUID) is not able to write to your ./data/archive dir (currently owned by $(stat -c '%u' "$DATA_DIR/archive"):$(stat -c '%g' "$DATA_DIR/archive")." > /dev/stderr
         echo -e "    Change ./data to be owned by PUID=$PUID PGID=$PGID on the host and retry:" > /dev/stderr
         echo -e "       \$ chown -R $PUID:$PGID ./data\n" > /dev/stderr
@@ -89,7 +89,8 @@ if ! chown $PUID:$PGID "$DATA_DIR"/* > /dev/null 2>&1; then
     find "$DATA_DIR" -type d -not -path "$DATA_DIR/archive*" -exec chown $PUID:$PGID {} \; > /dev/null 2>&1
     find "$DATA_DIR" -type f -not -path "$DATA_DIR/archive/*" -exec chown $PUID:$PGID {} \; > /dev/null 2>&1
 fi
-    
+mkdir -p /var/spool/cron/crontabs
+chown -R $PUID:$PGID /var/spool/cron/crontabs > /dev/null 2>&1 &
 
 # also chown BROWSERS_DIR because otherwise 'archivebox setup' wont be able to 'playwright install chromium' at runtime
 export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-/browsers}"
@@ -166,13 +167,13 @@ fi
 # symlink etc crontabs into place
 mkdir -p "$DATA_DIR/crontabs"
 if ! test -L /var/spool/cron/crontabs; then
-    # copy files from old location into new data dir location
-    for file in $(ls /var/spool/cron/crontabs); do
-        cp /var/spool/cron/crontabs/"$file" "$DATA_DIR/crontabs"
+    # move files from old location into new data dir location
+    for existing_file in /var/spool/cron/crontabs/*; do
+        mv "$existing_file" "$DATA_DIR/crontabs/"
     done
     # replace old system path with symlink to data dir location
     rm -Rf /var/spool/cron/crontabs
-    ln -s "$DATA_DIR/crontabs" /var/spool/cron/crontabs
+    ln -sf "$DATA_DIR/crontabs" /var/spool/cron/crontabs
 fi
 
 # set DBUS_SYSTEM_BUS_ADDRESS & DBUS_SESSION_BUS_ADDRESS
@@ -191,9 +192,11 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" ==
     #      "docker run archivebox /bin/bash -c '...'"
     #      "docker run archivebox cat /VERSION.txt"
     exec gosu "$PUID" /bin/bash -c "exec $(printf ' %q' "$@")"
+    # WARNING: make sure to test extensively if you change this line, there are many edge-cases with nested quotes, special character, etc.
     # printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113
     # gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
     # outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command
+    # - https://github.com/ArchiveBox/ArchiveBox/issues/1191
 else
     # handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand
     # e.g. "docker run archivebox help"

bin/docker_ipc_listener.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+# Allow another docker container to run commands on this container
+# This is the script to run on the server container.
+# The client can connect and run a command like so:
+#    $ echo whoami | nc servercontainername 2222
+#    root
+
+import socket
+import subprocess as sp
+from datetime import datetime
+
+LISTEN_PORT = 2222
+
+s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+s1.bind(("0.0.0.0", LISTEN_PORT))
+s1.listen(1)
+print("Listening for shell commands on 0.0.0.0:2222", flush=True)
+
+conn, addr = s1.accept()
+while True:
+    cmd = conn.recv(1024).decode()
+    if not cmd:
+        conn, addr = s1.accept()
+        continue
+
+    timestamp = datetime.now().isoformat()
+    client_ip, client_port = conn.getsockname()
+    print(f'\n[{timestamp}][{client_ip}:{client_port}] $', cmd)
+
+    with sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.STDOUT, stdin=sp.PIPE, bufsize=1, universal_newlines=True) as p:
+        for line in p.stdout:
+            print(line.strip(), flush=True)
+            conn.sendall(line.encode("utf-8"))
+
+    conn.close()
+    conn, addr = s1.accept()

bin/lock_pkgs.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+
+### Bash Environment Setup
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+# set -o xtrace
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+IFS=$'\n'
+
+REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
+
+cd "$REPO_DIR"
+
+py_version="$(grep 'version = ' pyproject.toml | awk '{print $3}' | jq -r)"
+js_version="$(jq -r '.version' package.json)"
+
+if [[ "$py_version" != "$js_version" ]]; then
+    echo "[❌] Version in pyproject.toml ($py_version) does not match version in package.json ($js_version)!"
+    exit 1
+fi
+
+echo "[🔒] Locking all ArchiveBox dependencies (pip, npm)"
+echo
+echo "pyproject.toml:              archivebox $py_version"
+echo "package.json:                archivebox $js_version"
+echo
+echo
+
+echo "[*] Cleaning up old lockfiles and build files"
+deactivate 2>/dev/null || true
+rm -Rf build dist
+rm -f pdm.lock
+rm -f pdm.dev.lock
+rm -f requirements.txt
+rm -f requirements-dev.txt
+rm -f package-lock.json
+rm -f archivebox/package.json
+rm -f archivebox/package-lock.json
+rm -Rf ./.venv
+rm -Rf ./node_modules
+rm -Rf ./archivebox/node_modules
+
+echo
+echo
+
+echo "[+] Generating dev & prod requirements.txt & pdm.lock from pyproject.toml..."
+pip install --upgrade pip setuptools
+pdm self update
+pdm venv create 3.12
+echo
+echo "pyproject.toml:    archivebox $(grep 'version = ' pyproject.toml | awk '{print $3}' | jq -r)"
+echo "$(which python):   $(python --version | head -n 1)"
+echo "$(which pdm):      $(pdm --version | head -n 1)"
+pdm info --env
+pdm info
+
+echo
+# https://pdm-project.org/latest/usage/lockfile/
+# prod
+pdm lock --group=':all' --production --lockfile pdm.lock --strategy="cross_platform"
+pdm sync --group=':all' --production --lockfile pdm.lock --clean
+pdm export --group=':all' --production --lockfile pdm.lock --without-hashes -o requirements.txt
+cp ./pdm.lock ./pip_dist/
+cp ./requirements.txt ./pip_dist/
+# dev
+pdm lock --group=':all' --dev --lockfile pdm.dev.lock --strategy="cross_platform" 
+pdm sync --group=':all' --dev --lockfile pdm.dev.lock --clean
+pdm export --group=':all' --dev --lockfile pdm.dev.lock --without-hashes -o requirements-dev.txt
+cp ./pdm.dev.lock ./pip_dist/
+cp ./requirements-dev.txt ./pip_dist/
+
+echo
+echo "[+]] Generating package-lock.json from package.json..."
+npm install -g npm
+echo
+echo "package.json:    archivebox $(jq -r '.version' package.json)"
+echo
+echo "$(which node):   $(node --version | head -n 1)"
+echo "$(which npm):    $(npm --version | head -n 1)"
+
+echo
+npm install --package-lock-only
+cp package.json archivebox/package.json
+cp package-lock.json archivebox/package-lock.json
+
+echo
+echo "[√] Finished. Don't forget to commit the new lockfiles:"
+echo
+ls "pyproject.toml" | cat
+ls "pdm.lock" | cat
+ls "pdm.dev.lock" | cat
+ls "requirements.txt" | cat
+ls "requirements-dev.txt" | cat
+echo
+ls "package.json" | cat
+ls "package-lock.json" | cat
+ls "archivebox/package.json" | cat
+ls "archivebox/package-lock.json" | cat

bin/setup.sh

@@ -27,9 +27,9 @@ if (which docker-compose > /dev/null && docker pull archivebox/archivebox:latest
     if [ -f "./index.sqlite3" ]; then
         mv -i ~/archivebox/* ~/archivebox/data/
     fi
-    curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/main/docker-compose.yml' > docker-compose.yml
+    curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/docker-compose.yml' > docker-compose.yml
     mkdir -p ./etc
-    curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/main/etc/sonic.cfg' > ./etc/sonic.cfg
+    curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/etc/sonic.cfg' > ./etc/sonic.cfg
     docker compose run --rm archivebox init --setup
     echo
     echo "[+] Starting ArchiveBox server using: docker compose up -d..."

docker-compose.yml

@@ -1,14 +1,12 @@
 # Usage:
-#     docker compose run archivebox init --setup
 #     docker compose up
-#     echo "https://example.com" | docker compose run archivebox archivebox add
-#     docker compose run archivebox add --depth=1 https://example.com/some/feed.rss
-#     docker compose run archivebox config --set MEDIA_MAX_SIZE=750m
+#     echo 'https://example.com' | docker compose run -T archivebox add
+#     docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+#     docker compose run archivebox config --set SAVE_ARCHIVE_DOT_ORG=False
 #     docker compose run archivebox help
 # Documentation:
 #     https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker-compose
 
-
 services:
     archivebox:
         image: archivebox/archivebox:latest
@@ -23,11 +21,11 @@ services:
             - PUBLIC_INDEX=True                 # set to False to prevent anonymous users from viewing snapshot list
             - PUBLIC_SNAPSHOTS=True             # set to False to prevent anonymous users from viewing snapshot content
             - PUBLIC_ADD_VIEW=False             # set to True to allow anonymous users to submit new URLs to archive
-            - SEARCH_BACKEND_ENGINE=sonic       # uncomment these and sonic container below for better full-text search
+            - SEARCH_BACKEND_ENGINE=sonic       # tells ArchiveBox to use sonic container below for fast full-text search
             - SEARCH_BACKEND_HOST_NAME=sonic
             - SEARCH_BACKEND_PASSWORD=SomeSecretPassword
             # - PUID=911                        # set to your host user's UID & GID if you encounter permissions issues
-            # - PGID=911
+            # - PGID=911                        # UID/GIDs <500 may clash with existing users and are not recommended
             # - MEDIA_MAX_SIZE=750m             # increase this filesize limit to allow archiving larger audio/video files
             # - TIMEOUT=60                      # increase this number to 120+ seconds if you see many slow downloads timing out
             # - CHECK_SSL_VALIDITY=True         # set to False to disable strict SSL checking (allows saving URLs w/ broken certs)
@@ -35,61 +33,58 @@ services:
             # ...
             # add further configuration options from archivebox/config.py as needed (to apply them only to this container)
             # or set using `docker compose run archivebox config --set SOME_KEY=someval` (to persist config across all containers)
-        
         # For ad-blocking during archiving, uncomment this section and pihole service section below
         # networks:
         #   - dns
         # dns:
         #   - 172.20.0.53
 
-
     ######## Optional Addons: tweak examples below as needed for your specific use case ########
 
-    ### Enable ability to run regularly scheduled archiving tasks by uncommenting this container
-    #   $ docker compose run archivebox schedule --every=day --depth=1 'https://example.com/some/rss/feed.xml'
-    # then restart the scheduler container to apply the changes to the schedule
+    ### This optional container runs any scheduled tasks in the background, add new tasks like so:
+    #   $ docker compose run archivebox schedule --add --every=day --depth=1 'https://example.com/some/rss/feed.xml'
+    # then restart the scheduler container to apply any changes to the scheduled task list:
     #   $ docker compose restart archivebox_scheduler
 
     archivebox_scheduler:
-       image: archivebox/archivebox:latest
-       command: schedule --foreground
-       environment:
-           - TIMEOUT=120                       # increase if you see timeouts often during archiving / on slow networks
-           - ONLY_NEW=True                     # set to False to retry previously failed URLs when re-adding instead of skipping them
-           # - PUID=502                        # set to your host user's UID & GID if you encounter permissions issues
-           # - PGID=20
-       volumes:
-           - ./data:/data
-       # cpus: 2                               # uncomment / edit these values to limit container resource consumption
-       # mem_limit: 2048m
-       # shm_size: 1024m
+        image: archivebox/archivebox:latest
+        command: schedule --foreground --update --every=day
+        environment:
+            - TIMEOUT=120                       # use a higher timeout than the main container to give slow tasks more time when retrying
+            # - PUID=502                        # set to your host user's UID & GID if you encounter permissions issues
+            # - PGID=20
+        volumes:
+            - ./data:/data
+        # cpus: 2                               # uncomment / edit these values to limit scheduler container resource consumption
+        # mem_limit: 2048m
+        # restart: always
 
 
-    ### Runs the Sonic full-text search backend, config file is auto-downloaded into sonic.cfg:
-    #   After starting, backfill any existing Snapshots into the full-text index:
+    ### This runs the optional Sonic full-text search backend (much faster than default rg backend).
+    # If Sonic is ever started after not running for a while, update its full-text index by running:
     #   $ docker-compose run archivebox update --index-only
 
     sonic:
         image: valeriansaliou/sonic:latest
         build:
             # custom build just auto-downloads archivebox's default sonic.cfg as a convenience
-            # not needed if you have already have /etc/sonic.cfg
+            # not needed after first run / if you have already have ./etc/sonic.cfg present
             dockerfile_inline: |
-                FROM quay.io/curl/curl:latest AS setup
-                RUN curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/main/etc/sonic.cfg' > /tmp/sonic.cfg
+                FROM quay.io/curl/curl:latest AS config_downloader
+                RUN curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/etc/sonic.cfg' > /tmp/sonic.cfg
                 FROM valeriansaliou/sonic:latest
-                COPY --from=setup /tmp/sonic.cfg /etc/sonic.cfg
+                COPY --from=config_downloader /tmp/sonic.cfg /etc/sonic.cfg
         expose:
             - 1491
         environment:
             - SEARCH_BACKEND_PASSWORD=SomeSecretPassword
         volumes:
-            - ./etc/sonic.cfg:/etc/sonic.cfg
+            - ./sonic.cfg:/etc/sonic.cfg
             - ./data/sonic:/var/lib/sonic/store
 
 
-    ### Example: Watch the ArchiveBox browser in realtime as it archives things,
-    # or remote control it to set up logins and credentials for sites you want to archive.
+    ### This container runs xvfb+noVNC so you can watch the ArchiveBox browser as it archives things,
+    # or remote control it to set up a chrome profile w/ login credentials for sites you want to archive.
     # https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile
 
     novnc:
@@ -99,11 +94,13 @@ services:
             - DISPLAY_HEIGHT=1080
             - RUN_XTERM=no
         ports:
-            # to view/control ArchiveBox's browser, visit: http://localhost:8080/vnc.html
-            - "8080:8080"
+            # to view/control ArchiveBox's browser, visit: http://127.0.0.1:8080/vnc.html
+            # restricted to access from localhost by default because it has no authentication
+            - 127.0.0.1:8080:8080
 
-    
-    ### Example: Put Nginx in front of the ArchiveBox server for SSL termination
+
+    ### Example: Put Nginx in front of the ArchiveBox server for SSL termination and static file serving.
+    # You can also any other ingress provider for SSL like Apache, Caddy, Traefik, Cloudflare Tunnels, etc.
 
     # nginx:
     #     image: nginx:alpine
@@ -121,7 +118,8 @@ services:
     # pihole:
     #   image: pihole/pihole:latest
     #   ports:
-    #     - 127.0.0.1:8090:80       # uncomment to access the admin HTTP interface on http://localhost:8090
+    #     # access the admin HTTP interface on http://localhost:8090
+    #     - 127.0.0.1:8090:80
     #   environment:
     #     - WEBPASSWORD=SET_THIS_TO_SOME_SECRET_PASSWORD_FOR_ADMIN_DASHBOARD
     #     - DNSMASQ_LISTENING=all
@@ -136,7 +134,44 @@ services:
     #     - ./etc/dnsmasq:/etc/dnsmasq.d
 
 
-    ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel
+    ### Example: Enable ability to run regularly scheduled archiving tasks by uncommenting this container
+    #   $ docker compose run archivebox schedule --every=day --depth=1 'https://example.com/some/rss/feed.xml'
+    # then restart the scheduler container to apply the changes to the schedule
+    #   $ docker compose restart archivebox_scheduler
+
+    # archivebox_scheduler:
+    #    image: archivebox/archivebox:latest
+    #    command: schedule --foreground
+    #    environment:
+    #        - MEDIA_MAX_SIZE=750m               # increase this number to allow archiving larger audio/video files
+    #        # - TIMEOUT=60                      # increase if you see timeouts often during archiving / on slow networks
+    #        # - ONLY_NEW=True                   # set to False to retry previously failed URLs when re-adding instead of skipping them
+    #        # - CHECK_SSL_VALIDITY=True         # set to False to allow saving URLs w/ broken SSL certs
+    #        # - SAVE_ARCHIVE_DOT_ORG=True       # set to False to disable submitting URLs to Archive.org when archiving
+    #        # - PUID=502                        # set to your host user's UID & GID if you encounter permissions issues
+    #        # - PGID=20
+    #    volumes:
+    #        - ./data:/data
+    #        - ./etc/crontabs:/var/spool/cron/crontabs
+    #    # cpus: 2                               # uncomment / edit these values to limit container resource consumption
+    #    # mem_limit: 2048m
+    #    # shm_size: 1024m
+
+
+    ### Example: Put Nginx in front of the ArchiveBox server for SSL termination
+
+    # nginx:
+    #     image: nginx:alpine
+    #     ports:
+    #         - 443:443
+    #         - 80:80
+    #     volumes:
+    #         - ./etc/nginx.conf:/etc/nginx/nginx.conf
+    #         - ./data:/var/www
+
+
+    ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel to avoid IP blocks.
+    # You can also use any other VPN that works at the docker IP level, e.g. Tailscale, OpenVPN, etc.
 
     # wireguard:
     #   image: linuxserver/wireguard:latest
@@ -152,6 +187,13 @@ services:
     #     - ./wireguard.conf:/config/wg0.conf:ro
 
 
+    ### Example: Run browsertrix in parallel with ArchiveBox
+
+    # browsertrix:
+    #     image: webrecorder/browsertrix-crawler:latest
+    #     volumes:
+    #         - ./browsertrix:/crawls:z
+
     ### Example: Run PYWB in parallel and auto-import WARCs from ArchiveBox
 
     # pywb:
@@ -167,10 +209,30 @@ services:
 
 
 networks:
-
-    # network needed for pihole container to offer :53 dns resolving on fixed ip for archivebox container
+    # network just used for pihole container to offer :53 dns resolving on fixed ip for archivebox container
     dns:
         ipam:
             driver: default
             config:
                 - subnet: 172.20.0.0/24
+
+
+# To use remote storage for your ./data/archive (e.g. Amazon S3, Backblaze B2, Google Drive, OneDrive, SFTP, etc.)
+#   Follow the steps here to set up the Docker RClone Plugin https://rclone.org/docker/
+#     $ docker plugin install rclone/docker-volume-rclone:amd64 --grant-all-permissions --alias rclone
+#     $ nano /var/lib/docker-plugins/rclone/config/rclone.conf
+#     [examplegdrive]
+#     type = drive
+#     scope = drive
+#     drive_id = 1234567...
+#     root_folder_id = 0Abcd...
+#     token = {"access_token":...}
+
+# volumes:
+#     archive:
+#         driver: rclone
+#         driver_opts:
+#             remote: 'examplegdrive:archivebox'
+#             allow_other: 'true'
+#             vfs_cache_mode: full
+#             poll_interval: 0

etc/uwsgi.ini

@@ -7,7 +7,12 @@ wsgi-file = archivebox/core/wsgi.py
 processes = 4
 threads = 1
 stats = 127.0.0.1:9191
-static-map /static=./archivebox/templates/static
+static-map = /static=./archivebox/templates/static
+static-map = /static=./archivebox/plugins/replaywebpage/static
+static-map = /archive=$(PWD)/archive
+static=index = index.html
 harakiri = 172800
 post-buffering = 1
 disable-logging = True
+check-static 
+honour-range = True

package-lock.json

@@ -1,23 +1,33 @@
 {
   "name": "archivebox",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "archivebox",
-      "version": "0.7.3",
+      "version": "0.8.0",
       "license": "MIT",
       "dependencies": {
         "@postlight/parser": "^2.2.3",
         "readability-extractor": "github:ArchiveBox/readability-extractor",
-        "single-file-cli": "^1.1.46"
+        "single-file-cli": "^1.1.54"
+      }
+    },
+    "node_modules/@asamuzakjp/dom-selector": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-2.0.2.tgz",
+      "integrity": "sha512-x1KXOatwofR6ZAYzXRBL5wrdV0vwNxlTCK9NCuLqAzQYARqGcvFwiJA6A1ERuh+dgeA4Dxm3JBYictIes+SqUQ==",
+      "dependencies": {
+        "bidi-js": "^1.0.3",
+        "css-tree": "^2.3.1",
+        "is-potential-custom-element-name": "^1.0.1"
       }
     },
     "node_modules/@babel/runtime-corejs2": {
-      "version": "7.23.7",
-      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.23.7.tgz",
-      "integrity": "sha512-JmMk2t1zGDNkvsY2MsLLksocjY+ufGzSk8UlcNcxzfrzAPu4nMx0HRFakzIg2bhcqQq6xBI2nUaW/sHoaYIHdQ==",
+      "version": "7.24.4",
+      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.24.4.tgz",
+      "integrity": "sha512-ZCKqyUKt/Coimg+3Kafu43yNetgYnTXzNbEGAgxc81J5sI0qFNbQ613w7PNny+SmijAmGVroL0GDvx5rG/JI5Q==",
       "dependencies": {
         "core-js": "^2.6.12",
         "regenerator-runtime": "^0.14.0"
@@ -168,9 +178,9 @@
       }
     },
     "node_modules/@puppeteer/browsers": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.8.0.tgz",
-      "integrity": "sha512-TkRHIV6k2D8OlUe8RtG+5jgOF/H98Myx0M6AOafC8DdNVOFiBSFa5cpRDtpm8LXOa9sVwe0+e6Q3FC56X/DZfg==",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.0.0.tgz",
+      "integrity": "sha512-3PS82/5+tnpEaUWonjAFFvlf35QHF15xqyGd34GBa5oP5EPVfFXRsbSxIGYf1M+vZlqBZ3oxT1kRg9OYhtt8ng==",
       "dependencies": {
         "debug": "4.3.4",
         "extract-zip": "2.0.1",
@@ -184,7 +194,7 @@
         "browsers": "lib/cjs/main-cli.js"
       },
       "engines": {
-        "node": ">=16.3.0"
+        "node": ">=18"
       }
     },
     "node_modules/@tootallnate/quickjs-emscripten": {
@@ -193,9 +203,9 @@
       "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA=="
     },
     "node_modules/@types/node": {
-      "version": "20.10.6",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.6.tgz",
-      "integrity": "sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==",
+      "version": "20.12.7",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz",
+      "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==",
       "optional": true,
       "dependencies": {
         "undici-types": "~5.26.4"
@@ -211,9 +221,9 @@
       }
     },
     "node_modules/agent-base": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz",
-      "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
+      "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==",
       "dependencies": {
         "debug": "^4.3.4"
       },
@@ -304,14 +314,15 @@
       "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg=="
     },
     "node_modules/b4a": {
-      "version": "1.6.4",
-      "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz",
-      "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw=="
+      "version": "1.6.6",
+      "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz",
+      "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg=="
     },
-    "node_modules/balanced-match": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+    "node_modules/bare-events": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.2.2.tgz",
+      "integrity": "sha512-h7z00dWdG0PYOQEvChhOSWvOfkIKsdZGkWr083FgN/HyoQuebSew/cgirYqh9SCuy/hRvxc5Vy6Fw8xAmYHLkQ==",
+      "optional": true
     },
     "node_modules/base64-js": {
       "version": "1.5.1",
@@ -333,9 +344,9 @@
       ]
     },
     "node_modules/basic-ftp": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.4.tgz",
-      "integrity": "sha512-8PzkB0arJFV4jJWSGOYR+OEic6aeKMu/osRhBULN6RY0ykby6LKhbmuQ5ublvaas5BOwboah5D87nrHyuh8PPA==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz",
+      "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==",
       "engines": {
         "node": ">=10.0.0"
       }
@@ -348,6 +359,14 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "node_modules/bidi-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
+      "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==",
+      "dependencies": {
+        "require-from-string": "^2.0.2"
+      }
+    },
     "node_modules/bluebird": {
       "version": "2.11.0",
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz",
@@ -358,15 +377,6 @@
       "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
       "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
     },
-    "node_modules/brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dependencies": {
-        "balanced-match": "^1.0.0",
-        "concat-map": "0.0.1"
-      }
-    },
     "node_modules/brotli": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz",
@@ -446,12 +456,12 @@
       }
     },
     "node_modules/chromium-bidi": {
-      "version": "0.4.33",
-      "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.4.33.tgz",
-      "integrity": "sha512-IxoFM5WGQOIAd95qrSXzJUv4eXIrh+RvU3rwwqIiwYuvfE7U/Llj4fejbsJnjJMUYCuGtVQsY2gv7oGl4aTNSQ==",
+      "version": "0.5.8",
+      "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.5.8.tgz",
+      "integrity": "sha512-blqh+1cEQbHBKmok3rVJkBlBxt9beKBgOsxbFgs7UJcoVbbeZ+K7+6liAsjgpc8l1Xd55cQUy14fXZdGSb4zIw==",
       "dependencies": {
         "mitt": "3.0.1",
-        "urlpattern-polyfill": "9.0.0"
+        "urlpattern-polyfill": "10.0.0"
       },
       "peerDependencies": {
         "devtools-protocol": "*"
@@ -497,11 +507,6 @@
         "node": ">= 0.8"
       }
     },
-    "node_modules/concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
-    },
     "node_modules/core-js": {
       "version": "2.6.12",
       "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
@@ -533,6 +538,18 @@
         "nth-check": "~1.0.1"
       }
     },
+    "node_modules/css-tree": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz",
+      "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==",
+      "dependencies": {
+        "mdn-data": "2.0.30",
+        "source-map-js": "^1.0.1"
+      },
+      "engines": {
+        "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
+      }
+    },
     "node_modules/css-what": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
@@ -542,14 +559,14 @@
       }
     },
     "node_modules/cssstyle": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz",
-      "integrity": "sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.0.1.tgz",
+      "integrity": "sha512-8ZYiJ3A/3OkDd093CBT/0UKDWry7ak4BdPTFP2+QEP7cmhouyq/Up709ASSj2cK02BbZiMgk7kYjZNS4QP5qrQ==",
       "dependencies": {
         "rrweb-cssom": "^0.6.0"
       },
       "engines": {
-        "node": ">=14"
+        "node": ">=18"
       }
     },
     "node_modules/dashdash": {
@@ -564,9 +581,9 @@
       }
     },
     "node_modules/data-uri-to-buffer": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.1.tgz",
-      "integrity": "sha512-MZd3VlchQkp8rdend6vrx7MmVDJzSNTBvghvKjirLkD+WTChA3KUf0jkE68Q4UyctNqI11zZO9/x2Yx+ub5Cvg==",
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
+      "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==",
       "engines": {
         "node": ">= 14"
       }
@@ -657,9 +674,9 @@
       }
     },
     "node_modules/devtools-protocol": {
-      "version": "0.0.1203626",
-      "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1203626.tgz",
-      "integrity": "sha512-nEzHZteIUZfGCZtTiS1fRpC8UZmsfD1SiyPvaUNvS13dvKf666OAm8YTi0+Ca3n1nLEyu49Cy4+dPWpaHFJk9g=="
+      "version": "0.0.1232444",
+      "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1232444.tgz",
+      "integrity": "sha512-pM27vqEfxSxRkTMnF+XCmxSEb6duO5R+t8A9DEEJgy4Wz2RVanje2mmj99B6A3zv2r/qGfYlOvYznUhuokizmg=="
     },
     "node_modules/difflib": {
       "version": "0.2.6",
@@ -696,9 +713,9 @@
       "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
     },
     "node_modules/dompurify": {
-      "version": "3.0.7",
-      "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.7.tgz",
-      "integrity": "sha512-BViYTZoqP3ak/ULKOc101y+CtHDUvBsVgSxIF1ku0HmK6BRf+C03MC+tArMvOPtVtZp83DDh5puywKDu4sbVjQ=="
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.0.tgz",
+      "integrity": "sha512-yoU4rhgPKCo+p5UrWWWNKiIq+ToGqmVVhk0PmMYBK4kRsR3/qhemNFL8f6CFmBd4gMwm3F4T7HBoydP5uY07fA=="
     },
     "node_modules/domutils": {
       "version": "1.5.1",
@@ -726,6 +743,11 @@
         "safer-buffer": "^2.1.0"
       }
     },
+    "node_modules/ecc-jsbn/node_modules/jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg=="
+    },
     "node_modules/ellipsize": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/ellipsize/-/ellipsize-0.1.0.tgz",
@@ -750,9 +772,9 @@
       "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
     },
     "node_modules/escalade": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
-      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
+      "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==",
       "engines": {
         "node": ">=6"
       }
@@ -890,31 +912,26 @@
       }
     },
     "node_modules/fs-extra": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
-      "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
+      "version": "11.2.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz",
+      "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==",
       "dependencies": {
         "graceful-fs": "^4.2.0",
-        "jsonfile": "^4.0.0",
-        "universalify": "^0.1.0"
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
       },
       "engines": {
-        "node": ">=6 <7 || >=8"
+        "node": ">=14.14"
       }
     },
     "node_modules/fs-extra/node_modules/universalify": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
-      "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
       "engines": {
-        "node": ">= 4.0.0"
+        "node": ">= 10.0.0"
       }
     },
-    "node_modules/fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
-    },
     "node_modules/get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -938,14 +955,14 @@
       }
     },
     "node_modules/get-uri": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.2.tgz",
-      "integrity": "sha512-5KLucCJobh8vBY1K07EFV4+cPZH3mrV9YeAruUseCQKHB58SGjjT2l9/eA9LD082IiuMjSlFJEcdJ27TXvbZNw==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz",
+      "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==",
       "dependencies": {
         "basic-ftp": "^5.0.2",
-        "data-uri-to-buffer": "^6.0.0",
+        "data-uri-to-buffer": "^6.0.2",
         "debug": "^4.3.4",
-        "fs-extra": "^8.1.0"
+        "fs-extra": "^11.2.0"
       },
       "engines": {
         "node": ">= 14"
@@ -959,25 +976,6 @@
         "assert-plus": "^1.0.0"
       }
     },
-    "node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
-      "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
-      },
-      "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/graceful-fs": {
       "version": "4.2.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -1034,9 +1032,9 @@
       }
     },
     "node_modules/http-proxy-agent": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.0.tgz",
-      "integrity": "sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
+      "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
       "dependencies": {
         "agent-base": "^7.1.0",
         "debug": "^4.3.4"
@@ -1059,9 +1057,9 @@
       }
     },
     "node_modules/https-proxy-agent": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz",
-      "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==",
+      "version": "7.0.4",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz",
+      "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==",
       "dependencies": {
         "agent-base": "^7.0.2",
         "debug": "4"
@@ -1105,24 +1103,22 @@
       "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
       "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="
     },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
-      "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
-      }
-    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
-    "node_modules/ip": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz",
-      "integrity": "sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg=="
+    "node_modules/ip-address": {
+      "version": "9.0.5",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
+      "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==",
+      "dependencies": {
+        "jsbn": "1.1.0",
+        "sprintf-js": "^1.1.3"
+      },
+      "engines": {
+        "node": ">= 12"
+      }
     },
     "node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
@@ -1153,16 +1149,17 @@
       "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g=="
     },
     "node_modules/jsbn": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
-      "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg=="
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz",
+      "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A=="
     },
     "node_modules/jsdom": {
-      "version": "23.0.1",
-      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-23.0.1.tgz",
-      "integrity": "sha512-2i27vgvlUsGEBO9+/kJQRbtqtm+191b5zAZrU/UezVmnC2dlDAFLgDYJvAEi94T4kjsRKkezEtLQTgsNEsW2lQ==",
+      "version": "23.2.0",
+      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-23.2.0.tgz",
+      "integrity": "sha512-L88oL7D/8ufIES+Zjz7v0aes+oBMh2Xnh3ygWvL0OaICOomKEPKuPnIfBJekiXr+BHbbMjrWn/xqrDQuxFTeyA==",
       "dependencies": {
-        "cssstyle": "^3.0.0",
+        "@asamuzakjp/dom-selector": "^2.0.1",
+        "cssstyle": "^4.0.1",
         "data-urls": "^5.0.0",
         "decimal.js": "^10.4.3",
         "form-data": "^4.0.0",
@@ -1170,7 +1167,6 @@
         "http-proxy-agent": "^7.0.0",
         "https-proxy-agent": "^7.0.2",
         "is-potential-custom-element-name": "^1.0.1",
-        "nwsapi": "^2.2.7",
         "parse5": "^7.1.2",
         "rrweb-cssom": "^0.6.0",
         "saxes": "^6.0.0",
@@ -1181,7 +1177,7 @@
         "whatwg-encoding": "^3.1.1",
         "whatwg-mimetype": "^4.0.0",
         "whatwg-url": "^14.0.0",
-        "ws": "^8.14.2",
+        "ws": "^8.16.0",
         "xml-name-validator": "^5.0.0"
       },
       "engines": {
@@ -1235,13 +1231,24 @@
       "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="
     },
     "node_modules/jsonfile": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
-      "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
       "optionalDependencies": {
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/jsonfile/node_modules/universalify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/jsprim": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz",
@@ -1375,6 +1382,11 @@
         "node": ">=12"
       }
     },
+    "node_modules/mdn-data": {
+      "version": "2.0.30",
+      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
+      "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA=="
+    },
     "node_modules/mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@@ -1394,17 +1406,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
-      "dependencies": {
-        "brace-expansion": "^1.1.7"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/mitt": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
@@ -1461,9 +1462,9 @@
       }
     },
     "node_modules/nwsapi": {
-      "version": "2.2.7",
-      "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz",
-      "integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ=="
+      "version": "2.2.9",
+      "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.9.tgz",
+      "integrity": "sha512-2f3F0SEEer8bBu0dsNCFF50N0cTThV1nWFYcEYFZttdW0lDAoybv9cQoK7X7/68Z89S7FoRrVjP1LPX4XRf9vg=="
     },
     "node_modules/oauth-sign": {
       "version": "0.9.0",
@@ -1500,12 +1501,11 @@
       }
     },
     "node_modules/pac-resolver": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.0.tgz",
-      "integrity": "sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg==",
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz",
+      "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==",
       "dependencies": {
         "degenerator": "^5.0.0",
-        "ip": "^1.1.8",
         "netmask": "^2.0.2"
       },
       "engines": {
@@ -1539,14 +1539,6 @@
         "url": "https://github.com/fb55/entities?sponsor=1"
       }
     },
-    "node_modules/path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/pend": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
@@ -1648,39 +1640,19 @@
       }
     },
     "node_modules/puppeteer-core": {
-      "version": "21.5.2",
-      "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-21.5.2.tgz",
-      "integrity": "sha512-v4T0cWnujSKs+iEfmb8ccd7u4/x8oblEyKqplqKnJ582Kw8PewYAWvkH4qUWhitN3O2q9RF7dzkvjyK5HbzjLA==",
+      "version": "22.0.0",
+      "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.0.0.tgz",
+      "integrity": "sha512-S3s91rLde0A86PWVeNY82h+P0fdS7CTiNWAicCVH/bIspRP4nS2PnO5j+VTFqCah0ZJizGzpVPAmxVYbLxTc9w==",
       "dependencies": {
-        "@puppeteer/browsers": "1.8.0",
-        "chromium-bidi": "0.4.33",
+        "@puppeteer/browsers": "2.0.0",
+        "chromium-bidi": "0.5.8",
         "cross-fetch": "4.0.0",
         "debug": "4.3.4",
-        "devtools-protocol": "0.0.1203626",
-        "ws": "8.14.2"
+        "devtools-protocol": "0.0.1232444",
+        "ws": "8.16.0"
       },
       "engines": {
-        "node": ">=16.13.2"
-      }
-    },
-    "node_modules/puppeteer-core/node_modules/ws": {
-      "version": "8.14.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
-      "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
+        "node": ">=18"
       }
     },
     "node_modules/qs": {
@@ -1703,8 +1675,7 @@
     },
     "node_modules/readability-extractor": {
       "version": "0.0.11",
-      "resolved": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#2fb4689a65c6433036453dcbee7a268840604eb9",
-      "license": "MIT",
+      "resolved": "git+ssh://git@github.com/ArchiveBox/readability-extractor.git#057f2046f9535cfc6df7b8d551aaad32a9e6226c",
       "dependencies": {
         "@mozilla/readability": "^0.5.0",
         "dompurify": "^3.0.6",
@@ -1740,25 +1711,19 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/require-from-string": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+      "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/requires-port": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
       "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
     },
-    "node_modules/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
-      "dependencies": {
-        "glob": "^7.1.3"
-      },
-      "bin": {
-        "rimraf": "bin.js"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/rrweb-cssom": {
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz",
@@ -1800,9 +1765,9 @@
       }
     },
     "node_modules/selenium-webdriver": {
-      "version": "4.15.0",
-      "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.15.0.tgz",
-      "integrity": "sha512-BNG1bq+KWiBGHcJ/wULi0eKY0yaDqFIbEmtbsYJmfaEghdCkXBsx1akgOorhNwjBipOr0uwpvNXqT6/nzl+zjg==",
+      "version": "4.17.0",
+      "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.17.0.tgz",
+      "integrity": "sha512-e2E+2XBlGepzwgFbyQfSwo9Cbj6G5fFfs9MzAS00nC99EewmcS2rwn2MwtgfP7I5p1e7DYv4HQJXtWedsu6DvA==",
       "dependencies": {
         "jszip": "^3.10.1",
         "tmp": "^0.2.1",
@@ -1818,16 +1783,16 @@
       "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="
     },
     "node_modules/single-file-cli": {
-      "version": "1.1.46",
-      "resolved": "https://registry.npmjs.org/single-file-cli/-/single-file-cli-1.1.46.tgz",
-      "integrity": "sha512-+vFj0a5Y4ESqpMwH0T6738pg8ZA9KVhhl6OlIOsicamGNU9DnMa+q9dL1S2KnLWHoauKjU0BThhR/YKUleJSxw==",
+      "version": "1.1.54",
+      "resolved": "https://registry.npmjs.org/single-file-cli/-/single-file-cli-1.1.54.tgz",
+      "integrity": "sha512-wnVPg7BklhswwFVrtuFXbmluI4piHxg2dC0xATxYTeXAld6PnRPlnp7ufallRKArjFBZdP2u+ihMkOIp7A38XA==",
       "dependencies": {
         "file-url": "3.0.0",
         "iconv-lite": "0.6.3",
-        "jsdom": "23.0.0",
-        "puppeteer-core": "21.5.2",
-        "selenium-webdriver": "4.15.0",
-        "single-file-core": "1.3.15",
+        "jsdom": "24.0.0",
+        "puppeteer-core": "22.0.0",
+        "selenium-webdriver": "4.17.0",
+        "single-file-core": "1.3.24",
         "strong-data-uri": "1.0.6",
         "yargs": "17.7.2"
       },
@@ -1847,11 +1812,11 @@
       }
     },
     "node_modules/single-file-cli/node_modules/jsdom": {
-      "version": "23.0.0",
-      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-23.0.0.tgz",
-      "integrity": "sha512-cbL/UCtohJguhFC7c2/hgW6BeZCNvP7URQGnx9tSJRYKCdnfbfWOrtuLTMfiB2VxKsx5wPHVsh/J0aBy9lIIhQ==",
+      "version": "24.0.0",
+      "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-24.0.0.tgz",
+      "integrity": "sha512-UDS2NayCvmXSXVP6mpTj+73JnNQadZlr9N68189xib2tx5Mls7swlTNao26IoHv46BZJFvXygyRtyXd1feAk1A==",
       "dependencies": {
-        "cssstyle": "^3.0.0",
+        "cssstyle": "^4.0.1",
         "data-urls": "^5.0.0",
         "decimal.js": "^10.4.3",
         "form-data": "^4.0.0",
@@ -1870,14 +1835,14 @@
         "whatwg-encoding": "^3.1.1",
         "whatwg-mimetype": "^4.0.0",
         "whatwg-url": "^14.0.0",
-        "ws": "^8.14.2",
+        "ws": "^8.16.0",
         "xml-name-validator": "^5.0.0"
       },
       "engines": {
         "node": ">=18"
       },
       "peerDependencies": {
-        "canvas": "^3.0.0"
+        "canvas": "^2.11.2"
       },
       "peerDependenciesMeta": {
         "canvas": {
@@ -1909,9 +1874,9 @@
       }
     },
     "node_modules/single-file-core": {
-      "version": "1.3.15",
-      "resolved": "https://registry.npmjs.org/single-file-core/-/single-file-core-1.3.15.tgz",
-      "integrity": "sha512-/YNpHBwASWNxmSmZXz0xRolmXf0+PGAbwpVrwn6A8tYeuAdezxxde5RYTTQ7V4Zv68+H4JMhE2DwCRV0sVUGNA=="
+      "version": "1.3.24",
+      "resolved": "https://registry.npmjs.org/single-file-core/-/single-file-core-1.3.24.tgz",
+      "integrity": "sha512-1B256mKBbNV8jXAV+hRyEv0aMa7tn0C0Ci+zx7Ya4ZXZB3b9/1MgKsB/fxVwDiL28WJSU0pxzh8ftIYubCNn9w=="
     },
     "node_modules/smart-buffer": {
       "version": "4.2.0",
@@ -1923,24 +1888,24 @@
       }
     },
     "node_modules/socks": {
-      "version": "2.7.1",
-      "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz",
-      "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+      "version": "2.8.3",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz",
+      "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==",
       "dependencies": {
-        "ip": "^2.0.0",
+        "ip-address": "^9.0.5",
         "smart-buffer": "^4.2.0"
       },
       "engines": {
-        "node": ">= 10.13.0",
+        "node": ">= 10.0.0",
         "npm": ">= 3.0.0"
       }
     },
     "node_modules/socks-proxy-agent": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz",
-      "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==",
+      "version": "8.0.3",
+      "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.3.tgz",
+      "integrity": "sha512-VNegTZKhuGq5vSD6XNKlbqWhyt/40CgoEw8XxD6dhnm8Jq9IEa3nIa4HwnM8XOqU0CdB0BwWVXusqiFXfHB3+A==",
       "dependencies": {
-        "agent-base": "^7.0.2",
+        "agent-base": "^7.1.1",
         "debug": "^4.3.4",
         "socks": "^2.7.1"
       },
@@ -1948,11 +1913,6 @@
         "node": ">= 14"
       }
     },
-    "node_modules/socks/node_modules/ip": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
-      "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ=="
-    },
     "node_modules/source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -1962,6 +1922,19 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/source-map-js": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz",
+      "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/sprintf-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
+      "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="
+    },
     "node_modules/sshpk": {
       "version": "1.18.0",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@@ -1986,6 +1959,11 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/sshpk/node_modules/jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg=="
+    },
     "node_modules/stream-length": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/stream-length/-/stream-length-1.0.2.tgz",
@@ -1995,12 +1973,15 @@
       }
     },
     "node_modules/streamx": {
-      "version": "2.15.6",
-      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.6.tgz",
-      "integrity": "sha512-q+vQL4AAz+FdfT137VF69Cc/APqUbxy+MDOImRrMvchJpigHj9GksgDU2LYbO9rx7RX6osWgxJB2WxhYv4SZAw==",
+      "version": "2.16.1",
+      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.16.1.tgz",
+      "integrity": "sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==",
       "dependencies": {
         "fast-fifo": "^1.1.0",
         "queue-tick": "^1.0.1"
+      },
+      "optionalDependencies": {
+        "bare-events": "^2.2.0"
       }
     },
     "node_modules/string_decoder": {
@@ -2067,9 +2048,9 @@
       }
     },
     "node_modules/tar-stream": {
-      "version": "3.1.6",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz",
-      "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==",
+      "version": "3.1.7",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz",
+      "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==",
       "dependencies": {
         "b4a": "^1.6.4",
         "fast-fifo": "^1.2.0",
@@ -2082,14 +2063,11 @@
       "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
     },
     "node_modules/tmp": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
-      "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
-      "dependencies": {
-        "rimraf": "^3.0.0"
-      },
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
+      "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
       "engines": {
-        "node": ">=8.17.0"
+        "node": ">=14.14"
       }
     },
     "node_modules/tough-cookie": {
@@ -2125,9 +2103,9 @@
       "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
     "node_modules/turndown": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.2.tgz",
-      "integrity": "sha512-ntI9R7fcUKjqBP6QU8rBK2Ehyt8LAzt3UBT9JR9tgo6GtuKvyUzpayWmeMKJw1DPdXzktvtIT8m2mVXz+bL/Qg==",
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.1.3.tgz",
+      "integrity": "sha512-Z3/iJ6IWh8VBiACWQJaA5ulPQE5E1QwvBHj00uGzdQxdRnd8fh1DPqNOJqzQDu6DkOstORrtXzf/9adB+vMtEA==",
       "dependencies": {
         "domino": "^2.1.6"
       }
@@ -2178,9 +2156,9 @@
       }
     },
     "node_modules/urlpattern-polyfill": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-9.0.0.tgz",
-      "integrity": "sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g=="
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz",
+      "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg=="
     },
     "node_modules/util-deprecate": {
       "version": "1.0.2",

package.json

@@ -1,13 +1,14 @@
 {
-  "name": "archivebox",
-  "version": "0.7.3",
-  "description": "ArchiveBox: The self-hosted internet archive",
-  "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
-  "repository": "github:ArchiveBox/ArchiveBox",
-  "license": "MIT",
-  "dependencies": {
-    "@postlight/parser": "^2.2.3",
-    "readability-extractor": "github:ArchiveBox/readability-extractor",
-    "single-file-cli": "^1.1.54"
-  }
+    "name": "archivebox",
+    "version": "0.7.3",
+    "description": "ArchiveBox: The self-hosted internet archive",
+    "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
+    "repository": "github:ArchiveBox/ArchiveBox",
+    "license": "MIT",
+    "dependencies":
+    {
+        "@postlight/parser": "^2.2.3",
+        "readability-extractor": "github:ArchiveBox/readability-extractor",
+        "single-file-cli": "^1.1.54"
+    }
 }

pyproject.toml

@@ -1,32 +1,50 @@
 [project]
 name = "archivebox"
-version = "0.7.3"
+version = "0.8.0"
+package-dir = "archivebox"
+requires-python = ">=3.10,<3.13"
+platform = "py3-none-any"
 description = "Self-hosted internet archiving solution."
-authors = [
-    {name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"},
-]
+authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
 license = {text = "MIT"}
 readme = "README.md"
-package-dir = "archivebox"
-requires-python = ">=3.10,<3.12"
+
+# pdm install
+# pdm update --unconstrained
 dependencies = [
-    # pdm update [--unconstrained] 
-    "croniter>=0.3.34",
-    "dateparser>=1.0.0",
-    "django-extensions>=3.2.3",
+    # Base Framework and Language Dependencies
+    "setuptools>=69.5.1",
     "django>=4.2.0,<5.0",
-    "setuptools>=69.0.3",
+    "django-ninja>=1.1.0",
+    "django-solo>=2.0.0",
+    "django-extensions>=3.2.3",
+    "mypy-extensions>=1.0.0",
+
+    # Python Helper Libraries
+    "requests>=2.31.0",
+    "dateparser>=1.0.0",
     "feedparser>=6.0.11",
-    "ipython>5.0.0",
-    "mypy-extensions>=0.4.3",
-    "python-crontab>=2.5.1",
-    "requests>=2.24.0",
     "w3lib>=1.22.0",
-    "yt-dlp>=2024.3.10",
-    #  dont add playwright becuase packages without sdists cause trouble on many build systems that refuse to install wheel-only packages
-    "playwright>=1.39.0; platform_machine != 'armv7l'",
+
+    # Feature-Specific Dependencies
+    "python-crontab>=2.5.1",          # for: archivebox schedule
+    "croniter>=0.3.34",               # for: archivebox schedule
+    "ipython>5.0.0",                  # for: archivebox shell
+
+    # Extractor Dependencies
+    "yt-dlp>=2024.4.9",               # for: media
+    "playwright>=1.43.0; platform_machine != 'armv7l'",  # WARNING: playwright doesn't have any sdist, causes trouble on build systems that refuse to install wheel-only packages
+    
+    # TODO: add more extractors
+    #  - gallery-dl
+    #  - scihubdl
+    #  - See Github issues for more...
 ]
 
+homepage = "https://github.com/ArchiveBox/ArchiveBox"
+repository = "https://github.com/ArchiveBox/ArchiveBox"
+documentation = "https://github.com/ArchiveBox/ArchiveBox/wiki"
+keywords = ["internet archiving", "web archiving", "digipres", "warc", "preservation", "backups", "archiving", "web", "bookmarks", "puppeteer", "browser", "download"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Environment :: Console",
@@ -59,50 +77,81 @@ classifiers = [
     "Topic :: Utilities",
     "Typing :: Typed",
 ]
+# dynamic = ["version"]  # TODO: programatticaly fetch version from package.json at build time
 
+# pdm lock --group=':all' 
+# pdm install -G:all
+# pdm update --group=':all' --unconstrained
 [project.optional-dependencies]
-# pdm update [--group=':all'] [--unconstrained] 
 sonic = [
     # echo "deb [signed-by=/usr/share/keyrings/valeriansaliou_sonic.gpg] https://packagecloud.io/valeriansaliou/sonic/debian/ bookworm main" > /etc/apt/sources.list.d/valeriansaliou_sonic.list
     # curl -fsSL https://packagecloud.io/valeriansaliou/sonic/gpgkey | gpg --dearmor -o /usr/share/keyrings/valeriansaliou_sonic.gpg
     # apt install sonic
-    "sonic-client>=0.0.5",
+    "sonic-client>=1.0.0",
 ]
 ldap = [
     # apt install libldap2-dev libsasl2-dev python3-ldap
     "python-ldap>=3.4.3",
     "django-auth-ldap>=4.1.0",
 ]
-# playwright = [
-#     platform_machine isnt respected by pdm export -o requirements.txt, this breaks arm/v7
-#     "playwright>=1.39.0; platform_machine != 'armv7l'",
-# ]
 
 
+# pdm lock --group=':all' --dev
 # pdm install -G:all --dev
-# pdm update --dev [--unconstrained] 
+# pdm update --dev --unconstrained
 [tool.pdm.dev-dependencies]
-dev = [
-    # building
+build = [
+    "setuptools>=69.5.1",
+    "pip",
     "wheel",
     "pdm",
-    "homebrew-pypi-poet>=0.10.0",
-    # documentation
+    "homebrew-pypi-poet>=0.10.0",      # for: generating archivebox.rb brewfile list of python packages
+]
+docs = [
     "recommonmark",
     "sphinx",
     "sphinx-rtd-theme",
-    # debugging
+]
+debug = [
     "django-debug-toolbar",
     "djdt_flamegraph",
     "ipdb",
-    # testing
+]
+test = [
+    "pdm[pytest]",
     "pytest",
-    # linting
+]
+lint = [
     "flake8",
     "mypy",
-    "django-stubs",
+    "django-stubs[compatible-mypy]>=4.2.7",
+    "types-requests>=2.31.0.20240125",
+    "pudb>=2024.1",
 ]
 
+[build-system]
+requires = ["pdm-backend"]
+build-backend = "pdm.backend"
+
+[project.scripts]
+archivebox = "archivebox.cli:main"
+
+[tool.pyright]
+include = ["archivebox"]
+exclude = ["data", "data2", "data3", "data4", "data5", "pip_dist", "brew_dist", "dist", "vendor", "migrations", "tests"]
+
+[tool.mypy]
+mypy_path = "archivebox"
+explicit_package_bases = true
+check_untyped_defs = true
+plugins = ["mypy_django_plugin.main"]
+# TODO: remove this eventually https://github.com/hauntsaninja/no_implicit_optional
+implicit_optional = true
+
+[tool.django-stubs]
+django_settings_module = "core.settings"
+strict_settings = false
+
 [tool.pdm.scripts]
 lint = "./bin/lint.sh"
 test = "./bin/test.sh"
@@ -111,13 +160,6 @@ test = "./bin/test.sh"
 [tool.pytest.ini_options]
 testpaths = [ "tests" ]
 
-[project.scripts]
-archivebox = "archivebox.cli:main"
-
-[build-system]
-requires = ["pdm-backend"]
-build-backend = "pdm.backend"
-
 
 [project.urls]
 Homepage = "https://github.com/ArchiveBox/ArchiveBox"

requirements.txt

@@ -1,54 +1,64 @@
 # This file is @generated by PDM.
 # Please do not edit it manually.
 
-asgiref==3.7.2
+annotated-types==0.6.0
+asgiref==3.8.1
 asttokens==2.4.1
 brotli==1.1.0; implementation_name == "cpython"
 brotlicffi==1.1.0.0; implementation_name != "cpython"
-certifi==2023.11.17
+certifi==2024.2.2
 cffi==1.16.0; implementation_name != "cpython"
 charset-normalizer==3.3.2
 colorama==0.4.6; sys_platform == "win32"
-croniter==2.0.1
+croniter==2.0.5
 dateparser==1.2.0
 decorator==5.1.1
-django==3.1.14
-django-auth-ldap==4.1.0
-django-extensions==3.1.5
-exceptiongroup==1.2.0; python_version < "3.11"
+django==4.2.11
+django-auth-ldap==4.8.0
+django-extensions==3.2.3
+django-ninja==1.1.0
+exceptiongroup==1.2.1; python_version < "3.11"
 executing==2.0.1
-idna==3.6
-ipython==8.18.1
+feedparser==6.0.11
+greenlet==3.0.3; platform_machine != "armv7l"
+idna==3.7
+ipython==8.23.0
 jedi==0.19.1
-matplotlib-inline==0.1.6
+matplotlib-inline==0.1.7
 mutagen==1.47.0
 mypy-extensions==1.0.0
-parso==0.8.3
-pexpect==4.9.0; sys_platform != "win32"
+parso==0.8.4
+pexpect==4.9.0; sys_platform != "win32" and sys_platform != "emscripten"
+playwright==1.43.0; platform_machine != "armv7l"
 prompt-toolkit==3.0.43
-ptyprocess==0.7.0; sys_platform != "win32"
+ptyprocess==0.7.0; sys_platform != "win32" and sys_platform != "emscripten"
 pure-eval==0.2.2
-pyasn1==0.5.1
-pyasn1-modules==0.3.0
-pycparser==2.21; implementation_name != "cpython"
-pycryptodomex==3.19.1
+pyasn1==0.6.0
+pyasn1-modules==0.4.0
+pycparser==2.22; implementation_name != "cpython"
+pycryptodomex==3.20.0
+pydantic==2.7.1
+pydantic-core==2.18.2
+pyee==11.1.0; platform_machine != "armv7l"
 pygments==2.17.2
 python-crontab==3.0.0
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
 python-ldap==3.4.4
-pytz==2023.3.post1
-regex==2023.12.25
+pytz==2024.1
+regex==2024.4.16
 requests==2.31.0
+setuptools==69.5.1
+sgmllib3k==1.0.0
 six==1.16.0
 sonic-client==1.0.0
-sqlparse==0.4.4
+sqlparse==0.5.0
 stack-data==0.6.3
-traitlets==5.14.1
-typing-extensions==4.9.0; python_version < "3.11"
-tzdata==2023.4; platform_system == "Windows"
+traitlets==5.14.3
+typing-extensions==4.11.0
+tzdata==2024.1; sys_platform == "win32" or platform_system == "Windows"
 tzlocal==5.2
-urllib3==2.1.0
+urllib3==2.2.1
 w3lib==2.1.2
-wcwidth==0.2.12
+wcwidth==0.2.13
 websockets==12.0
-yt-dlp==2023.12.30
+yt-dlp==2024.4.9