Merge branch 'dev' into plugins-browsertrix

Signed-off-by: longzai <437172242@qq.com>

.dockerignore

@@ -5,22 +5,28 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .github/
-.git/
 .pdm-build/
-.pdm-python/
+.pdm-python
 .eggs/
+.git/
+!.git/HEAD
+!.git/refs/heads/*
 
 venv/
 .venv/
+.venv-old/
 .docker-venv/
 node_modules/
 
+docs/
 build/
 dist/
 brew_dist/
 deb_dist/
 pip_dist/
 assets/
+docker/
 
 data/
+data*/
 output/

.github/FUNDING.yml

@@ -1,3 +1,3 @@
 github: pirate
 patreon: theSquashSH
-custom: ["https://twitter.com/ArchiveBoxApp", "https://paypal.me/NicholasSweeting", "https://www.blockchain.com/eth/address/0x5D4c34D4a121Fe08d1dDB7969F07550f2dB9f471", "https://www.blockchain.com/btc/address/1HuxXriPE2Bbnag3jJrqa3bkNHrs297dYH"]
+custom: ["https://hcb.hackclub.com/donations/start/archivebox", "https://paypal.me/NicholasSweeting"]

.github/ISSUE_TEMPLATE/documentation_change.md

@@ -6,6 +6,7 @@ labels: ''
 assignees: ''
 
 ---
+<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
 
 ## Wiki Page URL
 <!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/"
+    target-branch: "dev"
+    schedule:
+      interval: "weekly"

.github/workflows/codeql-analysis.yml

@@ -1,32 +0,0 @@
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ dev ]
-  pull_request:
-    branches: [ dev ]
-  schedule:
-    - cron: '43 1 * * 2'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'python' ]
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        queries: security-extended
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+  schedule:
+    - cron: '33 17 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/debian.yml

@@ -25,36 +25,23 @@ jobs:
             dh-python debhelper devscripts dput software-properties-common \
             python3-distutils python3-setuptools python3-wheel python3-stdeb
 
-      - name: Build Debian/Apt sdist_dsc
-        run: |
-          rm -Rf deb_dist/*
-          python3 setup.py --command-packages=stdeb.command sdist_dsc
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
 
-      - name: Build Debian/Apt bdist_deb
-        run: |
-          python3 setup.py --command-packages=stdeb.command bdist_deb
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
 
-      - name: Install archivebox from deb
-        run: |
-          cd deb_dist/
-          sudo apt-get install ./archivebox*.deb
-          cd ..
-          python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > ./requirements.txt
-          python3 -m pip install -r ./requirements.txt
-
-      - name: Check ArchiveBox version
-        run: |
-          # must create dir needed for snaps to run as non-root on github actions
-          sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-          mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-          archivebox --version
-          archivebox init --setup
-
-      - name: Add some links to test
-        run: |
-          cd "${{ github.workspace }}/data"
-          archivebox add 'https://example.com'
-          archivebox status
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
 
       # - name: Commit built package
       #   run: |

.github/workflows/docker.yml

@@ -12,7 +12,6 @@ on:
 env:
   DOCKER_IMAGE: archivebox-ci
 
-
 jobs:
   buildx:
     runs-on: ubuntu-latest
@@ -25,21 +24,21 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-      
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
-      
+          platforms: linux/amd64,linux/arm64
+
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
-      
+
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
-      
+
       - name: Cache Docker layers
         uses: actions/cache@v3
         with:
@@ -52,22 +51,27 @@ jobs:
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
-           username: ${{ secrets.DOCKER_USERNAME }}
-           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Collect Docker tags
+        # https://github.com/docker/metadata-action
         id: docker_meta
         uses: docker/metadata-action@v5
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          flavor: |
-              latest=auto
           tags: |
+              # :stable
               type=ref,event=branch
+              # :0.7.3
               type=semver,pattern={{version}}
+              # :0.7
               type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
               type=sha
-      
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+
       - name: Build and push
         id: docker_build
         uses: docker/build-push-action@v5
@@ -78,8 +82,25 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          platforms: linux/amd64,linux/arm64
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
+
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/jekyll-gh-pages.yml

@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/pip.yml

@@ -19,19 +19,26 @@ jobs:
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: '3.11'
           architecture: x64
 
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
       - uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
       - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --group :all --no-self
+        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
 
       - name: Build package
         run: |
-          rm ./dist/archivebox-*.whl
           pdm build
 
       - name: Install from build
@@ -45,8 +52,8 @@ jobs:
           archivebox version
           archivebox status
 
-      - name: Publish package distributions to PyPI
-        run: pdm publish --no-build
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
 
       # - name: Push build to PyPI
       #   run: |

.github/workflows/test.yml

@@ -13,11 +13,12 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
-        python: [3.9]
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
@@ -29,13 +30,16 @@ jobs:
           python-version: ${{ matrix.python }}
           architecture: x64
 
-      - name: Set up Node JS 14.7.0
-        uses: actions/setup-node@v3
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          node-version: 18.12.0
+          node-version: 20.10.0
 
       - name: Setup PDM
         uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
       ### Install Python & JS Dependencies
       - name: Get pip cache dir
@@ -44,7 +48,7 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-pip
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -52,11 +56,16 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
       - name: Install pip dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel pytest bottle build
-          ./bin/build_pip.sh
-          pdm install
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
 
       - name: Get npm cache dir
         id: npm-cache
@@ -64,7 +73,7 @@ jobs:
           echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
 
       - name: Cache npm
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-npm
         with:
           path: ${{ steps.npm-cache.outputs.dir }}
@@ -99,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1

.gitignore

@@ -12,6 +12,10 @@ venv/
 .docker-venv/
 node_modules/
 
+# Ignore dev lockfiles (should always be built fresh)
+requirements-dev.txt
+pdm.dev.lock
+
 # Packaging artifacts
 .pdm-python
 .pdm-build
@@ -22,10 +26,9 @@ dist/
 
 # Data folders
 data/
-data1/
-data2/
-data3/
+data*/
 output/
 
 # vim
 *.sw?
+.vscode/

.readthedocs.yaml

@@ -0,0 +1,33 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

Dockerfile

@@ -10,20 +10,34 @@
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
 #
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
 
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
 FROM python:3.11-slim-bookworm
+# Uses Debian 12 w/ faster-updating apt-lists added below: https://packages.debian.org/bookworm-backports/
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one personal internet archiving container" \
+    description="All-in-one self-hosted internet archiving solution" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
-
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
+    org.opencontainers.image.title="ArchiveBox" \
+    org.opencontainers.image.vendor="ArchiveBox" \
+    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
+    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'
+    
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
@@ -45,7 +59,7 @@ ENV TZ=UTC \
 
 # Version config
 ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=21
+    NODE_VERSION=20
 
 # User config
 ENV ARCHIVEBOX_USER="archivebox" \
@@ -73,7 +87,8 @@ COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
 RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
 
 # Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
-RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
 
 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
 RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
@@ -112,9 +127,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep \
+        zlib1g-dev dumb-init gosu cron unzip grep ncat \
         # 3. frivolous CLI helpers to make debugging failed archiving easier
-        # nano iputils-ping dnsutils htop procps jq yq
+        # nano iputils-ping dnsutils htop procps jq yq \
     && rm -rf /var/lib/apt/lists/*
 
 ######### Language Environments ####################################
@@ -123,7 +138,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
     && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
     && apt-get update -qq \
     && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
         nodejs libatomic1 python3-minimal \
@@ -166,15 +181,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         curl wget git yt-dlp ffmpeg ripgrep \
         # Packages we have also needed in the past:
         # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-        # fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
     && rm -rf /var/lib/apt/lists/* \
     # Save version info
     && ( \
         which curl && curl --version | head -n1 \
-        && which wget && wget --version | head -n1 \
-        && which yt-dlp && yt-dlp --version | head -n1 \
-        && which git && git --version | head -n1 \
-        && which rg && rg --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
@@ -182,6 +196,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
     && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
     && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
         # install Chromium using playwright
         pip install playwright \
@@ -189,25 +208,28 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         && playwright install --with-deps chromium \
         && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
     else \
-        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.) 
-        apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-            chromium fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        && export CHROME_BINARY="$(which chromium)"; \
+        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.)
+        # apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        #     chromium \
+        # && export CHROME_BINARY="$(which chromium)"; \
+        echo 'armv7 no longer supported in versions after v0.7.3' \
+        exit 1; \
     fi \
     && rm -rf /var/lib/apt/lists/* \
     && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
     && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
     && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
     && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
     # Save version info
     && ( \
-        which chromium-browser && /usr/bin/chromium-browser --version \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
 # Install Node dependencies
 WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
 RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
     && npm ci --prefer-offline --no-audit --cache /root/.npm \
@@ -221,9 +243,9 @@ RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGE
 
 # Install ArchiveBox Python dependencies
 WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR/"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \ 
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
     && apt-get update -qq \
     && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
         build-essential \
@@ -238,7 +260,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && pip install -r requirements.txt \
     && apt-get purge -y \
         build-essential \
-        # these are only needed to build CPython libs, we discard after build phase to shrink layer size
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
@@ -246,24 +267,30 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    && apt-get update -qq \
+    # && apt-get update -qq \
     # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-        build-essential  \
+    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    #     build-essential  \
     # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
     && pip install -e "$CODE_DIR"[sonic,ldap] \
     # save docker image size and always remove compilers / build tools after building is complete
-    && apt-get purge -y build-essential \
-    && apt-get autoremove -y \
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
 ####################################################
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chromium \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no \
+    ALLOWED_HOSTS=*
     ## No need to set explicitly, these values will be autodetected by archivebox in docker:
-    # CHROME_SANDBOX=False \
     # WGET_BINARY="wget" \
     # YOUTUBEDL_BINARY="yt-dlp" \
     # CHROME_BINARY="/usr/bin/chromium-browser" \
@@ -276,11 +303,10 @@ ENV IN_DOCKER=True
 
 # Print version for nice docker finish summary
 RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ}\n\n" \
-    && "$CODE_DIR/bin/docker_entrypoint.sh" \
-        archivebox version 2>&1 \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
 
 ####################################################
 
@@ -289,9 +315,8 @@ WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
-# Optional:
-# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-#     CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
+HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
 CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

archivebox/api/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class APIConfig(AppConfig):
+    name = 'api'

archivebox/api/archive.py

@@ -0,0 +1,184 @@
+# archivebox_api.py
+from typing import List, Optional
+from enum import Enum
+from pydantic import BaseModel
+from ninja import Router
+from main import (
+    add,
+    remove,
+    update,
+    list_all,
+    ONLY_NEW,
+)  # Assuming these functions are defined in main.py
+
+
+# Schemas
+
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'
+
+
+class AddURLSchema(BaseModel):
+    urls: List[str]
+    tag: str = ""
+    depth: int = 0
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
+    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+
+class RemoveURLSchema(BaseModel):
+    yes: bool = False
+    delete: bool = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    filter_type: str = "exact"
+    filter_patterns: Optional[List[str]] = None
+
+
+class UpdateSchema(BaseModel):
+    resume: Optional[float] = None
+    only_new: Optional[bool] = None
+    index_only: Optional[bool] = False
+    overwrite: Optional[bool] = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    status: Optional[StatusChoices] = None
+    filter_type: Optional[str] = 'exact'
+    filter_patterns: Optional[List[str]] = None
+    extractors: Optional[str] = ""
+
+
+class ListAllSchema(BaseModel):
+    filter_patterns: Optional[List[str]] = None
+    filter_type: str = 'exact'
+    status: Optional[StatusChoices] = None
+    after: Optional[float] = None
+    before: Optional[float] = None
+    sort: Optional[str] = None
+    csv: Optional[str] = None
+    json: bool = False
+    html: bool = False
+    with_headers: bool = False
+
+
+# API Router
+router = Router()
+
+
+@router.post("/add", response={200: dict})
+def api_add(request, payload: AddURLSchema):
+    try:
+        result = add(
+            urls=payload.urls,
+            tag=payload.tag,
+            depth=payload.depth,
+            update=payload.update,
+            update_all=payload.update_all,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            init=payload.init,
+            extractors=payload.extractors,
+            parser=payload.parser,
+        )
+        # Currently the add function returns a list of ALL items in the DB, ideally only return new items
+        return {
+            "status": "success",
+            "message": "URLs added successfully.",
+            "result": str(result),
+        }
+    except Exception as e:
+        # Handle exceptions raised by the add function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/remove", response={200: dict})
+def api_remove(request, payload: RemoveURLSchema):
+    try:
+        result = remove(
+            yes=payload.yes,
+            delete=payload.delete,
+            before=payload.before,
+            after=payload.after,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+        )
+        return {
+            "status": "success",
+            "message": "URLs removed successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the remove function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/update", response={200: dict})
+def api_update(request, payload: UpdateSchema):
+    try:
+        result = update(
+            resume=payload.resume,
+            only_new=payload.only_new,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            before=payload.before,
+            after=payload.after,
+            status=payload.status,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+            extractors=payload.extractors,
+        )
+        return {
+            "status": "success",
+            "message": "Archive updated successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the update function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/list_all", response={200: dict})
+def api_list_all(request, payload: ListAllSchema):
+    try:
+        result = list_all(
+            filter_patterns=payload.filter_patterns,
+            filter_type=payload.filter_type,
+            status=payload.status,
+            after=payload.after,
+            before=payload.before,
+            sort=payload.sort,
+            csv=payload.csv,
+            json=payload.json,
+            html=payload.html,
+            with_headers=payload.with_headers,
+        )
+        # TODO: This is kind of bad, make the format a choice field
+        if payload.json:
+            return {"status": "success", "format": "json", "data": result}
+        elif payload.html:
+            return {"status": "success", "format": "html", "data": result}
+        elif payload.csv:
+            return {"status": "success", "format": "csv", "data": result}
+        else:
+            return {
+                "status": "success",
+                "message": "List generated successfully.",
+                "data": result,
+            }
+    except Exception as e:
+        # Handle exceptions raised by the list_all function or during processing
+        return {"status": "error", "message": str(e)}

archivebox/api/auth.py

@@ -0,0 +1,48 @@
+from django.contrib.auth import authenticate
+from ninja import Form, Router, Schema
+from ninja.security import HttpBearer
+
+from api.models import Token
+
+router = Router()
+
+
+class GlobalAuth(HttpBearer):
+    def authenticate(self, request, token):
+        try:
+            return Token.objects.get(token=token).user
+        except Token.DoesNotExist:
+            pass
+
+
+class AuthSchema(Schema):
+    email: str
+    password: str
+
+
+@router.post("/authenticate", auth=None)  # overriding global auth
+def get_token(request, auth_data: AuthSchema):
+    user = authenticate(username=auth_data.email, password=auth_data.password)
+    if user:
+        # Assuming a user can have multiple tokens and you want to create a new one every time
+        new_token = Token.objects.create(user=user)
+        return {"token": new_token.token, "expires": new_token.expiry_as_iso8601}
+    else:
+        return {"error": "Invalid credentials"}
+
+
+class TokenValidationSchema(Schema):
+    token: str
+
+
+@router.post("/validate_token", auth=None) # No authentication required for this endpoint
+def validate_token(request, token_data: TokenValidationSchema):
+    try:
+        # Attempt to authenticate using the provided token
+        user = GlobalAuth().authenticate(request, token_data.token)
+        if user:
+            return {"status": "valid"}
+        else:
+            return {"status": "invalid"}
+    except Token.DoesNotExist:
+        return {"status": "invalid"}

archivebox/api/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.14 on 2024-04-09 18:52
+
+import api.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Token',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('token', models.CharField(default=auth.models.hex_uuid, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expiry', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tokens', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

archivebox/api/models.py

@@ -0,0 +1,30 @@
+import uuid
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+def hex_uuid():
+    return uuid.uuid4().hex
+
+
+class Token(models.Model):
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tokens"
+    )
+    token = models.CharField(max_length=32, default=hex_uuid, unique=True)
+    created = models.DateTimeField(auto_now_add=True)
+    expiry = models.DateTimeField(null=True, blank=True)
+
+    @property
+    def expiry_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = (
+            self.expiry if self.expiry else timezone.now() + timedelta(days=365 * 100)
+        )
+        return expiry_date.isoformat()
+
+    def __str__(self):
+        return self.token

archivebox/api/tests.py

@@ -0,0 +1,27 @@
+from django.test import TestCase
+from ninja.testing import TestClient
+from archivebox.api.archive import router as archive_router
+
+class ArchiveBoxAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(archive_router)
+
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "test"})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue("success" in response.json()["status"])

archivebox/cli/archivebox_schedule.py

@@ -39,6 +39,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=None,
         help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
     )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
     parser.add_argument(
         '--depth', # '-d',
         type=int,
@@ -97,6 +103,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         run_all=command.run_all,
         quiet=command.quiet,
         every=command.every,
+        tag=command.tag,
         depth=command.depth,
         overwrite=command.overwrite,
         update=command.update,

archivebox/config.py

@@ -30,6 +30,7 @@ import inspect
 import getpass
 import platform
 import shutil
+import requests
 import django
 from sqlite3 import dbapi2 as sqlite3
 
@@ -37,7 +38,7 @@ from hashlib import md5
 from pathlib import Path
 from datetime import datetime, timezone
 from typing import Optional, Type, Tuple, Dict, Union, List
-from subprocess import run, PIPE, DEVNULL
+from subprocess import run, PIPE, STDOUT, DEVNULL
 from configparser import ConfigParser
 from collections import defaultdict
 import importlib.metadata
@@ -51,25 +52,6 @@ from .config_stubs import (
 )
 
 
-### Pre-Fetch Minimal System Config
-
-SYSTEM_USER = getpass.getuser() or os.getlogin()
-
-try:
-    import pwd
-    SYSTEM_USER = pwd.getpwuid(os.geteuid()).pw_name or SYSTEM_USER
-except KeyError:
-    # Process' UID might not map to a user in cases such as running the Docker image
-    # (where `archivebox` is 999) as a different UID.
-    pass
-except ModuleNotFoundError:
-    # pwd is only needed for some linux systems, doesn't exist on windows
-    pass
-except Exception:
-    # this should never happen, uncomment to debug
-    # raise
-    pass
-
 ############################### Config Schema ##################################
 
 CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
@@ -81,7 +63,6 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'IN_QEMU':                  {'type': bool,  'default': False},
         'PUID':                     {'type': int,   'default': os.getuid()},
         'PGID':                     {'type': int,   'default': os.getgid()},
-        # TODO: 'SHOW_HINTS':       {'type:  bool,  'default': True},
     },
 
     'GENERAL_CONFIG': {
@@ -131,6 +112,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'LDAP_FIRSTNAME_ATTR':       {'type': str,   'default': None},
         'LDAP_LASTNAME_ATTR':        {'type': str,   'default': None},
         'LDAP_EMAIL_ATTR':           {'type': str,   'default': None},
+        'LDAP_CREATE_SUPERUSER':      {'type': bool,  'default': False},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -141,6 +123,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SAVE_SINGLEFILE':          {'type': bool,  'default': True, 'aliases': ('FETCH_SINGLEFILE',)},
         'SAVE_READABILITY':         {'type': bool,  'default': True, 'aliases': ('FETCH_READABILITY',)},
         'SAVE_MERCURY':             {'type': bool,  'default': True, 'aliases': ('FETCH_MERCURY',)},
+        'SAVE_HTMLTOTEXT':          {'type': bool,  'default': True, 'aliases': ('FETCH_HTMLTOTEXT',)},
         'SAVE_PDF':                 {'type': bool,  'default': True, 'aliases': ('FETCH_PDF',)},
         'SAVE_SCREENSHOT':          {'type': bool,  'default': True, 'aliases': ('FETCH_SCREENSHOT',)},
         'SAVE_DOM':                 {'type': bool,  'default': True, 'aliases': ('FETCH_DOM',)},
@@ -154,14 +137,15 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
     },
 
     'ARCHIVE_METHOD_OPTIONS': {
-        'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION',)},
-        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com'},
+        'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION','WINDOW_SIZE')},
+        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht'},
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
         'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'USER_AGENT':               {'type': str,   'default': None},
+        'CURL_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
@@ -169,7 +153,11 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CHROME_TIMEOUT':           {'type': int,   'default': 0},
         'CHROME_HEADLESS':          {'type': bool,  'default': True},
         'CHROME_SANDBOX':           {'type': bool,  'default': lambda c: not c['IN_DOCKER']},
+        'CHROME_EXTRA_ARGS':        {'type': list,  'default': None},
+
         'YOUTUBEDL_ARGS':           {'type': list,  'default': lambda c: [
+                                                                '--restrict-filenames',
+                                                                '--trim-filenames', '128',
                                                                 '--write-description',
                                                                 '--write-info-json',
                                                                 '--write-annotations',
@@ -191,6 +179,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--add-metadata',
                                                                 '--format=(bv*+ba/b)[filesize<={}][filesize_approx<=?{}]/(bv*+ba/b)'.format(c['MEDIA_MAX_SIZE'], c['MEDIA_MAX_SIZE']),
                                                                 ]},
+        'YOUTUBEDL_EXTRA_ARGS':     {'type': list,  'default': None},
 
 
         'WGET_ARGS':                {'type': list,  'default': ['--no-verbose',
@@ -202,12 +191,17 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--no-parent',
                                                                 '-e', 'robots=off',
                                                                 ]},
+        'WGET_EXTRA_ARGS':          {'type': list,  'default': None},
         'CURL_ARGS':                {'type': list,  'default': ['--silent',
                                                                 '--location',
                                                                 '--compressed'
                                                                ]},
+        'CURL_EXTRA_ARGS':          {'type': list,  'default': None},
         'GIT_ARGS':                 {'type': list,  'default': ['--recursive']},
-        'SINGLEFILE_ARGS':          {'type': list,  'default' : None},
+        'SINGLEFILE_ARGS':          {'type': list,  'default': None},
+        'SINGLEFILE_EXTRA_ARGS':    {'type': list,  'default': None},
+        'MERCURY_ARGS':             {'type': list,  'default': ['--format=text']},
+        'MERCURY_EXTRA_ARGS':       {'type': list,  'default': None},
         'FAVICON_PROVIDER':         {'type': str,   'default': 'https://www.google.com/s2/favicons?domain={}'},
     },
 
@@ -218,6 +212,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SEARCH_BACKEND_HOST_NAME': {'type': str,   'default': 'localhost'},
         'SEARCH_BACKEND_PORT':      {'type': int,   'default': 1491},
         'SEARCH_BACKEND_PASSWORD':  {'type': str,   'default': 'SecretPassword'},
+        'SEARCH_PROCESS_HTML':      {'type': bool,  'default': True},
         # SONIC
         'SONIC_COLLECTION':         {'type': str,   'default': 'archivebox'},
         'SONIC_BUCKET':             {'type': str,   'default': 'snapshots'},
@@ -255,7 +250,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'POCKET_CONSUMER_KEY':      {'type': str,   'default': None},
         'POCKET_ACCESS_TOKENS':     {'type': dict,  'default': {}},
 
-        'READWISE_READER_TOKENS':     {'type': dict,  'default': {}},
+        'READWISE_READER_TOKENS':   {'type': dict,  'default': {}},
     },
 }
 
@@ -286,6 +281,7 @@ TEMPLATES_DIR_NAME = 'templates'
 ARCHIVE_DIR_NAME = 'archive'
 SOURCES_DIR_NAME = 'sources'
 LOGS_DIR_NAME = 'logs'
+PERSONAS_DIR_NAME = 'personas'
 SQL_INDEX_FILENAME = 'index.sqlite3'
 JSON_INDEX_FILENAME = 'index.json'
 HTML_INDEX_FILENAME = 'index.html'
@@ -359,9 +355,11 @@ ALLOWED_IN_OUTPUT_DIR = {
     'static',
     'sonic',
     'search.sqlite3',
+    'crontabs',
     ARCHIVE_DIR_NAME,
     SOURCES_DIR_NAME,
     LOGS_DIR_NAME,
+    PERSONAS_DIR_NAME,
     SQL_INDEX_FILENAME,
     f'{SQL_INDEX_FILENAME}-wal',
     f'{SQL_INDEX_FILENAME}-shm',
@@ -374,23 +372,134 @@ ALLOWED_IN_OUTPUT_DIR = {
     'static_index.json',
 }
 
-def get_version(config):
-    return importlib.metadata.version(__package__ or 'archivebox')
-
-def get_commit_hash(config):
-    try:
-        return list((config['PACKAGE_DIR'] / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
-    except Exception:
-        return None
-
-############################## Derived Config ##################################
-
 
 ALLOWDENYLIST_REGEX_FLAGS: int = re.IGNORECASE | re.UNICODE | re.MULTILINE
 
+
+############################## Version Config ##################################
+
+def get_system_user() -> str:
+    # some host OS's are unable to provide a username (k3s, Windows), making this complicated
+    # uid 999 is especially problematic and breaks many attempts
+    SYSTEM_USER = None
+    FALLBACK_USER_PLACHOLDER = f'user_{os.getuid()}'
+
+    # Option 1
+    try:
+        import pwd
+        SYSTEM_USER = SYSTEM_USER or pwd.getpwuid(os.geteuid()).pw_name
+    except (ModuleNotFoundError, Exception):
+        pass
+
+    # Option 2
+    try:
+        SYSTEM_USER = SYSTEM_USER or getpass.getuser()
+    except Exception:
+        pass
+
+    # Option 3
+    try:
+        SYSTEM_USER = SYSTEM_USER or os.getlogin()
+    except Exception:
+        pass
+
+    return SYSTEM_USER or FALLBACK_USER_PLACHOLDER
+
+def get_version(config):
+    try:
+        return importlib.metadata.version(__package__ or 'archivebox')
+    except importlib.metadata.PackageNotFoundError:
+        try:
+            pyproject_config = (config['PACKAGE_DIR'] / 'pyproject.toml').read_text()
+            for line in pyproject_config:
+                if line.startswith('version = '):
+                    return line.split(' = ', 1)[-1].strip('"')
+        except FileNotFoundError:
+            # building docs, pyproject.toml is not available
+            return 'dev'
+
+    raise Exception('Failed to detect installed archivebox version!')
+
+def get_commit_hash(config) -> Optional[str]:
+    try:
+        git_dir = config['PACKAGE_DIR'] / '../.git'
+        ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
+        commit_hash = git_dir.joinpath(ref).read_text().strip()
+        return commit_hash
+    except Exception:
+        pass
+
+    try:
+        return list((config['PACKAGE_DIR'] / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
+    except Exception:
+        pass
+    
+    return None
+
+def get_build_time(config) -> str:
+    if config['IN_DOCKER']:
+        docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
+        return docker_build_end_time
+
+    src_last_modified_unix_timestamp = (config['PACKAGE_DIR'] / 'config.py').stat().st_mtime
+    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
+
+def get_versions_available_on_github(config):
+    """
+    returns a dictionary containing the ArchiveBox GitHub release info for
+    the recommended upgrade version and the currently installed version
+    """
+    
+    # we only want to perform the (relatively expensive) check for new versions
+    # when its most relevant, e.g. when the user runs a long-running command
+    subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
+    long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
+    if subcommand_run_by_user not in long_running_commands:
+        return None
+    
+    github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
+    response = requests.get(github_releases_api)
+    if response.status_code != 200:
+        stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
+        return None
+    all_releases = response.json()
+
+    installed_version = parse_version_string(config['VERSION'])
+
+    # find current version or nearest older version (to link to)
+    current_version = None
+    for idx, release in enumerate(all_releases):
+        release_version = parse_version_string(release['tag_name'])
+        if release_version <= installed_version:
+            current_version = release
+            break
+
+    current_version = current_version or all_releases[-1]
+    
+    # recommended version is whatever comes after current_version in the release list
+    # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
+    try:
+        recommended_version = all_releases[idx+1]
+    except IndexError:
+        recommended_version = None
+
+    return {'recommended_version': recommended_version, 'current_version': current_version}
+
+def can_upgrade(config):
+    if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
+        recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
+        current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
+        return recommended_version > current_version
+    return False
+
+
+############################## Derived Config ##################################
+
+# These are derived/computed values calculated *after* all user-provided config values are ingested
+# they appear in `archivebox config` output and are intended to be read-only for the user
 DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'TERM_WIDTH':               {'default': lambda c: lambda: shutil.get_terminal_size((100, 10)).columns},
-    'USER':                     {'default': lambda c: SYSTEM_USER},
+    'USER':                     {'default': lambda c: get_system_user()},
     'ANSI':                     {'default': lambda c: DEFAULT_CLI_COLORS if c['USE_COLOR'] else {k: '' for k in DEFAULT_CLI_COLORS.keys()}},
 
     'PACKAGE_DIR':              {'default': lambda c: Path(__file__).resolve().parent},
@@ -401,17 +510,23 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'ARCHIVE_DIR':              {'default': lambda c: c['OUTPUT_DIR'] / ARCHIVE_DIR_NAME},
     'SOURCES_DIR':              {'default': lambda c: c['OUTPUT_DIR'] / SOURCES_DIR_NAME},
     'LOGS_DIR':                 {'default': lambda c: c['OUTPUT_DIR'] / LOGS_DIR_NAME},
+    'PERSONAS_DIR':             {'default': lambda c: c['OUTPUT_DIR'] / PERSONAS_DIR_NAME},
     'CONFIG_FILE':              {'default': lambda c: Path(c['CONFIG_FILE']).resolve() if c['CONFIG_FILE'] else c['OUTPUT_DIR'] / CONFIG_FILENAME},
     'COOKIES_FILE':             {'default': lambda c: c['COOKIES_FILE'] and Path(c['COOKIES_FILE']).resolve()},
-    'CHROME_USER_DATA_DIR':     {'default': lambda c: find_chrome_data_dir() if c['CHROME_USER_DATA_DIR'] is None else (Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None)},   # None means unset, so we autodetect it with find_chrome_Data_dir(), but emptystring '' means user manually set it to '', and we should store it as None
+    'CHROME_USER_DATA_DIR':     {'default': lambda c: Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None},
     'URL_DENYLIST_PTN':         {'default': lambda c: c['URL_DENYLIST'] and re.compile(c['URL_DENYLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
     'URL_ALLOWLIST_PTN':        {'default': lambda c: c['URL_ALLOWLIST'] and re.compile(c['URL_ALLOWLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
-    'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},
+    'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},  # exec is always needed to list directories
 
     'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0] or bin_path('archivebox')},
-    'VERSION':                  {'default': lambda c: get_version(c)},
-    'COMMIT_HASH':              {'default': lambda c: get_commit_hash(c)},
+
+    'VERSION':                  {'default': lambda c: get_version(c).split('+', 1)[0]},     # remove +editable from user-displayed version string
+    'COMMIT_HASH':              {'default': lambda c: get_commit_hash(c)},                  # short git commit hash of codebase HEAD commit
+    'BUILD_TIME':               {'default': lambda c: get_build_time(c)},                   # docker build completed time or python src last modified time
     
+    'VERSIONS_AVAILABLE':       {'default': lambda c: get_versions_available_on_github(c)},
+    'CAN_UPGRADE':              {'default': lambda c: can_upgrade(c)},
+
     'PYTHON_BINARY':            {'default': lambda c: sys.executable},
     'PYTHON_ENCODING':          {'default': lambda c: sys.stdout.encoding.upper()},
     'PYTHON_VERSION':           {'default': lambda c: '{}.{}.{}'.format(*sys.version_info[:3])},
@@ -421,13 +536,14 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     
     'SQLITE_BINARY':            {'default': lambda c: inspect.getfile(sqlite3)},
     'SQLITE_VERSION':           {'default': lambda c: sqlite3.version},
-    #'SQLITE_JOURNAL_MODE':      {'default': lambda c: 'wal'},         # set at runtime below, interesting but unused for now
+    #'SQLITE_JOURNAL_MODE':      {'default': lambda c: 'wal'},         # set at runtime below, interesting if changed later but unused for now because its always expected to be wal
     #'SQLITE_OPTIONS':           {'default': lambda c: ['JSON1']},     # set at runtime below
 
     'USE_CURL':                 {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])},
     'CURL_VERSION':             {'default': lambda c: bin_version(c['CURL_BINARY']) if c['USE_CURL'] else None},
     'CURL_USER_AGENT':          {'default': lambda c: c['CURL_USER_AGENT'].format(**c)},
     'CURL_ARGS':                {'default': lambda c: c['CURL_ARGS'] or []},
+    'CURL_EXTRA_ARGS':          {'default': lambda c: c['CURL_EXTRA_ARGS'] or []},
     'SAVE_FAVICON':             {'default': lambda c: c['USE_CURL'] and c['SAVE_FAVICON']},
     'SAVE_ARCHIVE_DOT_ORG':     {'default': lambda c: c['USE_CURL'] and c['SAVE_ARCHIVE_DOT_ORG']},
 
@@ -438,18 +554,22 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'SAVE_WGET':                {'default': lambda c: c['USE_WGET'] and c['SAVE_WGET']},
     'SAVE_WARC':                {'default': lambda c: c['USE_WGET'] and c['SAVE_WARC']},
     'WGET_ARGS':                {'default': lambda c: c['WGET_ARGS'] or []},
+    'WGET_EXTRA_ARGS':          {'default': lambda c: c['WGET_EXTRA_ARGS'] or []},
 
     'RIPGREP_VERSION':          {'default': lambda c: bin_version(c['RIPGREP_BINARY']) if c['USE_RIPGREP'] else None},
 
     'USE_SINGLEFILE':           {'default': lambda c: c['USE_SINGLEFILE'] and c['SAVE_SINGLEFILE']},
     'SINGLEFILE_VERSION':       {'default': lambda c: bin_version(c['SINGLEFILE_BINARY']) if c['USE_SINGLEFILE'] else None},
     'SINGLEFILE_ARGS':          {'default': lambda c: c['SINGLEFILE_ARGS'] or []},
+    'SINGLEFILE_EXTRA_ARGS':    {'default': lambda c: c['SINGLEFILE_EXTRA_ARGS'] or []},
 
     'USE_READABILITY':          {'default': lambda c: c['USE_READABILITY'] and c['SAVE_READABILITY']},
     'READABILITY_VERSION':      {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None},
 
     'USE_MERCURY':              {'default': lambda c: c['USE_MERCURY'] and c['SAVE_MERCURY']},
     'MERCURY_VERSION':          {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None},  # mercury doesnt expose version info until this is merged https://github.com/postlight/parser/pull/750
+    'MERCURY_ARGS':             {'default': lambda c: c['MERCURY_ARGS'] or []},
+    'MERCURY_EXTRA_ARGS':       {'default': lambda c: c['MERCURY_EXTRA_ARGS'] or []},
 
     'USE_GIT':                  {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']},
     'GIT_VERSION':              {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None},
@@ -459,10 +579,12 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'YOUTUBEDL_VERSION':        {'default': lambda c: bin_version(c['YOUTUBEDL_BINARY']) if c['USE_YOUTUBEDL'] else None},
     'SAVE_MEDIA':               {'default': lambda c: c['USE_YOUTUBEDL'] and c['SAVE_MEDIA']},
     'YOUTUBEDL_ARGS':           {'default': lambda c: c['YOUTUBEDL_ARGS'] or []},
+    'YOUTUBEDL_EXTRA_ARGS':     {'default': lambda c: c['YOUTUBEDL_EXTRA_ARGS'] or []},
 
     'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] or find_chrome_binary()},
     'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and c['CHROME_BINARY'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
     'CHROME_VERSION':           {'default': lambda c: bin_version(c['CHROME_BINARY']) if c['USE_CHROME'] else None},
+    'CHROME_USER_AGENT':        {'default': lambda c: c['CHROME_USER_AGENT'].format(**c)},
 
     'SAVE_PDF':                 {'default': lambda c: c['USE_CHROME'] and c['SAVE_PDF']},
     'SAVE_SCREENSHOT':          {'default': lambda c: c['USE_CHROME'] and c['SAVE_SCREENSHOT']},
@@ -479,8 +601,9 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'EXTERNAL_LOCATIONS':       {'default': lambda c: get_external_locations(c)},
     'DATA_LOCATIONS':           {'default': lambda c: get_data_locations(c)},
     'CHROME_OPTIONS':           {'default': lambda c: get_chrome_info(c)},
+    'CHROME_EXTRA_ARGS':        {'default': lambda c: c['CHROME_EXTRA_ARGS'] or []},
     'SAVE_ALLOWLIST_PTN':       {'default': lambda c: c['SAVE_ALLOWLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_ALLOWLIST'].items()}},
-    'SAVE_DENYLIST_PTN':       {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
+    'SAVE_DENYLIST_PTN':        {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
 }
 
 
@@ -496,47 +619,60 @@ def load_config_val(key: str,
                     config_file_vars: Optional[Dict[str, str]]=None) -> ConfigValue:
     """parse bool, int, and str key=value pairs from env"""
 
+    assert isinstance(config, dict)
 
+    is_read_only = type is None
+    if is_read_only:
+        if callable(default):
+            return default(config)
+        return default
+
+    # get value from environment variables or config files
     config_keys_to_check = (key, *(aliases or ()))
+    val = None
     for key in config_keys_to_check:
         if env_vars:
             val = env_vars.get(key)
             if val:
                 break
+
         if config_file_vars:
             val = config_file_vars.get(key)
             if val:
                 break
 
-    if type is None or val is None:
+    is_unset = val is None
+    if is_unset:
         if callable(default):
-            assert isinstance(config, dict)
             return default(config)
-
         return default
 
-    elif type is bool:
-        if val.lower() in ('true', 'yes', '1'):
+    # calculate value based on expected type
+    BOOL_TRUEIES = ('true', 'yes', '1')
+    BOOL_FALSEIES = ('false', 'no', '0')
+
+    if type is bool:
+        if val.lower() in BOOL_TRUEIES:
             return True
-        elif val.lower() in ('false', 'no', '0'):
+        elif val.lower() in BOOL_FALSEIES:
             return False
         else:
             raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)')
 
     elif type is str:
-        if val.lower() in ('true', 'false', 'yes', 'no', '1', '0'):
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a string)')
+        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
+            raise ValueError(f'Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)')
         return val.strip()
 
     elif type is int:
-        if not val.isdigit():
+        if not val.strip().isdigit():
             raise ValueError(f'Invalid configuration option {key}={val} (expected an integer)')
-        return int(val)
+        return int(val.strip())
 
     elif type is list or type is dict:
         return json.loads(val)
 
-    raise Exception('Config values can only be str, bool, int or json')
+    raise Exception('Config values can only be str, bool, int, or json')
 
 
 def load_config_file(out_dir: str=None) -> Optional[Dict[str, str]]:
@@ -678,9 +814,11 @@ def load_config(defaults: ConfigDefaultDict,
 
     return extended_config
 
-# def write_config(config: ConfigDict):
 
-#     with open(os.path.join(config['OUTPUT_DIR'], CONFIG_FILENAME), 'w+') as f:
+def parse_version_string(version: str) -> Tuple[int, int, int]:
+    """parses a version tag string formatted like 'vx.x.x' into (major, minor, patch) ints"""
+    base = version.split('+')[0].split('v')[-1] # remove 'v' prefix and '+editable' suffix
+    return tuple(int(part) for part in base.split('.'))[:3]
 
 
 # Logging Helpers
@@ -716,7 +854,7 @@ def hint(text: Union[Tuple[str, ...], List[str], str], prefix='    ', config: Op
 
 
 # Dependency Metadata Helpers
-def bin_version(binary: Optional[str]) -> Optional[str]:
+def bin_version(binary: Optional[str], cmd=None) -> Optional[str]:
     """check the presence and return valid version line of a specified binary"""
 
     abspath = bin_path(binary)
@@ -725,11 +863,21 @@ def bin_version(binary: Optional[str]) -> Optional[str]:
 
     try:
         bin_env = os.environ | {'LANG': 'C'}
-        version_str = run([abspath, "--version"], stdout=PIPE, env=bin_env).stdout.strip().decode()
+        is_cmd_str = cmd and isinstance(cmd, str)
+        version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT, env=bin_env).stdout.strip().decode()
         if not version_str:
-            version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
-        # take first 3 columns of first line of version info
-        return ' '.join(version_str.split('\n')[0].strip().split()[:3])
+            version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT).stdout.strip().decode()
+        
+        version_ptn = re.compile(r"\d+?\.\d+?\.?\d*", re.MULTILINE)
+        try:
+            version_nums = version_ptn.findall(version_str.split('\n')[0])[0]
+            if version_nums:
+                return version_nums
+            else:
+                raise IndexError
+        except IndexError:
+            # take first 3 columns of first line of version info
+            return ' '.join(version_str.split('\n')[0].strip().split()[:3])
     except OSError:
         pass
         # stderr(f'[X] Unable to find working version of dependency: {binary}', color='red')
@@ -769,6 +917,7 @@ def find_chrome_binary() -> Optional[str]:
     # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
     # make sure data dir finding precedence order always matches binary finding order
     default_executable_paths = (
+        # '~/Library/Caches/ms-playwright/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium',
         'chromium-browser',
         'chromium',
         '/Applications/Chromium.app/Contents/MacOS/Chromium',
@@ -791,27 +940,36 @@ def find_chrome_binary() -> Optional[str]:
 
 def find_chrome_data_dir() -> Optional[str]:
     """find any installed chrome user data directories in the default locations"""
-    # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
-    # make sure data dir finding precedence order always matches binary finding order
-    default_profile_paths = (
-        '~/.config/chromium',
-        '~/Library/Application Support/Chromium',
-        '~/AppData/Local/Chromium/User Data',
-        '~/.config/chrome',
-        '~/.config/google-chrome',
-        '~/Library/Application Support/Google/Chrome',
-        '~/AppData/Local/Google/Chrome/User Data',
-        '~/.config/google-chrome-stable',
-        '~/.config/google-chrome-beta',
-        '~/Library/Application Support/Google/Chrome Canary',
-        '~/AppData/Local/Google/Chrome SxS/User Data',
-        '~/.config/google-chrome-unstable',
-        '~/.config/google-chrome-dev',
-    )
-    for path in default_profile_paths:
-        full_path = Path(path).resolve()
-        if full_path.exists():
-            return full_path
+    # deprecated because this is DANGEROUS, do not re-implement/uncomment this behavior.
+
+    # Going forward we want to discourage people from using their main chrome profile for archiving.
+    # Session tokens, personal data, and cookies are often returned in server responses,
+    # when they get archived, they are essentially burned as anyone who can view the archive
+    # can use that data to masquerade as the logged-in user that did the archiving.
+    # For this reason users should always create dedicated burner profiles for archiving and not use
+    # their daily driver main accounts.
+
+    # # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
+    # # make sure data dir finding precedence order always matches binary finding order
+    # default_profile_paths = (
+    #     '~/.config/chromium',
+    #     '~/Library/Application Support/Chromium',
+    #     '~/AppData/Local/Chromium/User Data',
+    #     '~/.config/chrome',
+    #     '~/.config/google-chrome',
+    #     '~/Library/Application Support/Google/Chrome',
+    #     '~/AppData/Local/Google/Chrome/User Data',
+    #     '~/.config/google-chrome-stable',
+    #     '~/.config/google-chrome-beta',
+    #     '~/Library/Application Support/Google/Chrome Canary',
+    #     '~/AppData/Local/Google/Chrome SxS/User Data',
+    #     '~/.config/google-chrome-unstable',
+    #     '~/.config/google-chrome-dev',
+    # )
+    # for path in default_profile_paths:
+    #     full_path = Path(path).resolve()
+    #     if full_path.exists():
+    #         return full_path
     return None
 
 def wget_supports_compression(config):
@@ -882,6 +1040,11 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'enabled': True,
             'is_valid': config['LOGS_DIR'].exists(),
         },
+        'PERSONAS_DIR': {
+            'path': config['PERSONAS_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['PERSONAS_DIR'].exists(),
+        },
         'ARCHIVE_DIR': {
             'path': config['ARCHIVE_DIR'].resolve(),
             'enabled': True,
@@ -1059,9 +1222,9 @@ globals().update(CONFIG)
 
 
 # Set timezone to UTC and umask to OUTPUT_PERMISSIONS
-assert TIMEZONE == 'UTC', 'The server timezone should always be set to UTC'  # we may allow this to change later
-os.environ["TZ"] = TIMEZONE
-os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8))  # noqa: F821
+assert TIMEZONE == 'UTC', 'The server timezone should always be set to UTC'  # noqa: F821
+os.environ["TZ"] = TIMEZONE                                                  # noqa: F821
+os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8))                        # noqa: F821
 
 # add ./node_modules/.bin to $PATH so we can use node scripts in extractors
 NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))
@@ -1078,7 +1241,6 @@ sys.path.append(NODE_BIN_PATH)
 # disable stderr "you really shouldnt disable ssl" warnings with library config
 if not CONFIG['CHECK_SSL_VALIDITY']:
     import urllib3
-    import requests
     requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -1095,14 +1257,25 @@ if not CONFIG['CHECK_SSL_VALIDITY']:
 
 def check_system_config(config: ConfigDict=CONFIG) -> None:
     ### Check system environment
-    if config['USER'] == 'root':
+    if config['USER'] == 'root' or str(config['PUID']) == "0":
         stderr('[!] ArchiveBox should never be run as root!', color='red')
         stderr('    For more information, see the security overview documentation:')
         stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root')
+        
+        if config['IN_DOCKER']:
+            attempted_command = ' '.join(sys.argv[:3])
+            stderr('')
+            stderr('    {lightred}Hint{reset}: When using Docker, you must run commands with {green}docker run{reset} instead of {lightyellow}docker exec{reset}, e.g.:'.format(**config['ANSI']))
+            stderr(f'        docker compose run archivebox {attempted_command}')
+            stderr(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}')
+            stderr('        or:')
+            stderr(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"')
+            stderr(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"')
+        
         raise SystemExit(2)
 
     ### Check Python environment
-    if sys.version_info[:3] < (3, 6, 0):
+    if sys.version_info[:3] < (3, 7, 0):
         stderr(f'[X] Python version is not new enough: {config["PYTHON_VERSION"]} (>3.6 is required)', color='red')
         stderr('    See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.')
         raise SystemExit(2)
@@ -1178,7 +1351,7 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None:
 
     if config['USE_YOUTUBEDL'] and config['MEDIA_TIMEOUT'] < 20:
         stderr(f'[!] Warning: MEDIA_TIMEOUT is set too low! (currently set to MEDIA_TIMEOUT={config["MEDIA_TIMEOUT"]} seconds)', color='red')
-        stderr('    Youtube-dl will fail to archive all media if set to less than ~20 seconds.')
+        stderr('    youtube-dl/yt-dlp will fail to archive any media if set to less than ~20 seconds.')
         stderr('    (Setting it somewhere over 60 seconds is recommended)')
         stderr()
         stderr('    If you want to disable media archiving entirely, set SAVE_MEDIA=False instead:')
@@ -1219,6 +1392,8 @@ def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CO
 
     (Path(output_dir) / SOURCES_DIR_NAME).mkdir(exist_ok=True)
     (Path(output_dir) / LOGS_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / PERSONAS_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / PERSONAS_DIR_NAME / 'Default').mkdir(exist_ok=True)
 
 
 
@@ -1266,8 +1441,7 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
         with open(settings.ERROR_LOG, "a", encoding='utf-8') as f:
             command = ' '.join(sys.argv)
             ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
-            f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
-
+            f.write(f"\n> {command}; TS={ts} VERSION={config['VERSION']} IN_DOCKER={config['IN_DOCKER']} IS_TTY={config['IS_TTY']}\n")
 
         if check_db:
             # Enable WAL mode in sqlite3

archivebox/config_stubs.py

@@ -9,6 +9,8 @@ SimpleConfigValueDict = Dict[str, SimpleConfigValue]
 SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
 ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
 
+SHArgs = List[str]   # shell command args list e.g. ["--something=1", "--someotherarg"]
+
 
 class BaseConfig(TypedDict):
     pass
@@ -16,10 +18,10 @@ class BaseConfig(TypedDict):
 class ConfigDict(BaseConfig, total=False):
     """
     # Regenerate by pasting this quine into `archivebox shell` 🥚
-    from archivebox.config import ConfigDict, CONFIG_DEFAULTS
+    from archivebox.config import ConfigDict, CONFIG_SCHEMA
     print('class ConfigDict(BaseConfig, total=False):')
     print('    ' + '"'*3 + ConfigDict.__doc__ + '"'*3)
-    for section, configs in CONFIG_DEFAULTS.items():
+    for section, configs in CONFIG_SCHEMA.items():
         for key, attrs in configs.items():
             Type, default = attrs['type'], attrs['default']
             if default is None:
@@ -32,16 +34,23 @@ class ConfigDict(BaseConfig, total=False):
     USE_COLOR: bool
     SHOW_PROGRESS: bool
     IN_DOCKER: bool
+    IN_QEMU: bool
+    PUID: int
+    PGID: int
 
-    PACKAGE_DIR: Path
-    OUTPUT_DIR: Path
-    CONFIG_FILE: Path
+    OUTPUT_DIR: Optional[str]
+    CONFIG_FILE: Optional[str]
     ONLY_NEW: bool
     TIMEOUT: int
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
     RESTRICT_FILE_NAMES: str
     URL_DENYLIST: str
+    URL_ALLOWLIST: Optional[str]
+    ADMIN_USERNAME: Optional[str]
+    ADMIN_PASSWORD: Optional[str]
+    ENFORCE_ATOMIC_WRITES: bool
+    TAG_SEPARATOR_PATTERN: str
 
     SECRET_KEY: Optional[str]
     BIND_ADDR: str
@@ -49,7 +58,27 @@ class ConfigDict(BaseConfig, total=False):
     DEBUG: bool
     PUBLIC_INDEX: bool
     PUBLIC_SNAPSHOTS: bool
+    PUBLIC_ADD_VIEW: bool
     FOOTER_INFO: str
+    SNAPSHOTS_PER_PAGE: int
+    CUSTOM_TEMPLATES_DIR: Optional[str]
+    TIME_ZONE: str
+    TIMEZONE: str
+    REVERSE_PROXY_USER_HEADER: str
+    REVERSE_PROXY_WHITELIST: str
+    LOGOUT_REDIRECT_URL: str
+    PREVIEW_ORIGINALS: bool
+    LDAP: bool
+    LDAP_SERVER_URI: Optional[str]
+    LDAP_BIND_DN: Optional[str]
+    LDAP_BIND_PASSWORD: Optional[str]
+    LDAP_USER_BASE: Optional[str]
+    LDAP_USER_FILTER: Optional[str]
+    LDAP_USERNAME_ATTR: Optional[str]
+    LDAP_FIRSTNAME_ATTR: Optional[str]
+    LDAP_LASTNAME_ATTR: Optional[str]
+    LDAP_EMAIL_ATTR: Optional[str]
+    LDAP_CREATE_SUPERUSER: bool
 
     SAVE_TITLE: bool
     SAVE_FAVICON: bool
@@ -58,25 +87,50 @@ class ConfigDict(BaseConfig, total=False):
     SAVE_SINGLEFILE: bool
     SAVE_READABILITY: bool
     SAVE_MERCURY: bool
+    SAVE_HTMLTOTEXT: bool
     SAVE_PDF: bool
     SAVE_SCREENSHOT: bool
     SAVE_DOM: bool
+    SAVE_HEADERS: bool
     SAVE_WARC: bool
     SAVE_GIT: bool
     SAVE_MEDIA: bool
     SAVE_ARCHIVE_DOT_ORG: bool
+    SAVE_ALLOWLIST: dict
+    SAVE_DENYLIST: dict
 
     RESOLUTION: str
     GIT_DOMAINS: str
     CHECK_SSL_VALIDITY: bool
+    MEDIA_MAX_SIZE: str
     CURL_USER_AGENT: str
     WGET_USER_AGENT: str
     CHROME_USER_AGENT: str
-    COOKIES_FILE: Union[str, Path, None]
-    CHROME_USER_DATA_DIR: Union[str, Path, None]
+    COOKIES_FILE: Optional[str]
+    CHROME_USER_DATA_DIR: Optional[str]
     CHROME_TIMEOUT: int
     CHROME_HEADLESS: bool
     CHROME_SANDBOX: bool
+    YOUTUBEDL_ARGS: list
+    WGET_ARGS: list
+    CURL_ARGS: list
+    GIT_ARGS: list
+    SINGLEFILE_ARGS: Optional[list]
+    FAVICON_PROVIDER: str
+
+    USE_INDEXING_BACKEND: bool
+    USE_SEARCHING_BACKEND: bool
+    SEARCH_BACKEND_ENGINE: str
+    SEARCH_BACKEND_HOST_NAME: str
+    SEARCH_BACKEND_PORT: int
+    SEARCH_BACKEND_PASSWORD: str
+    SEARCH_PROCESS_HTML: bool
+    SONIC_COLLECTION: str
+    SONIC_BUCKET: str
+    SEARCH_BACKEND_TIMEOUT: int
+    FTS_SEPARATE_DATABASE: bool
+    FTS_TOKENIZERS: str
+    FTS_SQLITE_MAX_LENGTH: int
 
     USE_CURL: bool
     USE_WGET: bool
@@ -85,7 +139,9 @@ class ConfigDict(BaseConfig, total=False):
     USE_MERCURY: bool
     USE_GIT: bool
     USE_CHROME: bool
+    USE_NODE: bool
     USE_YOUTUBEDL: bool
+    USE_RIPGREP: bool
     CURL_BINARY: str
     GIT_BINARY: str
     WGET_BINARY: str
@@ -93,13 +149,12 @@ class ConfigDict(BaseConfig, total=False):
     READABILITY_BINARY: str
     MERCURY_BINARY: str
     YOUTUBEDL_BINARY: str
+    NODE_BINARY: str
+    RIPGREP_BINARY: str
     CHROME_BINARY: Optional[str]
-
-    YOUTUBEDL_ARGS: List[str]
-    WGET_ARGS: List[str]
-    CURL_ARGS: List[str]
-    GIT_ARGS: List[str]
-    TAG_SEPARATOR_PATTERN: str
+    POCKET_CONSUMER_KEY: Optional[str]
+    POCKET_ACCESS_TOKENS: dict
+    READWISE_READER_TOKENS: dict
 
 
 ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

archivebox/core/__init__.py

@@ -1 +1,3 @@
 __package__ = 'archivebox.core'
+
+# default_app_config = 'core.apps.CoreAppConfig'

archivebox/core/admin.py

@@ -6,11 +6,13 @@ from contextlib import redirect_stdout
 from datetime import datetime, timezone
 
 from django.contrib import admin
+from django.db.models import Count
 from django.urls import path
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group, Permission
 from django import forms
 
 from ..util import htmldecode, urldecode, ansi_to_html
@@ -23,8 +25,16 @@ from core.mixins import SearchResultsAdminMixin
 from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
-from config import OUTPUT_DIR, SNAPSHOTS_PER_PAGE
 from extractors import archive_links
+from config import (
+    OUTPUT_DIR,
+    SNAPSHOTS_PER_PAGE,
+    VERSION,
+    VERSIONS_AVAILABLE,
+    CAN_UPGRADE
+)
+
+GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
 
 # Admin URLs
 # /admin/
@@ -39,6 +49,60 @@ from extractors import archive_links
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
 
+class ArchiveBoxAdmin(admin.AdminSite):
+    site_header = 'ArchiveBox'
+    index_title = 'Links'
+    site_title = 'Index'
+    namespace = 'admin'
+
+    def get_urls(self):
+        return [
+            path('core/snapshot/add/', self.add_view, name='Add'),
+        ] + super().get_urls()
+
+    def add_view(self, request):
+        if not request.user.is_authenticated:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        request.current_app = self.name
+        context = {
+            **self.each_context(request),
+            'title': 'Add URLs',
+        }
+
+        if request.method == 'GET':
+            context['form'] = AddLinkForm()
+
+        elif request.method == 'POST':
+            form = AddLinkForm(request.POST)
+            if form.is_valid():
+                url = form.cleaned_data["url"]
+                print(f'[+] Adding URL: {url}')
+                depth = 0 if form.cleaned_data["depth"] == "0" else 1
+                input_kwargs = {
+                    "urls": url,
+                    "depth": depth,
+                    "update_all": False,
+                    "out_dir": OUTPUT_DIR,
+                }
+                add_stdout = StringIO()
+                with redirect_stdout(add_stdout):
+                   add(**input_kwargs)
+                print(add_stdout.getvalue())
+
+                context.update({
+                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
+                    "form": AddLinkForm()
+                })
+            else:
+                context["form"] = form
+
+        return render(template_name='add.html', request=request, context=context)
+
+archivebox_admin = ArchiveBoxAdmin()
+archivebox_admin.register(get_user_model())
+archivebox_admin.disable_action('delete_selected')
+
 class ArchiveResultInline(admin.TabularInline):
     model = ArchiveResult
 
@@ -51,6 +115,8 @@ from django.contrib.admin.widgets import AutocompleteSelectMultiple
 class AutocompleteTags:
     model = Tag
     search_fields = ['name']
+    name = 'tags'
+    remote_field = TagInline
 
 class AutocompleteTagsAdminStub:
     name = 'admin'
@@ -78,6 +144,7 @@ class SnapshotActionForm(ActionForm):
     # )
 
 
+@admin.register(Snapshot, site=archivebox_admin)
 class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     list_display = ('added', 'title_str', 'files', 'size', 'url_str')
     sort_fields = ('title_str', 'url_str', 'added', 'files')
@@ -93,6 +160,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
     action_form = SnapshotActionForm
 
+
+    def changelist_view(self, request, extra_context=None):
+        extra_context = extra_context or {}
+        return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
+
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
@@ -102,7 +174,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
     def get_queryset(self, request):
         self.request = request
-        return super().get_queryset(request).prefetch_related('tags')
+        return super().get_queryset(request).prefetch_related('tags').annotate(archiveresult_count=Count('archiveresult'))
 
     def tag_list(self, obj):
         return ', '.join(obj.tags.values_list('name', flat=True))
@@ -160,6 +232,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             obj.id,
         )
 
+    @admin.display(
+        description='Title',
+        ordering='title',
+    )
     def title_str(self, obj):
         canon = obj.as_link().canonical_outputs()
         tags = ''.join(
@@ -181,12 +257,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
         ) + mark_safe(f' <span class="tags">{tags}</span>')
 
+    @admin.display(
+        description='Files Saved',
+        ordering='archiveresult_count',
+    )
     def files(self, obj):
         return snapshot_icons(obj)
 
-    files.admin_order_field = 'updated'
-    files.short_description = 'Files Saved'
 
+    @admin.display(
+        ordering='archiveresult_count'
+    )
     def size(self, obj):
         archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
         if archive_size:
@@ -201,8 +282,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             size_txt,
         )
 
-    size.admin_order_field = 'archiveresult__count'
 
+    @admin.display(
+        description='Original URL',
+        ordering='url',
+    )
     def url_str(self, obj):
         return format_html(
             '<a href="{}"><code style="user-select: all;">{}</code></a>',
@@ -239,65 +323,76 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     #     print('[*] Got request', request.method, request.POST)
     #     return super().changelist_view(request, extra_context=None)
 
+    @admin.action(
+        description="Pull"
+    )
     def update_snapshots(self, request, queryset):
         archive_links([
             snapshot.as_link()
             for snapshot in queryset
         ], out_dir=OUTPUT_DIR)
-    update_snapshots.short_description = "Pull"
 
+    @admin.action(
+        description="⬇️ Title"
+    )
     def update_titles(self, request, queryset):
         archive_links([
             snapshot.as_link()
             for snapshot in queryset
         ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
-    update_titles.short_description = "⬇️ Title"
 
+    @admin.action(
+        description="Re-Snapshot"
+    )
     def resnapshot_snapshot(self, request, queryset):
         for snapshot in queryset:
             timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
             new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
             add(new_url, tag=snapshot.tags_str())
-    resnapshot_snapshot.short_description = "Re-Snapshot"
 
+    @admin.action(
+        description="Reset"
+    )
     def overwrite_snapshots(self, request, queryset):
         archive_links([
             snapshot.as_link()
             for snapshot in queryset
         ], overwrite=True, out_dir=OUTPUT_DIR)
-    overwrite_snapshots.short_description = "Reset"
 
+    @admin.action(
+        description="Delete"
+    )
     def delete_snapshots(self, request, queryset):
         remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
 
-    delete_snapshots.short_description = "Delete"
 
+    @admin.action(
+        description="+"
+    )
     def add_tags(self, request, queryset):
         tags = request.POST.getlist('tags')
         print('[+] Adding tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.add(*tags)
 
-    add_tags.short_description = "+"
 
+    @admin.action(
+        description="–"
+    )
     def remove_tags(self, request, queryset):
         tags = request.POST.getlist('tags')
         print('[-] Removing tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.remove(*tags)
 
-    remove_tags.short_description = "–"
 
         
 
-    title_str.short_description = 'Title'
-    url_str.short_description = 'Original URL'
-
-    title_str.admin_order_field = 'title'
-    url_str.admin_order_field = 'url'
 
 
 
+
+@admin.register(Tag, site=archivebox_admin)
 class TagAdmin(admin.ModelAdmin):
     list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
     sort_fields = ('id', 'name', 'slug')
@@ -328,6 +423,7 @@ class TagAdmin(admin.ModelAdmin):
         ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={obj.id}">and {total_count-10} more...<a>' if obj.snapshot_set.count() > 10 else ''))
 
 
+@admin.register(ArchiveResult, site=archivebox_admin)
 class ArchiveResultAdmin(admin.ModelAdmin):
     list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
@@ -340,6 +436,9 @@ class ArchiveResultAdmin(admin.ModelAdmin):
     ordering = ['-start_ts']
     list_per_page = SNAPSHOTS_PER_PAGE
 
+    @admin.display(
+        description='snapshot'
+    )
     def snapshot_str(self, obj):
         return format_html(
             '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
@@ -349,6 +448,9 @@ class ArchiveResultAdmin(admin.ModelAdmin):
             obj.snapshot.url[:128],
         )
 
+    @admin.display(
+        description='tags'
+    )
     def tags_str(self, obj):
         return obj.snapshot.tags_str()
 
@@ -365,62 +467,3 @@ class ArchiveResultAdmin(admin.ModelAdmin):
             obj.output if (obj.status == 'succeeded') and obj.extractor not in ('title', 'archive_org') else 'index.html',
             obj.output,
         )
-
-    tags_str.short_description = 'tags'
-    snapshot_str.short_description = 'snapshot'
-
-class ArchiveBoxAdmin(admin.AdminSite):
-    site_header = 'ArchiveBox'
-    index_title = 'Links'
-    site_title = 'Index'
-
-    def get_urls(self):
-        return [
-            path('core/snapshot/add/', self.add_view, name='Add'),
-        ] + super().get_urls()
-
-    def add_view(self, request):
-        if not request.user.is_authenticated:
-            return redirect(f'/admin/login/?next={request.path}')
-
-        request.current_app = self.name
-        context = {
-            **self.each_context(request),
-            'title': 'Add URLs',
-        }
-
-        if request.method == 'GET':
-            context['form'] = AddLinkForm()
-
-        elif request.method == 'POST':
-            form = AddLinkForm(request.POST)
-            if form.is_valid():
-                url = form.cleaned_data["url"]
-                print(f'[+] Adding URL: {url}')
-                depth = 0 if form.cleaned_data["depth"] == "0" else 1
-                input_kwargs = {
-                    "urls": url,
-                    "depth": depth,
-                    "update_all": False,
-                    "out_dir": OUTPUT_DIR,
-                }
-                add_stdout = StringIO()
-                with redirect_stdout(add_stdout):
-                   add(**input_kwargs)
-                print(add_stdout.getvalue())
-
-                context.update({
-                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
-                    "form": AddLinkForm()
-                })
-            else:
-                context["form"] = form
-
-        return render(template_name='add.html', request=request, context=context)
-
-admin.site = ArchiveBoxAdmin()
-admin.site.register(get_user_model())
-admin.site.register(Snapshot, SnapshotAdmin)
-admin.site.register(Tag, TagAdmin)
-admin.site.register(ArchiveResult, ArchiveResultAdmin)
-admin.site.disable_action('delete_selected')

archivebox/core/apps.py

@@ -1,6 +1,17 @@
 from django.apps import AppConfig
 
 
-class CoreConfig(AppConfig):
+class CoreAppConfig(AppConfig):
     name = 'core'
-    default_auto_field = 'django.db.models.UUIDField'
+
+    # label = 'Archive Data'
+    verbose_name = "Archive Data"
+
+    # WIP: broken by Django 3.1.2 -> 4.0 migration
+    # default_auto_field = 'django.db.models.UUIDField'
+
+
+    def ready(self):
+        from .auth import register_signals
+
+        register_signals()

archivebox/core/auth.py

@@ -0,0 +1,15 @@
+__package__ = 'archivebox.core'
+
+
+
+from ..config import (
+    LDAP
+)
+
+def register_signals():
+
+    if LDAP:
+        import django_auth_ldap.backend
+        from .auth_ldap import create_user
+
+        django_auth_ldap.backend.populate_user.connect(create_user)

archivebox/core/auth_ldap.py

@@ -0,0 +1,12 @@
+from django.conf import settings
+from ..config import (
+    LDAP_CREATE_SUPERUSER
+)
+
+def create_user(sender, user=None, ldap_user=None, **kwargs):
+
+    if not user.id and LDAP_CREATE_SUPERUSER:
+        user.is_superuser = True
+
+    user.is_staff = True
+    print(f'[!] WARNING: Creating new user {user} based on LDAP user {ldap_user} (is_staff={user.is_staff}, is_superuser={user.is_superuser})')

archivebox/core/migrations/0022_auto_20231023_2008.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.14 on 2023-10-23 20:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0021_auto_20220914_0934'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('htmltotext', 'htmltotext'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+        ),
+    ]

archivebox/core/mixins.py

@@ -18,4 +18,4 @@ class SearchResultsAdminMixin:
             print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
             messages.add_message(request, messages.WARNING, f'Error from the search backend, only showing results from default admin search fields - Error: {err}')
         
-        return qs, use_distinct
+        return qs.distinct(), use_distinct

archivebox/core/models.py

@@ -50,7 +50,7 @@ class Tag(models.Model):
 
     class Meta:
         verbose_name = "Tag"
-        verbose_name_plural = "Tags"
+        verbose_name_plural = "🏷️ Tags"
 
     def __str__(self):
         return self.name
@@ -98,6 +98,10 @@ class Snapshot(models.Model):
 
     keys = ('url', 'timestamp', 'title', 'tags', 'updated')
 
+    class Meta:
+        verbose_name = "Snapshot"
+        verbose_name_plural = "⭐️ Archived Webpages (Snapshots)"
+
     def __repr__(self) -> str:
         title = self.title or '-'
         return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
@@ -282,5 +286,9 @@ class ArchiveResult(models.Model):
 
     objects = ArchiveResultManager()
 
+    class Meta:
+        verbose_name = "ArchiveResult"
+        verbose_name_plural = "📑 Logs (ArchiveResults)"
+
     def __str__(self):
         return self.extractor

archivebox/core/settings.py

@@ -1,5 +1,10 @@
 __package__ = 'archivebox.core'
 
+# TODO: add this after we upgrade to Django >=3.2
+# https://github.com/typeddjango/django-stubs
+# import django_stubs_ext
+# django_stubs_ext.monkeypatch()
+
 import os
 import sys
 import re
@@ -59,12 +64,88 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
+    'solo',
+
 
     'core',
+    'api',
+    
+    # Plugins
+    
+    'plugins.defaults',
+    'plugins.system',
+    # 'plugins.replaywebpage',           # provides UI to view WARC files
+    # 'plugins.gallerydl',               # provides gallerydl dependency + extractor
+    # 'plugins.browsertrix',             # provides browsertrix dependency + extractor
+    # 'plugins.playwright',              # provides playwright dependency
+    # ...
+    # someday we may have enough plugins to justify dynamic loading:
+    # *(path.parent.name for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/apps.py')),,
 
     'django_extensions',
 ]
 
+################################################################################
+### Staticfile and Template Settings
+################################################################################
+
+STATIC_URL = '/static/'
+
+STATIC_ROOT = Path(PACKAGE_DIR) / 'collected_static'
+
+STATICFILES_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
+
+    # Plugins
+    # str(Path(PACKAGE_DIR) / 'plugins/defaults/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/gallerydl/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/browsertrix/static'),
+    # str(Path(PACKAGE_DIR) / 'plugins/playwright/static'),
+    # ...
+    # someday if there are many more plugins / user-addable plugins:
+    # *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/static')),
+]
+
+MEDIA_URL = '/archive/'
+MEDIA_ROOT = OUTPUT_DIR / 'archive'
+
+
+TEMPLATE_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
+    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
+
+    # Plugins
+    # added by plugins.<PluginName>.apps.<AppName>.ready -> .settings.register_plugin_settings
+    # str(Path(PACKAGE_DIR) / 'plugins/defaults/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/gallerydl/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/browsertrix/templates'),
+    # str(Path(PACKAGE_DIR) / 'plugins/playwright/templates'),
+    # ...
+    #
+    # someday if there are many more plugins / user-addable plugins:
+    # *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/templates')),
+]
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': TEMPLATE_DIRS,
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
 
 # For usage with https://www.jetadmin.io/integrations/django
 # INSTALLED_APPS += ['jet_django']
@@ -124,6 +205,7 @@ if LDAP:
         }
 
         AUTHENTICATION_BACKENDS = [
+            'django.contrib.auth.backends.ModelBackend',
             'django_auth_ldap.backend.LDAPBackend',
         ]
     except ModuleNotFoundError:
@@ -161,7 +243,7 @@ if DEBUG_TOOLBAR:
         'debug_toolbar.panels.request.RequestPanel',
         'debug_toolbar.panels.sql.SQLPanel',
         'debug_toolbar.panels.staticfiles.StaticFilesPanel',
-        # 'debug_toolbar.panels.templates.TemplatesPanel',
+        # 'debug_toolbar.panels.templates.TemplatesPanel',    # buggy/slow
         'debug_toolbar.panels.cache.CachePanel',
         'debug_toolbar.panels.signals.SignalsPanel',
         'debug_toolbar.panels.logging.LoggingPanel',
@@ -171,39 +253,6 @@ if DEBUG_TOOLBAR:
     ]
     MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
 
-################################################################################
-### Staticfile and Template Settings
-################################################################################
-
-STATIC_URL = '/static/'
-
-STATICFILES_DIRS = [
-    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
-]
-
-TEMPLATE_DIRS = [
-    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
-    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
-]
-
-TEMPLATES = [
-    {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': TEMPLATE_DIRS,
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
-            ],
-        },
-    },
-]
 
 
 ################################################################################
@@ -268,7 +317,6 @@ AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
 ]
 
-
 ################################################################################
 ### Shell Settings
 ################################################################################
@@ -287,7 +335,6 @@ if IS_SHELL:
 
 LANGUAGE_CODE = 'en-us'
 USE_I18N = True
-USE_L10N = True
 USE_TZ = True
 DATETIME_FORMAT = 'Y-m-d g:iA'
 SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
@@ -312,21 +359,21 @@ IGNORABLE_404_URLS = [
 ]
 
 class NoisyRequestsFilter(logging.Filter):
-    def filter(self, record):
+    def filter(self, record) -> bool:
         logline = record.getMessage()
 
         # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
         for ignorable_url_pattern in IGNORABLE_404_URLS:
             ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
             if ignorable_log_pattern.match(logline):
-                return 0
+                return False
 
         # ignore staticfile requests that 200 or 30*
         ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M)
         if ignoreable_200_log_pattern.match(logline):
-            return 0
+            return False
 
-        return 1
+        return True
 
 if LOGS_DIR.exists():
     ERROR_LOG = (LOGS_DIR / 'errors.log')

archivebox/core/urls.py

@@ -1,4 +1,4 @@
-from django.contrib import admin
+from .admin import archivebox_admin
 
 from django.urls import path, include
 from django.views import static
@@ -8,6 +8,17 @@ from django.views.generic.base import RedirectView
 
 from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
 
+from ninja import NinjaAPI
+from api.auth import GlobalAuth
+
+api = NinjaAPI(auth=GlobalAuth())
+api.add_router("/auth/", "api.auth.router")
+api.add_router("/archive/", "api.archive.router")
+
+# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
+# from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
+# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
+
 
 # print('DEBUG', settings.DEBUG)
 
@@ -21,6 +32,10 @@ urlpatterns = [
 
     path('archive/', RedirectView.as_view(url='/')),
     path('archive/<path:path>', SnapshotView.as_view(), name='Snapshot'),
+    path('web/<path:path>', SnapshotView.as_view()),   # support archive.org-style URLs
+
+    path('plugins/replaywebpage/', include('plugins.replaywebpage.urls')),
+    # ... dynamic load these someday if there are more of them
 
     path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
     path('add/', AddView.as_view(), name='add'),
@@ -30,7 +45,9 @@ urlpatterns = [
 
 
     path('accounts/', include('django.contrib.auth.urls')),
-    path('admin/', admin.site.urls),
+    path('admin/', archivebox_admin.urls),
+    
+    path("api/", api.urls),
 
     path('health/', HealthCheckView.as_view(), name='healthcheck'),
     path('error/', lambda _: 1/0),

archivebox/core/views.py

@@ -23,6 +23,7 @@ from ..config import (
     PUBLIC_SNAPSHOTS,
     PUBLIC_ADD_VIEW,
     VERSION,
+    COMMIT_HASH,
     FOOTER_INFO,
     SNAPSHOTS_PER_PAGE,
 )
@@ -55,12 +56,18 @@ class SnapshotView(View):
             slug, archivefile = path.split('/', 1)[0], 'index.html'
 
         # slug is a timestamp
-        if slug.replace('.','').isdigit():
+        if slug.replace('.', '').isdigit():
 
             # missing trailing slash -> redirect to index
             if '/' not in path:
                 return redirect(f'{path}/index.html')
 
+            # TODO: add support for archive.org-style URLs where timestamp may be a human-readable date
+            # https://web.archivebox.io / web / 2022-01 / https://example.com
+            # https://web.archivebox.io / web / 20220505103616 / https://example.com
+            # https://web.archivebox.io / web / 2022-05-05__0:36:16 / https://example.com
+            # use archivebox.util.parse_date (supports unix timestamps, iso date strings, and lots more etc.)
+
             try:
                 try:
                     snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
@@ -217,6 +224,7 @@ class PublicIndexView(ListView):
         return {
             **super().get_context_data(**kwargs),
             'VERSION': VERSION,
+            'COMMIT_HASH': COMMIT_HASH,
             'FOOTER_INFO': FOOTER_INFO,
         }
 
@@ -229,7 +237,7 @@ class PublicIndexView(ListView):
                 qs = qs | query_search_index(query)
             except Exception as err:
                 print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
-        return qs
+        return qs.distinct()
 
     def get(self, *args, **kwargs):
         if PUBLIC_INDEX or self.request.user.is_authenticated:

archivebox/extractors/__init__.py

@@ -37,6 +37,7 @@ from .wget import should_save_wget, save_wget
 from .singlefile import should_save_singlefile, save_singlefile
 from .readability import should_save_readability, save_readability
 from .mercury import should_save_mercury, save_mercury
+from .htmltotext import should_save_htmltotext, save_htmltotext
 from .pdf import should_save_pdf, save_pdf
 from .screenshot import should_save_screenshot, save_screenshot
 from .dom import should_save_dom, save_dom
@@ -59,14 +60,26 @@ def get_default_archive_methods() -> List[ArchiveMethodEntry]:
         ('screenshot', should_save_screenshot, save_screenshot),
         ('dom', should_save_dom, save_dom),
         ('wget', should_save_wget, save_wget),
-        ('title', should_save_title, save_title),                   # keep title and readability below wget and singlefile, as it depends on them
+        # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
+        ('title', should_save_title, save_title),
         ('readability', should_save_readability, save_readability),
         ('mercury', should_save_mercury, save_mercury),
+        ('htmltotext', should_save_htmltotext, save_htmltotext),
         ('git', should_save_git, save_git),
         ('media', should_save_media, save_media),
         ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
+ARCHIVE_METHODS_INDEXING_PRECEDENCE = [
+    ('readability', 1),
+    ('mercury', 2),
+    ('htmltotext', 3),
+    ('singlefile', 4),
+    ('dom', 5),
+    ('wget', 6)
+]
+
+
 @enforce_types
 def get_archive_methods_for_link(link: Link) -> Iterable[ArchiveMethodEntry]:
     DEFAULT_METHODS = get_default_archive_methods()
@@ -86,8 +99,6 @@ def get_archive_methods_for_link(link: Link) -> Iterable[ArchiveMethodEntry]:
 
     return (m for m in DEFAULT_METHODS if m[0] in allowed_methods)
 
-ARCHIVE_METHODS_INDEXING_PRECEDENCE = [('readability', 1), ('singlefile', 2), ('dom', 3), ('wget', 4)]
-
 @enforce_types
 def ignore_methods(to_ignore: List[str]) -> Iterable[str]:
     ARCHIVE_METHODS = get_default_archive_methods()
@@ -120,7 +131,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
 
         link = load_link_details(link, out_dir=out_dir)
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
-        log_link_archiving_started(link, out_dir, is_new)
+        log_link_archiving_started(link, str(out_dir), is_new)
         link = link.overwrite(updated=datetime.now(timezone.utc))
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
         start_ts = datetime.now(timezone.utc)
@@ -154,16 +165,6 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
                     stats['skipped'] += 1
             except Exception as e:
-                # Disabled until https://github.com/ArchiveBox/ArchiveBox/issues/984
-                # and https://github.com/ArchiveBox/ArchiveBox/issues/1014
-                # are fixed.
-                """
-                raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
-                    method_name,
-                    link.url,
-                )) from e
-                """
-                # Instead, use the kludgy workaround from
                 # https://github.com/ArchiveBox/ArchiveBox/issues/984#issuecomment-1150541627
                 with open(ERROR_LOG, "a", encoding='utf-8') as f:
                     command = ' '.join(sys.argv)
@@ -173,8 +174,15 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                         link.url,
                         command,
                         ts
-                    ) + "\n"))
+                    ) + "\n" + str(e) + "\n"))
                     #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
+               
+                # print(f'        ERROR: {method_name} {e.__class__.__name__}: {e} {getattr(e, "hints", "")}', ts, link.url, command)
+                raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
+                    method_name,
+                    link.url,
+                )) from e
+
 
         # print('    ', stats)
 
@@ -207,7 +215,7 @@ def archive_links(all_links: Union[Iterable[Link], QuerySet], overwrite: bool=Fa
 
     if type(all_links) is QuerySet:
         num_links: int = all_links.count()
-        get_link = lambda x: x.as_link()
+        get_link = lambda x: x.as_link_with_details()
         all_links = all_links.iterator()
     else:
         num_links: int = len(all_links)

archivebox/extractors/archive_org.py

@@ -10,10 +10,12 @@ from ..system import run, chmod_file
 from ..util import (
     enforce_types,
     is_static_file,
+    dedupe,
 )
 from ..config import (
     TIMEOUT,
     CURL_ARGS,
+    CURL_EXTRA_ARGS,
     CHECK_SSL_VALIDITY,
     SAVE_ARCHIVE_DOT_ORG,
     CURL_BINARY,
@@ -44,13 +46,18 @@ def save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, timeout: int=
     output: ArchiveOutput = 'archive.org.txt'
     archive_org_url = None
     submit_url = 'https://web.archive.org/save/{}'.format(link.url)
-    cmd = [
-        CURL_BINARY,
+    # later options take precedence
+    options = [
         *CURL_ARGS,
+        *CURL_EXTRA_ARGS,
         '--head',
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
+    ]
+    cmd = [
+        CURL_BINARY,
+        *dedupe(options),
         submit_url,
     ]
     status = 'succeeded'

archivebox/extractors/favicon.py

@@ -6,13 +6,18 @@ from typing import Optional
 
 from ..index.schema import Link, ArchiveResult, ArchiveOutput
 from ..system import chmod_file, run
-from ..util import enforce_types, domain
+from ..util import (
+    enforce_types,
+     domain,
+     dedupe,
+)
 from ..config import (
     TIMEOUT,
     SAVE_FAVICON,
     FAVICON_PROVIDER,
     CURL_BINARY,
     CURL_ARGS,
+    CURL_EXTRA_ARGS,
     CURL_VERSION,
     CHECK_SSL_VALIDITY,
     CURL_USER_AGENT,
@@ -34,13 +39,18 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
 
     out_dir = out_dir or link.link_dir
     output: ArchiveOutput = 'favicon.ico'
-    cmd = [
-        CURL_BINARY,
+    # later options take precedence
+    options = [
         *CURL_ARGS,
+        *CURL_EXTRA_ARGS,
         '--max-time', str(timeout),
         '--output', str(output),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
+    ]
+    cmd = [
+        CURL_BINARY,
+        *dedupe(options),
         FAVICON_PROVIDER.format(domain(link.url)),
     ]
     status = 'failed'

archivebox/extractors/headers.py

@@ -9,11 +9,13 @@ from ..system import atomic_write
 from ..util import (
     enforce_types,
     get_headers,
+    dedupe,
 )
 from ..config import (
     TIMEOUT,
     CURL_BINARY,
     CURL_ARGS,
+    CURL_EXTRA_ARGS,
     CURL_USER_AGENT,
     CURL_VERSION,
     CHECK_SSL_VALIDITY,
@@ -40,14 +42,18 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
-
-    cmd = [
-        CURL_BINARY,
+    # later options take precedence
+    options = [
         *CURL_ARGS,
+        *CURL_EXTRA_ARGS,
         '--head',
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
+    ]
+    cmd = [
+        CURL_BINARY,
+        *dedupe(options),
         link.url,
     ]
     try:

archivebox/extractors/htmltotext.py

@@ -0,0 +1,155 @@
+__package__ = 'archivebox.extractors'
+
+from html.parser import HTMLParser
+import io
+from pathlib import Path
+from typing import Optional
+
+from ..config import (
+    SAVE_HTMLTOTEXT,
+    TIMEOUT,
+    VERSION,
+)
+from ..index.schema import Link, ArchiveResult, ArchiveError
+from ..logging_util import TimedProgress
+from ..system import atomic_write
+from ..util import (
+    enforce_types,
+    is_static_file,
+)
+from .title import get_html
+
+class HTMLTextExtractor(HTMLParser):
+    TEXT_ATTRS = [
+        "alt", "cite", "href", "label",
+        "list", "placeholder", "title", "value"
+    ]
+    NOTEXT_TAGS = ["script", "style", "template"]
+    NOTEXT_HREF = ["data:", "javascript:", "#"]
+
+    def __init__(self):
+        super().__init__()
+
+        self.output = io.StringIO()
+        self._tag_stack = []
+
+    def _is_text_attr(self, name, value):
+        if not isinstance(value, str):
+            return False
+        if name == "href" and any(map(lambda p: value.startswith(p), self.NOTEXT_HREF)):
+            return False
+
+        if name in self.TEXT_ATTRS:
+            return True
+
+        return False
+
+    def _parent_tag(self):
+        try:
+            return self._tag_stack[-1]
+        except IndexError:
+            return None
+
+    def _in_notext_tag(self):
+        return any([t in self._tag_stack for t in self.NOTEXT_TAGS])
+
+    def handle_starttag(self, tag, attrs):
+        self._tag_stack.append(tag)
+
+        # Don't write out attribute values if any ancestor
+        # is in NOTEXT_TAGS
+        if self._in_notext_tag():
+            return
+
+        for name, value in attrs:
+            if self._is_text_attr(name, value):
+                self.output.write(f"({value.strip()}) ")
+
+    def handle_endtag(self, tag):
+        orig_stack = self._tag_stack.copy()
+        try:
+            # Keep popping tags until we find the nearest
+            # ancestor matching this end tag
+            while tag != self._tag_stack.pop():
+                pass
+            # Write a space after every tag, to ensure that tokens
+            # in tag text aren't concatenated. This may result in
+            # excess spaces, which should be ignored by search tokenizers.
+            if not self._in_notext_tag() and tag not in self.NOTEXT_TAGS:
+                self.output.write(" ")
+        except IndexError:
+            # Got to the top of the stack, but somehow missed
+            # this end tag -- maybe malformed markup -- restore the
+            # stack
+            self._tag_stack = orig_stack
+
+    def handle_data(self, data):
+        # Don't output text data if any ancestor is in NOTEXT_TAGS
+        if self._in_notext_tag():
+            return
+
+        data = data.lstrip()
+        len_before_rstrip = len(data)
+        data = data.rstrip()
+        spaces_rstripped = len_before_rstrip - len(data)
+        if data:
+            self.output.write(data)
+            if spaces_rstripped:
+                # Add back a single space if 1 or more
+                # whitespace characters were stripped
+                self.output.write(' ')
+
+    def __str__(self):
+        return self.output.getvalue()
+
+
+@enforce_types
+def should_save_htmltotext(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
+        return False
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'htmltotext.txt').exists():
+        return False
+
+    return SAVE_HTMLTOTEXT
+
+
+@enforce_types
+def save_htmltotext(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+    """extract search-indexing-friendly text from an HTML document"""
+
+    out_dir = Path(out_dir or link.link_dir)
+    output = "htmltotext.txt"
+    cmd = ['(internal) archivebox.extractors.htmltotext', './{singlefile,dom}.html']
+
+    timer = TimedProgress(timeout, prefix='      ')
+    extracted_text = None
+    status = 'failed'
+    try:
+        extractor = HTMLTextExtractor()
+        document = get_html(link, out_dir)
+
+        if not document:
+            raise ArchiveError('htmltotext could not find HTML to parse for article text')
+
+        extractor.feed(document)
+        extractor.close()
+        extracted_text = str(extractor)
+
+        atomic_write(str(out_dir / output), extracted_text)
+        status = 'succeeded'
+    except (Exception, OSError) as err:
+        output = err
+    finally:
+        timer.end()
+
+    return ArchiveResult(
+        cmd=cmd,
+        pwd=str(out_dir),
+        cmd_version=VERSION,
+        output=output,
+        status=status,
+        index_texts=[extracted_text] if extracted_text else [],
+        **timer.stats,  
+    )

archivebox/extractors/media.py

@@ -8,11 +8,13 @@ from ..system import run, chmod_file
 from ..util import (
     enforce_types,
     is_static_file,
+    dedupe,
 )
 from ..config import (
     MEDIA_TIMEOUT,
     SAVE_MEDIA,
     YOUTUBEDL_ARGS,
+    YOUTUBEDL_EXTRA_ARGS,
     YOUTUBEDL_BINARY,
     YOUTUBEDL_VERSION,
     CHECK_SSL_VALIDITY
@@ -39,11 +41,16 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
     output: ArchiveOutput = 'media'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
-    cmd = [
-        YOUTUBEDL_BINARY,
+    # later options take precedence
+    options = [
         *YOUTUBEDL_ARGS,
+        *YOUTUBEDL_EXTRA_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
         # TODO: add --cookies-from-browser={CHROME_USER_DATA_DIR}
+    ]
+    cmd = [
+        YOUTUBEDL_BINARY,
+        *dedupe(options),
         link.url,
     ]
     status = 'succeeded'

archivebox/extractors/mercury.py

@@ -11,13 +11,15 @@ from ..system import run, atomic_write
 from ..util import (
     enforce_types,
     is_static_file,
-
+    dedupe,
 )
 from ..config import (
     TIMEOUT,
     SAVE_MERCURY,
     DEPENDENCIES,
     MERCURY_VERSION,
+    MERCURY_ARGS,
+    MERCURY_EXTRA_ARGS,
 )
 from ..logging_util import TimedProgress
 
@@ -60,12 +62,16 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
     timer = TimedProgress(timeout, prefix='      ')
     try:
         output_folder.mkdir(exist_ok=True)
-
-        # Get plain text version of article
+        # later options take precedence
+        options = [
+            *MERCURY_ARGS,
+            *MERCURY_EXTRA_ARGS,
+        ]
+        # By default, get plain text version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
             link.url,
-            "--format=text"
+            *dedupe(options)
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:

archivebox/extractors/readability.py

@@ -67,13 +67,12 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
             temp_doc.name,
             link.url,
         ]
-
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:
             result_json = json.loads(result.stdout)
             assert result_json and 'content' in result_json, 'Readability output is not valid JSON'
         except json.JSONDecodeError:
-            raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)
+            raise ArchiveError('Readability was not able to archive the page (invalid JSON)', result.stdout + result.stderr)
 
         output_folder.mkdir(exist_ok=True)
         readability_content = result_json.pop("textContent") 
@@ -81,8 +80,6 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         atomic_write(str(output_folder / "content.txt"), readability_content)
         atomic_write(str(output_folder / "article.json"), result_json)
 
-        # parse out number of files downloaded from last line of stderr:
-        #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
         output_tail = [
             line.strip()
             for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
@@ -95,11 +92,13 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
 
         # Check for common failure cases
         if (result.returncode > 0):
-            raise ArchiveError('Readability was not able to archive the page', hints)
+            raise ArchiveError(f'Readability was not able to archive the page (status={result.returncode})', hints)
     except (Exception, OSError) as err:
         status = 'failed'
         output = err
-        cmd = [cmd[0], './{singlefile,dom}.html']
+
+        # prefer Chrome dom output to singlefile because singlefile often contains huge url(data:image/...base64) strings that make the html too long to parse with readability
+        cmd = [cmd[0], './{dom,singlefile}.html']
     finally:
         timer.end()
 

archivebox/extractors/singlefile.py

@@ -11,6 +11,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    dedupe,
 )
 from ..config import (
     TIMEOUT,
@@ -18,7 +19,9 @@ from ..config import (
     DEPENDENCIES,
     SINGLEFILE_VERSION,
     SINGLEFILE_ARGS,
+    SINGLEFILE_EXTRA_ARGS,
     CHROME_BINARY,
+    COOKIES_FILE,
 )
 from ..logging_util import TimedProgress
 
@@ -46,37 +49,24 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
 
     # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
     browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:]))
+    # later options take precedence
     options = [
-        *SINGLEFILE_ARGS,
         '--browser-executable-path={}'.format(CHROME_BINARY),
+        *(["--browser-cookies-file={}".format(COOKIES_FILE)] if COOKIES_FILE else []),
         browser_args,
+        *SINGLEFILE_ARGS,
+        *SINGLEFILE_EXTRA_ARGS,
     ]
-
-    # Deduplicate options (single-file doesn't like when you use the same option two times)
-    #
-    # NOTE: Options names that come first clobber conflicting names that come later
-    # My logic is SINGLEFILE_ARGS is the option that affects the singlefile command with most 
-    # specificity, therefore the user sets it with a lot intent, therefore it should take precedence 
-    # kind of like the ergonomic principle of lexical scope in programming languages.
-    seen_option_names = []
-    def test_seen(argument):
-        option_name = argument.split("=")[0]
-        if option_name in seen_option_names:
-            return False
-        else:
-            seen_option_names.append(option_name)
-            return True
-    deduped_options = list(filter(test_seen, options))
-
     cmd = [
         DEPENDENCIES['SINGLEFILE_BINARY']['path'],
-        *deduped_options,
+        *dedupe(options),
         link.url,
         output,
     ]
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
+    result = None
     try:
         result = run(cmd, cwd=str(out_dir), timeout=timeout)
 
@@ -84,7 +74,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
         #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
         output_tail = [
             line.strip()
-            for line in (result.stdout + result.stderr).decode().rsplit('\n', 3)[-3:]
+            for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
             if line.strip()
         ]
         hints = (
@@ -94,12 +84,13 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
 
         # Check for common failure cases
         if (result.returncode > 0) or not (out_dir / output).is_file():
-            raise ArchiveError('SingleFile was not able to archive the page', hints)
+            raise ArchiveError(f'SingleFile was not able to archive the page (status={result.returncode})', hints)
         chmod_file(output, cwd=str(out_dir))
     except (Exception, OSError) as err:
         status = 'failed'
         # TODO: Make this prettier. This is necessary to run the command (escape JSON internal quotes).
         cmd[2] = browser_args.replace('"', "\\\"")
+        err.hints = (result.stdout + result.stderr).decode().split('\n')
         output = err
     finally:
         timer.end()

archivebox/extractors/title.py

@@ -10,6 +10,7 @@ from ..util import (
     enforce_types,
     download_url,
     htmldecode,
+    dedupe,
 )
 from ..config import (
     TIMEOUT,
@@ -17,6 +18,7 @@ from ..config import (
     SAVE_TITLE,
     CURL_BINARY,
     CURL_ARGS,
+    CURL_EXTRA_ARGS,
     CURL_VERSION,
     CURL_USER_AGENT,
 )
@@ -66,14 +68,16 @@ def get_html(link: Link, path: Path, timeout: int=TIMEOUT) -> str:
     """
     canonical = link.canonical_outputs()
     abs_path = path.absolute()
-    sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
+
+    # prefer chrome-generated DOM dump to singlefile as singlefile output often includes HUGE url(data:image/...base64) strings that crash parsers
+    sources = [canonical["dom_path"], canonical["singlefile_path"], canonical["wget_path"]]
     document = None
     for source in sources:
         try:
             with open(abs_path / source, "r", encoding="utf-8") as f:
                 document = f.read()
                 break
-        except (FileNotFoundError, TypeError):
+        except (FileNotFoundError, TypeError, UnicodeDecodeError):
             continue
     if document is None:
         return download_url(link.url, timeout=timeout)
@@ -100,12 +104,17 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
     from core.models import Snapshot
 
     output: ArchiveOutput = None
-    cmd = [
-        CURL_BINARY,
+    # later options take precedence
+    options = [
         *CURL_ARGS,
+        *CURL_EXTRA_ARGS,
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
+    ]
+    cmd = [
+        CURL_BINARY,
+        *dedupe(options),
         link.url,
     ]
     status = 'succeeded'

archivebox/extractors/wget.py

@@ -15,9 +15,11 @@ from ..util import (
     path,
     domain,
     urldecode,
+    dedupe,
 )
 from ..config import (
     WGET_ARGS,
+    WGET_EXTRA_ARGS,
     TIMEOUT,
     SAVE_WGET,
     SAVE_WARC,
@@ -55,10 +57,10 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
 
     # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
     output: ArchiveOutput = None
-    cmd = [
-        WGET_BINARY,
-        # '--server-response',  # print headers for better error parsing
+    # later options take precedence
+    options = [
         *WGET_ARGS,
+        *WGET_EXTRA_ARGS,
         '--timeout={}'.format(timeout),
         *(['--restrict-file-names={}'.format(RESTRICT_FILE_NAMES)] if RESTRICT_FILE_NAMES else []),
         *(['--warc-file={}'.format(str(warc_path))] if SAVE_WARC else []),
@@ -68,6 +70,11 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
         *(['--compression=auto'] if WGET_AUTO_COMPRESSION else []),
         *([] if SAVE_WARC else ['--timestamping']),
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),
+        # '--server-response',  # print headers for better error parsing
+    ]
+    cmd = [
+        WGET_BINARY,
+        *dedupe(options),
         link.url,
     ]
 
@@ -202,4 +209,9 @@ def wget_output_path(link: Link) -> Optional[str]:
     if search_dir.is_dir():
         return domain(link.url).replace(":", "+")
 
+    # fallback to just the domain dir without port
+    search_dir = Path(link.link_dir) / domain(link.url).split(":", 1)[0]
+    if search_dir.is_dir():
+        return domain(link.url).split(":", 1)[0]
+
     return None

archivebox/index/__init__.py

@@ -250,7 +250,7 @@ def load_main_index(out_dir: Path=OUTPUT_DIR, warn: bool=True) -> List[Link]:
     """parse and load existing index with any new links from import_path merged in"""
     from core.models import Snapshot
     try:
-        return Snapshot.objects.all()
+        return Snapshot.objects.all().only('id')
 
     except (KeyboardInterrupt, SystemExit):
         raise SystemExit(0)
@@ -407,7 +407,7 @@ def snapshot_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type
 
 def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """indexed links without checking archive status or data directory validity"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    links = (snapshot.as_link() for snapshot in snapshots.iterator())
     return {
         link.link_dir: link
         for link in links
@@ -415,7 +415,7 @@ def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Option
 
 def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """indexed links that are archived with a valid data directory"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    links = (snapshot.as_link() for snapshot in snapshots.iterator())
     return {
         link.link_dir: link
         for link in filter(is_archived, links)
@@ -423,7 +423,7 @@ def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optio
 
 def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """indexed links that are unarchived with no data directory or an empty data directory"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    links = (snapshot.as_link() for snapshot in snapshots.iterator())
     return {
         link.link_dir: link
         for link in filter(is_unarchived, links)

archivebox/index/html.py

@@ -143,7 +143,7 @@ def snapshot_icons(snapshot) -> str:
             "mercury": "🅼",
             "warc": "📦"
         }
-        exclude = ["favicon", "title", "headers", "archive_org"]
+        exclude = ["favicon", "title", "headers", "htmltotext", "archive_org"]
         # Missing specific entry for WARC
 
         extractor_outputs = defaultdict(lambda: None)

archivebox/index/schema.py

@@ -379,11 +379,15 @@ class Link:
 
         output_paths = (
             domain(self.url),
+            'output.html',
             'output.pdf',
             'screenshot.png',
-            'output.html',
+            'singlefile.html',
+            'readability/content.html',
+            'mercury/content.html',
+            'htmltotext.txt',
             'media',
-            'singlefile.html'
+            'git',
         )
 
         return any(
@@ -429,6 +433,7 @@ class Link:
             'singlefile_path': 'singlefile.html',
             'readability_path': 'readability/content.html',
             'mercury_path': 'mercury/content.html',
+            'htmltotext_path': 'htmltotext.txt',
             'pdf_path': 'output.pdf',
             'screenshot_path': 'screenshot.png',
             'dom_path': 'output.html',
@@ -452,6 +457,7 @@ class Link:
                 'singlefile_path': static_path,
                 'readability_path': static_path,
                 'mercury_path': static_path,
+                'htmltotext_path': static_path,
             })
         return canonical
 

archivebox/index/sql.py

@@ -109,11 +109,13 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
         snap = Snapshot.objects.get(url=link.url)
     except Snapshot.DoesNotExist:
         snap = write_link_to_sql_index(link)
+
     snap.title = link.title
 
-    tag_list = list(dict.fromkeys(
-        tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')
-    ))
+    tag_list = list(
+        {tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')}
+        | set(snap.tags.values_list('name', flat=True))
+    )
 
     snap.save()
     snap.save_tags(tag_list)

archivebox/logging_util.py

@@ -393,7 +393,11 @@ def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats
     else:
         _LAST_RUN_STATS.succeeded += 1
 
-    size = get_dir_size(link_dir)
+    try:
+        size = get_dir_size(link_dir)
+    except FileNotFoundError:
+        size = (0, None, '0')
+
     end_ts = datetime.now(timezone.utc)
     duration = str(end_ts - start_ts).split('.')[0]
     print('        {black}{} files ({}) in {}s {reset}'.format(size[2], printable_filesize(size[0]), duration, **ANSI))
@@ -409,7 +413,7 @@ def log_archive_method_finished(result: "ArchiveResult"):
     """
     # Prettify CMD string and make it safe to copy-paste by quoting arguments
     quoted_cmd = ' '.join(
-        '"{}"'.format(arg) if ' ' in arg else arg
+        '"{}"'.format(arg) if (' ' in arg) or (':' in arg) else arg
         for arg in result.cmd
     )
 
@@ -428,12 +432,14 @@ def log_archive_method_finished(result: "ArchiveResult"):
                     **ANSI,
                 ),
             ]
+        
+        # import pudb; pudb.set_trace()
 
         # Prettify error output hints string and limit to five lines
         hints = getattr(result.output, 'hints', None) or ()
         if hints:
             if isinstance(hints, (list, tuple, type(_ for _ in ()))):
-                hints = [hint.decode() for hint in hints if isinstance(hint, bytes)]
+                hints = [hint.decode() if isinstance(hint, bytes) else str(hint) for hint in hints]
             else:
                 if isinstance(hints, bytes):
                     hints = hints.decode()
@@ -444,12 +450,18 @@ def log_archive_method_finished(result: "ArchiveResult"):
                 for line in list(hints)[:5] if line.strip()
             )
 
+        docker_hints = ()
+        if IN_DOCKER:
+            docker_hints = (
+                '  docker run -it -v $PWD/data:/data archivebox/archivebox /bin/bash',
+            )
 
         # Collect and prefix output lines with indentation
         output_lines = [
             *hint_header,
             *hints,
             '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']),
+            *docker_hints,
             *(['    cd {};'.format(result.pwd)] if result.pwd else []),
             '    {}'.format(quoted_cmd),
         ]
@@ -482,12 +494,12 @@ def log_removal_started(links: List["Link"], yes: bool, delete: bool):
     if delete:
         file_counts = [link.num_outputs for link in links if Path(link.link_dir).exists()]
         print(
-            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
+            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n' +
             f'    ({len(file_counts)} data folders with {sum(file_counts)} archived files will be deleted!)'
         )
     else:
         print(
-            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
+            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n' +
             '    (Pass --delete if you also want to permanently delete the data folders)'
         )
 
@@ -517,8 +529,8 @@ def log_shell_welcome_msg():
     from .cli import list_subcommands
 
     print('{green}# ArchiveBox Imports{reset}'.format(**ANSI))
-    print('{green}from core.models import Snapshot, User{reset}'.format(**ANSI))
-    print('{green}from archivebox import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
+    print('{green}from archivebox.core.models import Snapshot, ArchiveResult, Tag, User{reset}'.format(**ANSI))
+    print('{green}from archivebox.cli import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
     print()
     print('[i] Welcome to the ArchiveBox Shell!')
     print('    https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage')
@@ -626,17 +638,15 @@ def printable_folder_status(name: str, folder: Dict) -> str:
 
 @enforce_types
 def printable_dependency_version(name: str, dependency: Dict) -> str:
-    version = None
+    color, symbol, note, version = 'red', 'X', 'invalid', '?'
+
     if dependency['enabled']:
         if dependency['is_valid']:
-            color, symbol, note, version = 'green', '√', 'valid', ''
+            color, symbol, note = 'green', '√', 'valid'
 
             parsed_version_num = re.search(r'[\d\.]+', dependency['version'])
             if parsed_version_num:
                 version = f'v{parsed_version_num[0]}'
-
-        if not version:
-            color, symbol, note, version = 'red', 'X', 'invalid', '?'
     else:
         color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
 

archivebox/main.py

@@ -93,11 +93,16 @@ from .config import (
     SQL_INDEX_FILENAME,
     ALLOWED_IN_OUTPUT_DIR,
     SEARCH_BACKEND_ENGINE,
+    LDAP,
+    get_version,
     check_dependencies,
     check_data_folder,
     write_config_file,
     VERSION,
+    VERSIONS_AVAILABLE,
+    CAN_UPGRADE,
     COMMIT_HASH,
+    BUILD_TIME,
     CODE_LOCATIONS,
     EXTERNAL_LOCATIONS,
     DATA_LOCATIONS,
@@ -217,32 +222,40 @@ def version(quiet: bool=False,
     print(VERSION)
     
     if not quiet:
-        # 0.6.3
-        # ArchiveBox v0.6.3 Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY)
-        # DEBUG=False IN_DOCKER=True IN_QEMU=False IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 FS_USER=501:20 SEARCH_BACKEND=ripgrep
+        # 0.7.1
+        # ArchiveBox v0.7.1+editable COMMIT_HASH=951bba5 BUILD_TIME=2023-12-17 16:46:05 1702860365
+        # IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-14.2-arm64-arm-64bit PYTHON=Cpython
+        # FS_ATOMIC=True FS_REMOTE=False FS_USER=501:20 FS_PERMS=644
+        # DEBUG=False IS_TTY=True TZ=UTC SEARCH_BACKEND=ripgrep LDAP=False
         
         p = platform.uname()
         print(
-            'ArchiveBox v{}'.format(VERSION),
-            *((COMMIT_HASH[:7],) if COMMIT_HASH else ()),
-            sys.implementation.name.title(),
-            p.system,
-            platform.platform(),
-            p.machine,
+            'ArchiveBox v{}'.format(get_version(CONFIG)),
+            *((f'COMMIT_HASH={COMMIT_HASH[:7]}',) if COMMIT_HASH else ()),
+            f'BUILD_TIME={BUILD_TIME}',
+        )
+        print(
+            f'IN_DOCKER={IN_DOCKER}',
+            f'IN_QEMU={IN_QEMU}',
+            f'ARCH={p.machine}',
+            f'OS={p.system}',
+            f'PLATFORM={platform.platform()}',
+            f'PYTHON={sys.implementation.name.title()}',
         )
         OUTPUT_IS_REMOTE_FS = DATA_LOCATIONS['OUTPUT_DIR']['is_mount'] or DATA_LOCATIONS['ARCHIVE_DIR']['is_mount']
         print(
-            f'DEBUG={DEBUG}',
-            f'IN_DOCKER={IN_DOCKER}',
-            f'IN_QEMU={IN_QEMU}',
-            f'IS_TTY={IS_TTY}',
-            f'TZ={TIMEZONE}',
-            #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
             f'FS_ATOMIC={ENFORCE_ATOMIC_WRITES}',
             f'FS_REMOTE={OUTPUT_IS_REMOTE_FS}',
             f'FS_USER={PUID}:{PGID}',
             f'FS_PERMS={OUTPUT_PERMISSIONS}',
+        )
+        print(
+            f'DEBUG={DEBUG}',
+            f'IS_TTY={IS_TTY}',
+            f'TZ={TIMEZONE}',
             f'SEARCH_BACKEND={SEARCH_BACKEND_ENGINE}',
+            f'LDAP={LDAP}',
+            #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
         )
         print()
 
@@ -271,7 +284,7 @@ def version(quiet: bool=False,
                 print(printable_folder_status(name, path))
         else:
             print()
-            print('{white}[i] Data locations:{reset}'.format(**ANSI))
+            print('{white}[i] Data locations:{reset} (not in a data directory)'.format(**ANSI))
 
         print()
         check_dependencies()
@@ -591,7 +604,7 @@ def add(urls: Union[str, List[str]],
         out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Add a new URL or list of URLs to your archive"""
 
-    from core.models import Tag
+    from core.models import Snapshot, Tag
 
     assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)'
 
@@ -635,6 +648,19 @@ def add(urls: Union[str, List[str]],
     write_main_index(links=new_links, out_dir=out_dir)
     all_links = load_main_index(out_dir=out_dir)
 
+    tags = [
+        Tag.objects.get_or_create(name=name.strip())[0]
+        for name in tag.split(',')
+        if name.strip()
+    ]
+    if tags:
+        for link in imported_links:
+            snapshot = Snapshot.objects.get(url=link.url)
+            snapshot.tags.add(*tags)
+            snapshot.tags_str(nocache=True)
+            snapshot.save()
+        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
+
     if index_only:
         # mock archive all the links using the fake index_only extractor method in order to update their state
         if overwrite:
@@ -666,21 +692,8 @@ def add(urls: Union[str, List[str]],
             stderr(f'[*] [{ts}] Archiving {len(new_links)}/{len(all_links)} URLs from added set...', color='green')
             archive_links(new_links, overwrite=False, **archive_kwargs)
 
-
-    # add any tags to imported links
-    tags = [
-        Tag.objects.get_or_create(name=name.strip())[0]
-        for name in tag.split(',')
-        if name.strip()
-    ]
-    if tags:
-        for link in imported_links:
-            snapshot = link.as_snapshot()
-            snapshot.tags.add(*tags)
-            snapshot.tags_str(nocache=True)
-            snapshot.save()
-        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
-
+    if CAN_UPGRADE:
+        hint(f"There's a new version of ArchiveBox available! Your current version is {VERSION}. You can upgrade to {VERSIONS_AVAILABLE['recommended_version']['tag_name']} ({VERSIONS_AVAILABLE['recommended_version']['html_url']}). For more on how to upgrade: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives\n")
 
     return all_links
 
@@ -778,6 +791,8 @@ def update(resume: Optional[float]=None,
            out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Import any new links from subscriptions and retry any previously failed/skipped links"""
 
+    from core.models import ArchiveResult
+
     check_data_folder(out_dir=out_dir)
     check_dependencies()
     new_links: List[Link] = [] # TODO: Remove input argument: only_new
@@ -785,19 +800,23 @@ def update(resume: Optional[float]=None,
     extractors = extractors.split(",") if extractors else []
 
     # Step 1: Filter for selected_links
+    print('[*] Finding matching Snapshots to update...')
+    print(f'    - Filtering by {" ".join(filter_patterns)} ({filter_type}) {before=} {after=} {status=}...')
     matching_snapshots = list_links(
         filter_patterns=filter_patterns,
         filter_type=filter_type,
         before=before,
         after=after,
     )
-
+    print(f'    - Checking {matching_snapshots.count()} snapshot folders for existing data with {status=}...')
     matching_folders = list_folders(
         links=matching_snapshots,
         status=status,
         out_dir=out_dir,
     )
-    all_links = [link for link in matching_folders.values() if link]
+    all_links = (link for link in matching_folders.values() if link)
+    print('    - Sorting by most unfinished -> least unfinished + date archived...')
+    all_links = sorted(all_links, key=lambda link: (ArchiveResult.objects.filter(snapshot__url=link.url).count(), link.timestamp))
 
     if index_only:
         for link in all_links:
@@ -823,6 +842,7 @@ def update(resume: Optional[float]=None,
     if extractors:
         archive_kwargs["methods"] = extractors
 
+
     archive_links(to_archive, overwrite=overwrite, **archive_kwargs)
 
     # Step 4: Re-write links index with updated titles, icons, and resources
@@ -1005,9 +1025,9 @@ def setup(out_dir: Path=OUTPUT_DIR) -> None:
 
     stderr('\n    Installing SINGLEFILE_BINARY, READABILITY_BINARY, MERCURY_BINARY automatically using npm...')
     if not NODE_VERSION:
-        stderr('[X] You must first install node using your system package manager', color='red')
+        stderr('[X] You must first install node & npm using your system package manager', color='red')
         hint([
-            'curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -',
+            'https://github.com/nodesource/distributions#table-of-contents',
             'or to disable all node-based modules run: archivebox config --set USE_NODE=False',
         ])
         raise SystemExit(1)
@@ -1155,6 +1175,7 @@ def schedule(add: bool=False,
              run_all: bool=False,
              quiet: bool=False,
              every: Optional[str]=None,
+             tag: str='',
              depth: int=0,
              overwrite: bool=False,
              update: bool=not ONLY_NEW,
@@ -1188,6 +1209,7 @@ def schedule(add: bool=False,
                 'add',
                 *(['--overwrite'] if overwrite else []),
                 *(['--update'] if update else []),
+                *([f'--tag={tag}'] if tag else []),
                 f'--depth={depth}',
                 f'"{import_path}"',
             ] if import_path else ['update']),
@@ -1270,6 +1292,9 @@ def schedule(add: bool=False,
                 print('\n{green}[√] Stopped.{reset}'.format(**ANSI))
                 raise SystemExit(1)
 
+    if CAN_UPGRADE:
+        hint(f"There's a new version of ArchiveBox available! Your current version is {VERSION}. You can upgrade to {VERSIONS_AVAILABLE['recommended_version']['tag_name']} ({VERSIONS_AVAILABLE['recommended_version']['html_url']}). For more on how to upgrade: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives\n")
+
     
 @enforce_types
 def server(runserver_args: Optional[List[str]]=None,

archivebox/manage.py

@@ -7,7 +7,7 @@ if __name__ == '__main__':
     # versions of ./manage.py commands whenever possible. When that's not possible
     # (e.g. makemigrations), you can comment out this check temporarily
 
-    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv):
+    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'collectstatic' in sys.argv):
         print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
         print()
         print('    Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')

archivebox/mypy.ini

@@ -1,3 +0,0 @@
-[mypy]
-plugins =
-    mypy_django_plugin.main

archivebox/package.json

@@ -1,13 +1,13 @@
 {
   "name": "archivebox",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "ArchiveBox: The self-hosted internet archive",
   "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {
     "@postlight/parser": "^2.2.3",
-    "readability-extractor": "git+https://github.com/ArchiveBox/readability-extractor.git",
-    "single-file-cli": "^1.1.12"
+    "readability-extractor": "github:ArchiveBox/readability-extractor",
+    "single-file-cli": "^1.1.54"
   }
 }

archivebox/parsers/__init__.py

@@ -44,6 +44,7 @@ from . import medium_rss
 from . import netscape_html
 from . import generic_rss
 from . import generic_json
+from . import generic_jsonl
 from . import generic_html
 from . import generic_txt
 from . import url_list
@@ -63,6 +64,7 @@ PARSERS = {
     netscape_html.KEY:  (netscape_html.NAME,    netscape_html.PARSER),
     generic_rss.KEY:    (generic_rss.NAME,      generic_rss.PARSER),
     generic_json.KEY:   (generic_json.NAME,     generic_json.PARSER),
+    generic_jsonl.KEY:  (generic_jsonl.NAME,    generic_jsonl.PARSER),
     generic_html.KEY:   (generic_html.NAME,     generic_html.PARSER),
 
     # Catchall fallback parser

archivebox/parsers/generic_json.py

@@ -11,6 +11,60 @@ from ..util import (
     enforce_types,
 )
 
+# This gets used by generic_jsonl, too
+def jsonObjectToLink(link: str, source: str):
+    json_date = lambda s: datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')
+
+    # example line
+    # {"href":"http:\/\/www.reddit.com\/r\/example","description":"title here","extended":"","meta":"18a973f09c9cc0608c116967b64e0419","hash":"910293f019c2f4bb1a749fb937ba58e3","time":"2014-06-14T15:51:42Z","shared":"no","toread":"no","tags":"reddit android"}]
+    # Parse URL
+    url = link.get('href') or link.get('url') or link.get('URL')
+    if not url:
+        raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
+
+    # Parse the timestamp
+    ts_str = str(datetime.now(timezone.utc).timestamp())
+    if link.get('timestamp'):
+        # chrome/ff histories use a very precise timestamp
+        ts_str = str(link['timestamp'] / 10000000)
+    elif link.get('time'):
+        ts_str = str(json_date(link['time'].split(',', 1)[0]).timestamp())
+    elif link.get('created_at'):
+        ts_str = str(json_date(link['created_at']).timestamp())
+    elif link.get('created'):
+        ts_str = str(json_date(link['created']).timestamp())
+    elif link.get('date'):
+        ts_str = str(json_date(link['date']).timestamp())
+    elif link.get('bookmarked'):
+        ts_str = str(json_date(link['bookmarked']).timestamp())
+    elif link.get('saved'):
+        ts_str = str(json_date(link['saved']).timestamp())
+
+    # Parse the title
+    title = None
+    if link.get('title'):
+        title = link['title'].strip()
+    elif link.get('description'):
+        title = link['description'].replace(' — Readability', '').strip()
+    elif link.get('name'):
+        title = link['name'].strip()
+
+    # if we have a list, join it with commas
+    tags = link.get('tags')
+    if type(tags) == list:
+        tags = ','.join(tags)
+    elif type(tags) == str:
+        # if there's no comma, assume it was space-separated
+        if ',' not in tags:
+            tags = tags.replace(' ', ',')
+
+    return Link(
+        url=htmldecode(url),
+        timestamp=ts_str,
+        title=htmldecode(title) or None,
+        tags=htmldecode(tags),
+        sources=[source],
+    )
 
 @enforce_types
 def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
@@ -18,55 +72,21 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
 
     json_file.seek(0)
 
-    # sometimes the first line is a comment or filepath, so we get everything after the first {
-    json_file_json_str = '{' + json_file.read().split('{', 1)[-1]
-    links = json.loads(json_file_json_str)
-    json_date = lambda s: datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')
+    try:
+        links = json.load(json_file)
+        if type(links) != list:
+            raise Exception('JSON parser expects list of objects, maybe this is JSONL?')
+    except json.decoder.JSONDecodeError:
+        # sometimes the first line is a comment or other junk, so try without
+        json_file.seek(0)
+        first_line = json_file.readline()
+        #print('      > Trying JSON parser without first line: "', first_line.strip(), '"', sep= '')
+        links = json.load(json_file)
+        # we may fail again, which means we really don't know what to do
 
     for link in links:
-        # example line
-        # {"href":"http:\/\/www.reddit.com\/r\/example","description":"title here","extended":"","meta":"18a973f09c9cc0608c116967b64e0419","hash":"910293f019c2f4bb1a749fb937ba58e3","time":"2014-06-14T15:51:42Z","shared":"no","toread":"no","tags":"reddit android"}]
         if link:
-            # Parse URL
-            url = link.get('href') or link.get('url') or link.get('URL')
-            if not url:
-                raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
-
-            # Parse the timestamp
-            ts_str = str(datetime.now(timezone.utc).timestamp())
-            if link.get('timestamp'):
-                # chrome/ff histories use a very precise timestamp
-                ts_str = str(link['timestamp'] / 10000000)  
-            elif link.get('time'):
-                ts_str = str(json_date(link['time'].split(',', 1)[0]).timestamp())
-            elif link.get('created_at'):
-                ts_str = str(json_date(link['created_at']).timestamp())
-            elif link.get('created'):
-                ts_str = str(json_date(link['created']).timestamp())
-            elif link.get('date'):
-                ts_str = str(json_date(link['date']).timestamp())
-            elif link.get('bookmarked'):
-                ts_str = str(json_date(link['bookmarked']).timestamp())
-            elif link.get('saved'):
-                ts_str = str(json_date(link['saved']).timestamp())
-            
-            # Parse the title
-            title = None
-            if link.get('title'):
-                title = link['title'].strip()
-            elif link.get('description'):
-                title = link['description'].replace(' — Readability', '').strip()
-            elif link.get('name'):
-                title = link['name'].strip()
-
-            yield Link(
-                url=htmldecode(url),
-                timestamp=ts_str,
-                title=htmldecode(title) or None,
-                tags=htmldecode(link.get('tags')) or '',
-                sources=[json_file.name],
-            )
-
+            yield jsonObjectToLink(link,json_file.name)
 
 KEY = 'json'
 NAME = 'Generic JSON'

archivebox/parsers/generic_jsonl.py

@@ -0,0 +1,34 @@
+__package__ = 'archivebox.parsers'
+
+import json
+
+from typing import IO, Iterable
+from datetime import datetime, timezone
+
+from ..index.schema import Link
+from ..util import (
+    htmldecode,
+    enforce_types,
+)
+
+from .generic_json import jsonObjectToLink
+
+def parse_line(line: str):
+    if line.strip() != "":
+        return json.loads(line)
+
+@enforce_types
+def parse_generic_jsonl_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
+    """Parse JSONL format bookmarks export files"""
+
+    json_file.seek(0)
+
+    links = [ parse_line(line) for line in json_file ]
+
+    for link in links:
+        if link:
+            yield jsonObjectToLink(link,json_file.name)
+
+KEY = 'jsonl'
+NAME = 'Generic JSONL'
+PARSER = parse_generic_jsonl_export

archivebox/parsers/generic_rss.py

@@ -2,13 +2,13 @@ __package__ = 'archivebox.parsers'
 
 
 from typing import IO, Iterable
-from datetime import datetime
+from time import mktime
+from feedparser import parse as feedparser
 
 from ..index.schema import Link
 from ..util import (
     htmldecode,
-    enforce_types,
-    str_between,
+    enforce_types
 )
 
 @enforce_types
@@ -16,35 +16,27 @@ def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse RSS XML-format files into links"""
 
     rss_file.seek(0)
-    items = rss_file.read().split('<item>')
-    items = items[1:] if items else []
-    for item in items:
-        # example item:
-        # <item>
-        # <title><![CDATA[How JavaScript works: inside the V8 engine]]></title>
-        # <category>Unread</category>
-        # <link>https://blog.sessionstack.com/how-javascript-works-inside</link>
-        # <guid>https://blog.sessionstack.com/how-javascript-works-inside</guid>
-        # <pubDate>Mon, 21 Aug 2017 14:21:58 -0500</pubDate>
-        # </item>
+    feed = feedparser(rss_file.read())
+    for item in feed.entries:
+        url = item.link
+        title = item.title
+        time = mktime(item.updated_parsed)
 
-        trailing_removed = item.split('</item>', 1)[0]
-        leading_removed = trailing_removed.split('<item>', 1)[-1].strip()
-        rows = leading_removed.split('\n')
+        try:
+            tags = ','.join(map(lambda tag: tag.term, item.tags))
+        except AttributeError:
+            tags = ''
 
-        def get_row(key):
-            return [r for r in rows if r.strip().startswith('<{}>'.format(key))][0]
-
-        url = str_between(get_row('link'), '<link>', '</link>')
-        ts_str = str_between(get_row('pubDate'), '<pubDate>', '</pubDate>')
-        time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z")
-        title = str_between(get_row('title'), '<![CDATA[', ']]').strip()
+        if url is None:
+            # Yielding a Link with no URL will
+            # crash on a URL validation assertion
+            continue
 
         yield Link(
             url=htmldecode(url),
-            timestamp=str(time.timestamp()),
+            timestamp=str(time),
             title=htmldecode(title) or None,
-            tags=None,
+            tags=tags,
             sources=[rss_file.name],
         )
 

archivebox/parsers/pinboard_rss.py

@@ -2,50 +2,41 @@ __package__ = 'archivebox.parsers'
 
 
 from typing import IO, Iterable
-from datetime import datetime, timezone
-
-from xml.etree import ElementTree
+from time import mktime
+from feedparser import parse as feedparser
 
 from ..index.schema import Link
 from ..util import (
     htmldecode,
-    enforce_types,
+    enforce_types
 )
 
-
 @enforce_types
 def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse Pinboard RSS feed files into links"""
 
     rss_file.seek(0)
-    root = ElementTree.parse(rss_file).getroot()
-    items = root.findall("{http://purl.org/rss/1.0/}item")
-    for item in items:
-        find = lambda p: item.find(p).text.strip() if item.find(p) is not None else None    # type: ignore
+    feed = feedparser(rss_file.read())
+    for item in feed.entries:
+        url = item.link
+        # title will start with "[priv] " if pin was marked private. useful?
+        title = item.title
+        time = mktime(item.updated_parsed)
 
-        url = find("{http://purl.org/rss/1.0/}link")
-        tags = find("{http://purl.org/dc/elements/1.1/}subject")
-        title = find("{http://purl.org/rss/1.0/}title")
-        ts_str = find("{http://purl.org/dc/elements/1.1/}date")
+        # all tags are in one entry.tags with spaces in it. annoying!
+        try:
+            tags = item.tags[0].term.replace(' ', ',')
+        except AttributeError:
+            tags = ''
         
         if url is None:
             # Yielding a Link with no URL will
             # crash on a URL validation assertion
             continue
 
-        # Pinboard includes a colon in its date stamp timezone offsets, which
-        # Python can't parse. Remove it:
-        if ts_str and ts_str[-3:-2] == ":":
-            ts_str = ts_str[:-3]+ts_str[-2:]
-
-        if ts_str:
-            time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
-        else:
-            time = datetime.now(timezone.utc)
-
         yield Link(
             url=htmldecode(url),
-            timestamp=str(time.timestamp()),
+            timestamp=str(time),
             title=htmldecode(title) or None,
             tags=htmldecode(tags) or None,
             sources=[rss_file.name],

archivebox/plugins/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins'
+
+

archivebox/plugins/defaults/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins.defaults'
+
+default_app_config = 'plugins.defaults.apps.DefaultsPluginAppConfig'

archivebox/plugins/defaults/admin.py

@@ -0,0 +1,20 @@
+from django.contrib import admin
+from solo.admin import SingletonModelAdmin
+
+from .models import (
+	ArchiveBoxDefaultDependency,
+	ArchiveBoxDefaultExtractor,
+)
+
+
+class DependencyAdmin(SingletonModelAdmin):
+	readonly_fields = ('id', 'NAME', 'LABEL', 'REQUIRED', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
+
+class ExtractorAdmin(SingletonModelAdmin):
+	readonly_fields = ('id', 'NAME', 'LABEL', 'DEFAULT_ENABLED', 'DEFAULT_CMD', 'CMD', 'ARGS', 'TIMEOUT', 'dependency', 'is_valid', 'is_enabled')
+
+print('DefaultsPluginConfig.admin')
+
+
+admin.site.register(ArchiveBoxDefaultDependency, DependencyAdmin)
+admin.site.register(ArchiveBoxDefaultExtractor, ExtractorAdmin)

archivebox/plugins/defaults/apps.py

@@ -0,0 +1,24 @@
+# __package__ = 'archivebox.plugins.defaults'
+
+
+
+from django.apps import AppConfig
+
+
+class DefaultsPluginAppConfig(AppConfig):
+    name = "plugins.defaults"
+
+    # label = "ArchiveBox Defaults"
+    verbose_name = "Plugin Configuration Defaults"
+    
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        print('plugins.defaults.apps.DefaultsPluginConfig.ready')
+
+        from django.conf import settings
+
+        from .settings import register_plugin_settings
+
+        register_plugin_settings(settings, name=self.name)
+        

archivebox/plugins/defaults/migrations/0001_initial.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.1.14 on 2024-01-24 08:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ArchiveBoxDefaultDependency',
+            fields=[
+                ('ENABLED', models.BooleanField(default=True, editable=False)),
+                ('BINARY', models.CharField(default='/bin/false', max_length=255)),
+                ('ARGS', models.CharField(default='', max_length=255)),
+                ('id', models.AutoField(default=1, primary_key=True, serialize=False)),
+            ],
+            options={
+                'verbose_name': 'Dependency Configuration Defaults',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ArchiveBoxDefaultExtractor',
+            fields=[
+                ('ENABLED', models.BooleanField(default=True)),
+                ('CMD', models.CharField(default=['{DEPENDENCY.BINARY}', '{ARGS}', '{url}'], max_length=255)),
+                ('ARGS', models.CharField(default=['--timeout={TIMEOUT}'], max_length=255)),
+                ('TIMEOUT', models.CharField(default='{TIMEOUT}', max_length=255)),
+                ('id', models.AutoField(default=1, primary_key=True, serialize=False)),
+            ],
+            options={
+                'verbose_name': 'Extractor Configuration Defaults',
+                'abstract': False,
+            },
+        ),
+    ]

archivebox/plugins/defaults/migrations/0002_auto_20240124_0943.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.1.14 on 2024-01-24 09:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('defaults', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='archiveboxdefaultdependency',
+            options={'verbose_name': 'Default Configuration: Dependencies'},
+        ),
+        migrations.AlterModelOptions(
+            name='archiveboxdefaultextractor',
+            options={'verbose_name': 'Default Configuration: Extractors'},
+        ),
+        migrations.AlterField(
+            model_name='archiveboxdefaultdependency',
+            name='BINARY',
+            field=models.CharField(default='/bin/bash', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='archiveboxdefaultdependency',
+            name='ENABLED',
+            field=models.BooleanField(default=True),
+        ),
+    ]

archivebox/plugins/defaults/models.py

@@ -0,0 +1,385 @@
+__package__ = 'archivebox.plugins.defaults'
+
+# import shutil
+
+import re
+
+from typing import List, Dict, Any
+from pathlib import Path
+
+from django.db import models, transaction
+from django.utils.functional import cached_property
+
+from solo.models import SingletonModel                        # type: ignore[import-untyped]
+
+
+from config import bin_path, bin_version
+
+ConfigDict = Dict[str, Any]
+
+
+# def bin_path(binary: str) -> str | None:
+#     return shutil.which(str(Path(binary).expanduser())) or shutil.which(str(binary)) or binary
+
+# def bin_version(bin_path: str, cmd: str | None=None) -> str | None:
+#     return '0.0.0'
+
+# def pretty_path(path: Path) -> str:
+#     """take a Path object and return the path as a string relative to the current directory"""
+
+#     if not path:
+#         return ''
+
+#     return str(path.expanduser().resolve().relative_to(Path.cwd().resolve()))
+
+
+class ArchiveBoxBaseDependency(models.Model):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    NAME = 'DEFAULT'
+    LABEL = "Default"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES: List[str] = []
+
+    BIN_DEPENDENCIES: List[str] = []
+    APT_DEPENDENCIES: List[str] = []
+    BREW_DEPENDENCIES: List[str] = []
+    PIP_DEPENDENCIES: List[str] = []
+    NPM_DEPENDENCIES: List[str] = []
+
+    DEFAULT_BINARY: str | None              = '/bin/bash'
+    DEFAULT_START_CMD: str | None           = '/bin/bash -c "while true; do sleep 1; done"'
+    DEFAULT_PID_FILE: str | None            = 'logs/{NAME}_WORKER.pid'
+    DEFAULT_STOP_CMD: str | None            = 'kill "$(<{PID_FILE})"'
+    DEFAULT_VERSION_COMMAND: str | None     = '{BINARY} --version'
+    DEFAULT_ARGS: str | None                = ''
+
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True, editable=False)
+    BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+    
+    # START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
+    # WORKERS = models.IntegerField(default=1)
+
+    class Meta:
+        abstract = True
+        app_label = 'defaults'
+
+    def __str__(self):
+        return f"{self.LABEL} Dependency Configuration"
+
+    def __json__(self):
+        return {
+            'type': 'ArchiveBoxDependency',
+            '__class__': self.__class__.__name__,
+            'NAME': self.NAME,
+            'LABEL': self.LABEL,
+            'ENABLED': self.ENABLED,
+            'BINARY': self.BINARY,
+            'ARGS': self.ARGS,
+            # 'START_CMD': self.START_CMD,
+            # 'WORKERS': self.WORKERS,
+        }
+
+    @cached_property
+    def bin_path(self) -> str:
+        return bin_path(self.BINARY or self.DEFAULT_BINARY)
+
+    @cached_property
+    def bin_version(self) -> str | None:
+        print(f'ArchiveBoxBaseDependency.bin_version({self.bin_path}, cmd={self.VERSION_CMD.format(BINARY=self.BINARY)})')
+        return bin_version(self.bin_path, cmd=self.VERSION_CMD.format(BINARY=self.BINARY))
+        # return bin_version(self.bin_path, cmd=self.VERSION_CMD)
+
+    @cached_property
+    def is_valid(self) -> bool:
+        return bool(self.bin_path and self.bin_version)
+
+    @cached_property
+    def is_enabled(self) -> bool:
+        return bool(self.ENABLED and self.is_valid)
+
+    @cached_property
+    def pretty_version(self) -> str:
+        if self.is_enabled:
+            if self.is_valid:
+                color, symbol, note, version = 'green', '√', 'valid', ''
+
+                parsed_version_num = re.search(r'[\d\.]+', self.bin_version)
+                if parsed_version_num:
+                    version = f'v{parsed_version_num[0]}'
+
+            if not self.bin_version:
+                color, symbol, note, version = 'red', 'X', 'invalid', '?'
+        else:
+            color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
+
+        path = pretty_path(self.bin_path)
+
+        return ' '.join((
+            ANSI[color],
+            symbol,
+            ANSI['reset'],
+            name.ljust(21),
+            version.ljust(14),
+            ANSI[color],
+            note.ljust(8),
+            ANSI['reset'],
+            path.ljust(76),
+        ))
+
+    # @helper
+    def install_parents(self, config):
+        return {
+            # parent_dependency.NAME: parent_dependency.get_solo().install_self()
+            parent_dependency: parent_dependency
+            for parent_dependency in self.PARENT_DEPENDENCIES
+        }
+
+    # @helper
+    def install_self(self, config):
+        assert all(self.install_parents(config=config).values())
+
+        BashEnvironmentDependency.get_solo().install_pkgs(self.BIN_DEPENDENCIES)
+        AptEnvironmentDependency.get_solo().install_pkgs(self.APT_DEPENDENCIES)
+        BrewEnvironmentDependency.get_solo().install_pkgs(self.BREW_DEPENDENCIES)
+        PipEnvironmentDependency.get_solo().install_pkgs(self.PIP_DEPENDENCIES)
+        NPMEnvironmentDependency.get_solo().install_pkgs(self.NPM_DEPENDENCIES)
+
+        assert self.is_valid
+        return self.bin_version
+
+    # @task
+    def run(args, pwd, timeout):
+        errors = None
+        timer = TimedProgress(timeout, prefix='      ')
+        try:
+            proc = run(cmd=[self.bin_path, *args], pwd=pwd, timeout=timeout)
+
+        except Exception as err:
+            errors = err
+        finally:
+            timer.end()
+
+        return proc, timer, errors
+
+class ArchiveBoxDefaultDependency(ArchiveBoxBaseDependency, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+
+    class Meta:                 # pyright: ignore [reportIncompatibleVariableOverride]
+        abstract = False
+        app_label = 'defaults'
+        verbose_name = 'Default Configuration: Dependencies'
+
+
+class ArchiveBoxBaseExtractor(models.Model):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    NAME = 'DEFAULT'
+    LABEL = 'Default'
+
+    DEFAULT_DEPENDENCY = ArchiveBoxDefaultDependency
+    DEPENDENCY = DEFAULT_DEPENDENCY
+
+
+    DEFAULT_ENABLED = True
+    DEFAULT_CMD = ['{DEPENDENCY.BINARY}', '{ARGS}', '{url}']
+    DEFAULT_ARGS = ['--timeout={TIMEOUT}']
+    DEFAULT_TIMEOUT = '{TIMEOUT}'
+    # DEFAULT_USER_AGENT = '{USER_AGENT}'
+    # DEFAULT_COOKIES_TXT = '{COOKIES_TXT}'
+
+    ENABLED = models.BooleanField(default=DEFAULT_ENABLED, editable=True)
+
+    CMD = models.CharField(max_length=255, default=DEFAULT_CMD)
+    ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
+    TIMEOUT = models.CharField(max_length=255, default=DEFAULT_TIMEOUT)
+    
+    ALIASES = {
+        'ENABLED': (f'SAVE_{NAME}', f'USE_{NAME}', f'FETCH_{NAME}'),
+    }
+
+    def __str__(self):
+        return f"{self.LABEL} Extractor Configuration"
+
+    class Meta:             # pyright: ignore [reportIncompatibleVariableOverride]
+        abstract = True
+        verbose_name = "Default Extractor Configuration"
+        app_label = 'defaults'
+
+    @cached_property
+    def dependency(self):
+        return self.DEPENDENCY.get_solo()
+
+    def __json__(self):
+        return {
+            'type': 'ArchiveBoxExtractor',
+            '__class__': self.__class__.__name__,
+            'NAME': self.NAME,
+            'LABEL': self.LABEL,
+            'ENABLED': self.ENABLED,
+            'DEPENDENCY': self.dependency.__json__(),
+            'ARGS': self.ARGS,
+            'CMD': self.CMD,
+            'TIMEOUT': self.TIMEOUT,
+            'is_valid': self.is_valid,
+            'is_enabled': self.is_enabled,
+        }
+
+
+    def format_args(self, csv: List[str], **config):
+        un_prefixed_config = {**self.__json__()}          # e.g. ENABLED=True
+        prefixed_config = {                               # e.g. GALLERYDL_ENABLED=True
+            f'{self.NAME}_{key}': value
+            for key, value in un_prefixed_config.items()
+        }
+
+        merged_config = {
+            **config,                  # e.g. TIMEOUT=60
+            **un_prefixed_config,      # e.g. ENABLED=True
+            **prefixed_config,         # e.g. GALLERYDL_ENABLED=True
+        }
+        formatted_config = [
+            arg.format(**merged_config)
+            for arg in csv
+        ]
+
+        return formatted_config
+
+    @cached_property
+    def is_valid(self):
+        if not self.dependency.is_valid:
+            return False
+
+        # TIMEOUT must be at least 5 seconds
+        # if self.TIMEOUT < 5:
+        #     return False
+
+        # assert Path(self.COOKIES_TXT).exists()
+        # TODO: validate user agent with uaparser
+        # TODO: validate args, cookies.txt?
+        return True
+
+    @cached_property
+    def is_enabled(self):
+        return self.ENABLED and self.is_valid and self.dependency.is_enabled
+
+
+    def save(self, *args, **kwargs):
+        # assert self.is_valid
+
+        with transaction.atomic():
+            result = super().save(*args, **kwargs)
+            # post to message bus:
+            print({
+                'type': f'{self.__class__.__name__}.save',
+                'diff': self.__json__(),
+                'kwargs': kwargs,
+            })
+            # potential consumers of this event:
+            #    - event logger: write to events.log
+            #    - config file updater: writes to ArchiveBox.conf
+            #    - supervisor: restarts relevant dependencies/extractors
+            #    - etc...
+
+        return result
+
+    def out_dir(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        return (snapshot_dir / self.NAME)
+
+    def create_out_dir(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        out_dir = self.out_dir(url=url, snapshot_dir=snapshot_dir, config=config)
+        return out_dir.mkdir(exist_ok=True)
+
+    def should_extract(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        # return False if extractor is disabled
+        if not self.is_enabled:
+            return False
+
+        out_dir = self.out_dir(url=url, snapshot_dir=snapshot_dir, config=config)
+        
+        if has_existing_output := out_dir.glob('*'):
+            return False
+
+        if not (has_write_access := os.access(out_dir, os.W_OK | os.X_OK)):
+            return False
+
+        return True
+
+
+    def get_dependency_cmd(self, url: str, extractor_dir: Path, config: ConfigDict):
+        return [
+            self.format_args(self.CMD, **config),
+            url,
+            *self.format_args(self.ARGS, **config),   # TODO: split and requote this properly
+        ]
+
+    # @requires_config('HOSTNAME', 'TIMEOUT', 'USER_AGENT', 'CHECK_SSL_VALIDITY')
+    def extract(self, url: str, snapshot_dir: Path, config: ConfigDict):
+        if not self.ENABLED:
+            return
+
+        extractor_dir = self.create_extractor_directory(snapshot_dir)
+
+        cmd = self.get_dependency_cmd(url=url, extractor_dir=extractor_dir, config=config)
+
+        status, stdout, stderr, output_path = 'failed', '', '', None
+        try:
+            proc, timer, errors = self.dependency.run(cmd, cwd=extractor_dir, timeout=self.TIMEOUT)
+            stdout, stderr = proc.stdout, proc.stderr
+            
+            if 'ERROR: Unsupported URL' in stderr:
+                hints = ('gallery-dl doesnt support this type of url yet',)
+                raise ArchiveError('Failed to save gallerydl', hints)
+
+            if proc.returncode == 0 and 'finished' in stdout:
+                output_path = extractor_dir / 'index.html'
+                status = 'succeeded'
+        except Exception as err:
+            stderr += err
+
+        num_bytes, num_dirs, num_files = get_dir_size(extractor_dir)
+
+        return ArchiveResult(
+            cmd=cmd,
+            pwd=str(out_dir),
+            cmd_version=self.dependency.bin_version,
+            cmd_path=self.dependency.bin_path,
+            cmd_hostname=config.HOSTNAME,
+
+            output_path=output_path,
+            stdout=stdout,
+            stderr=stderr,
+            status=status,
+
+            num_bytes=num_bytes,
+            num_files=num_files,
+            num_dirs=num_dirs,
+            **timer.stats,
+        )
+
+
+class ArchiveBoxDefaultExtractor(ArchiveBoxBaseExtractor, SingletonModel):
+    singleton_instance_id = 1
+
+    id = models.AutoField(default=singleton_instance_id, primary_key=True)
+
+    DEPENDENCY = ArchiveBoxDefaultDependency
+
+    ENABLED = models.BooleanField(default=True, editable=True)
+
+    class Meta:
+        abstract = False
+        app_label = 'defaults'
+        verbose_name = 'Default Configuration: Extractors'

archivebox/plugins/defaults/settings.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from django.conf import settings
+
+def register_plugin_settings(settings=settings, name='defaults'):
+
+	settings.STATICFILES_DIRS += [
+		str(Path(settings.PACKAGE_DIR) / f'plugins/{name}/static'),
+	]
+
+	settings.TEMPLATE_DIRS += [
+		str(Path(settings.PACKAGE_DIR) / f'plugins/{name}/templates'),
+	]
+
+	print('REGISTERED PLUGIN SETTINGS', name)

archivebox/plugins/gallerydl/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.plugins.replaywebpage'

archivebox/plugins/gallerydl/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+from solo.admin import SingletonModelAdmin
+
+from .models import GalleryDLDependency, GalleryDLExtractor
+
+
+admin.site.register(GalleryDLDependency, SingletonModelAdmin)
+admin.site.register(GalleryDLExtractor, SingletonModelAdmin)

archivebox/plugins/gallerydl/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+
+
+class GalleryDLAppConfig(AppConfig):
+    label = "Gallery-DL"
+    name = "plugin_gallerydl"
+    
+    default_auto_field = "django.db.models.BigAutoField"
+
+    def ready(self):
+        # querying models is ok, but don't fetch rows from DB or perform stateful actions here
+
+        print('√ Loaded GalleryDL Plugin')

archivebox/plugins/gallerydl/extractors.py

@@ -0,0 +1,50 @@
+# browsertrix extractor
+
+def save_browsertrix(link, out_dir, timeout, config):
+
+
+	browsertrix_dir = out_dir / 'browsertrix'
+	browsertrix_dir.mkdir(exist_ok=True)
+
+	crawl_id = link.timestamp
+
+	browsertrix_crawler_cmd = [
+		'crawl',
+		f'--url', link.url,
+		f'--collection={crawl_id}',
+		'--scopeType=page',
+		'--generateWACZ',
+		'--text=final-to-warc',
+		'--timeLimit=60',
+	]
+
+	remote_cmd = """
+	rm /tmp/dump.rdb;
+	rm -rf /crawls/collections;
+	mkdir /crawls/collections;
+	env CRAWL_ID={crawl_id} 
+	"""
+
+	local_cmd = ['nc', 'browsertrix', '2222']
+
+	status = 'succeeded'
+    timer = TimedProgress(timeout, prefix='      ')
+    try:
+        result = run(local_cmd, cwd=str(out_dir), input=remote_cmd, timeout=timeout)
+		
+		cmd_output = result.stdout.decode()
+
+		wacz_output_file = Path('/browsertrix/crawls') / crawl_id / f'{crawl_id}'.wacz
+
+		copy_and_overwrite(wacz_output_file, browsertrix_dir / wacz_output_file.name)
+
+
+
+TEMPLATE = """
+
+"""
+
+# rm /tmp/dump.rdb;
+# rm -rf /crawls/collections;
+# mkdir /crawls/collections;
+# env CRAWL_ID=tec2342 crawl --url 'https://example.com' --scopeType page --generateWACZ --collection tec2342 --text final-to-warc --timeLimit 60

archivebox/plugins/gallerydl/models.py

@@ -0,0 +1,121 @@
+from django.db import models
+from django.utils.functional import cached_property
+
+from solo.models import SingletonModel
+
+from archivebox.plugins.defaults.models import (
+    ArchiveBoxDefaultDependency,
+    ArchiveBoxDefaultExtractor,
+    BashEnvironmentDependency,
+    PipEnvironmentDependency,
+)
+
+
+class GalleryDLDependency(ArchiveBoxDefaultDependency, SingletonModel):
+    NAME = 'GALLERYDL'
+    LABEL = "GalleryDL"
+    REQUIRED = False
+
+    PARENT_DEPENDENCIES = [
+        BashEnvironmentDependency,
+        PipEnvironmentDependency,
+    ]
+
+    BIN_DEPENDENCIES = ['gallery-dl']
+    APT_DEPENDENCIES = []
+    BREW_DEPENDENCIES = []
+    PIP_PACKAGES = ['gallery-dl']
+    NPM_PACKAGES = []
+
+    DEFAULT_BINARY = 'gallery-dl'
+    DEFAULT_START_CMD = None
+    DEFAULT_ARGS = []
+    VERSION_CMD = '{BINARY} --version'
+
+    ENABLED = models.BooleanField(default=True)
+    BINARY = models.CharField(max_length=255, default='gallery-dl')
+
+    WORKERS = models.IntegerField(default='1')
+
+
+class GalleryDLExtractor(ArchiveBoxDefaultExtractor, SingletonModel):
+    NAME = 'GALLERYDL'
+    LABEL = 'gallery-dl'
+
+    DEPENDENCY = GalleryDLDependency.get_solo()
+
+    # https://github.com/mikf/gallery-dl
+    DEFAULT_CMD = [
+        '{DEPENDENCY.BINARY}',
+        '{ARGS}'
+        '{url}',
+    ]
+    DEFAULT_ARGS = [
+        '--timeout', self.TIMEOUT.format(**config),
+        '--cookies', self.COOKIES_TXT.format(**config),
+        '--user-agent', self.COOKIES_TXT.format(**config),
+        '--verify', self.CHECK_SSL_VALIDITY.format(**config),
+    ]
+
+    ENABLED = models.BooleanField(default=True)
+
+    CMD = models.CharField(max_length=255, default=DEFAULT_CMD)
+    ARGS = models.CSVField(max_length=255, default=DEFAULT_ARGS)
+    
+    TIMEOUT = models.CharField(max_length=255, default='{TIMEOUT}')
+    USER_AGENT = models.CharField(max_length=255, default='{USER_AGENT}')
+    COOKIES_TXT = models.CharField(max_length=255, default='{COOKIES_TXT}')
+    CHECK_SSL_VALIDITY = models.CharField(default='{CHECK_SSL_VALIDITY}')
+
+    # @task
+    # @requires_config('HOSTNAME', 'TIMEOUT', 'USER_AGENT', 'CHECK_SSL_VALIDITY')
+    def extract(self, url: str, out_dir: Path, config: ConfigDict):
+        if not self.ENABLED:
+            return
+
+        extractor_dir = self.create_extractor_directory(out_dir)
+
+        cmd = [
+            self.CMD,
+            url,
+            '--timeout', self.TIMEOUT.format(**config),
+            '--cookies', self.COOKIES_TXT.format(**config),
+            '--user-agent', self.COOKIES_TXT.format(**config),
+            '--verify', self.CHECK_SSL_VALIDITY.format(**config),
+            *split_args(self.ARGS.format(**config)),
+        ]
+
+        status, stdout, stderr, output_path = 'failed', '', '', None
+        try:
+            proc, timer, errors = self.DEPENDENCY.run(cmd, cwd=extractor_dir, timeout=self.GALLERYDL_TIMEOUT)
+            stdout, stderr = proc.stdout, proc.stderr
+            
+            if 'ERROR: Unsupported URL' in stderr:
+                hints = ('gallery-dl doesnt support this type of url yet',)
+                raise ArchiveError('Failed to save gallerydl', hints)
+
+            if proc.returncode == 0 and 'finished' in stdout:
+                output_path = extractor_dir / 'index.html'
+                status = 'succeeded'
+        except Exception as err:
+            stderr += err
+
+        num_bytes, num_dirs, num_files = get_dir_size(extractor_dir)
+
+        return ArchiveResult(
+            cmd=cmd,
+            pwd=str(out_dir),
+            cmd_version=self.DEPENDENCY.bin_version,
+            cmd_path=self.DEPENDENCY.bin_path,
+            cmd_hostname=config.HOSTNAME,
+
+            output_path=output_path,
+            stdout=stdout,
+            stderr=stderr,
+            status=status,
+
+            num_bytes=num_bytes,
+            num_files=num_files,
+            num_dirs=num_dirs,
+            **timer.stats,
+        )

archivebox/plugins/gallerydl/plugin.yaml

@@ -0,0 +1,59 @@
+dependencies:
+	GalleryDLDependency:
+		ID: gallerydl
+		LABEL: GalleryDL
+		REQUIRED: false
+
+		PARENT_DEPENDENCIES:
+			- BashEnvironmentDependency
+			- PipEnvironmentDependency
+
+		PIP_DEPENDENCIES:
+			- gallery-dl
+
+		USER_CONFIG:
+			ENABLED: models.BooleanField(max_length=255, default={DEFAULT_CONFIG.ENABLED})
+			BINARY: models.CharField(max_length=255, default={DEFAULT_CONFIG.BINARY})
+
+		DEFAULT_CONFIG:
+			ENABLED: true
+			BINARY: 'gallery-dl'
+
+		CONFIG_ALIASES:
+			- SAVE_GALLERYDL: ENABLED
+			- USE_GALLERYDL: ENABLED
+			- GALLERYDL_ENABLED: ENABLED
+			- GALLERYDL_BINARY: BINARY
+
+		TASKS:
+			# plugins.GalleryDLDependency
+			run_dependency: plugins.gallerydl.models.GalleryDLDependency.run_dependency
+
+
+extractors:
+	GalleryDLExtractor:
+		ID: GALLERYDL
+		LABEL: GalleryDL
+		ENABLED: true
+
+		DEPENDENCY: GalleryDLDependency
+
+		CONFIG:
+			ENABLED: models.BooleanField(default={DEFAULT_CONFIG.ENABLED})
+			CMD: models.CharField(max_length=255, default={DEFAULT_CONFIG.CMD})
+			ARGS: models.CharField(max_length=255, default={DEFAULT_CONFIG.ARGS})
+			USER_AGENT: models.CharField(max_length=255, default={DEFAULT_CONFIG.USER_AGENT})
+			CHECK_SSL_VALIDITY: models.CharField(max_length=255, default={DEFAULT_CONFIG.CHECK_SSL_VALIDITY})
+
+		DEFAULT_CONFIG:
+			ENABLED: true
+			CMD: gallery-dl {args} {url}
+			ARGS: --user-agent={USER_AGENT} --check-ssl={CHECK_SSL_VALIDITY}
+      CHECK_SSL_VALIDITY: {CHECK_SSL_VALIDITY}
+      USER_AGENT: {USER_AGENT}
+
+
+		TASKS:
+			CREATE_OUT_DIR: plugins.gallerydl.tasks.create_out_dir
+			SHOULD_EXTRACT: plugins.gallerydl.tasks.should_extract
+			EXTRACT: plugins.gallerydl.tasks.extract

archivebox/plugins/gallerydl/static/sw.js

@@ -0,0 +1,124 @@
+/**
+ * Skipped minification because the original files appears to be already minified.
+ * Original file: /npm/replaywebpage@1.8.14/sw.js
+ *
+ * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files
+ */
+/*! sw.js (wabac.js 2.16.13) is part of Webrecorder project. Copyright (C) 2020-2023, Webrecorder Software. Licensed under the Affero General Public License v3. */(()=>{var t={9742:(t,e)=>{"use strict";e.byteLength=function(t){var e=c(t),r=e[0],i=e[1];return 3*(r+i)/4-i},e.toByteArray=function(t){var e,r,s=c(t),o=s[0],a=s[1],l=new n(function(t,e,r){return 3*(e+r)/4-r}(0,o,a)),h=0,u=a>0?o-4:o;for(r=0;r<u;r+=4)e=i[t.charCodeAt(r)]<<18|i[t.charCodeAt(r+1)]<<12|i[t.charCodeAt(r+2)]<<6|i[t.charCodeAt(r+3)],l[h++]=e>>16&255,l[h++]=e>>8&255,l[h++]=255&e;return 2===a&&(e=i[t.charCodeAt(r)]<<2|i[t.charCodeAt(r+1)]>>4,l[h++]=255&e),1===a&&(e=i[t.charCodeAt(r)]<<10|i[t.charCodeAt(r+1)]<<4|i[t.charCodeAt(r+2)]>>2,l[h++]=e>>8&255,l[h++]=255&e),l},e.fromByteArray=function(t){for(var e,i=t.length,n=i%3,s=[],o=16383,a=0,c=i-n;a<c;a+=o)s.push(l(t,a,a+o>c?c:a+o));return 1===n?(e=t[i-1],s.push(r[e>>2]+r[e<<4&63]+"==")):2===n&&(e=(t[i-2]<<8)+t[i-1],s.push(r[e>>10]+r[e>>4&63]+r[e<<2&63]+"=")),s.join("")};for(var r=[],i=[],n="undefined"!=typeof Uint8Array?Uint8Array:Array,s="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",o=0,a=s.length;o<a;++o)r[o]=s[o],i[s.charCodeAt(o)]=o;function c(t){var e=t.length;if(e%4>0)throw new Error("Invalid string. Length must be a multiple of 4");var r=t.indexOf("=");return-1===r&&(r=e),[r,r===e?0:4-r%4]}function l(t,e,i){for(var n,s,o=[],a=e;a<i;a+=3)n=(t[a]<<16&16711680)+(t[a+1]<<8&65280)+(255&t[a+2]),o.push(r[(s=n)>>18&63]+r[s>>12&63]+r[s>>6&63]+r[63&s]);return o.join("")}i["-".charCodeAt(0)]=62,i["_".charCodeAt(0)]=63},4181:t=>{var e=4096,r=new Uint32Array([0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535,131071,262143,524287,1048575,2097151,4194303,8388607,16777215]);function i(t){this.buf_=new Uint8Array(8224),this.input_=t,this.reset()}i.READ_SIZE=e,i.IBUF_MASK=8191,i.prototype.reset=function(){this.buf_ptr_=0,this.val_=0,this.pos_=0,this.bit_pos_=0,this.bit_end_pos_=0,this.eos_=0,this.readMoreInput();for(var t=0;t<4;t++)this.val_|=this.buf_[this.pos_]<<8*t,++this.pos_;return this.bit_end_pos_>0},i.prototype.readMoreInput=function(){if(!(this.bit_end_pos_>256))if(this.eos_){if(this.bit_pos_>this.bit_end_pos_)throw new Error("Unexpected end of input "+this.bit_pos_+" "+this.bit_end_pos_)}else{var t=this.buf_ptr_,r=this.input_.read(this.buf_,t,e);if(r<0)throw new Error("Unexpected end of input");if(r<e){this.eos_=1;for(var i=0;i<32;i++)this.buf_[t+r+i]=0}if(0===t){for(i=0;i<32;i++)this.buf_[8192+i]=this.buf_[i];this.buf_ptr_=e}else this.buf_ptr_=0;this.bit_end_pos_+=r<<3}},i.prototype.fillBitWindow=function(){for(;this.bit_pos_>=8;)this.val_>>>=8,this.val_|=this.buf_[8191&this.pos_]<<24,++this.pos_,this.bit_pos_=this.bit_pos_-8>>>0,this.bit_end_pos_=this.bit_end_pos_-8>>>0},i.prototype.readBits=function(t){32-this.bit_pos_<t&&this.fillBitWindow();var e=this.val_>>>this.bit_pos_&r[t];return this.bit_pos_+=t,e},t.exports=i},7080:(t,e)=>{e.lookup=new Uint8Array([0,0,0,0,0,0,0,0,0,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,12,16,12,12,20,12,16,24,28,12,12,32,12,36,12,44,44,44,44,44,44,44,44,44,44,32,32,24,40,28,12,12,48,52,52,52,48,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,24,12,28,12,12,12,56,60,60,60,56,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,24,12,28,12,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,0,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,32,32,32,32,32,32,32,32,32,32,32,32,32,Line truncated
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */const i=r(9742),n=r(645),s="function"==typeof Symbol&&"function"==typeof Symbol.for?Symbol.for("nodejs.util.inspect.custom"):null;e.Buffer=c,e.SlowBuffer=function(t){return+t!=t&&(t=0),c.alloc(+t)},e.INSPECT_MAX_BYTES=50;const o=2147483647;function a(t){if(t>o)throw new RangeError('The value "'+t+'" is invalid for option "size"');const e=new Uint8Array(t);return Object.setPrototypeOf(e,c.prototype),e}function c(t,e,r){if("number"==typeof t){if("string"==typeof e)throw new TypeError('The "string" argument must be of type string. Received type number');return u(t)}return l(t,e,r)}function l(t,e,r){if("string"==typeof t)return function(t,e){if("string"==typeof e&&""!==e||(e="utf8"),!c.isEncoding(e))throw new TypeError("Unknown encoding: "+e);const r=0|g(t,e);let i=a(r);const n=i.write(t,e);return n!==r&&(i=i.slice(0,n)),i}(t,e);if(ArrayBuffer.isView(t))return function(t){if(Y(t,Uint8Array)){const e=new Uint8Array(t);return d(e.buffer,e.byteOffset,e.byteLength)}return p(t)}(t);if(null==t)throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t);if(Y(t,ArrayBuffer)||t&&Y(t.buffer,ArrayBuffer))return d(t,e,r);if("undefined"!=typeof SharedArrayBuffer&&(Y(t,SharedArrayBuffer)||t&&Y(t.buffer,SharedArrayBuffer)))return d(t,e,r);if("number"==typeof t)throw new TypeError('The "value" argument must not be of type number. Received type number');const i=t.valueOf&&t.valueOf();if(null!=i&&i!==t)return c.from(i,e,r);const n=function(t){if(c.isBuffer(t)){const e=0|f(t.length),r=a(e);return 0===r.length||t.copy(r,0,0,e),r}return void 0!==t.length?"number"!=typeof t.length||X(t.length)?a(0):p(t):"Buffer"===t.type&&Array.isArray(t.data)?p(t.data):void 0}(t);if(n)return n;if("undefined"!=typeof Symbol&&null!=Symbol.toPrimitive&&"function"==typeof t[Symbol.toPrimitive])return c.from(t[Symbol.toPrimitive]("string"),e,r);throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t)}function h(t){if("number"!=typeof t)throw new TypeError('"size" argument must be of type number');if(t<0)throw new RangeError('The value "'+t+'" is invalid for option "size"')}function u(t){return h(t),a(t<0?0:0|f(t))}function p(t){const e=t.length<0?0:0|f(t.length),r=a(e);for(let i=0;i<e;i+=1)r[i]=255&t[i];return r}function d(t,e,r){if(e<0||t.byteLength<e)throw new RangeError('"offset" is outside of buffer bounds');if(t.byteLength<e+(r||0))throw new RangeError('"length" is outside of buffer bounds');let i;return i=void 0===e&&void 0===r?new Uint8Array(t):void 0===r?new Uint8Array(t,e):new Uint8Array(t,e,r),Object.setPrototypeOf(i,c.prototype),i}function f(t){if(t>=o)throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+o.toString(16)+" bytes");return 0|t}function g(t,e){if(c.isBuffer(t))return t.length;if(ArrayBuffer.isView(t)||Y(t,ArrayBuffer))return t.byteLength;if("string"!=typeof t)throw new TypeError('The "string" argument must be one of type string, Buffer, or ArrayBuffer. Received type '+typeof t);const r=t.length,i=arguments.length>2&&!0===arguments[2];if(!i&&0===r)return 0;let n=!1;for(;;)switch(e){case"ascii":case"latin1":case"binary":return r;case"utf8":case"utf-8":return q(t).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*r;case"hex":return r>>>1;case"base64":return Q(t).length;default:if(n)return i?-1:q(t).length;e=(""+e).toLowerCase(),n=!0}}function m(t,e,r){let i=!1;if((void 0===e||e<0)&&(e=0),e>this.length)return"";if((void 0===r||r>this.length)&&(r=this.length),r<=0)return"";if((r>>>=0)<=(e>>>=0))return"";for(t||(t="utf8");;)switch(t){case"hex":return k(this,e,r);case"utf8":case"utf-8":return x(this,e,r);case"ascii":return I(this,e,r);case"latin1":case"binary":return R(this,e,r);case"base64":return T(this,e,r);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return N(this,e,r);default:if(i)throw new TypeError("Unknown encoding: "+t);t=(t+"").toLowerCase(),i=!0}}function w(t,e,r){const i=t[e];t[e]=t[r],t[r]=i}function y(t,e,r,i,n){if(0===t.length)return-1;if("string"==typeof r?(i=r,r=0):r>2147483647?r=2147483647:r<-2147483648&&(r=-2147483648),X(r=+r)&&(r=n?0:t.length-1),r<0&&(r=t.length+r),r>=t.length){if(n)return-1;r=t.length-1}else if(r<0){if(!n)return-1;r=0}if("string"==typeof e&&(e=c.from(e,i)),c.isBuffer(e))return 0===e.length?-1:b(t,e,r,i,n);if("number"==typeof e)return e&=255,"function"==typeof Uint8Array.prototype.indexOf?n?Uint8Array.prototype.indexOf.call(t,e,r):Uint8Array.prototype.lastIndexOf.call(t,e,r):b(t,[e],r,i,n);throw new TypeError("val must be string, number or Buffer")}function b(t,e,r,i,n){let s,o=1,a=t.length,c=e.length;if(void 0!==i&&("ucs2"===(i=String(i).toLowerCase())||"ucs-2"===i||"utf16le"===i||"utf-16le"===i)){if(t.length<2||e.length<2)return-1;o=2,a/=2,c/=2,r/=2}function l(t,e){return 1===o?t[e]:t.readUInt16BE(e*o)}if(n){let i=-1;for(s=r;s<a;s++)if(l(t,s)===l(e,-1===i?0:s-i)){if(-1===i&&(i=Line truncated
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+e.read=function(t,e,r,i,n){var s,o,a=8*n-i-1,c=(1<<a)-1,l=c>>1,h=-7,u=r?n-1:0,p=r?-1:1,d=t[e+u];for(u+=p,s=d&(1<<-h)-1,d>>=-h,h+=a;h>0;s=256*s+t[e+u],u+=p,h-=8);for(o=s&(1<<-h)-1,s>>=-h,h+=i;h>0;o=256*o+t[e+u],u+=p,h-=8);if(0===s)s=1-l;else{if(s===c)return o?NaN:1/0*(d?-1:1);o+=Math.pow(2,i),s-=l}return(d?-1:1)*o*Math.pow(2,s-i)},e.write=function(t,e,r,i,n,s){var o,a,c,l=8*s-n-1,h=(1<<l)-1,u=h>>1,p=23===n?Math.pow(2,-24)-Math.pow(2,-77):0,d=i?0:s-1,f=i?1:-1,g=e<0||0===e&&1/e<0?1:0;for(e=Math.abs(e),isNaN(e)||e===1/0?(a=isNaN(e)?1:0,o=h):(o=Math.floor(Math.log(e)/Math.LN2),e*(c=Math.pow(2,-o))<1&&(o--,c*=2),(e+=o+u>=1?p/c:p*Math.pow(2,1-u))*c>=2&&(o++,c/=2),o+u>=h?(a=0,o=h):o+u>=1?(a=(e*c-1)*Math.pow(2,n),o+=u):(a=e*Math.pow(2,u-1)*Math.pow(2,n),o=0));n>=8;t[r+d]=255&a,d+=f,a/=256,n-=8);for(o=o<<n|a,l+=n;l>0;t[r+d]=255&o,d+=f,o/=256,l-=8);t[r+d-f]|=128*g}},5717:t=>{"function"==typeof Object.create?t.exports=function(t,e){e&&(t.super_=e,t.prototype=Object.create(e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}))}:t.exports=function(t,e){if(e){t.super_=e;var r=function(){};r.prototype=e.prototype,t.prototype=new r,t.prototype.constructor=t}}},6512:function(t){!function(e){"use strict";const r="(0?\\d+|0x[a-f0-9]+)",i={fourOctet:new RegExp(`^${r}\\.${r}\\.${r}\\.${r}$`,"i"),threeOctet:new RegExp(`^${r}\\.${r}\\.${r}$`,"i"),twoOctet:new RegExp(`^${r}\\.${r}$`,"i"),longValue:new RegExp(`^${r}$`,"i")},n=new RegExp("^0[0-7]+$","i"),s=new RegExp("^0x[a-f0-9]+$","i"),o="%[0-9a-z]{1,}",a="(?:[0-9a-f]+::?)+",c={zoneIndex:new RegExp(o,"i"),native:new RegExp(`^(::)?(${a})?([0-9a-f]+)?(::)?(${o})?$`,"i"),deprecatedTransitional:new RegExp(`^(?:::)(${r}\\.${r}\\.${r}\\.${r}(${o})?)$`,"i"),transitional:new RegExp(`^((?:${a})|(?:::)(?:${a})?)${r}\\.${r}\\.${r}\\.${r}(${o})?$`,"i")};function l(t,e){if(t.indexOf("::")!==t.lastIndexOf("::"))return null;let r,i,n=0,s=-1,o=(t.match(c.zoneIndex)||[])[0];for(o&&(o=o.substring(1),t=t.replace(/%.+$/,""));(s=t.indexOf(":",s+1))>=0;)n++;if("::"===t.substr(0,2)&&n--,"::"===t.substr(-2,2)&&n--,n>e)return null;for(i=e-n,r=":";i--;)r+="0:";return":"===(t=t.replace("::",r))[0]&&(t=t.slice(1)),":"===t[t.length-1]&&(t=t.slice(0,-1)),{parts:e=function(){const e=t.split(":"),r=[];for(let t=0;t<e.length;t++)r.push(parseInt(e[t],16));return r}(),zoneId:o}}function h(t,e,r,i){if(t.length!==e.length)throw new Error("ipaddr: cannot match CIDR for objects with different lengths");let n,s=0;for(;i>0;){if(n=r-i,n<0&&(n=0),t[s]>>n!=e[s]>>n)return!1;i-=r,s+=1}return!0}function u(t){if(s.test(t))return parseInt(t,16);if("0"===t[0]&&!isNaN(parseInt(t[1],10))){if(n.test(t))return parseInt(t,8);throw new Error(`ipaddr: cannot parse ${t} as octal`)}return parseInt(t,10)}function p(t,e){for(;t.length<e;)t=`0${t}`;return t}const d={};d.IPv4=function(){function t(t){if(4!==t.length)throw new Error("ipaddr: ipv4 octet count should be 4");let e,r;for(e=0;e<t.length;e++)if(r=t[e],!(0<=r&&r<=255))throw new Error("ipaddr: ipv4 octet should fit in 8 bits");this.octets=t}return t.prototype.SpecialRanges={unspecified:[[new t([0,0,0,0]),8]],broadcast:[[new t([255,255,255,255]),32]],multicast:[[new t([224,0,0,0]),4]],linkLocal:[[new t([169,254,0,0]),16]],loopback:[[new t([127,0,0,0]),8]],carrierGradeNat:[[new t([100,64,0,0]),10]],private:[[new t([10,0,0,0]),8],[new t([172,16,0,0]),12],[new t([192,168,0,0]),16]],reserved:[[new t([192,0,0,0]),24],[new t([192,0,2,0]),24],[new t([192,88,99,0]),24],[new t([198,51,100,0]),24],[new t([203,0,113,0]),24],[new t([240,0,0,0]),4]]},t.prototype.kind=function(){return"ipv4"},t.prototype.match=function(t,e){let r;if(void 0===e&&(r=t,t=r[0],e=r[1]),"ipv4"!==t.kind())throw new Error("ipaddr: cannot match ipv4 address with non-ipv4 one");return h(this.octets,t.octets,8,e)},t.prototype.prefixLengthFromSubnetMask=function(){let t=0,e=!1;const r={0:8,128:7,192:6,224:5,240:4,248:3,252:2,254:1,255:0};let i,n,s;for(i=3;i>=0;i-=1){if(n=this.octets[i],!(n in r))return null;if(s=r[n],e&&0!==s)return null;8!==s&&(e=!0),t+=s}return 32-t},t.prototype.range=function(){return d.subnetMatch(this,this.SpecialRanges)},t.prototype.toByteArray=function(){return this.octets.slice(0)},t.prototype.toIPv4MappedAddress=function(){return d.IPv6.parse(`::ffff:${this.toString()}`)},t.prototype.toNormalizedString=function(){return this.toString()},t.prototype.toString=function(){return this.octets.join(".")},t}(),d.IPv4.broadcastAddressFromCIDR=function(t){try{const e=this.parseCIDR(t),r=e[0].toByteArray(),i=this.subnetMaskFromPrefixLength(e[1]).toByteArray(),n=[];let s=0;for(;s<4;)n.push(parseInt(r[s],10)|255^parseInt(i[s],10)),s++;return new this(n)}catch(t){throw new Error("ipaddr: the address does not have IPv4 CIDR format")}},d.IPv4.isIPv4=function(t){return null!==this.parser(t)},d.IPv4.isValid=function(t){try{return new this(this.parser(t)),!0}catch(t){return!1}},d.IPv4.isValidFourPartDecimal=function(t){return!(!d.IPv4.isValid(t)||!t.match(/^(0|[1-9]\d*)(\.(0|[1-9]\d*)){3}$/))},d.Line truncated
+/*! *****************************************************************************
+Copyright (C) Microsoft. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */!function(t){!function(e){var i="object"==typeof r.g?r.g:"object"==typeof self?self:"object"==typeof this?this:Function("return this;")(),s=o(t);function o(t,e){return function(r,i){"function"!=typeof t[r]&&Object.defineProperty(t,r,{configurable:!0,writable:!0,value:i}),e&&e(r,i)}}void 0===i.Reflect?i.Reflect=t:s=o(i.Reflect,s),function(t){var e=Object.prototype.hasOwnProperty,r="function"==typeof Symbol,i=r&&void 0!==Symbol.toPrimitive?Symbol.toPrimitive:"@@toPrimitive",s=r&&void 0!==Symbol.iterator?Symbol.iterator:"@@iterator",o="function"==typeof Object.create,a={__proto__:[]}instanceof Array,c=!o&&!a,l={create:o?function(){return M(Object.create(null))}:a?function(){return M({__proto__:null})}:function(){return M({})},has:c?function(t,r){return e.call(t,r)}:function(t,e){return e in t},get:c?function(t,r){return e.call(t,r)?t[r]:void 0}:function(t,e){return t[e]}},h=Object.getPrototypeOf(Function),u="object"==typeof n&&n.env&&"true"===n.env.REFLECT_METADATA_USE_MAP_POLYFILL,p=u||"function"!=typeof Map||"function"!=typeof Map.prototype.entries?function(){var t={},e=[],r=function(){function t(t,e,r){this._index=0,this._keys=t,this._values=e,this._selector=r}return t.prototype["@@iterator"]=function(){return this},t.prototype[s]=function(){return this},t.prototype.next=function(){var t=this._index;if(t>=0&&t<this._keys.length){var r=this._selector(this._keys[t],this._values[t]);return t+1>=this._keys.length?(this._index=-1,this._keys=e,this._values=e):this._index++,{value:r,done:!1}}return{value:void 0,done:!0}},t.prototype.throw=function(t){throw this._index>=0&&(this._index=-1,this._keys=e,this._values=e),t},t.prototype.return=function(t){return this._index>=0&&(this._index=-1,this._keys=e,this._values=e),{value:t,done:!0}},t}();return function(){function e(){this._keys=[],this._values=[],this._cacheKey=t,this._cacheIndex=-2}return Object.defineProperty(e.prototype,"size",{get:function(){return this._keys.length},enumerable:!0,configurable:!0}),e.prototype.has=function(t){return this._find(t,!1)>=0},e.prototype.get=function(t){var e=this._find(t,!1);return e>=0?this._values[e]:void 0},e.prototype.set=function(t,e){var r=this._find(t,!0);return this._values[r]=e,this},e.prototype.delete=function(e){var r=this._find(e,!1);if(r>=0){for(var i=this._keys.length,n=r+1;n<i;n++)this._keys[n-1]=this._keys[n],this._values[n-1]=this._values[n];return this._keys.length--,this._values.length--,e===this._cacheKey&&(this._cacheKey=t,this._cacheIndex=-2),!0}return!1},e.prototype.clear=function(){this._keys.length=0,this._values.length=0,this._cacheKey=t,this._cacheIndex=-2},e.prototype.keys=function(){return new r(this._keys,this._values,i)},e.prototype.values=function(){return new r(this._keys,this._values,n)},e.prototype.entries=function(){return new r(this._keys,this._values,o)},e.prototype["@@iterator"]=function(){return this.entries()},e.prototype[s]=function(){return this.entries()},e.prototype._find=function(t,e){return this._cacheKey!==t&&(this._cacheIndex=this._keys.indexOf(this._cacheKey=t)),this._cacheIndex<0&&e&&(this._cacheIndex=this._keys.length,this._keys.push(t),this._values.push(void 0)),this._cacheIndex},e}();function i(t,e){return t}function n(t,e){return e}function o(t,e){return[t,e]}}():Map,d=u||"function"!=typeof Set||"function"!=typeof Set.prototype.entries?function(){function t(){this._map=new p}return Object.defineProperty(t.prototype,"size",{get:function(){return this._map.size},enumerable:!0,configurable:!0}),t.prototype.has=function(t){return this._map.has(t)},t.prototype.add=function(t){return this._map.set(t,t),this},t.prototype.delete=function(t){return this._map.delete(t)},t.prototype.clear=function(){this._map.clear()},t.prototype.keys=function(){return this._map.keys()},t.prototype.values=function(){return this._map.values()},t.prototype.entries=function(){return this._map.entries()},t.prototype["@@iterator"]=function(){return this.keys()},t.prototype[s]=function(){return this.keys()},t}():Set,f=new(u||"function"!=typeof WeakMap?function(){var t=l.create(),r=i();return function(){function t(){this._key=i()}return t.prototype.has=function(t){var e=n(t,!1);return void 0!==e&&l.has(e,this._key)},t.prototype.get=function(t){var e=n(t,!1);return void 0!==e?l.get(e,this._key):void 0},t.prototype.set=function(t,e){return n(t,!0)[this._key]=e,this},t.prototype.delete=function(t){var e=n(t,!1);return void 0!==e&&delete e[this._key]},t.prototype.clear=function(){this._key=i()},t}();function i(){var e;do{e="@@WeakMap@@"+o()}while(l.has(t,e));return t[e]=!0,e}function n(t,i){if(!e.call(t,r)){if(!i)return;Object.defineProperty(t,r,{value:l.create()})}return t[r]}function s(t,e){for(var r=0;r<e;++r)t[r]=255*Math.random()|0;return t}function o(){var t=function(t){return"function"==typeof Uint8Array?"undefined"!=typeof crypto?crypto.getRandomValues(new Uint8Array(t)):"undefined"!=typeof msCrypto?msCrypto.getRandomValues(nLine truncated
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+var i=r(8764),n=i.Buffer;function s(t,e){for(var r in t)e[r]=t[r]}function o(t,e,r){return n(t,e,r)}n.from&&n.alloc&&n.allocUnsafe&&n.allocUnsafeSlow?t.exports=i:(s(i,e),e.Buffer=o),o.prototype=Object.create(n.prototype),s(n,o),o.from=function(t,e,r){if("number"==typeof t)throw new TypeError("Argument must not be a number");return n(t,e,r)},o.alloc=function(t,e,r){if("number"!=typeof t)throw new TypeError("Argument must be a number");var i=n(t);return void 0!==e?"string"==typeof r?i.fill(e,r):i.fill(e):i.fill(0),i},o.allocUnsafe=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return n(t)},o.allocUnsafeSlow=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return i.SlowBuffer(t)}},2830:(t,e,r)=>{t.exports=n;var i=r(7187).EventEmitter;function n(){i.call(this)}r(5717)(n,i),n.Readable=r(9481),n.Writable=r(4229),n.Duplex=r(6753),n.Transform=r(4605),n.PassThrough=r(2725),n.finished=r(8610),n.pipeline=r(9946),n.Stream=n,n.prototype.pipe=function(t,e){var r=this;function n(e){t.writable&&!1===t.write(e)&&r.pause&&r.pause()}function s(){r.readable&&r.resume&&r.resume()}r.on("data",n),t.on("drain",s),t._isStdio||e&&!1===e.end||(r.on("end",a),r.on("close",c));var o=!1;function a(){o||(o=!0,t.end())}function c(){o||(o=!0,"function"==typeof t.destroy&&t.destroy())}function l(t){if(h(),0===i.listenerCount(this,"error"))throw t}function h(){r.removeListener("data",n),t.removeListener("drain",s),r.removeListener("end",a),r.removeListener("close",c),r.removeListener("error",l),t.removeListener("error",l),r.removeListener("end",h),r.removeListener("close",h),t.removeListener("close",h)}return r.on("error",l),t.on("error",l),r.on("end",h),r.on("close",h),t.on("close",h),t.emit("pipe",r),t}},2553:(t,e,r)=>{"use strict";var i=r(9509).Buffer,n=i.isEncoding||function(t){switch((t=""+t)&&t.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function s(t){var e;switch(this.encoding=function(t){var e=function(t){if(!t)return"utf8";for(var e;;)switch(t){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return t;default:if(e)return;t=(""+t).toLowerCase(),e=!0}}(t);if("string"!=typeof e&&(i.isEncoding===n||!n(t)))throw new Error("Unknown encoding: "+t);return e||t}(t),this.encoding){case"utf16le":this.text=c,this.end=l,e=4;break;case"utf8":this.fillLast=a,e=4;break;case"base64":this.text=h,this.end=u,e=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=i.allocUnsafe(e)}function o(t){return t<=127?0:t>>5==6?2:t>>4==14?3:t>>3==30?4:t>>6==2?-1:-2}function a(t){var e=this.lastTotal-this.lastNeed,r=function(t,e,r){if(128!=(192&e[0]))return t.lastNeed=0,"�";if(t.lastNeed>1&&e.length>1){if(128!=(192&e[1]))return t.lastNeed=1,"�";if(t.lastNeed>2&&e.length>2&&128!=(192&e[2]))return t.lastNeed=2,"�"}}(this,t);return void 0!==r?r:this.lastNeed<=t.length?(t.copy(this.lastChar,e,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(t.copy(this.lastChar,e,0,t.length),void(this.lastNeed-=t.length))}function c(t,e){if((t.length-e)%2==0){var r=t.toString("utf16le",e);if(r){var i=r.charCodeAt(r.length-1);if(i>=55296&&i<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1],r.slice(0,-1)}return r}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=t[t.length-1],t.toString("utf16le",e,t.length-1)}function l(t){var e=t&&t.length?this.write(t):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return e+this.lastChar.toString("utf16le",0,r)}return e}function h(t,e){var r=(t.length-e)%3;return 0===r?t.toString("base64",e):(this.lastNeed=3-r,this.lastTotal=3,1===r?this.lastChar[0]=t[t.length-1]:(this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1]),t.toString("base64",e,t.length-r))}function u(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+this.lastChar.toString("base64",0,3-this.lastNeed):e}function p(t){return t.toString(this.encoding)}function d(t){return t&&t.length?this.write(t):""}e.s=s,s.prototype.write=function(t){if(0===t.length)return"";var e,r;if(this.lastNeed){if(void 0===(e=this.fillLast(t)))return"";r=this.lastNeed,this.lastNeed=0}else r=0;return r<t.length?e?e+this.text(t,r):this.text(t,r):e||""},s.prototype.end=function(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+"�":e},s.prototype.text=function(t,e){var r=function(t,e,r){var i=e.length-1;if(i<r)return 0;var n=o(e[i]);return n>=0?(n>0&&(t.lastNeed=n-1),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(t.lastNeed=n-2),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(2===n?n=0:t.lastNeed=n-3),n):0}(this,t,e);if(!this.lastNeed)return t.toString("utf8",e);this.lastTotal=r;var i=t.length-(r-this.lastNeed);return t.copy(tLine truncated
+/*!
+ * hash-wasm (https://www.npmjs.com/package/hash-wasm)
+ * (c) Dani Biro
+ * @license MIT
+ */
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */function B(t,e,r,i){return new(r||(r=Promise))((function(n,s){function o(t){try{c(i.next(t))}catch(t){s(t)}}function a(t){try{c(i.throw(t))}catch(t){s(t)}}function c(t){var e;t.done?n(t.value):(e=t.value,e instanceof r?e:new r((function(t){t(e)}))).then(o,a)}c((i=i.apply(t,e||[])).next())}))}class P{constructor(){this.mutex=Promise.resolve()}lock(){let t=()=>{};return this.mutex=this.mutex.then((()=>new Promise(t))),new Promise((e=>{t=e}))}dispatch(t){return B(this,void 0,void 0,(function*(){const e=yield this.lock();try{return yield Promise.resolve(t())}finally{e()}}))}}const L="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:r.g,D=null!==(I=L.Buffer)&&void 0!==I?I:null,U=L.TextEncoder?new L.TextEncoder:null;function M(t,e){return(15&t)+(t>>6|t>>3&8)<<4|(15&e)+(e>>6|e>>3&8)}const F="a".charCodeAt(0)-10,H="0".charCodeAt(0);function W(t,e,r){let i=0;for(let n=0;n<r;n++){let r=e[n]>>>4;t[i++]=r>9?r+F:r+H,r=15&e[n],t[i++]=r>9?r+F:r+H}return String.fromCharCode.apply(null,t)}const j=null!==D?t=>{if("string"==typeof t){const e=D.from(t,"utf8");return new Uint8Array(e.buffer,e.byteOffset,e.length)}if(D.isBuffer(t))return new Uint8Array(t.buffer,t.byteOffset,t.length);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")}:t=>{if("string"==typeof t)return U.encode(t);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")},V="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",G=new Uint8Array(256);for(let t=0;t<V.length;t++)G[V.charCodeAt(t)]=t;function z(t){const e=function(t){let e=Math.floor(.75*t.length);const r=t.length;return"="===t[r-1]&&(e-=1,"="===t[r-2]&&(e-=1)),e}(t),r=t.length,i=new Uint8Array(e);let n=0;for(let e=0;e<r;e+=4){const r=G[t.charCodeAt(e)],s=G[t.charCodeAt(e+1)],o=G[t.charCodeAt(e+2)],a=G[t.charCodeAt(e+3)];i[n]=r<<2|s>>4,n+=1,i[n]=(15&s)<<4|o>>2,n+=1,i[n]=(3&o)<<6|63&a,n+=1}return i}const q=16384,Q=new P,K=new Map;function Y(t,e){return B(this,void 0,void 0,(function*(){let r=null,i=null,n=!1;if("undefined"==typeof WebAssembly)throw new Error("WebAssembly is not supported in this environment!");const s=()=>new DataView(r.exports.memory.buffer).getUint32(r.exports.STATE_SIZE,!0),o=Q.dispatch((()=>B(this,void 0,void 0,(function*(){if(!K.has(t.name)){const e=z(t.data),r=WebAssembly.compile(e);K.set(t.name,r)}const e=yield K.get(t.name);r=yield WebAssembly.instantiate(e,{})})))),a=(t=null)=>{n=!0,r.exports.Hash_Init(t)},c=t=>{if(!n)throw new Error("update() called before init()");(t=>{let e=0;for(;e<t.length;){const n=t.subarray(e,e+q);e+=n.length,i.set(n),r.exports.Hash_Update(n.length)}})(j(t))},l=new Uint8Array(2*e),h=(t,s=null)=>{if(!n)throw new Error("digest() called before init()");return n=!1,r.exports.Hash_Final(s),"binary"===t?i.slice(0,e):W(l,i,e)},u=t=>"string"==typeof t?t.length<4096:t.byteLength<q;let p=u;switch(t.name){case"argon2":case"scrypt":p=()=>!0;break;case"blake2b":case"blake2s":p=(t,e)=>e<=512&&u(t);break;case"blake3":p=(t,e)=>0===e&&u(t);break;case"xxhash64":case"xxhash3":case"xxhash128":p=()=>!1}return yield(()=>B(this,void 0,void 0,(function*(){r||(yield o);const t=r.exports.Hash_GetBuffer(),e=r.exports.memory.buffer;i=new Uint8Array(e,t,q)})))(),{getMemory:()=>i,writeMemory:(t,e=0)=>{i.set(t,e)},getExports:()=>r.exports,setMemorySize:t=>{r.exports.Hash_SetMemorySize(t);const e=r.exports.Hash_GetBuffer(),n=r.exports.memory.buffer;i=new Uint8Array(n,e,t)},init:a,update:c,digest:h,save:()=>{if(!n)throw new Error("save() can only be called after init() and before digest()");const e=r.exports.Hash_GetState(),i=s(),o=r.exports.memory.buffer,a=new Uint8Array(o,e,i),c=new Uint8Array(4+i);return function(t,e){const r=e.length>>1;for(let i=0;i<r;i++){const r=i<<1;t[i]=M(e.charCodeAt(r),e.charCodeAt(r+1))}}(c,t.hash),c.set(a,4),c},load:e=>{if(!(e instanceof Uint8Array))throw new Error("load() expects an Uint8Array generated by save()");const i=r.exports.Hash_GetState(),o=s(),a=4+o,c=r.exports.memory.buffer;if(e.length!==a)throw new Error(`Bad state length (expected ${a} bytes, got ${e.length})`);if(!function(t,e){if(t.length!==2*e.length)return!1;for(let r=0;r<e.length;r++){const i=r<<1;if(e[r]!==M(t.charCodeAt(i),t.charCodeAt(i+1)))return!1}return!0}(t.hash,e.subarray(0,4)))throw new Error("This state was written by an incompatible hash implementation");const l=e.subarray(4);new Uint8Array(c,i,o).set(l),n=!0},calculate:(t,n=null,s=null)=>{if(!p(t,n))return a(n),c(t),h("hex",s);const o=j(t);return i.set(o),r.exports.Hash_Calculate(o.length,n,s),W(l,i,e)},hashLength:e}}))}new P,new P,new DataView(new ArrayBuffer(4)),new P,new P,new P,new P,new P,new P,new P,new P,new P;var X={name:"sha256",data:"AGFzbQEAAAABEQRgAAF/YAF/AGACf38AYAAAAwgHAAEBAgMAAgQFAXABAQEFBAEBAgIGDgJ/AUHwiQULfwBBgAgLB3AIBm1lbW9yeQIADLine truncated
+/*! js-yaml 4.1.0 https://github.com/nodeca/js-yaml @license MIT */function so(t){return null==t}var oo={isNothing:so,isObject:function(t){return"object"==typeof t&&null!==t},toArray:function(t){return Array.isArray(t)?t:so(t)?[]:[t]},repeat:function(t,e){var r,i="";for(r=0;r<e;r+=1)i+=t;return i},isNegativeZero:function(t){return 0===t&&Number.NEGATIVE_INFINITY===1/t},extend:function(t,e){var r,i,n,s;if(e)for(r=0,i=(s=Object.keys(e)).length;r<i;r+=1)t[n=s[r]]=e[n];return t}};function ao(t,e){var r="",i=t.reason||"(unknown reason)";return t.mark?(t.mark.name&&(r+='in "'+t.mark.name+'" '),r+="("+(t.mark.line+1)+":"+(t.mark.column+1)+")",!e&&t.mark.snippet&&(r+="\n\n"+t.mark.snippet),i+" "+r):i}function co(t,e){Error.call(this),this.name="YAMLException",this.reason=t,this.mark=e,this.message=ao(this,!1),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack||""}co.prototype=Object.create(Error.prototype),co.prototype.constructor=co,co.prototype.toString=function(t){return this.name+": "+ao(this,t)};var lo=co;function ho(t,e,r,i,n){var s="",o="",a=Math.floor(n/2)-1;return i-e>a&&(e=i-a+(s=" ... ").length),r-i>a&&(r=i+a-(o=" ...").length),{str:s+t.slice(e,r).replace(/\t/g,"→")+o,pos:i-e+s.length}}function uo(t,e){return oo.repeat(" ",e-t.length)+t}var po=["kind","multi","resolve","construct","instanceOf","predicate","represent","representName","defaultStyle","styleAliases"],fo=["scalar","sequence","mapping"],go=function(t,e){if(e=e||{},Object.keys(e).forEach((function(e){if(-1===po.indexOf(e))throw new lo('Unknown option "'+e+'" is met in definition of "'+t+'" YAML type.')})),this.options=e,this.tag=t,this.kind=e.kind||null,this.resolve=e.resolve||function(){return!0},this.construct=e.construct||function(t){return t},this.instanceOf=e.instanceOf||null,this.predicate=e.predicate||null,this.represent=e.represent||null,this.representName=e.representName||null,this.defaultStyle=e.defaultStyle||null,this.multi=e.multi||!1,this.styleAliases=function(t){var e={};return null!==t&&Object.keys(t).forEach((function(r){t[r].forEach((function(t){e[String(t)]=r}))})),e}(e.styleAliases||null),-1===fo.indexOf(this.kind))throw new lo('Unknown kind "'+this.kind+'" is specified for "'+t+'" YAML type.')};function mo(t,e){var r=[];return t[e].forEach((function(t){var e=r.length;r.forEach((function(r,i){r.tag===t.tag&&r.kind===t.kind&&r.multi===t.multi&&(e=i)})),r[e]=t})),r}function wo(t){return this.extend(t)}wo.prototype.extend=function(t){var e=[],r=[];if(t instanceof go)r.push(t);else if(Array.isArray(t))r=r.concat(t);else{if(!t||!Array.isArray(t.implicit)&&!Array.isArray(t.explicit))throw new lo("Schema.extend argument should be a Type, [ Type ], or a schema definition ({ implicit: [...], explicit: [...] })");t.implicit&&(e=e.concat(t.implicit)),t.explicit&&(r=r.concat(t.explicit))}e.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.");if(t.loadKind&&"scalar"!==t.loadKind)throw new lo("There is a non-scalar type in the implicit list of a schema. Implicit resolving of such types is not supported.");if(t.multi)throw new lo("There is a multi type in the implicit list of a schema. Multi tags can only be listed as explicit.")})),r.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.")}));var i=Object.create(wo.prototype);return i.implicit=(this.implicit||[]).concat(e),i.explicit=(this.explicit||[]).concat(r),i.compiledImplicit=mo(i,"implicit"),i.compiledExplicit=mo(i,"explicit"),i.compiledTypeMap=function(){var t,e,r={scalar:{},sequence:{},mapping:{},fallback:{},multi:{scalar:[],sequence:[],mapping:[],fallback:[]}};function i(t){t.multi?(r.multi[t.kind].push(t),r.multi.fallback.push(t)):r[t.kind][t.tag]=r.fallback[t.tag]=t}for(t=0,e=arguments.length;t<e;t+=1)arguments[t].forEach(i);return r}(i.compiledImplicit,i.compiledExplicit),i};var yo=wo,bo=new go("tag:yaml.org,2002:str",{kind:"scalar",construct:function(t){return null!==t?t:""}}),Ao=new go("tag:yaml.org,2002:seq",{kind:"sequence",construct:function(t){return null!==t?t:[]}}),_o=new go("tag:yaml.org,2002:map",{kind:"mapping",construct:function(t){return null!==t?t:{}}}),vo=new yo({explicit:[bo,Ao,_o]}),Eo=new go("tag:yaml.org,2002:null",{kind:"scalar",resolve:function(t){if(null===t)return!0;var e=t.length;return 1===e&&"~"===t||4===e&&("null"===t||"Null"===t||"NULL"===t)},construct:function(){return null},predicate:function(t){return null===t},represent:{canonical:function(){return"~"},lowercase:function(){return"null"},uppercase:function(){return"NULL"},camelcase:function(){return"Null"},empty:function(){return""}},defaultStyle:"lowercase"}),So=new go("tag:yaml.org,2002:bool",{kind:"scalar",resolve:function(t){if(null===t)return!1;var e=t.length;return 4===e&&("true"===t||"True"===t||"TRUE"===t)||5===e&&("false"===t||"False"===t||"FALSE"===t)},construct:fuLine truncated
+/*!
+ * Copyright (c) 2014, GMO GlobalSign
+ * Copyright (c) 2015-2022, Peculiar Ventures
+ * All rights reserved.
+ * 
+ * Author 2014-2019, Yury Strozhevsky
+ * 
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ * 
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * 
+ */
+function lc(){if("undefined"==typeof BigInt)throw new Error("BigInt is not defined. Your environment doesn't implement BigInt.")}function hc(t){let e=0,r=0;for(let r=0;r<t.length;r++)e+=t[r].byteLength;const i=new Uint8Array(e);for(let e=0;e<t.length;e++){const n=t[e];i.set(new Uint8Array(n),r),r+=n.byteLength}return i.buffer}function uc(t,e,r,i){return e instanceof Uint8Array?e.byteLength?r<0?(t.error="Wrong parameter: inputOffset less than zero",!1):i<0?(t.error="Wrong parameter: inputLength less than zero",!1):!(e.byteLength-r-i<0&&(t.error="End of input reached before message was fully decoded (inconsistent offset and length values)",1)):(t.error="Wrong parameter: inputBuffer has zero length",!1):(t.error="Wrong parameter: inputBuffer must be 'Uint8Array'",!1)}Math.log(2);class pc{constructor(){this.items=[]}write(t){this.items.push(t)}final(){return hc(this.items)}}const dc=[new Uint8Array([1])],fc="0123456789",gc="name",mc="valueHexView",wc="",yc=new ArrayBuffer(0),bc=new Uint8Array(0),Ac="EndOfContent",_c="OCTET STRING",vc="BIT STRING";function Ec(t){var e;return(e=class extends t{constructor(...t){var e;super(...t);const r=t[0]||{};this.isHexOnly=null!==(e=r.isHexOnly)&&void 0!==e&&e,this.valueHexView=r.valueHex?ic.vJ.toUint8Array(r.valueHex):bc}get valueHex(){return this.valueHexView.slice().buffer}set valueHex(t){this.valueHexView=new Uint8Array(t)}fromBER(t,e,r){const i=t instanceof ArrayBuffer?new Uint8Array(t):t;if(!uc(this,i,e,r))return-1;const n=e+r;return this.valueHexView=i.subarray(e,n),this.valueHexView.length?(this.blockLength=r,n):(this.warnings.push("Zero buffer length"),e)}toBER(t=!1){return this.isHexOnly?t?new ArrayBuffer(this.valueHexView.byteLength):this.valueHexView.byteLength===this.valueHexView.buffer.byteLength?this.valueHexView.buffer:this.valueHexView.slice().buffer:(this.error="Flag 'isHexOnly' is not set, abort",yc)}toJSON(){return{...super.toJSON(),isHexOnly:this.isHexOnly,valueHex:ic.ep.ToHex(this.valueHexView)}}}).NAME="hexBlock",e}class Sc{constructor({blockLength:t=0,error:e="",warnings:r=[],valueBeforeDecode:i=bc}={}){this.blockLength=t,this.error=e,this.warnings=r,this.valueBeforeDecodeView=ic.vJ.toUint8Array(i)}static blockName(){return this.NAME}get valueBeforeDecode(){return this.valueBeforeDecodeView.slice().buffer}set valueBeforeDecode(t){this.valueBeforeDecodeView=new Uint8Array(t)}toJSON(){return{blockName:this.constructor.NAME,blockLength:this.blockLength,error:this.error,warnings:this.warnings,valueBeforeDecode:ic.ep.ToHex(this.valueBeforeDecodeView)}}}Sc.NAME="baseBlock";class Tc extends Sc{fromBER(t,e,r){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}toBER(t,e){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}}Tc.NAME="valueBlock";class xc extends(Ec(Sc)){constructor({idBlock:t={}}={}){var e,r,i,n;super(),t?(this.isHexOnly=null!==(e=t.isHexOnly)&&void 0!==e&&e,this.valueHexView=t.valueHex?ic.vJ.toUint8Array(t.valueHex):bc,this.tagClass=null!==(r=t.tagClass)&&void 0!==r?r:-1,this.tagNumber=null!==(i=t.tagNumber)&&void 0!==i?i:-1,this.isConstructed=null!==(n=t.isConstructed)&&void 0!==n&&n):(this.tagClass=-1,this.tagNumber=-1,this.isConstructed=!1)}toBER(t=!1){let e=0;switch(this.tagClass){case 1:e|=0;break;case 2:e|=64;break;case 3:e|=128;break;case 4:e|=192;break;default:return this.error="Unknown tag class",yc}if(this.isConstructed&&(e|=32),this.tagNumber<31&&!this.isHexOnly){const r=new Uint8Array(1);if(!t){let t=this.tagNumber;t&=31,e|=t,r[0]=e}return r.buffer}if(!this.isHexOnly){const r=sc(this.tagNumber,7),i=new Uint8Array(r),n=r.byteLength,s=new Uint8Array(n+1);if(s[0]=31|e,!t){for(let t=0;t<n-1;t++)s[t+1]=128|i[t];s[n]=i[n-1]}return s.buffer}const r=new Uint8Array(this.valueHexView.byteLength+1);if(r[0]=31|e,!t){const t=this.valueHexView;for(let e=0;e<t.length-1;e++)r[e+1]=128|t[e];r[this.valueHexView.byteLength]=t[t.length-1]}return r.buffer}fromBER(t,e,r){const i=ic.vJ.toUint8Array(t);if(!uc(this,i,e,r))return-1;const n=i.subarray(e,e+r);if(0===n.length)return this.error="Zero buffer length",-1;switch(192&n[0]){case 0:this.tagClass=1;break;case 64:this.tagClass=2;break;case 128:this.tagClass=3;break;case 192:this.tagClass=4;break;default:return this.error="Unknown tag class",-1}this.isConstructed=32==(32&n[0]),this.isHexOnly=!1;const s=31&n[0];if(31!==s)this.tagNumber=s,this.blockLength=1;else{let t=1,e=this.valueHexView=new Uint8Array(255),r=255;for(;128&n[t];){if(e[t-1]=127&n[t],t++,t>=n.length)return this.error="End of input reached before message was fully decoded",-1;if(t===r){r+=255;const t=new Uint8Array(r);for(let r=0;r<e.length;r++)t[r]=e[r];e=this.valueHexView=new Uint8Array(r)}}this.blockLength=t+1,e[t-1]=127&n[t];const i=new Uint8Array(t);for(let r=0;r<t;r++)i[r]=e[r];e=this.valueHexView=new Uint8Array(t),e.set(i),this.blockLength<=9?this.tagNumber=nc(e,7):(this.isHexOnly=!0,this.warnings.push("Tag too long, represented as hex-coded"))}if(1=Line truncated
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */var vg=function(t,e){return vg=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])},vg(t,e)};function Eg(t,e){function r(){this.constructor=t}vg(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}function Sg(t){var e="function"==typeof Symbol&&Symbol.iterator,r=e&&t[e],i=0;if(r)return r.call(t);if(t&&"number"==typeof t.length)return{next:function(){return t&&i>=t.length&&(t=void 0),{value:t&&t[i++],done:!t}}};throw new TypeError(e?"Object is not iterable.":"Symbol.iterator is not defined.")}function Tg(t,e){var r="function"==typeof Symbol&&t[Symbol.iterator];if(!r)return t;var i,n,s=r.call(t),o=[];try{for(;(void 0===e||e-- >0)&&!(i=s.next()).done;)o.push(i.value)}catch(t){n={error:t}}finally{try{i&&!i.done&&(r=s.return)&&r.call(s)}finally{if(n)throw n.error}}return o}function xg(){for(var t=[],e=0;e<arguments.length;e++)t=t.concat(Tg(arguments[e]));return t}function Cg(t){return!!t.useClass}function Ig(t){return!!t.useFactory}var Rg=function(){function t(t){this.wrap=t,this.reflectMethods=["get","getPrototypeOf","setPrototypeOf","getOwnPropertyDescriptor","defineProperty","has","set","deleteProperty","apply","construct","ownKeys"]}return t.prototype.createProxy=function(t){var e,r=this,i=!1;return new Proxy({},this.createHandler((function(){return i||(e=t(r.wrap()),i=!0),e})))},t.prototype.createHandler=function(t){var e={};return this.reflectMethods.forEach((function(r){e[r]=function(){for(var e=[],i=0;i<arguments.length;i++)e[i]=arguments[i];e[0]=t();var n=Reflect[r];return n.apply(void 0,xg(e))}})),e},t}();function kg(t){return"string"==typeof t||"symbol"==typeof t}function Ng(t){return"object"==typeof t&&"token"in t&&"transform"in t}function Og(t){return!!t.useToken}function Bg(t){return null!=t.useValue}const Pg=function(){function t(){this._registryMap=new Map}return t.prototype.entries=function(){return this._registryMap.entries()},t.prototype.getAll=function(t){return this.ensure(t),this._registryMap.get(t)},t.prototype.get=function(t){this.ensure(t);var e=this._registryMap.get(t);return e[e.length-1]||null},t.prototype.set=function(t,e){this.ensure(t),this._registryMap.get(t).push(e)},t.prototype.setAll=function(t,e){this._registryMap.set(t,e)},t.prototype.has=function(t){return this.ensure(t),this._registryMap.get(t).length>0},t.prototype.clear=function(){this._registryMap.clear()},t.prototype.ensure=function(t){this._registryMap.has(t)||this._registryMap.set(t,[])},t}(),Lg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Dg=function(){this.scopedResolutions=new Map};var Ug=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Mg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg);const Fg=function(){this.preResolution=new Ug,this.postResolution=new Mg};var Hg=new Map,Wg=function(){function t(t){this.parent=t,this._registry=new Lg,this.interceptors=new Fg,this.disposed=!1,this.disposables=new Set}return t.prototype.register=function(t,e,r){var i;if(void 0===r&&(r={lifecycle:_g.Transient}),this.ensureNotDisposed(),i=function(t){return Cg(t)||Bg(t)||Og(t)||Ig(t)}(e)?e:{useClass:e},Og(i))for(var n=[t],s=i;null!=s;){var o=s.useToken;if(n.includes(o))throw new Error("Token registration cycle detected! "+xg(n,[o]).join(" -> "));n.push(o);var a=this._registry.get(o);s=a&&Og(a.provider)?a.provider:null}if((r.lifecycle===_g.Singleton||r.lifecycle==_g.ContainerScoped||r.lifecycle==_g.ResolutionScoped)&&(Bg(i)||Ig(i)))throw new Error('Cannot use lifecycle "'+_g[r.lifecycle]+'" with ValueProviders or FactoryProviders');return this._registry.set(t,{provider:i,options:r}),this},t.prototype.registerType=function(t,e){return this.ensureNotDisposed(),kg(e)?this.register(t,{useToken:e}):this.register(t,{useClass:e})},t.prototype.registerInstance=function(t,e){return this.ensureNotDisposed(),this.register(t,{useValue:e})},t.prototype.registerSingleton=function(t,e){if(this.ensureNotDisposed(),kg(t)){if(kg(e))return this.register(t,{useToken:e},{lifecycle:_g.Singleton});if(e)return this.register(t,{useClass:e},{lifecycle:_g.Singleton});throw new Error('Cannot register a type name as a singleton without a "to" token')}var r=t;return e&&!kg(e)&&(r=e),this.register(t,{useClass:r},{lifecycle:_g.Singleton})},t.prototype.resolve=function(t,e){void 0===e&&(e=new Dg),this.ensureNotDisposed();var r=this.getRegistration(t);if(!r&&kg(t))throw new Error('Attempted to resolve unregistered dependency token: "'+t.toString()+'"');if(this.executePreResolutionInterceptor(t,"Single"),r){var i=this.resolveRegistration(r,e);return this.executePostResolutionInterceptor(t,i,"Single"),i}if(function(t){return"function"==typeof t||t instanceof Rg}(t))return i=this.construct(t,e),this.executePostResolutionInterceptor(t,i,"Line truncated
+/*!
+ * MIT License
+ * 
+ * Copyright (c) Peculiar Ventures. All rights reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ * 
+ */
+const zm="crypto.algorithm",qm="crypto.algorithmProvider";var Qm;jg.registerSingleton(qm,class{getAlgorithms(){return jg.resolveAll(zm)}toAsnAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toAsnAlgorithm(t);if(r)return r}if(/[0-9.]+/.test(t.name)){const e=new Fp({algorithm:t.name});if("parameters"in t){const r=t;e.parameters=r.parameters}return e}throw new Error("Cannot convert WebCrypto algorithm to ASN.1 algorithm")}toWebAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toWebAlgorithm(t);if(r)return r}return{name:t.algorithm,parameters:t.parameters}}});const Km="1.3.36.3.3.2.8.1.1",Ym=`${Km}.1`,Xm=`${Km}.2`,Jm=`${Km}.3`,Zm=`${Km}.4`,$m=`${Km}.5`,tw=`${Km}.6`,ew=`${Km}.7`,rw=`${Km}.8`,iw=`${Km}.9`,nw=`${Km}.10`,sw=`${Km}.11`,ow=`${Km}.12`,aw=`${Km}.13`,cw=`${Km}.14`,lw="brainpoolP160r1",hw="brainpoolP160t1",uw="brainpoolP192r1",pw="brainpoolP192t1",dw="brainpoolP224r1",fw="brainpoolP224t1",gw="brainpoolP256r1",mw="brainpoolP256t1",ww="brainpoolP320r1",yw="brainpoolP320t1",bw="brainpoolP384r1",Aw="brainpoolP384t1",_w="brainpoolP512r1",vw="brainpoolP512t1",Ew="ECDSA";let Sw=Qm=class{toAsnAlgorithm(t){if(t.name.toLowerCase()===Ew.toLowerCase())if("hash"in t)switch(("string"==typeof t.hash?t.hash:t.hash.name).toLowerCase()){case"sha-1":return Pf;case"sha-256":return Lf;case"sha-384":return Df;case"sha-512":return Uf}else if("namedCurve"in t){let e="";switch(t.namedCurve){case"P-256":e=kf;break;case"K-256":e=Qm.SECP256K1;break;case"P-384":e=Nf;break;case"P-521":e=Of;break;case lw:e=Ym;break;case hw:e=Xm;break;case uw:e=Jm;break;case pw:e=Zm;break;case dw:e=$m;break;case fw:e=tw;break;case gw:e=ew;break;case mw:e=rw;break;case ww:e=iw;break;case yw:e=nw;break;case bw:e=sw;break;case Aw:e=ow;break;case _w:e=aw;break;case vw:e=cw}if(e)return new Fp({algorithm:Sf,parameters:iu.serialize(new Mf({namedCurve:e}))})}return null}toWebAlgorithm(t){switch(t.algorithm){case Tf:return{name:Ew,hash:{name:"SHA-1"}};case Cf:return{name:Ew,hash:{name:"SHA-256"}};case If:return{name:Ew,hash:{name:"SHA-384"}};case Rf:return{name:Ew,hash:{name:"SHA-512"}};case Sf:if(!t.parameters)throw new TypeError("Cannot get required parameters from EC algorithm");switch(iu.parse(t.parameters,Mf).namedCurve){case kf:return{name:Ew,namedCurve:"P-256"};case Qm.SECP256K1:return{name:Ew,namedCurve:"K-256"};case Nf:return{name:Ew,namedCurve:"P-384"};case Of:return{name:Ew,namedCurve:"P-521"};case Ym:return{name:Ew,namedCurve:lw};case Xm:return{name:Ew,namedCurve:hw};case Jm:return{name:Ew,namedCurve:uw};case Zm:return{name:Ew,namedCurve:pw};case $m:return{name:Ew,namedCurve:dw};case tw:return{name:Ew,namedCurve:fw};case ew:return{name:Ew,namedCurve:gw};case rw:return{name:Ew,namedCurve:mw};case iw:return{name:Ew,namedCurve:ww};case nw:return{name:Ew,namedCurve:yw};case sw:return{name:Ew,namedCurve:bw};case ow:return{name:Ew,namedCurve:Aw};case aw:return{name:Ew,namedCurve:_w};case cw:return{name:Ew,namedCurve:vw}}}return null}};Sw.SECP256K1="1.3.132.0.10",Sw=Qm=bg([Vg()],Sw),jg.registerSingleton(zm,Sw);const Tw=Symbol("name"),xw=Symbol("value");class Cw{constructor(t,e={},r=""){this[Tw]=t,this[xw]=r;for(const t in e)this[t]=e[t]}}Cw.NAME=Tw,Cw.VALUE=xw;class Iw{static toString(t){return this.items[t]||t}}Iw.items={[eg]:"sha1",[rg]:"sha224",[ig]:"sha256",[ng]:"sha384",[sg]:"sha512",[jf]:"rsaEncryption",[Kf]:"sha1WithRSAEncryption",[Yf]:"sha224WithRSAEncryption",[Xf]:"sha256WithRSAEncryption",[Jf]:"sha384WithRSAEncryption",[Zf]:"sha512WithRSAEncryption",[Sf]:"ecPublicKey",[Tf]:"ecdsaWithSHA1",[xf]:"ecdsaWithSHA224",[Cf]:"ecdsaWithSHA256",[If]:"ecdsaWithSHA384",[Rf]:"ecdsaWithSHA512",[np]:"TLS WWW server authentication",[sp]:"TLS WWW client authentication",[op]:"Code Signing",[ap]:"E-mail Protection",[cp]:"Time Stamping",[lp]:"OCSP Signing","1.2.840.113549.1.7.2":"Signed Data"};class Rw{static serialize(t){return this.serializeObj(t).join("\n")}static pad(t=0){return"".padStart(2*t," ")}static serializeObj(t,e=0){const r=[];let i=this.pad(e++),n="";const s=t[Cw.VALUE];s&&(n=` ${s}`),r.push(`${i}${t[Cw.NAME]}:${n}`),i=this.pad(e);for(const n in t){if("symbol"==typeof n)continue;const s=t[n],o=n?`${n}: `:"";if("string"==typeof s||"number"==typeof s||"boolean"==typeof s)r.push(`${i}${o}${s}`);else if(s instanceof Date)r.push(`${i}${o}${s.toUTCString()}`);else if(Array.isArray(s))for(const t of s)t[Cw.NAME]=n,r.push(...this.serializeObj(t,e));else if(s instanceof Cw)s[Cw.NAME]=n,r.push(...this.serializeObj(s,e));else if(ic.vJ.isBufferSource(s))n?(r.push(`${i}${o}`),r.push(...this.serializeBufferSource(s,e+1))):r.push(...this.serializeBufferSource(s,e));else{if(!("toTextObject"in s))throw new TypeError("Cannot serialize data in text format. Unsupported type.");{const t=s.toTextObject();t[Cw.NAME]=n,r.push(...this.serializeObj(t,e))}}}return r}static serializeBufferSource(t,e=0){const r=this.pad(e),i=ic.vJ.toUint8Array(t),n=[];for(let t=0;t<i.length;){const e=[];for(let r=0;r<16&&t<i.length;r++){8===r&&e.push("");const n=i[t++].toStrinLine truncated

archivebox/plugins/gallerydl/static/test.txt

@@ -0,0 +1 @@
+test content this should be visible

archivebox/plugins/gallerydl/templates/plugin_replaywebpage__viewer.html

@@ -0,0 +1,40 @@
+{% load tz core_tags static %}
+
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>{{title}}</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+      
+        </style>
+        <style>
+            html, body {
+                width: 100%;
+                height: 100%;
+                background-color: #ddd;
+            }
+        </style>
+    </head>
+    <body>
+        ReplayWeb.page for: {{snapshot.url}} ({{timestamp}}) /{{warc_filename}}
+
+        {{snapshot}}
+
+        <script>
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/sw.min.js
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/ui.min.js
+        </script>
+
+        <style>
+        </style>
+        <script src="/static/ui.js"></script>
+
+        <replay-web-page
+            style="height: 600px"
+            embed="replay"
+            replayBase="/static/"
+            source="/static/test.wacz"
+            url="https://example.com/">
+        </replay-web-page>
+    </body>
+</html>

archivebox/plugins/gallerydl/urls.py

@@ -0,0 +1,12 @@
+from django.urls import path
+
+from .views import GalleryDLIconView, GalleryDLEmbedView, GalleryDLOutputView, GalleryDLDependencyView, GalleryDLExtractorView
+
+urlpatterns = [
+	path('/plugins/gallerydl/icon/<path:path>', GalleryDLIconView(.as_view), name='gallerydl_icon'),
+	path('/plugins/gallerydl/embed/<path:path>', GalleryDLEmbedView.as_view(), name='gallerydl_embed'),
+	path('/plugins/gallerydl/output/<path:path>', GalleryDLOutputView.as_view(), name='gallerydl_output'),
+
+	path('/plugins/gallerydl/dependency/', GalleryDLDependencyView.as_view(), name='gallerydl_dependency'),
+	path('/plugins/gallerydl/extractor/', GalleryDLExtractorView.as_view(), name='gallerydl_extractor'),
+]

archivebox/plugins/gallerydl/views.py

@@ -0,0 +1,78 @@
+import os
+import sys
+from pathlib import Path
+
+from django.views import View
+from django.shortcuts import render
+from django.db.models import Q
+
+from core.models import Snapshot
+
+# from archivebox.config import PUBLIC_SNAPSHOTS
+PUBLIC_SNAPSHOTS = True
+
+
+class GalleryDLIconView(View):
+    template_name = 'plugin_gallerydl__icon.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        # ...
+        return render(template_name=self.template_name, request=self.request, context=context)
+
+
+class GalleryDLEmbedView(View):
+    template_name = 'plugin_gallerydl__embed.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        # ...
+        return render(template_name=self.template_name, request=self.request, context=context)
+
+
+class GalleryDLOutputView(View):
+    template_name = 'plugin_gallerydl__output.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        # ...
+        return render(template_name=self.template_name, request=self.request, context=context)

archivebox/plugins/replaywebpage/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.plugins.replaywebpage'

archivebox/plugins/replaywebpage/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class ReplayWebPageConfig(AppConfig):
+    label = "ReplayWeb.Page"
+    name = "plugin_replaywebpage"
+    
+    default_auto_field = "django.db.models.BigAutoField"

archivebox/plugins/replaywebpage/extractors.py

@@ -0,0 +1,50 @@
+# browsertrix extractor
+
+def save_browsertrix(link, out_dir, timeout, config):
+
+
+	browsertrix_dir = out_dir / 'browsertrix'
+	browsertrix_dir.mkdir(exist_ok=True)
+
+	crawl_id = link.timestamp
+
+	browsertrix_crawler_cmd = [
+		'crawl',
+		f'--url', link.url,
+		f'--collection={crawl_id}',
+		'--scopeType=page',
+		'--generateWACZ',
+		'--text=final-to-warc',
+		'--timeLimit=60',
+	]
+
+	remote_cmd = """
+	rm /tmp/dump.rdb;
+	rm -rf /crawls/collections;
+	mkdir /crawls/collections;
+	env CRAWL_ID={crawl_id} 
+	"""
+
+	local_cmd = ['nc', 'browsertrix', '2222']
+
+	status = 'succeeded'
+    timer = TimedProgress(timeout, prefix='      ')
+    try:
+        result = run(local_cmd, cwd=str(out_dir), input=remote_cmd, timeout=timeout)
+		
+		cmd_output = result.stdout.decode()
+
+		wacz_output_file = Path('/browsertrix/crawls') / crawl_id / f'{crawl_id}'.wacz
+
+		copy_and_overwrite(wacz_output_file, browsertrix_dir / wacz_output_file.name)
+
+
+
+TEMPLATE = """
+
+"""
+
+# rm /tmp/dump.rdb;
+# rm -rf /crawls/collections;
+# mkdir /crawls/collections;
+# env CRAWL_ID=tec2342 crawl --url 'https://example.com' --scopeType page --generateWACZ --collection tec2342 --text final-to-warc --timeLimit 60

archivebox/plugins/replaywebpage/models.py

@@ -0,0 +1,12 @@
+# from solo.models import SingletonModel
+
+
+# class ReplayWebPageConfiguration(SingletonModel):
+#     site_name = models.CharField(max_length=255, default='Site Name')
+#     maintenance_mode = models.BooleanField(default=False)
+
+#     def __str__(self):
+#         return "Site Configuration"
+
+#     class Meta:
+#         verbose_name = "Site Configuration"

archivebox/plugins/replaywebpage/static/sw.js

@@ -0,0 +1,124 @@
+/**
+ * Skipped minification because the original files appears to be already minified.
+ * Original file: /npm/replaywebpage@1.8.14/sw.js
+ *
+ * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files
+ */
+/*! sw.js (wabac.js 2.16.13) is part of Webrecorder project. Copyright (C) 2020-2023, Webrecorder Software. Licensed under the Affero General Public License v3. */(()=>{var t={9742:(t,e)=>{"use strict";e.byteLength=function(t){var e=c(t),r=e[0],i=e[1];return 3*(r+i)/4-i},e.toByteArray=function(t){var e,r,s=c(t),o=s[0],a=s[1],l=new n(function(t,e,r){return 3*(e+r)/4-r}(0,o,a)),h=0,u=a>0?o-4:o;for(r=0;r<u;r+=4)e=i[t.charCodeAt(r)]<<18|i[t.charCodeAt(r+1)]<<12|i[t.charCodeAt(r+2)]<<6|i[t.charCodeAt(r+3)],l[h++]=e>>16&255,l[h++]=e>>8&255,l[h++]=255&e;return 2===a&&(e=i[t.charCodeAt(r)]<<2|i[t.charCodeAt(r+1)]>>4,l[h++]=255&e),1===a&&(e=i[t.charCodeAt(r)]<<10|i[t.charCodeAt(r+1)]<<4|i[t.charCodeAt(r+2)]>>2,l[h++]=e>>8&255,l[h++]=255&e),l},e.fromByteArray=function(t){for(var e,i=t.length,n=i%3,s=[],o=16383,a=0,c=i-n;a<c;a+=o)s.push(l(t,a,a+o>c?c:a+o));return 1===n?(e=t[i-1],s.push(r[e>>2]+r[e<<4&63]+"==")):2===n&&(e=(t[i-2]<<8)+t[i-1],s.push(r[e>>10]+r[e>>4&63]+r[e<<2&63]+"=")),s.join("")};for(var r=[],i=[],n="undefined"!=typeof Uint8Array?Uint8Array:Array,s="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",o=0,a=s.length;o<a;++o)r[o]=s[o],i[s.charCodeAt(o)]=o;function c(t){var e=t.length;if(e%4>0)throw new Error("Invalid string. Length must be a multiple of 4");var r=t.indexOf("=");return-1===r&&(r=e),[r,r===e?0:4-r%4]}function l(t,e,i){for(var n,s,o=[],a=e;a<i;a+=3)n=(t[a]<<16&16711680)+(t[a+1]<<8&65280)+(255&t[a+2]),o.push(r[(s=n)>>18&63]+r[s>>12&63]+r[s>>6&63]+r[63&s]);return o.join("")}i["-".charCodeAt(0)]=62,i["_".charCodeAt(0)]=63},4181:t=>{var e=4096,r=new Uint32Array([0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535,131071,262143,524287,1048575,2097151,4194303,8388607,16777215]);function i(t){this.buf_=new Uint8Array(8224),this.input_=t,this.reset()}i.READ_SIZE=e,i.IBUF_MASK=8191,i.prototype.reset=function(){this.buf_ptr_=0,this.val_=0,this.pos_=0,this.bit_pos_=0,this.bit_end_pos_=0,this.eos_=0,this.readMoreInput();for(var t=0;t<4;t++)this.val_|=this.buf_[this.pos_]<<8*t,++this.pos_;return this.bit_end_pos_>0},i.prototype.readMoreInput=function(){if(!(this.bit_end_pos_>256))if(this.eos_){if(this.bit_pos_>this.bit_end_pos_)throw new Error("Unexpected end of input "+this.bit_pos_+" "+this.bit_end_pos_)}else{var t=this.buf_ptr_,r=this.input_.read(this.buf_,t,e);if(r<0)throw new Error("Unexpected end of input");if(r<e){this.eos_=1;for(var i=0;i<32;i++)this.buf_[t+r+i]=0}if(0===t){for(i=0;i<32;i++)this.buf_[8192+i]=this.buf_[i];this.buf_ptr_=e}else this.buf_ptr_=0;this.bit_end_pos_+=r<<3}},i.prototype.fillBitWindow=function(){for(;this.bit_pos_>=8;)this.val_>>>=8,this.val_|=this.buf_[8191&this.pos_]<<24,++this.pos_,this.bit_pos_=this.bit_pos_-8>>>0,this.bit_end_pos_=this.bit_end_pos_-8>>>0},i.prototype.readBits=function(t){32-this.bit_pos_<t&&this.fillBitWindow();var e=this.val_>>>this.bit_pos_&r[t];return this.bit_pos_+=t,e},t.exports=i},7080:(t,e)=>{e.lookup=new Uint8Array([0,0,0,0,0,0,0,0,0,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,12,16,12,12,20,12,16,24,28,12,12,32,12,36,12,44,44,44,44,44,44,44,44,44,44,32,32,24,40,28,12,12,48,52,52,52,48,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,48,52,52,52,52,52,24,12,28,12,12,12,56,60,60,60,56,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,56,60,60,60,60,60,24,12,28,12,0,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,2,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,0,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,32,32,32,32,32,32,32,32,32,32,32,32,32,Line truncated
+/*!
+ * The buffer module from node.js, for the browser.
+ *
+ * @author   Feross Aboukhadijeh <https://feross.org>
+ * @license  MIT
+ */const i=r(9742),n=r(645),s="function"==typeof Symbol&&"function"==typeof Symbol.for?Symbol.for("nodejs.util.inspect.custom"):null;e.Buffer=c,e.SlowBuffer=function(t){return+t!=t&&(t=0),c.alloc(+t)},e.INSPECT_MAX_BYTES=50;const o=2147483647;function a(t){if(t>o)throw new RangeError('The value "'+t+'" is invalid for option "size"');const e=new Uint8Array(t);return Object.setPrototypeOf(e,c.prototype),e}function c(t,e,r){if("number"==typeof t){if("string"==typeof e)throw new TypeError('The "string" argument must be of type string. Received type number');return u(t)}return l(t,e,r)}function l(t,e,r){if("string"==typeof t)return function(t,e){if("string"==typeof e&&""!==e||(e="utf8"),!c.isEncoding(e))throw new TypeError("Unknown encoding: "+e);const r=0|g(t,e);let i=a(r);const n=i.write(t,e);return n!==r&&(i=i.slice(0,n)),i}(t,e);if(ArrayBuffer.isView(t))return function(t){if(Y(t,Uint8Array)){const e=new Uint8Array(t);return d(e.buffer,e.byteOffset,e.byteLength)}return p(t)}(t);if(null==t)throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t);if(Y(t,ArrayBuffer)||t&&Y(t.buffer,ArrayBuffer))return d(t,e,r);if("undefined"!=typeof SharedArrayBuffer&&(Y(t,SharedArrayBuffer)||t&&Y(t.buffer,SharedArrayBuffer)))return d(t,e,r);if("number"==typeof t)throw new TypeError('The "value" argument must not be of type number. Received type number');const i=t.valueOf&&t.valueOf();if(null!=i&&i!==t)return c.from(i,e,r);const n=function(t){if(c.isBuffer(t)){const e=0|f(t.length),r=a(e);return 0===r.length||t.copy(r,0,0,e),r}return void 0!==t.length?"number"!=typeof t.length||X(t.length)?a(0):p(t):"Buffer"===t.type&&Array.isArray(t.data)?p(t.data):void 0}(t);if(n)return n;if("undefined"!=typeof Symbol&&null!=Symbol.toPrimitive&&"function"==typeof t[Symbol.toPrimitive])return c.from(t[Symbol.toPrimitive]("string"),e,r);throw new TypeError("The first argument must be one of type string, Buffer, ArrayBuffer, Array, or Array-like Object. Received type "+typeof t)}function h(t){if("number"!=typeof t)throw new TypeError('"size" argument must be of type number');if(t<0)throw new RangeError('The value "'+t+'" is invalid for option "size"')}function u(t){return h(t),a(t<0?0:0|f(t))}function p(t){const e=t.length<0?0:0|f(t.length),r=a(e);for(let i=0;i<e;i+=1)r[i]=255&t[i];return r}function d(t,e,r){if(e<0||t.byteLength<e)throw new RangeError('"offset" is outside of buffer bounds');if(t.byteLength<e+(r||0))throw new RangeError('"length" is outside of buffer bounds');let i;return i=void 0===e&&void 0===r?new Uint8Array(t):void 0===r?new Uint8Array(t,e):new Uint8Array(t,e,r),Object.setPrototypeOf(i,c.prototype),i}function f(t){if(t>=o)throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+o.toString(16)+" bytes");return 0|t}function g(t,e){if(c.isBuffer(t))return t.length;if(ArrayBuffer.isView(t)||Y(t,ArrayBuffer))return t.byteLength;if("string"!=typeof t)throw new TypeError('The "string" argument must be one of type string, Buffer, or ArrayBuffer. Received type '+typeof t);const r=t.length,i=arguments.length>2&&!0===arguments[2];if(!i&&0===r)return 0;let n=!1;for(;;)switch(e){case"ascii":case"latin1":case"binary":return r;case"utf8":case"utf-8":return q(t).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*r;case"hex":return r>>>1;case"base64":return Q(t).length;default:if(n)return i?-1:q(t).length;e=(""+e).toLowerCase(),n=!0}}function m(t,e,r){let i=!1;if((void 0===e||e<0)&&(e=0),e>this.length)return"";if((void 0===r||r>this.length)&&(r=this.length),r<=0)return"";if((r>>>=0)<=(e>>>=0))return"";for(t||(t="utf8");;)switch(t){case"hex":return k(this,e,r);case"utf8":case"utf-8":return x(this,e,r);case"ascii":return I(this,e,r);case"latin1":case"binary":return R(this,e,r);case"base64":return T(this,e,r);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return N(this,e,r);default:if(i)throw new TypeError("Unknown encoding: "+t);t=(t+"").toLowerCase(),i=!0}}function w(t,e,r){const i=t[e];t[e]=t[r],t[r]=i}function y(t,e,r,i,n){if(0===t.length)return-1;if("string"==typeof r?(i=r,r=0):r>2147483647?r=2147483647:r<-2147483648&&(r=-2147483648),X(r=+r)&&(r=n?0:t.length-1),r<0&&(r=t.length+r),r>=t.length){if(n)return-1;r=t.length-1}else if(r<0){if(!n)return-1;r=0}if("string"==typeof e&&(e=c.from(e,i)),c.isBuffer(e))return 0===e.length?-1:b(t,e,r,i,n);if("number"==typeof e)return e&=255,"function"==typeof Uint8Array.prototype.indexOf?n?Uint8Array.prototype.indexOf.call(t,e,r):Uint8Array.prototype.lastIndexOf.call(t,e,r):b(t,[e],r,i,n);throw new TypeError("val must be string, number or Buffer")}function b(t,e,r,i,n){let s,o=1,a=t.length,c=e.length;if(void 0!==i&&("ucs2"===(i=String(i).toLowerCase())||"ucs-2"===i||"utf16le"===i||"utf-16le"===i)){if(t.length<2||e.length<2)return-1;o=2,a/=2,c/=2,r/=2}function l(t,e){return 1===o?t[e]:t.readUInt16BE(e*o)}if(n){let i=-1;for(s=r;s<a;s++)if(l(t,s)===l(e,-1===i?0:s-i)){if(-1===i&&(i=Line truncated
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+e.read=function(t,e,r,i,n){var s,o,a=8*n-i-1,c=(1<<a)-1,l=c>>1,h=-7,u=r?n-1:0,p=r?-1:1,d=t[e+u];for(u+=p,s=d&(1<<-h)-1,d>>=-h,h+=a;h>0;s=256*s+t[e+u],u+=p,h-=8);for(o=s&(1<<-h)-1,s>>=-h,h+=i;h>0;o=256*o+t[e+u],u+=p,h-=8);if(0===s)s=1-l;else{if(s===c)return o?NaN:1/0*(d?-1:1);o+=Math.pow(2,i),s-=l}return(d?-1:1)*o*Math.pow(2,s-i)},e.write=function(t,e,r,i,n,s){var o,a,c,l=8*s-n-1,h=(1<<l)-1,u=h>>1,p=23===n?Math.pow(2,-24)-Math.pow(2,-77):0,d=i?0:s-1,f=i?1:-1,g=e<0||0===e&&1/e<0?1:0;for(e=Math.abs(e),isNaN(e)||e===1/0?(a=isNaN(e)?1:0,o=h):(o=Math.floor(Math.log(e)/Math.LN2),e*(c=Math.pow(2,-o))<1&&(o--,c*=2),(e+=o+u>=1?p/c:p*Math.pow(2,1-u))*c>=2&&(o++,c/=2),o+u>=h?(a=0,o=h):o+u>=1?(a=(e*c-1)*Math.pow(2,n),o+=u):(a=e*Math.pow(2,u-1)*Math.pow(2,n),o=0));n>=8;t[r+d]=255&a,d+=f,a/=256,n-=8);for(o=o<<n|a,l+=n;l>0;t[r+d]=255&o,d+=f,o/=256,l-=8);t[r+d-f]|=128*g}},5717:t=>{"function"==typeof Object.create?t.exports=function(t,e){e&&(t.super_=e,t.prototype=Object.create(e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}))}:t.exports=function(t,e){if(e){t.super_=e;var r=function(){};r.prototype=e.prototype,t.prototype=new r,t.prototype.constructor=t}}},6512:function(t){!function(e){"use strict";const r="(0?\\d+|0x[a-f0-9]+)",i={fourOctet:new RegExp(`^${r}\\.${r}\\.${r}\\.${r}$`,"i"),threeOctet:new RegExp(`^${r}\\.${r}\\.${r}$`,"i"),twoOctet:new RegExp(`^${r}\\.${r}$`,"i"),longValue:new RegExp(`^${r}$`,"i")},n=new RegExp("^0[0-7]+$","i"),s=new RegExp("^0x[a-f0-9]+$","i"),o="%[0-9a-z]{1,}",a="(?:[0-9a-f]+::?)+",c={zoneIndex:new RegExp(o,"i"),native:new RegExp(`^(::)?(${a})?([0-9a-f]+)?(::)?(${o})?$`,"i"),deprecatedTransitional:new RegExp(`^(?:::)(${r}\\.${r}\\.${r}\\.${r}(${o})?)$`,"i"),transitional:new RegExp(`^((?:${a})|(?:::)(?:${a})?)${r}\\.${r}\\.${r}\\.${r}(${o})?$`,"i")};function l(t,e){if(t.indexOf("::")!==t.lastIndexOf("::"))return null;let r,i,n=0,s=-1,o=(t.match(c.zoneIndex)||[])[0];for(o&&(o=o.substring(1),t=t.replace(/%.+$/,""));(s=t.indexOf(":",s+1))>=0;)n++;if("::"===t.substr(0,2)&&n--,"::"===t.substr(-2,2)&&n--,n>e)return null;for(i=e-n,r=":";i--;)r+="0:";return":"===(t=t.replace("::",r))[0]&&(t=t.slice(1)),":"===t[t.length-1]&&(t=t.slice(0,-1)),{parts:e=function(){const e=t.split(":"),r=[];for(let t=0;t<e.length;t++)r.push(parseInt(e[t],16));return r}(),zoneId:o}}function h(t,e,r,i){if(t.length!==e.length)throw new Error("ipaddr: cannot match CIDR for objects with different lengths");let n,s=0;for(;i>0;){if(n=r-i,n<0&&(n=0),t[s]>>n!=e[s]>>n)return!1;i-=r,s+=1}return!0}function u(t){if(s.test(t))return parseInt(t,16);if("0"===t[0]&&!isNaN(parseInt(t[1],10))){if(n.test(t))return parseInt(t,8);throw new Error(`ipaddr: cannot parse ${t} as octal`)}return parseInt(t,10)}function p(t,e){for(;t.length<e;)t=`0${t}`;return t}const d={};d.IPv4=function(){function t(t){if(4!==t.length)throw new Error("ipaddr: ipv4 octet count should be 4");let e,r;for(e=0;e<t.length;e++)if(r=t[e],!(0<=r&&r<=255))throw new Error("ipaddr: ipv4 octet should fit in 8 bits");this.octets=t}return t.prototype.SpecialRanges={unspecified:[[new t([0,0,0,0]),8]],broadcast:[[new t([255,255,255,255]),32]],multicast:[[new t([224,0,0,0]),4]],linkLocal:[[new t([169,254,0,0]),16]],loopback:[[new t([127,0,0,0]),8]],carrierGradeNat:[[new t([100,64,0,0]),10]],private:[[new t([10,0,0,0]),8],[new t([172,16,0,0]),12],[new t([192,168,0,0]),16]],reserved:[[new t([192,0,0,0]),24],[new t([192,0,2,0]),24],[new t([192,88,99,0]),24],[new t([198,51,100,0]),24],[new t([203,0,113,0]),24],[new t([240,0,0,0]),4]]},t.prototype.kind=function(){return"ipv4"},t.prototype.match=function(t,e){let r;if(void 0===e&&(r=t,t=r[0],e=r[1]),"ipv4"!==t.kind())throw new Error("ipaddr: cannot match ipv4 address with non-ipv4 one");return h(this.octets,t.octets,8,e)},t.prototype.prefixLengthFromSubnetMask=function(){let t=0,e=!1;const r={0:8,128:7,192:6,224:5,240:4,248:3,252:2,254:1,255:0};let i,n,s;for(i=3;i>=0;i-=1){if(n=this.octets[i],!(n in r))return null;if(s=r[n],e&&0!==s)return null;8!==s&&(e=!0),t+=s}return 32-t},t.prototype.range=function(){return d.subnetMatch(this,this.SpecialRanges)},t.prototype.toByteArray=function(){return this.octets.slice(0)},t.prototype.toIPv4MappedAddress=function(){return d.IPv6.parse(`::ffff:${this.toString()}`)},t.prototype.toNormalizedString=function(){return this.toString()},t.prototype.toString=function(){return this.octets.join(".")},t}(),d.IPv4.broadcastAddressFromCIDR=function(t){try{const e=this.parseCIDR(t),r=e[0].toByteArray(),i=this.subnetMaskFromPrefixLength(e[1]).toByteArray(),n=[];let s=0;for(;s<4;)n.push(parseInt(r[s],10)|255^parseInt(i[s],10)),s++;return new this(n)}catch(t){throw new Error("ipaddr: the address does not have IPv4 CIDR format")}},d.IPv4.isIPv4=function(t){return null!==this.parser(t)},d.IPv4.isValid=function(t){try{return new this(this.parser(t)),!0}catch(t){return!1}},d.IPv4.isValidFourPartDecimal=function(t){return!(!d.IPv4.isValid(t)||!t.match(/^(0|[1-9]\d*)(\.(0|[1-9]\d*)){3}$/))},d.Line truncated
+/*! *****************************************************************************
+Copyright (C) Microsoft. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at http://www.apache.org/licenses/LICENSE-2.0
+
+THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+MERCHANTABLITY OR NON-INFRINGEMENT.
+
+See the Apache Version 2.0 License for specific language governing permissions
+and limitations under the License.
+***************************************************************************** */!function(t){!function(e){var i="object"==typeof r.g?r.g:"object"==typeof self?self:"object"==typeof this?this:Function("return this;")(),s=o(t);function o(t,e){return function(r,i){"function"!=typeof t[r]&&Object.defineProperty(t,r,{configurable:!0,writable:!0,value:i}),e&&e(r,i)}}void 0===i.Reflect?i.Reflect=t:s=o(i.Reflect,s),function(t){var e=Object.prototype.hasOwnProperty,r="function"==typeof Symbol,i=r&&void 0!==Symbol.toPrimitive?Symbol.toPrimitive:"@@toPrimitive",s=r&&void 0!==Symbol.iterator?Symbol.iterator:"@@iterator",o="function"==typeof Object.create,a={__proto__:[]}instanceof Array,c=!o&&!a,l={create:o?function(){return M(Object.create(null))}:a?function(){return M({__proto__:null})}:function(){return M({})},has:c?function(t,r){return e.call(t,r)}:function(t,e){return e in t},get:c?function(t,r){return e.call(t,r)?t[r]:void 0}:function(t,e){return t[e]}},h=Object.getPrototypeOf(Function),u="object"==typeof n&&n.env&&"true"===n.env.REFLECT_METADATA_USE_MAP_POLYFILL,p=u||"function"!=typeof Map||"function"!=typeof Map.prototype.entries?function(){var t={},e=[],r=function(){function t(t,e,r){this._index=0,this._keys=t,this._values=e,this._selector=r}return t.prototype["@@iterator"]=function(){return this},t.prototype[s]=function(){return this},t.prototype.next=function(){var t=this._index;if(t>=0&&t<this._keys.length){var r=this._selector(this._keys[t],this._values[t]);return t+1>=this._keys.length?(this._index=-1,this._keys=e,this._values=e):this._index++,{value:r,done:!1}}return{value:void 0,done:!0}},t.prototype.throw=function(t){throw this._index>=0&&(this._index=-1,this._keys=e,this._values=e),t},t.prototype.return=function(t){return this._index>=0&&(this._index=-1,this._keys=e,this._values=e),{value:t,done:!0}},t}();return function(){function e(){this._keys=[],this._values=[],this._cacheKey=t,this._cacheIndex=-2}return Object.defineProperty(e.prototype,"size",{get:function(){return this._keys.length},enumerable:!0,configurable:!0}),e.prototype.has=function(t){return this._find(t,!1)>=0},e.prototype.get=function(t){var e=this._find(t,!1);return e>=0?this._values[e]:void 0},e.prototype.set=function(t,e){var r=this._find(t,!0);return this._values[r]=e,this},e.prototype.delete=function(e){var r=this._find(e,!1);if(r>=0){for(var i=this._keys.length,n=r+1;n<i;n++)this._keys[n-1]=this._keys[n],this._values[n-1]=this._values[n];return this._keys.length--,this._values.length--,e===this._cacheKey&&(this._cacheKey=t,this._cacheIndex=-2),!0}return!1},e.prototype.clear=function(){this._keys.length=0,this._values.length=0,this._cacheKey=t,this._cacheIndex=-2},e.prototype.keys=function(){return new r(this._keys,this._values,i)},e.prototype.values=function(){return new r(this._keys,this._values,n)},e.prototype.entries=function(){return new r(this._keys,this._values,o)},e.prototype["@@iterator"]=function(){return this.entries()},e.prototype[s]=function(){return this.entries()},e.prototype._find=function(t,e){return this._cacheKey!==t&&(this._cacheIndex=this._keys.indexOf(this._cacheKey=t)),this._cacheIndex<0&&e&&(this._cacheIndex=this._keys.length,this._keys.push(t),this._values.push(void 0)),this._cacheIndex},e}();function i(t,e){return t}function n(t,e){return e}function o(t,e){return[t,e]}}():Map,d=u||"function"!=typeof Set||"function"!=typeof Set.prototype.entries?function(){function t(){this._map=new p}return Object.defineProperty(t.prototype,"size",{get:function(){return this._map.size},enumerable:!0,configurable:!0}),t.prototype.has=function(t){return this._map.has(t)},t.prototype.add=function(t){return this._map.set(t,t),this},t.prototype.delete=function(t){return this._map.delete(t)},t.prototype.clear=function(){this._map.clear()},t.prototype.keys=function(){return this._map.keys()},t.prototype.values=function(){return this._map.values()},t.prototype.entries=function(){return this._map.entries()},t.prototype["@@iterator"]=function(){return this.keys()},t.prototype[s]=function(){return this.keys()},t}():Set,f=new(u||"function"!=typeof WeakMap?function(){var t=l.create(),r=i();return function(){function t(){this._key=i()}return t.prototype.has=function(t){var e=n(t,!1);return void 0!==e&&l.has(e,this._key)},t.prototype.get=function(t){var e=n(t,!1);return void 0!==e?l.get(e,this._key):void 0},t.prototype.set=function(t,e){return n(t,!0)[this._key]=e,this},t.prototype.delete=function(t){var e=n(t,!1);return void 0!==e&&delete e[this._key]},t.prototype.clear=function(){this._key=i()},t}();function i(){var e;do{e="@@WeakMap@@"+o()}while(l.has(t,e));return t[e]=!0,e}function n(t,i){if(!e.call(t,r)){if(!i)return;Object.defineProperty(t,r,{value:l.create()})}return t[r]}function s(t,e){for(var r=0;r<e;++r)t[r]=255*Math.random()|0;return t}function o(){var t=function(t){return"function"==typeof Uint8Array?"undefined"!=typeof crypto?crypto.getRandomValues(new Uint8Array(t)):"undefined"!=typeof msCrypto?msCrypto.getRandomValues(nLine truncated
+/*! safe-buffer. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */
+var i=r(8764),n=i.Buffer;function s(t,e){for(var r in t)e[r]=t[r]}function o(t,e,r){return n(t,e,r)}n.from&&n.alloc&&n.allocUnsafe&&n.allocUnsafeSlow?t.exports=i:(s(i,e),e.Buffer=o),o.prototype=Object.create(n.prototype),s(n,o),o.from=function(t,e,r){if("number"==typeof t)throw new TypeError("Argument must not be a number");return n(t,e,r)},o.alloc=function(t,e,r){if("number"!=typeof t)throw new TypeError("Argument must be a number");var i=n(t);return void 0!==e?"string"==typeof r?i.fill(e,r):i.fill(e):i.fill(0),i},o.allocUnsafe=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return n(t)},o.allocUnsafeSlow=function(t){if("number"!=typeof t)throw new TypeError("Argument must be a number");return i.SlowBuffer(t)}},2830:(t,e,r)=>{t.exports=n;var i=r(7187).EventEmitter;function n(){i.call(this)}r(5717)(n,i),n.Readable=r(9481),n.Writable=r(4229),n.Duplex=r(6753),n.Transform=r(4605),n.PassThrough=r(2725),n.finished=r(8610),n.pipeline=r(9946),n.Stream=n,n.prototype.pipe=function(t,e){var r=this;function n(e){t.writable&&!1===t.write(e)&&r.pause&&r.pause()}function s(){r.readable&&r.resume&&r.resume()}r.on("data",n),t.on("drain",s),t._isStdio||e&&!1===e.end||(r.on("end",a),r.on("close",c));var o=!1;function a(){o||(o=!0,t.end())}function c(){o||(o=!0,"function"==typeof t.destroy&&t.destroy())}function l(t){if(h(),0===i.listenerCount(this,"error"))throw t}function h(){r.removeListener("data",n),t.removeListener("drain",s),r.removeListener("end",a),r.removeListener("close",c),r.removeListener("error",l),t.removeListener("error",l),r.removeListener("end",h),r.removeListener("close",h),t.removeListener("close",h)}return r.on("error",l),t.on("error",l),r.on("end",h),r.on("close",h),t.on("close",h),t.emit("pipe",r),t}},2553:(t,e,r)=>{"use strict";var i=r(9509).Buffer,n=i.isEncoding||function(t){switch((t=""+t)&&t.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function s(t){var e;switch(this.encoding=function(t){var e=function(t){if(!t)return"utf8";for(var e;;)switch(t){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return t;default:if(e)return;t=(""+t).toLowerCase(),e=!0}}(t);if("string"!=typeof e&&(i.isEncoding===n||!n(t)))throw new Error("Unknown encoding: "+t);return e||t}(t),this.encoding){case"utf16le":this.text=c,this.end=l,e=4;break;case"utf8":this.fillLast=a,e=4;break;case"base64":this.text=h,this.end=u,e=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=i.allocUnsafe(e)}function o(t){return t<=127?0:t>>5==6?2:t>>4==14?3:t>>3==30?4:t>>6==2?-1:-2}function a(t){var e=this.lastTotal-this.lastNeed,r=function(t,e,r){if(128!=(192&e[0]))return t.lastNeed=0,"�";if(t.lastNeed>1&&e.length>1){if(128!=(192&e[1]))return t.lastNeed=1,"�";if(t.lastNeed>2&&e.length>2&&128!=(192&e[2]))return t.lastNeed=2,"�"}}(this,t);return void 0!==r?r:this.lastNeed<=t.length?(t.copy(this.lastChar,e,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(t.copy(this.lastChar,e,0,t.length),void(this.lastNeed-=t.length))}function c(t,e){if((t.length-e)%2==0){var r=t.toString("utf16le",e);if(r){var i=r.charCodeAt(r.length-1);if(i>=55296&&i<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1],r.slice(0,-1)}return r}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=t[t.length-1],t.toString("utf16le",e,t.length-1)}function l(t){var e=t&&t.length?this.write(t):"";if(this.lastNeed){var r=this.lastTotal-this.lastNeed;return e+this.lastChar.toString("utf16le",0,r)}return e}function h(t,e){var r=(t.length-e)%3;return 0===r?t.toString("base64",e):(this.lastNeed=3-r,this.lastTotal=3,1===r?this.lastChar[0]=t[t.length-1]:(this.lastChar[0]=t[t.length-2],this.lastChar[1]=t[t.length-1]),t.toString("base64",e,t.length-r))}function u(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+this.lastChar.toString("base64",0,3-this.lastNeed):e}function p(t){return t.toString(this.encoding)}function d(t){return t&&t.length?this.write(t):""}e.s=s,s.prototype.write=function(t){if(0===t.length)return"";var e,r;if(this.lastNeed){if(void 0===(e=this.fillLast(t)))return"";r=this.lastNeed,this.lastNeed=0}else r=0;return r<t.length?e?e+this.text(t,r):this.text(t,r):e||""},s.prototype.end=function(t){var e=t&&t.length?this.write(t):"";return this.lastNeed?e+"�":e},s.prototype.text=function(t,e){var r=function(t,e,r){var i=e.length-1;if(i<r)return 0;var n=o(e[i]);return n>=0?(n>0&&(t.lastNeed=n-1),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(t.lastNeed=n-2),n):--i<r||-2===n?0:(n=o(e[i]))>=0?(n>0&&(2===n?n=0:t.lastNeed=n-3),n):0}(this,t,e);if(!this.lastNeed)return t.toString("utf8",e);this.lastTotal=r;var i=t.length-(r-this.lastNeed);return t.copy(tLine truncated
+/*!
+ * hash-wasm (https://www.npmjs.com/package/hash-wasm)
+ * (c) Dani Biro
+ * @license MIT
+ */
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */function B(t,e,r,i){return new(r||(r=Promise))((function(n,s){function o(t){try{c(i.next(t))}catch(t){s(t)}}function a(t){try{c(i.throw(t))}catch(t){s(t)}}function c(t){var e;t.done?n(t.value):(e=t.value,e instanceof r?e:new r((function(t){t(e)}))).then(o,a)}c((i=i.apply(t,e||[])).next())}))}class P{constructor(){this.mutex=Promise.resolve()}lock(){let t=()=>{};return this.mutex=this.mutex.then((()=>new Promise(t))),new Promise((e=>{t=e}))}dispatch(t){return B(this,void 0,void 0,(function*(){const e=yield this.lock();try{return yield Promise.resolve(t())}finally{e()}}))}}const L="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:r.g,D=null!==(I=L.Buffer)&&void 0!==I?I:null,U=L.TextEncoder?new L.TextEncoder:null;function M(t,e){return(15&t)+(t>>6|t>>3&8)<<4|(15&e)+(e>>6|e>>3&8)}const F="a".charCodeAt(0)-10,H="0".charCodeAt(0);function W(t,e,r){let i=0;for(let n=0;n<r;n++){let r=e[n]>>>4;t[i++]=r>9?r+F:r+H,r=15&e[n],t[i++]=r>9?r+F:r+H}return String.fromCharCode.apply(null,t)}const j=null!==D?t=>{if("string"==typeof t){const e=D.from(t,"utf8");return new Uint8Array(e.buffer,e.byteOffset,e.length)}if(D.isBuffer(t))return new Uint8Array(t.buffer,t.byteOffset,t.length);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")}:t=>{if("string"==typeof t)return U.encode(t);if(ArrayBuffer.isView(t))return new Uint8Array(t.buffer,t.byteOffset,t.byteLength);throw new Error("Invalid data type!")},V="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",G=new Uint8Array(256);for(let t=0;t<V.length;t++)G[V.charCodeAt(t)]=t;function z(t){const e=function(t){let e=Math.floor(.75*t.length);const r=t.length;return"="===t[r-1]&&(e-=1,"="===t[r-2]&&(e-=1)),e}(t),r=t.length,i=new Uint8Array(e);let n=0;for(let e=0;e<r;e+=4){const r=G[t.charCodeAt(e)],s=G[t.charCodeAt(e+1)],o=G[t.charCodeAt(e+2)],a=G[t.charCodeAt(e+3)];i[n]=r<<2|s>>4,n+=1,i[n]=(15&s)<<4|o>>2,n+=1,i[n]=(3&o)<<6|63&a,n+=1}return i}const q=16384,Q=new P,K=new Map;function Y(t,e){return B(this,void 0,void 0,(function*(){let r=null,i=null,n=!1;if("undefined"==typeof WebAssembly)throw new Error("WebAssembly is not supported in this environment!");const s=()=>new DataView(r.exports.memory.buffer).getUint32(r.exports.STATE_SIZE,!0),o=Q.dispatch((()=>B(this,void 0,void 0,(function*(){if(!K.has(t.name)){const e=z(t.data),r=WebAssembly.compile(e);K.set(t.name,r)}const e=yield K.get(t.name);r=yield WebAssembly.instantiate(e,{})})))),a=(t=null)=>{n=!0,r.exports.Hash_Init(t)},c=t=>{if(!n)throw new Error("update() called before init()");(t=>{let e=0;for(;e<t.length;){const n=t.subarray(e,e+q);e+=n.length,i.set(n),r.exports.Hash_Update(n.length)}})(j(t))},l=new Uint8Array(2*e),h=(t,s=null)=>{if(!n)throw new Error("digest() called before init()");return n=!1,r.exports.Hash_Final(s),"binary"===t?i.slice(0,e):W(l,i,e)},u=t=>"string"==typeof t?t.length<4096:t.byteLength<q;let p=u;switch(t.name){case"argon2":case"scrypt":p=()=>!0;break;case"blake2b":case"blake2s":p=(t,e)=>e<=512&&u(t);break;case"blake3":p=(t,e)=>0===e&&u(t);break;case"xxhash64":case"xxhash3":case"xxhash128":p=()=>!1}return yield(()=>B(this,void 0,void 0,(function*(){r||(yield o);const t=r.exports.Hash_GetBuffer(),e=r.exports.memory.buffer;i=new Uint8Array(e,t,q)})))(),{getMemory:()=>i,writeMemory:(t,e=0)=>{i.set(t,e)},getExports:()=>r.exports,setMemorySize:t=>{r.exports.Hash_SetMemorySize(t);const e=r.exports.Hash_GetBuffer(),n=r.exports.memory.buffer;i=new Uint8Array(n,e,t)},init:a,update:c,digest:h,save:()=>{if(!n)throw new Error("save() can only be called after init() and before digest()");const e=r.exports.Hash_GetState(),i=s(),o=r.exports.memory.buffer,a=new Uint8Array(o,e,i),c=new Uint8Array(4+i);return function(t,e){const r=e.length>>1;for(let i=0;i<r;i++){const r=i<<1;t[i]=M(e.charCodeAt(r),e.charCodeAt(r+1))}}(c,t.hash),c.set(a,4),c},load:e=>{if(!(e instanceof Uint8Array))throw new Error("load() expects an Uint8Array generated by save()");const i=r.exports.Hash_GetState(),o=s(),a=4+o,c=r.exports.memory.buffer;if(e.length!==a)throw new Error(`Bad state length (expected ${a} bytes, got ${e.length})`);if(!function(t,e){if(t.length!==2*e.length)return!1;for(let r=0;r<e.length;r++){const i=r<<1;if(e[r]!==M(t.charCodeAt(i),t.charCodeAt(i+1)))return!1}return!0}(t.hash,e.subarray(0,4)))throw new Error("This state was written by an incompatible hash implementation");const l=e.subarray(4);new Uint8Array(c,i,o).set(l),n=!0},calculate:(t,n=null,s=null)=>{if(!p(t,n))return a(n),c(t),h("hex",s);const o=j(t);return i.set(o),r.exports.Hash_Calculate(o.length,n,s),W(l,i,e)},hashLength:e}}))}new P,new P,new DataView(new ArrayBuffer(4)),new P,new P,new P,new P,new P,new P,new P,new P,new P;var X={name:"sha256",data:"AGFzbQEAAAABEQRgAAF/YAF/AGACf38AYAAAAwgHAAEBAgMAAgQFAXABAQEFBAEBAgIGDgJ/AUHwiQULfwBBgAgLB3AIBm1lbW9yeQIADLine truncated
+/*! js-yaml 4.1.0 https://github.com/nodeca/js-yaml @license MIT */function so(t){return null==t}var oo={isNothing:so,isObject:function(t){return"object"==typeof t&&null!==t},toArray:function(t){return Array.isArray(t)?t:so(t)?[]:[t]},repeat:function(t,e){var r,i="";for(r=0;r<e;r+=1)i+=t;return i},isNegativeZero:function(t){return 0===t&&Number.NEGATIVE_INFINITY===1/t},extend:function(t,e){var r,i,n,s;if(e)for(r=0,i=(s=Object.keys(e)).length;r<i;r+=1)t[n=s[r]]=e[n];return t}};function ao(t,e){var r="",i=t.reason||"(unknown reason)";return t.mark?(t.mark.name&&(r+='in "'+t.mark.name+'" '),r+="("+(t.mark.line+1)+":"+(t.mark.column+1)+")",!e&&t.mark.snippet&&(r+="\n\n"+t.mark.snippet),i+" "+r):i}function co(t,e){Error.call(this),this.name="YAMLException",this.reason=t,this.mark=e,this.message=ao(this,!1),Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack||""}co.prototype=Object.create(Error.prototype),co.prototype.constructor=co,co.prototype.toString=function(t){return this.name+": "+ao(this,t)};var lo=co;function ho(t,e,r,i,n){var s="",o="",a=Math.floor(n/2)-1;return i-e>a&&(e=i-a+(s=" ... ").length),r-i>a&&(r=i+a-(o=" ...").length),{str:s+t.slice(e,r).replace(/\t/g,"→")+o,pos:i-e+s.length}}function uo(t,e){return oo.repeat(" ",e-t.length)+t}var po=["kind","multi","resolve","construct","instanceOf","predicate","represent","representName","defaultStyle","styleAliases"],fo=["scalar","sequence","mapping"],go=function(t,e){if(e=e||{},Object.keys(e).forEach((function(e){if(-1===po.indexOf(e))throw new lo('Unknown option "'+e+'" is met in definition of "'+t+'" YAML type.')})),this.options=e,this.tag=t,this.kind=e.kind||null,this.resolve=e.resolve||function(){return!0},this.construct=e.construct||function(t){return t},this.instanceOf=e.instanceOf||null,this.predicate=e.predicate||null,this.represent=e.represent||null,this.representName=e.representName||null,this.defaultStyle=e.defaultStyle||null,this.multi=e.multi||!1,this.styleAliases=function(t){var e={};return null!==t&&Object.keys(t).forEach((function(r){t[r].forEach((function(t){e[String(t)]=r}))})),e}(e.styleAliases||null),-1===fo.indexOf(this.kind))throw new lo('Unknown kind "'+this.kind+'" is specified for "'+t+'" YAML type.')};function mo(t,e){var r=[];return t[e].forEach((function(t){var e=r.length;r.forEach((function(r,i){r.tag===t.tag&&r.kind===t.kind&&r.multi===t.multi&&(e=i)})),r[e]=t})),r}function wo(t){return this.extend(t)}wo.prototype.extend=function(t){var e=[],r=[];if(t instanceof go)r.push(t);else if(Array.isArray(t))r=r.concat(t);else{if(!t||!Array.isArray(t.implicit)&&!Array.isArray(t.explicit))throw new lo("Schema.extend argument should be a Type, [ Type ], or a schema definition ({ implicit: [...], explicit: [...] })");t.implicit&&(e=e.concat(t.implicit)),t.explicit&&(r=r.concat(t.explicit))}e.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.");if(t.loadKind&&"scalar"!==t.loadKind)throw new lo("There is a non-scalar type in the implicit list of a schema. Implicit resolving of such types is not supported.");if(t.multi)throw new lo("There is a multi type in the implicit list of a schema. Multi tags can only be listed as explicit.")})),r.forEach((function(t){if(!(t instanceof go))throw new lo("Specified list of YAML types (or a single Type object) contains a non-Type object.")}));var i=Object.create(wo.prototype);return i.implicit=(this.implicit||[]).concat(e),i.explicit=(this.explicit||[]).concat(r),i.compiledImplicit=mo(i,"implicit"),i.compiledExplicit=mo(i,"explicit"),i.compiledTypeMap=function(){var t,e,r={scalar:{},sequence:{},mapping:{},fallback:{},multi:{scalar:[],sequence:[],mapping:[],fallback:[]}};function i(t){t.multi?(r.multi[t.kind].push(t),r.multi.fallback.push(t)):r[t.kind][t.tag]=r.fallback[t.tag]=t}for(t=0,e=arguments.length;t<e;t+=1)arguments[t].forEach(i);return r}(i.compiledImplicit,i.compiledExplicit),i};var yo=wo,bo=new go("tag:yaml.org,2002:str",{kind:"scalar",construct:function(t){return null!==t?t:""}}),Ao=new go("tag:yaml.org,2002:seq",{kind:"sequence",construct:function(t){return null!==t?t:[]}}),_o=new go("tag:yaml.org,2002:map",{kind:"mapping",construct:function(t){return null!==t?t:{}}}),vo=new yo({explicit:[bo,Ao,_o]}),Eo=new go("tag:yaml.org,2002:null",{kind:"scalar",resolve:function(t){if(null===t)return!0;var e=t.length;return 1===e&&"~"===t||4===e&&("null"===t||"Null"===t||"NULL"===t)},construct:function(){return null},predicate:function(t){return null===t},represent:{canonical:function(){return"~"},lowercase:function(){return"null"},uppercase:function(){return"NULL"},camelcase:function(){return"Null"},empty:function(){return""}},defaultStyle:"lowercase"}),So=new go("tag:yaml.org,2002:bool",{kind:"scalar",resolve:function(t){if(null===t)return!1;var e=t.length;return 4===e&&("true"===t||"True"===t||"TRUE"===t)||5===e&&("false"===t||"False"===t||"FALSE"===t)},construct:fuLine truncated
+/*!
+ * Copyright (c) 2014, GMO GlobalSign
+ * Copyright (c) 2015-2022, Peculiar Ventures
+ * All rights reserved.
+ * 
+ * Author 2014-2019, Yury Strozhevsky
+ * 
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ * 
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * 
+ */
+function lc(){if("undefined"==typeof BigInt)throw new Error("BigInt is not defined. Your environment doesn't implement BigInt.")}function hc(t){let e=0,r=0;for(let r=0;r<t.length;r++)e+=t[r].byteLength;const i=new Uint8Array(e);for(let e=0;e<t.length;e++){const n=t[e];i.set(new Uint8Array(n),r),r+=n.byteLength}return i.buffer}function uc(t,e,r,i){return e instanceof Uint8Array?e.byteLength?r<0?(t.error="Wrong parameter: inputOffset less than zero",!1):i<0?(t.error="Wrong parameter: inputLength less than zero",!1):!(e.byteLength-r-i<0&&(t.error="End of input reached before message was fully decoded (inconsistent offset and length values)",1)):(t.error="Wrong parameter: inputBuffer has zero length",!1):(t.error="Wrong parameter: inputBuffer must be 'Uint8Array'",!1)}Math.log(2);class pc{constructor(){this.items=[]}write(t){this.items.push(t)}final(){return hc(this.items)}}const dc=[new Uint8Array([1])],fc="0123456789",gc="name",mc="valueHexView",wc="",yc=new ArrayBuffer(0),bc=new Uint8Array(0),Ac="EndOfContent",_c="OCTET STRING",vc="BIT STRING";function Ec(t){var e;return(e=class extends t{constructor(...t){var e;super(...t);const r=t[0]||{};this.isHexOnly=null!==(e=r.isHexOnly)&&void 0!==e&&e,this.valueHexView=r.valueHex?ic.vJ.toUint8Array(r.valueHex):bc}get valueHex(){return this.valueHexView.slice().buffer}set valueHex(t){this.valueHexView=new Uint8Array(t)}fromBER(t,e,r){const i=t instanceof ArrayBuffer?new Uint8Array(t):t;if(!uc(this,i,e,r))return-1;const n=e+r;return this.valueHexView=i.subarray(e,n),this.valueHexView.length?(this.blockLength=r,n):(this.warnings.push("Zero buffer length"),e)}toBER(t=!1){return this.isHexOnly?t?new ArrayBuffer(this.valueHexView.byteLength):this.valueHexView.byteLength===this.valueHexView.buffer.byteLength?this.valueHexView.buffer:this.valueHexView.slice().buffer:(this.error="Flag 'isHexOnly' is not set, abort",yc)}toJSON(){return{...super.toJSON(),isHexOnly:this.isHexOnly,valueHex:ic.ep.ToHex(this.valueHexView)}}}).NAME="hexBlock",e}class Sc{constructor({blockLength:t=0,error:e="",warnings:r=[],valueBeforeDecode:i=bc}={}){this.blockLength=t,this.error=e,this.warnings=r,this.valueBeforeDecodeView=ic.vJ.toUint8Array(i)}static blockName(){return this.NAME}get valueBeforeDecode(){return this.valueBeforeDecodeView.slice().buffer}set valueBeforeDecode(t){this.valueBeforeDecodeView=new Uint8Array(t)}toJSON(){return{blockName:this.constructor.NAME,blockLength:this.blockLength,error:this.error,warnings:this.warnings,valueBeforeDecode:ic.ep.ToHex(this.valueBeforeDecodeView)}}}Sc.NAME="baseBlock";class Tc extends Sc{fromBER(t,e,r){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}toBER(t,e){throw TypeError("User need to make a specific function in a class which extends 'ValueBlock'")}}Tc.NAME="valueBlock";class xc extends(Ec(Sc)){constructor({idBlock:t={}}={}){var e,r,i,n;super(),t?(this.isHexOnly=null!==(e=t.isHexOnly)&&void 0!==e&&e,this.valueHexView=t.valueHex?ic.vJ.toUint8Array(t.valueHex):bc,this.tagClass=null!==(r=t.tagClass)&&void 0!==r?r:-1,this.tagNumber=null!==(i=t.tagNumber)&&void 0!==i?i:-1,this.isConstructed=null!==(n=t.isConstructed)&&void 0!==n&&n):(this.tagClass=-1,this.tagNumber=-1,this.isConstructed=!1)}toBER(t=!1){let e=0;switch(this.tagClass){case 1:e|=0;break;case 2:e|=64;break;case 3:e|=128;break;case 4:e|=192;break;default:return this.error="Unknown tag class",yc}if(this.isConstructed&&(e|=32),this.tagNumber<31&&!this.isHexOnly){const r=new Uint8Array(1);if(!t){let t=this.tagNumber;t&=31,e|=t,r[0]=e}return r.buffer}if(!this.isHexOnly){const r=sc(this.tagNumber,7),i=new Uint8Array(r),n=r.byteLength,s=new Uint8Array(n+1);if(s[0]=31|e,!t){for(let t=0;t<n-1;t++)s[t+1]=128|i[t];s[n]=i[n-1]}return s.buffer}const r=new Uint8Array(this.valueHexView.byteLength+1);if(r[0]=31|e,!t){const t=this.valueHexView;for(let e=0;e<t.length-1;e++)r[e+1]=128|t[e];r[this.valueHexView.byteLength]=t[t.length-1]}return r.buffer}fromBER(t,e,r){const i=ic.vJ.toUint8Array(t);if(!uc(this,i,e,r))return-1;const n=i.subarray(e,e+r);if(0===n.length)return this.error="Zero buffer length",-1;switch(192&n[0]){case 0:this.tagClass=1;break;case 64:this.tagClass=2;break;case 128:this.tagClass=3;break;case 192:this.tagClass=4;break;default:return this.error="Unknown tag class",-1}this.isConstructed=32==(32&n[0]),this.isHexOnly=!1;const s=31&n[0];if(31!==s)this.tagNumber=s,this.blockLength=1;else{let t=1,e=this.valueHexView=new Uint8Array(255),r=255;for(;128&n[t];){if(e[t-1]=127&n[t],t++,t>=n.length)return this.error="End of input reached before message was fully decoded",-1;if(t===r){r+=255;const t=new Uint8Array(r);for(let r=0;r<e.length;r++)t[r]=e[r];e=this.valueHexView=new Uint8Array(r)}}this.blockLength=t+1,e[t-1]=127&n[t];const i=new Uint8Array(t);for(let r=0;r<t;r++)i[r]=e[r];e=this.valueHexView=new Uint8Array(t),e.set(i),this.blockLength<=9?this.tagNumber=nc(e,7):(this.isHexOnly=!0,this.warnings.push("Tag too long, represented as hex-coded"))}if(1=Line truncated
+/*! *****************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */var vg=function(t,e){return vg=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])},vg(t,e)};function Eg(t,e){function r(){this.constructor=t}vg(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}function Sg(t){var e="function"==typeof Symbol&&Symbol.iterator,r=e&&t[e],i=0;if(r)return r.call(t);if(t&&"number"==typeof t.length)return{next:function(){return t&&i>=t.length&&(t=void 0),{value:t&&t[i++],done:!t}}};throw new TypeError(e?"Object is not iterable.":"Symbol.iterator is not defined.")}function Tg(t,e){var r="function"==typeof Symbol&&t[Symbol.iterator];if(!r)return t;var i,n,s=r.call(t),o=[];try{for(;(void 0===e||e-- >0)&&!(i=s.next()).done;)o.push(i.value)}catch(t){n={error:t}}finally{try{i&&!i.done&&(r=s.return)&&r.call(s)}finally{if(n)throw n.error}}return o}function xg(){for(var t=[],e=0;e<arguments.length;e++)t=t.concat(Tg(arguments[e]));return t}function Cg(t){return!!t.useClass}function Ig(t){return!!t.useFactory}var Rg=function(){function t(t){this.wrap=t,this.reflectMethods=["get","getPrototypeOf","setPrototypeOf","getOwnPropertyDescriptor","defineProperty","has","set","deleteProperty","apply","construct","ownKeys"]}return t.prototype.createProxy=function(t){var e,r=this,i=!1;return new Proxy({},this.createHandler((function(){return i||(e=t(r.wrap()),i=!0),e})))},t.prototype.createHandler=function(t){var e={};return this.reflectMethods.forEach((function(r){e[r]=function(){for(var e=[],i=0;i<arguments.length;i++)e[i]=arguments[i];e[0]=t();var n=Reflect[r];return n.apply(void 0,xg(e))}})),e},t}();function kg(t){return"string"==typeof t||"symbol"==typeof t}function Ng(t){return"object"==typeof t&&"token"in t&&"transform"in t}function Og(t){return!!t.useToken}function Bg(t){return null!=t.useValue}const Pg=function(){function t(){this._registryMap=new Map}return t.prototype.entries=function(){return this._registryMap.entries()},t.prototype.getAll=function(t){return this.ensure(t),this._registryMap.get(t)},t.prototype.get=function(t){this.ensure(t);var e=this._registryMap.get(t);return e[e.length-1]||null},t.prototype.set=function(t,e){this.ensure(t),this._registryMap.get(t).push(e)},t.prototype.setAll=function(t,e){this._registryMap.set(t,e)},t.prototype.has=function(t){return this.ensure(t),this._registryMap.get(t).length>0},t.prototype.clear=function(){this._registryMap.clear()},t.prototype.ensure=function(t){this._registryMap.has(t)||this._registryMap.set(t,[])},t}(),Lg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Dg=function(){this.scopedResolutions=new Map};var Ug=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg),Mg=function(t){function e(){return null!==t&&t.apply(this,arguments)||this}return Eg(e,t),e}(Pg);const Fg=function(){this.preResolution=new Ug,this.postResolution=new Mg};var Hg=new Map,Wg=function(){function t(t){this.parent=t,this._registry=new Lg,this.interceptors=new Fg,this.disposed=!1,this.disposables=new Set}return t.prototype.register=function(t,e,r){var i;if(void 0===r&&(r={lifecycle:_g.Transient}),this.ensureNotDisposed(),i=function(t){return Cg(t)||Bg(t)||Og(t)||Ig(t)}(e)?e:{useClass:e},Og(i))for(var n=[t],s=i;null!=s;){var o=s.useToken;if(n.includes(o))throw new Error("Token registration cycle detected! "+xg(n,[o]).join(" -> "));n.push(o);var a=this._registry.get(o);s=a&&Og(a.provider)?a.provider:null}if((r.lifecycle===_g.Singleton||r.lifecycle==_g.ContainerScoped||r.lifecycle==_g.ResolutionScoped)&&(Bg(i)||Ig(i)))throw new Error('Cannot use lifecycle "'+_g[r.lifecycle]+'" with ValueProviders or FactoryProviders');return this._registry.set(t,{provider:i,options:r}),this},t.prototype.registerType=function(t,e){return this.ensureNotDisposed(),kg(e)?this.register(t,{useToken:e}):this.register(t,{useClass:e})},t.prototype.registerInstance=function(t,e){return this.ensureNotDisposed(),this.register(t,{useValue:e})},t.prototype.registerSingleton=function(t,e){if(this.ensureNotDisposed(),kg(t)){if(kg(e))return this.register(t,{useToken:e},{lifecycle:_g.Singleton});if(e)return this.register(t,{useClass:e},{lifecycle:_g.Singleton});throw new Error('Cannot register a type name as a singleton without a "to" token')}var r=t;return e&&!kg(e)&&(r=e),this.register(t,{useClass:r},{lifecycle:_g.Singleton})},t.prototype.resolve=function(t,e){void 0===e&&(e=new Dg),this.ensureNotDisposed();var r=this.getRegistration(t);if(!r&&kg(t))throw new Error('Attempted to resolve unregistered dependency token: "'+t.toString()+'"');if(this.executePreResolutionInterceptor(t,"Single"),r){var i=this.resolveRegistration(r,e);return this.executePostResolutionInterceptor(t,i,"Single"),i}if(function(t){return"function"==typeof t||t instanceof Rg}(t))return i=this.construct(t,e),this.executePostResolutionInterceptor(t,i,"Line truncated
+/*!
+ * MIT License
+ * 
+ * Copyright (c) Peculiar Ventures. All rights reserved.
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ * 
+ */
+const zm="crypto.algorithm",qm="crypto.algorithmProvider";var Qm;jg.registerSingleton(qm,class{getAlgorithms(){return jg.resolveAll(zm)}toAsnAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toAsnAlgorithm(t);if(r)return r}if(/[0-9.]+/.test(t.name)){const e=new Fp({algorithm:t.name});if("parameters"in t){const r=t;e.parameters=r.parameters}return e}throw new Error("Cannot convert WebCrypto algorithm to ASN.1 algorithm")}toWebAlgorithm(t){for(const e of this.getAlgorithms()){const r=e.toWebAlgorithm(t);if(r)return r}return{name:t.algorithm,parameters:t.parameters}}});const Km="1.3.36.3.3.2.8.1.1",Ym=`${Km}.1`,Xm=`${Km}.2`,Jm=`${Km}.3`,Zm=`${Km}.4`,$m=`${Km}.5`,tw=`${Km}.6`,ew=`${Km}.7`,rw=`${Km}.8`,iw=`${Km}.9`,nw=`${Km}.10`,sw=`${Km}.11`,ow=`${Km}.12`,aw=`${Km}.13`,cw=`${Km}.14`,lw="brainpoolP160r1",hw="brainpoolP160t1",uw="brainpoolP192r1",pw="brainpoolP192t1",dw="brainpoolP224r1",fw="brainpoolP224t1",gw="brainpoolP256r1",mw="brainpoolP256t1",ww="brainpoolP320r1",yw="brainpoolP320t1",bw="brainpoolP384r1",Aw="brainpoolP384t1",_w="brainpoolP512r1",vw="brainpoolP512t1",Ew="ECDSA";let Sw=Qm=class{toAsnAlgorithm(t){if(t.name.toLowerCase()===Ew.toLowerCase())if("hash"in t)switch(("string"==typeof t.hash?t.hash:t.hash.name).toLowerCase()){case"sha-1":return Pf;case"sha-256":return Lf;case"sha-384":return Df;case"sha-512":return Uf}else if("namedCurve"in t){let e="";switch(t.namedCurve){case"P-256":e=kf;break;case"K-256":e=Qm.SECP256K1;break;case"P-384":e=Nf;break;case"P-521":e=Of;break;case lw:e=Ym;break;case hw:e=Xm;break;case uw:e=Jm;break;case pw:e=Zm;break;case dw:e=$m;break;case fw:e=tw;break;case gw:e=ew;break;case mw:e=rw;break;case ww:e=iw;break;case yw:e=nw;break;case bw:e=sw;break;case Aw:e=ow;break;case _w:e=aw;break;case vw:e=cw}if(e)return new Fp({algorithm:Sf,parameters:iu.serialize(new Mf({namedCurve:e}))})}return null}toWebAlgorithm(t){switch(t.algorithm){case Tf:return{name:Ew,hash:{name:"SHA-1"}};case Cf:return{name:Ew,hash:{name:"SHA-256"}};case If:return{name:Ew,hash:{name:"SHA-384"}};case Rf:return{name:Ew,hash:{name:"SHA-512"}};case Sf:if(!t.parameters)throw new TypeError("Cannot get required parameters from EC algorithm");switch(iu.parse(t.parameters,Mf).namedCurve){case kf:return{name:Ew,namedCurve:"P-256"};case Qm.SECP256K1:return{name:Ew,namedCurve:"K-256"};case Nf:return{name:Ew,namedCurve:"P-384"};case Of:return{name:Ew,namedCurve:"P-521"};case Ym:return{name:Ew,namedCurve:lw};case Xm:return{name:Ew,namedCurve:hw};case Jm:return{name:Ew,namedCurve:uw};case Zm:return{name:Ew,namedCurve:pw};case $m:return{name:Ew,namedCurve:dw};case tw:return{name:Ew,namedCurve:fw};case ew:return{name:Ew,namedCurve:gw};case rw:return{name:Ew,namedCurve:mw};case iw:return{name:Ew,namedCurve:ww};case nw:return{name:Ew,namedCurve:yw};case sw:return{name:Ew,namedCurve:bw};case ow:return{name:Ew,namedCurve:Aw};case aw:return{name:Ew,namedCurve:_w};case cw:return{name:Ew,namedCurve:vw}}}return null}};Sw.SECP256K1="1.3.132.0.10",Sw=Qm=bg([Vg()],Sw),jg.registerSingleton(zm,Sw);const Tw=Symbol("name"),xw=Symbol("value");class Cw{constructor(t,e={},r=""){this[Tw]=t,this[xw]=r;for(const t in e)this[t]=e[t]}}Cw.NAME=Tw,Cw.VALUE=xw;class Iw{static toString(t){return this.items[t]||t}}Iw.items={[eg]:"sha1",[rg]:"sha224",[ig]:"sha256",[ng]:"sha384",[sg]:"sha512",[jf]:"rsaEncryption",[Kf]:"sha1WithRSAEncryption",[Yf]:"sha224WithRSAEncryption",[Xf]:"sha256WithRSAEncryption",[Jf]:"sha384WithRSAEncryption",[Zf]:"sha512WithRSAEncryption",[Sf]:"ecPublicKey",[Tf]:"ecdsaWithSHA1",[xf]:"ecdsaWithSHA224",[Cf]:"ecdsaWithSHA256",[If]:"ecdsaWithSHA384",[Rf]:"ecdsaWithSHA512",[np]:"TLS WWW server authentication",[sp]:"TLS WWW client authentication",[op]:"Code Signing",[ap]:"E-mail Protection",[cp]:"Time Stamping",[lp]:"OCSP Signing","1.2.840.113549.1.7.2":"Signed Data"};class Rw{static serialize(t){return this.serializeObj(t).join("\n")}static pad(t=0){return"".padStart(2*t," ")}static serializeObj(t,e=0){const r=[];let i=this.pad(e++),n="";const s=t[Cw.VALUE];s&&(n=` ${s}`),r.push(`${i}${t[Cw.NAME]}:${n}`),i=this.pad(e);for(const n in t){if("symbol"==typeof n)continue;const s=t[n],o=n?`${n}: `:"";if("string"==typeof s||"number"==typeof s||"boolean"==typeof s)r.push(`${i}${o}${s}`);else if(s instanceof Date)r.push(`${i}${o}${s.toUTCString()}`);else if(Array.isArray(s))for(const t of s)t[Cw.NAME]=n,r.push(...this.serializeObj(t,e));else if(s instanceof Cw)s[Cw.NAME]=n,r.push(...this.serializeObj(s,e));else if(ic.vJ.isBufferSource(s))n?(r.push(`${i}${o}`),r.push(...this.serializeBufferSource(s,e+1))):r.push(...this.serializeBufferSource(s,e));else{if(!("toTextObject"in s))throw new TypeError("Cannot serialize data in text format. Unsupported type.");{const t=s.toTextObject();t[Cw.NAME]=n,r.push(...this.serializeObj(t,e))}}}return r}static serializeBufferSource(t,e=0){const r=this.pad(e),i=ic.vJ.toUint8Array(t),n=[];for(let t=0;t<i.length;){const e=[];for(let r=0;r<16&&t<i.length;r++){8===r&&e.push("");const n=i[t++].toStrinLine truncated

archivebox/plugins/replaywebpage/static/test.txt

@@ -0,0 +1 @@
+test content this should be visible

archivebox/plugins/replaywebpage/templates/plugin_replaywebpage__viewer.html

@@ -0,0 +1,40 @@
+{% load tz core_tags static %}
+
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>{{title}}</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+      
+        </style>
+        <style>
+            html, body {
+                width: 100%;
+                height: 100%;
+                background-color: #ddd;
+            }
+        </style>
+    </head>
+    <body>
+        ReplayWeb.page for: {{snapshot.url}} ({{timestamp}}) /{{warc_filename}}
+
+        {{snapshot}}
+
+        <script>
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/sw.min.js
+            // https://cdn.jsdelivr.net/npm/replaywebpage@1.8.14/ui.min.js
+        </script>
+
+        <style>
+        </style>
+        <script src="/static/ui.js"></script>
+
+        <replay-web-page
+            style="height: 600px"
+            embed="replay"
+            replayBase="/static/"
+            source="/static/test.wacz"
+            url="https://example.com/">
+        </replay-web-page>
+    </body>
+</html>

archivebox/plugins/replaywebpage/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from .views import ReplayWebPageViewer
+
+urlpatterns = [
+	path('<path:path>', ReplayWebPageViewer.as_view(), name='plugin_replaywebpage__viewer'),
+]

archivebox/plugins/replaywebpage/views.py

@@ -0,0 +1,47 @@
+import os
+import sys
+from pathlib import Path
+
+from django.views import View
+from django.shortcuts import render
+from django.db.models import Q
+
+from core.models import Snapshot
+
+# from archivebox.config import PUBLIC_SNAPSHOTS
+PUBLIC_SNAPSHOTS = True
+
+
+class ReplayWebPageViewer(View):
+    template_name = 'plugin_replaywebpage__viewer.html'
+
+    # render static html index from filesystem archive/<timestamp>/index.html
+
+    def get_context_data(self, **kwargs):
+        return {
+            # **super().get_context_data(**kwargs),
+            # 'VERSION': VERSION,
+            # 'COMMIT_HASH': COMMIT_HASH,
+            # 'FOOTER_INFO': FOOTER_INFO,
+        }
+
+
+    def get(self, request, path):
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        try:
+            timestamp, warc_filename = path.split('/', 1)
+        except (IndexError, ValueError):
+            timestamp, warc_filename = path.split('/', 1)[0], ''
+
+        snapshot = Snapshot.objects.get(Q(timestamp=timestamp) | Q(id__startswith=timestamp))
+
+        context = self.get_context_data()
+        context.update({
+            "snapshot": snapshot,
+            "timestamp": timestamp,
+            "warc_filename": warc_filename,
+        })
+        return render(template_name=self.template_name, request=self.request, context=context)
+

archivebox/plugins/system/__init__.py

@@ -0,0 +1,3 @@
+__package__ = 'archivebox.plugins.system'
+
+default_app_config = 'plugins.system.apps.SystemPluginAppConfig'

archivebox/plugins/system/admin.py

@@ -0,0 +1,49 @@
+from django.contrib import admin
+from solo.admin import SingletonModelAdmin
+
+from plugins.defaults.admin import DependencyAdmin, ExtractorAdmin
+
+from .models import (
+	BashEnvironmentDependency,
+	PythonEnvironmentDependency,
+	NodeJSEnvironmentDependency,
+
+	AptEnvironmentDependency,
+	BrewEnvironmentDependency,
+	PipEnvironmentDependency,
+	NPMEnvironmentDependency,
+
+	SQLiteDependency,
+	DjangoDependency,
+	ArchiveBoxDependency,
+
+	# ArchiveBoxDefaultExtractor,
+)
+
+
+print('DefaultsPluginConfig.admin')
+
+class MultiDependencyAdmin(admin.ModelAdmin):
+	readonly_fields = DependencyAdmin.readonly_fields
+	list_display = ('id', 'NAME', 'ENABLED', 'BINARY', 'ARGS', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
+
+class MultiExtractorAdmin(admin.ModelAdmin):
+	readonly_fields = DependencyAdmin.readonly_fields
+	list_display = ('id', 'NAME', 'CMD', 'ARGS', 'is_valid', 'is_enabled')
+
+
+# admin.site.register(BashEnvironmentDependency, DependencyAdmin)
+admin.site.register(BashEnvironmentDependency, MultiDependencyAdmin)
+admin.site.register(PythonEnvironmentDependency, DependencyAdmin)
+admin.site.register(NodeJSEnvironmentDependency, DependencyAdmin)
+
+admin.site.register(AptEnvironmentDependency, DependencyAdmin)
+admin.site.register(BrewEnvironmentDependency, DependencyAdmin)
+admin.site.register(PipEnvironmentDependency, DependencyAdmin)
+admin.site.register(NPMEnvironmentDependency, DependencyAdmin)
+
+admin.site.register(SQLiteDependency, DependencyAdmin)
+admin.site.register(DjangoDependency, DependencyAdmin)
+admin.site.register(ArchiveBoxDependency, DependencyAdmin)
+
+# admin.site.register(ArchiveBoxDefaultExtractor, ExtractorAdmin)

archivebox/plugins/system/apps.py

@@ -0,0 +1,21 @@
+__package__ = 'archivebox.plugins.system'
+
+
+from django.apps import AppConfig
+
+
+class SystemPluginAppConfig(AppConfig):
+    name = "plugins.system"
+    verbose_name = "Host System Configuration"
+    
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        print('plugins.system.apps.SystemPluginConfig.ready')
+
+        from django.conf import settings
+
+        from plugins.defaults.settings import register_plugin_settings
+
+        register_plugin_settings(settings, name=self.name)
+