fix docker release script

Co-authored-by: Nick Sweeting <git@sweeting.me>

.dockerignore

@@ -5,17 +5,27 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .github/
+.pdm-build/
+.pdm-python
+.eggs/
+.git/
+!.git/HEAD
+!.git/refs/heads/*
 
 venv/
 .venv/
+.venv-old/
 .docker-venv/
+node_modules/
 
+docs/
 build/
 dist/
-pip_dist/
-!pip_dist/archivebox.egg-info/requires.txt
 brew_dist/
+deb_dist/
+pip_dist/
 assets/
+docker/
 
 data/
 output/

.github/workflows/debian.yml

@@ -25,36 +25,23 @@ jobs:
             dh-python debhelper devscripts dput software-properties-common \
             python3-distutils python3-setuptools python3-wheel python3-stdeb
 
-      - name: Build Debian/Apt sdist_dsc
-        run: |
-          rm -Rf deb_dist/*
-          python3 setup.py --command-packages=stdeb.command sdist_dsc
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
 
-      - name: Build Debian/Apt bdist_deb
-        run: |
-          python3 setup.py --command-packages=stdeb.command bdist_deb
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
 
-      - name: Install archivebox from deb
-        run: |
-          cd deb_dist/
-          sudo apt-get install ./archivebox*.deb
-          cd ..
-          python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > ./requirements.txt
-          python3 -m pip install -r ./requirements.txt
-
-      - name: Check ArchiveBox version
-        run: |
-          # must create dir needed for snaps to run as non-root on github actions
-          sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-          mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-          archivebox --version
-          archivebox init --setup
-
-      - name: Add some links to test
-        run: |
-          cd "${{ github.workspace }}/data"
-          archivebox add 'https://example.com'
-          archivebox status
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
 
       # - name: Commit built package
       #   run: |

.github/workflows/docker.yml

@@ -11,27 +11,27 @@ on:
 
 env:
   DOCKER_IMAGE: archivebox-ci
-
-
+      
 jobs:
   buildx:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
       
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
@@ -40,7 +40,7 @@ jobs:
         run: echo ${{ steps.buildx.outputs.platforms }}
       
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -48,7 +48,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Docker Login
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
            username: ${{ secrets.DOCKER_USERNAME }}
@@ -56,20 +56,19 @@ jobs:
       
       - name: Collect Docker tags
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v5
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          flavor: |
-              latest=auto
           tags: |
               type=ref,event=branch
               type=semver,pattern={{version}}
               type=semver,pattern={{major}}.{{minor}}
               type=sha
+              type=raw,value=latest,enable={{is_default_branch}}
       
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
@@ -77,8 +76,18 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
           platforms: linux/amd64,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
+       
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/jekyll-gh-pages.yml

@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/pip.yml

@@ -7,29 +7,42 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
+
+    permissions:
+      id-token: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: '3.11'
           architecture: x64
 
-      - name: Build Python Package
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
+      - uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
+
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --group :all --no-self
+
+      - name: Build package
         run: |
-          pip3 install --upgrade pip setuptools wheel
-          rm -Rf pip_dist/*.whl
-          python3 setup.py \
-            sdist --dist-dir=./pip_dist \
-            bdist_wheel --dist-dir=./pip_dist \
-            egg_info --egg-base=./pip_dist
-          pip install pip_dist/archivebox-*.whl
+          pdm build
+
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl
 
       - name: Add some links to test
         run: |
@@ -39,6 +52,9 @@ jobs:
           archivebox version
           archivebox status
 
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
+
       # - name: Push build to PyPI
       #   run: |
       #     cd pip_dist/

.github/workflows/test.yml

@@ -13,26 +13,33 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
-        python: [3.7]
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
-      - name: Set up Node JS 14.7.0
-        uses: actions/setup-node@v1
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          node-version: 14.7.0
+          node-version: 20.10.0
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
       ### Install Python & JS Dependencies
       - name: Get pip cache dir
@@ -41,7 +48,7 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-pip
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -49,11 +56,16 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
       - name: Install pip dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel pytest bottle
-          ./bin/build_pip.sh
-          python -m pip install .
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
 
       - name: Get npm cache dir
         id: npm-cache
@@ -61,7 +73,7 @@ jobs:
           echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
 
       - name: Cache npm
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-npm
         with:
           path: ${{ steps.npm-cache.outputs.dir }}
@@ -96,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1

.gitignore

@@ -13,6 +13,8 @@ venv/
 node_modules/
 
 # Packaging artifacts
+.pdm-python
+.pdm-build
 archivebox.egg-info
 archivebox-*.tar.gz
 build/

.readthedocs.yaml

@@ -0,0 +1,34 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements.txt
+    - requirements: docs/requirements.txt

Dockerfile

@@ -12,122 +12,280 @@
 #     docker buildx create --use
 #     docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
 #
-# Read more about [developing
-# Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
 
-FROM python:3.11-slim-bullseye
+# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
+FROM python:3.11-slim-bookworm
 
 LABEL name="archivebox" \
-    maintainer="Nick Sweeting <archivebox-docker@sweeting.me>" \
+    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
     description="All-in-one personal internet archiving container" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
     documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
 
-# System-level base config
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+######### Environment Variables #################################
+
+# Global system-level config
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
     PYTHONIOENCODING=UTF-8 \
     PYTHONUNBUFFERED=1 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error
 
-# Application-level base config
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=20
+
+# User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911
+
+# Global paths
 ENV CODE_DIR=/app \
-    VENV_PATH=/venv \
     DATA_DIR=/data \
-    NODE_DIR=/node \
-    ARCHIVEBOX_USER="archivebox"
+    GLOBAL_VENV=/venv \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
+
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
+
+# Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
 
 # Create non-privileged user for archivebox and chrome
-RUN groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
 
-# Install system dependencies
-RUN apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        apt-transport-https ca-certificates gnupg2 zlib1g-dev \
-        dumb-init gosu cron unzip curl \
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        # nano iputils-ping dnsutils htop procps jq yq
     && rm -rf /var/lib/apt/lists/*
 
-# Install apt dependencies
-RUN apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        wget curl chromium git ffmpeg youtube-dl ripgrep \
-        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-    && ln -s /usr/bin/chromium /usr/bin/chromium-browser \
-    && rm -rf /var/lib/apt/lists/*
+######### Language Environments ####################################
 
 # Install Node environment
-RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
-    && echo 'deb https://deb.nodesource.com/node_18.x buster main' >> /etc/apt/sources.list \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
     && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        nodejs \
-    # && npm install -g npm \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        nodejs libatomic1 python3-minimal \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    && ( \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Extractor Dependencies ##################################
+
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        curl wget git yt-dlp ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+        # fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+    && apt-get update -qq \
+    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
+        # install Chromium using playwright
+        pip install playwright \
+        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+        && playwright install --with-deps chromium \
+        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
+    else \
+        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.) 
+        apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+            chromium fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        && export CHROME_BINARY="$(which chromium)"; \
+    fi \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # Save version info
+    && ( \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
 
 # Install Node dependencies
-WORKDIR "$NODE_DIR"
-ENV PATH="${PATH}:$NODE_DIR/node_modules/.bin" \
-    npm_config_loglevel=error
-ADD ./package.json ./package.json
-ADD ./package-lock.json ./package-lock.json
-RUN npm ci
-
-# Install Python dependencies
 WORKDIR "$CODE_DIR"
-ENV PATH="${PATH}:$VENV_PATH/bin"
-RUN python -m venv --clear --symlinks "$VENV_PATH" \
-    && pip install --upgrade --quiet pip setuptools \
-    && mkdir -p "$CODE_DIR/archivebox"
-ADD "./setup.py" "$CODE_DIR/"
-ADD "./package.json" "$CODE_DIR/archivebox/"
-RUN apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential python-dev python3-dev \
-    && echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \
-    && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
-    && pip install -r /tmp/requirements.txt \
-    && pip install --upgrade youtube-dl yt-dlp \
-    && apt-get purge -y build-essential python-dev python3-dev \
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
+        which node && node --version \
+        && which npm && npm version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Build Dependencies ####################################
+
+# Install ArchiveBox Python dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        build-essential \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
-# Install apt development dependencies
-# RUN apt-get install -qq \
-#     && apt-get install -qq -y --no-install-recommends \
-#         python3 python3-dev python3-pip python3-venv python3-all \
-#         dh-python debhelper devscripts dput software-properties-common \
-#         python3-distutils python3-setuptools python3-wheel python3-stdeb
-# RUN python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.extras_require["dev"]))' > /tmp/dev_requirements.txt \
-    # && pip install --quiet -r /tmp/dev_requirements.txt
+# Install ArchiveBox Python package from source
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    #     build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install ArchiveBox Python package and its dependencies
-WORKDIR "$CODE_DIR"
-ADD . "$CODE_DIR"
-RUN pip install -e .
+####################################################
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True \
-    CHROME_SANDBOX=False \
-    CHROME_BINARY="/usr/bin/chromium-browser" \
-    USE_SINGLEFILE=True \
-    SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \
-    USE_READABILITY=True \
-    READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor" \
-    USE_MERCURY=True \
-    MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser" \
-    YOUTUBEDL_BINARY="yt-dlp"
+ENV IN_DOCKER=True
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # CHROME_SANDBOX=False \
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
 
 # Print version for nice docker finish summary
-# RUN archivebox version
-RUN /app/bin/docker_entrypoint.sh archivebox version
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
 
 # Open up the interfaces to the outside world
+WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 

MANIFEST.in

@@ -1,6 +0,0 @@
-graft archivebox
-global-exclude .DS_Store
-global-exclude __pycache__
-global-exclude *.pyc
-
-prune tests/

Pipfile

@@ -1,12 +0,0 @@
-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
-verify_ssl = true
-
-[packages]
-# see setup.py for package dependency list
-"e1839a8" = {path = ".", editable = true}
-
-[dev-packages]
-# see setup.py for dev package dependency list
-"e1839a8" = {path = ".", extras = ["dev"], editable = true}

README.md

@@ -1,45 +1,77 @@
 <div align="center">
-<em><img src="https://i.imgur.com/5B48E3N.png" height="90px"></em>
+<em><img src="https://archivebox.io/icon.png" height="90px"></em>
 <h1>ArchiveBox<br/><sub>Open-source self-hosted web archiving.</sub></h1>
 
+<br/>
+
 ▶️ <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a> |
 <a href="https://demo.archivebox.io">Demo</a> |
 <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub</a> |
 <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a> |
 <a href="#background--motivation">Info & Motivation</a> |
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a> |
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap">Roadmap</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a>
 
-<pre lang="bash"><code style="white-space: pre-line">"Your own personal internet archive" (网站存档 / 爬虫)
-curl -sSL 'https://get.archivebox.io' | sh
-</code></pre>
+<br/>
 
 <!--<a href="http://webchat.freenode.net?channels=ArchiveBox&uio=d4"><img src="https://img.shields.io/badge/Community_chat-IRC-%2328A745.svg"/></a>-->
 
-<a href="https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a>
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a>
-<a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=active"/></a> &nbsp;
+<a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp;
 <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/badge/Python-yellow.svg?logo=python&logoColor=yellow"/></a>
 <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies"><img src="https://img.shields.io/badge/Chromium-orange.svg?logo=Google+Chrome&logoColor=orange"/></a>
 <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/badge/Docker-lightblue.svg?logo=docker&logoColor=lightblue"/></a>
 
 
+<!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
+curl -sSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instructions below)
+</code></pre>-->
+
 <hr/>
 </div>
 
-**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view sites you want to preserve offline.**
+**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**
 
-You can set it up as a [command-line tool](#quickstart), [web app](#quickstart), and [desktop app](https://github.com/ArchiveBox/electron-archivebox) (alpha), on Linux, macOS, and Windows.
+Without active preservation effort, everything on the internet eventually dissapears or degrades. Archive.org does a great job as a free central archive, but they require all archives to be public, and they can't save every type of content.
 
-**You can feed it URLs one at a time, or schedule regular imports** from browser bookmarks or history, feeds like RSS, bookmark services like Pocket/Pinboard, and more. See <a href="#input-formats">input formats</a> for a full list.
+*ArchiveBox is an open source tool that helps you archive web content on your own (or privately within an organization): save copies of browser bookmarks, preserve evidence for legal cases, backup photos from FB / Insta / Flickr, download your media from YT / Soundcloud / etc., snapshot research papers & academic citations, and more...*
 
-**It saves snapshots of the URLs you feed it in several formats:** HTML, PDF, PNG screenshots, WARC, and more out-of-the-box, with a wide variety of content extracted and preserved automatically (article text, audio/video, git repos, etc.). See <a href="#output-formats">output formats</a> for a full list.
+> ➡️ *Use ArchiveBox as a [command-line package](#quickstart) and/or [self-hosted web app](#quickstart) on Linux, macOS, or in [Docker](#quickstart).*
+
+<hr/>
+
+📥 **You can feed ArchiveBox URLs one at a time, or schedule regular imports** from browser bookmarks or history, feeds like RSS, bookmark services like Pocket/Pinboard, and more. See <a href="#input-formats">input formats</a> for a full list.
+
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/90f1ce3c-75bb-401d-88ed-6297694b76ae" alt="snapshot detail page" align="right" width="190px"/>
+
+💾 **It saves snapshots of the URLs you feed it in several redundant formats.**  
+It also detects any content featured *inside* each webpage & extracts it out into a folder:
+- `HTML/Generic websites -> HTML, PDF, PNG, WARC, Singlefile`
+- `YouTube/SoundCloud/etc. -> MP3/MP4 + subtitles, description, thumbnail`
+- `News articles -> article body TXT + title, author, featured images`
+- `Github/Gitlab/etc. links -> git cloned source code`
+- *[and more...](#output-formats)*
+
+It uses normal filesystem folders to organize archives (no complicated proprietary formats), and offers a CLI + web UI.  
+
+---
+
+🏛️ ArchiveBox is used by many *[professionals](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102) and [hobbyists](https://zulip.archivebox.io/#narrow/stream/158-development)* who save content off the web, for example:
+
+- **Individuals:**
+  `backing up browser bookmarks/history`, `saving FB/Insta/etc. content`, `shopping lists`  
+- **Journalists:**
+  `crawling and collecting research`, `preserving quoted material`, `fact-checking and review`  
+- **Lawyers:**
+  `evidence collection`, `hashing & integrity verifying`, `search, tagging, & review`  
+- **Researchers:**
+  `collecting AI training sets`, `feeding analysis / web crawling pipelines`
 
 The goal is to sleep soundly knowing the part of the internet you care about will be automatically preserved in durable, easily accessible formats [for decades](#background--motivation) after it goes down.
 
 <div align="center">
 <br/><br/>
-<img src="https://i.imgur.com/PAzXZE8.png" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://i.imgur.com/asPNk8n.png" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://i.imgur.com/PAzXZE8.png" height="70px" alt="bookshelf graphic">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
 <br/><br/>
 <small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
 <br/>
@@ -49,20 +81,24 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 
 <br/>
 
-**📦&nbsp; Get ArchiveBox with Docker / `apt` / `brew` / `pip3` / etc. ([see Quickstart below](#quickstart)).**
+**📦&nbsp; Get ArchiveBox with `docker` / `apt` / `brew` / `pip3` / `nix` / etc. ([see Quickstart below](#quickstart)).**
 
 ```bash
-# Follow the instructions for your package manager in the quickstart, e.g.:
+# Get ArchiveBox with Docker or Docker Compose (recommended)
+docker run -v $PWD/data:/data -it archivebox/archivebox:dev init --setup
+
+# Or install with your preferred package manager (see Quickstart below for apt, brew, and more)
 pip3 install archivebox
 
-# Or use the optional auto setup script to install it for you:
+# Or use the optional auto setup script to install it
 curl -sSL 'https://get.archivebox.io' | sh
 ```
 
 **🔢 Example usage: adding links to archive.**
 ```bash
-archivebox add 'https://example.com'                                   # add URLs one at a time via args / piped stdin
-archivebox schedule --every=day --depth=1 https://example.com/rss.xml  # or have it import URLs regularly on a schedule
+archivebox add 'https://example.com'                                   # add URLs one at a time
+archivebox add < ~/Downloads/bookmarks.json                            # or pipe in URLs in any text-based format
+archivebox schedule --every=day --depth=1 https://example.com/rss.xml  # or auto-import URLs regularly on a schedule
 ```
 **🔢 Example usage: viewing the archived content.**
 ```bash
@@ -73,36 +109,49 @@ ls ./archive/*/index.json                 # or browse directly via the filesyste
 
 <div align="center">
 <br/><br/>
-<img src="https://i.imgur.com/njxgSbl.png" width="22%" alt="cli init screenshot" align="top">
-<img src="https://i.imgur.com/lUuicew.png" width="22%" alt="cli init screenshot" align="top">
-<img src="https://i.imgur.com/p6wK6KM.png" width="22%" alt="server snapshot admin screenshot" align="top">
-<img src="https://i.imgur.com/xHvQfon.png" width="28.6%" alt="server snapshot details page screenshot" align="top"/>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/8d67382c-e0ce-4286-89f7-7915f09b930c" width="22%" alt="cli init screenshot" align="top">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dad2bc51-e7e5-484e-bb26-f956ed692d16" width="22%" alt="cli init screenshot" align="top">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2" width="22%" alt="server snapshot admin screenshot" align="top">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2" width="28.6%" alt="server snapshot details page screenshot" align="top"/>
 <br/><br/>
 </div>
 
 ## Key Features
 
-- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/master/LICENSE), doesn't require signing up online, stores all data locally
+- [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), doesn't require signing up online, stores all data locally
 - [**Powerful, intuitive command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular optional dependencies](#dependencies) 
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
-- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (youtube-dl or yt-dlp), articles (readability), code (git), etc.](#output-formats)
+- [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
-- [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, and WARC
+- [**Uses standard, durable, long-term formats**](#saves-lots-of-useful-stuff-for-each-imported-link) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
 - [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox) (ALPHA)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
-- Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)...
+- Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
-<br/><br/>
+<br/>
+
+## 🤝 Professional Integration
+
+*[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102) if your non-profit institution/org wants to use ArchiveBox professionally.*
+
+- setup & support, team permissioning, hashing, audit logging, backups, custom archiving etc.  
+- for **individuals**, **NGOs**, **academia**, **governments**, **journalism**, **law**, and more...
+
+*All our work is open-source and primarily geared towards non-profits.*  
+*Support/consulting pays for hosting and funds new ArchiveBox open-source development.*
+
+<br/>
 
 <div align="center">
 <br/>
-<img src="https://i.imgur.com/T2UAGUD.png" width="49%" alt="grass"/><img src="https://i.imgur.com/T2UAGUD.png" width="49%" alt="grass"/>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0db52ea7-4a2c-441d-b47f-5553a5d8fe96" width="49%" alt="grass"/><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0db52ea7-4a2c-441d-b47f-5553a5d8fe96" width="49%" alt="grass"/>
 </div>
 
 # Quickstart
 
-**🖥&nbsp; Supported OSs:** Linux/BSD, macOS, Windows (Docker/WSL) &nbsp; **👾&nbsp; CPUs:** amd64, x86, arm8, arm7 <sup>(raspi>=3)</sup>
+**🖥&nbsp; Supported OSs:** Linux/BSD, macOS, Windows (Docker) &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64` (`arm8`), `arm7` <sup>(raspi>=3)</sup><br/>
+<sub>Note: On `arm7` the `playwright` package is not available, so `chromium` must be installed manually if needed.</sub>
 
 <br/>
 
@@ -111,21 +160,21 @@ ls ./archive/*/index.json                 # or browse directly via the filesyste
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux/Windows) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
-<i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all the <a href="#dependencies">extras</a> working out-of-the-box.</i>
+<i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all the <a href="#dependencies">extras</a> out-of-the-box.</i>
 <br/><br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> and <a href="https://docs.docker.com/compose/install/#install-using-pip">Docker Compose</a> on your system (if not already installed).</li>
-<li>Download the <a href="https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml" download><code>docker-compose.yml</code></a> file into a new empty directory (can be anywhere).
+<li>Download the <a href="https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/docker-compose.yml" download><code>docker-compose.yml</code></a> file into a new empty directory (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir ~/archivebox && cd ~/archivebox
-curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml'
+curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/docker-compose.yml'
 </code></pre></li>
 <li>Run the initial setup and create an admin user.
-<pre lang="bash"><code style="white-space: pre-line">docker-compose run archivebox init --setup
+<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --setup
 </code></pre></li>
 <li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker-compose up
+<pre lang="bash"><code style="white-space: pre-line">docker compose up
 # completely optional, CLI can always be used without running a server
-# docker-compose run [-T] archivebox [subcommand] [--args]
+# docker compose run [-T] archivebox [subcommand] [--args]
 </code></pre></li>
 </ol>
 
@@ -134,7 +183,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker</code></b>  (macOS/Linux/Windows)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b>  (macOS/Linux/Windows)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
@@ -181,12 +230,7 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 <br/>
 <ol>
 <li>Add the ArchiveBox repository to your sources.<br/>
-<pre lang="bash"><code style="white-space: pre-line"># On Ubuntu == 20.04, add the sources automatically:
-sudo apt install software-properties-common
-sudo add-apt-repository -u ppa:archivebox/archivebox
-</code></pre>
-<pre lang="bash"><code style="white-space: pre-line"># On Ubuntu >= 20.10 or <= 19.10, or other Debian-style systems, add the sources manually:
-echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
+<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
 sudo apt update
 </code></pre>
@@ -244,11 +288,11 @@ See the <a href="https://github.com/ArchiveBox/homebrew-archivebox"><code>homebr
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/Windows)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.7</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v14</a> on your system (if not already installed).</li>
+<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.9</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v18</a> on your system (if not already installed).</li>
 <li>Install the ArchiveBox package using <code>pip3</code>.
 <pre lang="bash"><code style="white-space: pre-line">pip3 install archivebox
 </code></pre>
@@ -275,6 +319,10 @@ See the <a href="https://github.com/ArchiveBox/pip-archivebox"><code>pip-archive
 <details>
 <summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
+
+> [!WARNING]  
+> *These are contributed by external volunteers and may lag behind the official `pip` channel.*
+
 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
 <li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -sSL 'https://get.archivebox.io' | sh</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
@@ -302,7 +350,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 </ul>
 </li>
 </ol>
-<img src="https://i.imgur.com/QPHUS5C.png" width="320px">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/575ef92f-bb3e-4a7c-a4ba-986c1fd76ecf" width="320px">
 <br/>
 <i>✨ Alpha (contributors wanted!)</i>: for more info, see the: <a href="https://github.com/ArchiveBox/electron-archivebox">Electron ArchiveBox</a> repo.
   <br/>
@@ -312,13 +360,16 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <summary><img src="https://user-images.githubusercontent.com/511499/117448723-1663b180-af0d-11eb-837f-d43959227810.png" alt="paid" height="27px" align="top"/> Paid hosting solutions (cloud VPS)</summary>
 <br/>
 <ul>
+<li><a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">
+ <img src="https://img.shields.io/badge/Custom_Development-ArchiveBox.io-%231a1a1a.svg?style=flat" height="22px"/>
+</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directy from us</a>)</li>
 <li><a href="https://monadical.com">
- <img src="https://img.shields.io/badge/Custom_Managed_Solution-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://monadical.com/contact-us.html">for larger setups, get a quote</a>)</li>
+ <img src="https://img.shields.io/badge/General_Dev_Consulting-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
+</a> (<a href="https://monadical.com/contact-us.html">for a generalist software consultancy that helps with ArchiveBox maintainance</a>)</li>
 <br/>
-None of these hosting providers are officially endorsed:<br/>
-<sub><i>(most still require manual setup or manual periodic updating using the methods above)</i></sub>
+Other providers of paid ArchiveBox hosting (not officially endorsed):<br/>
 <br/><br/>
+<li><a href="https://elest.io/open-source/archivebox"><img src="https://img.shields.io/badge/Managed_Hosting-Elest.io-%23193f7e.svg?style=flat" height="22px"/></a></li>
 <li><a href="https://www.stellarhosted.com/archivebox/"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-StellarHosted.com-%23193f7e.svg?style=flat" height="22px"/></a> (USD $29-250/mo, <a href="https://www.stellarhosted.com/archivebox/#pricing">pricing</a>)</li>
 <li><a href="https://www.pikapods.com/pods?run=archivebox"><img src="https://img.shields.io/badge/Semi_Managed_Hosting-PikaPods.com-%2343a047.svg?style=flat" height="22px"/></a> (from USD $2.6/mo)</li>
 <li><a href="https://m.do.co/c/cbc4c0c17840">
@@ -391,7 +442,7 @@ ls ./archive/*/index.html  # or inspect snapshots on the filesystem
 
 <br/>
 <div align="center">
-<img src="https://i.imgur.com/6AmOGJT.png" width="49%" alt="grass"/><img src="https://i.imgur.com/6AmOGJT.png" width="49%" alt="grass"/>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/65f82532-18dd-49c5-86f1-02b1f3100e1e" width="49%" alt="grass"/><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/65f82532-18dd-49c5-86f1-02b1f3100e1e" width="49%" alt="grass"/>
 </div>
 <br/>
 
@@ -408,7 +459,7 @@ ls ./archive/*/index.html  # or inspect snapshots on the filesystem
 ---
 
 <div align="center">
-<img src="https://i.imgur.com/OUmgdlH.png" width="96%" alt="lego">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ac1f897a-8baa-4f8b-8ee8-7443611f258b" width="96%" alt="lego">
 </div>
 
 <br/>
@@ -422,12 +473,12 @@ ArchiveBox supports many input formats for URLs, including Pocket & Pinboard exp
 
 *Click these links for instructions on how to prepare your links from these sources:*
 
-- <img src="https://nicksweeting.com/images/rss.svg" height="22px"/> TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file)
-- <img src="https://nicksweeting.com/images/bookmarks.png" height="22px"/> [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](http://i.imgur.com/AtcvUZA.png), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](http://help.opera.com/Windows/12.10/en/importexport.html), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive))
-- <img src="https://i.imgur.com/AQyHbu8.png" height="22px"/> Browser extension [`archivebox-exporter`](https://github.com/tjhorner/archivebox-exporter) (realtime archiving from Chrome/Chromium/Firefox)
-- <img src="https://getpocket.com/favicon.ico" height="22px"/> [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user/export), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> TXT, RSS, XML, JSON, CSV, SQL, HTML, Markdown, or [any other text-based format...](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file)
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> [Browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (see instructions for: [Chrome](https://support.google.com/chrome/answer/96816?hl=en), [Firefox](https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer), [Safari](https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a), [IE](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows), [Opera](https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive))
+- <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ff20d251-5347-4b85-ae9b-83037d0ac01e" height="22px"/> Browser extension [`archivebox-exporter`](https://github.com/tjhorner/archivebox-exporter) (realtime archiving from Chrome/Chromium/Firefox)
+- <img src="https://getpocket.com/favicon.ico" height="22px"/> [Pocket](https://getpocket.com/export), [Pinboard](https://pinboard.in/export/), [Instapaper](https://www.instapaper.com/user), [Shaarli](https://shaarli.readthedocs.io/en/master/Usage/#importexport), [Delicious](https://www.groovypost.com/howto/howto/export-delicious-bookmarks-xml/), [Reddit Saved](https://github.com/csu/export-saved-reddit), [Wallabag](https://doc.wallabag.org/en/user/import/wallabagv2.html), [Unmark.it](http://help.unmark.it/import-export), [OneTab](https://www.addictivetips.com/web/onetab-save-close-all-chrome-tabs-to-restore-export-or-import/), [and more...](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 
-<img src="https://i.imgur.com/zM4z1aU.png" width="330px" align="right">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right">
 
 
 ```bash
@@ -438,10 +489,10 @@ archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any_text_with [urls](https://example.com) in it' | archivebox add
 
-# if using docker add -i when piping stdin:
+# if using Docker, add -i when piping stdin:
 # echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
-# if using docker-compose add -T when piping stdin / stdout:
-# echo 'https://example.com' | docker-compose run -T archivebox add
+# if using Docker Compose, add -T when piping stdin / stdout:
+# echo 'https://example.com' | docker compose run -T archivebox add
 ```
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
@@ -454,7 +505,7 @@ It also includes a built-in scheduled import feature with `archivebox schedule`
 
 Inside each Snapshot folder, ArchiveBox saves these different types of extractor outputs as plain files:
 
-<img src="https://i.imgur.com/xHvQfon.png" width="330px" align="right">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2" width="330px" align="right">
 
 `./archive/<timestamp>/*`
 
@@ -478,7 +529,7 @@ It does everything out-of-the-box by default, but you can disable or tweak [indi
 
 ## Configuration
 
-<img src="https://i.imgur.com/H08eaia.png" width="330px" align="right">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ea672e6b-4df5-49d8-b550-7f450951fd27" width="330px" align="right">
 
 ArchiveBox can be configured via environment variables, by using the `archivebox config` CLI, or by editing the `ArchiveBox.conf` config file directly.
 
@@ -495,7 +546,7 @@ env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
 
 <sup>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sup>
 
-**The config loading logic with all the options defined is here: [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/master/archivebox/config.py).**
+**The config loading logic with all the options defined is here: [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py).**
 
 Most options are also documented on the **[Configuration Wiki page](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**.
 
@@ -512,17 +563,29 @@ MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
 PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
+
+CHROME_USER_AGENT="Mozilla/5.0 ..."  # change these to get around bot blocking
+WGET_USER_AGENT="Mozilla/5.0 ..."
+CURL_USER_AGENT="Mozilla/5.0 ..."
 ```
 
 <br/>
 
 ## Dependencies
 
+To achieve high-fidelity archives in as many situations as possible, ArchiveBox depends on a variety of high-quality 3rd-party tools and libraries that specialize in extracting different types of content.
+
+<br/>
+<details>
+<summary><i>Expand to learn more about ArchiveBox's dependencies...</i></summary>
+<br/>
+
 For better security, easier updating, and to avoid polluting your host system with extra dependencies, **it is strongly recommended to use the official [Docker image](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker)** with everything pre-installed for the best experience.
 
-To achieve high-fidelity archives in as many situations as possible, ArchiveBox depends on a variety of 3rd-party tools and libraries that specialize in extracting different types of content. These optional dependencies used for archiving sites include:
+These optional dependencies used for archiving sites include:
+
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right">
 
-<img src="https://i.imgur.com/5vSBO2R.png" width="330px" align="right">
 
 - `chromium` / `chrome` (for screenshots, PDF, DOM HTML, and headless JS scripts)
 - `node` & `npm` (for readability, mercury, and singlefile)
@@ -546,21 +609,41 @@ archivebox --version   # see info and check validity of installed dependencies
 
 Installing directly on **Windows without Docker or WSL/WSL2/Cygwin is not officially supported** (I cannot respond to Windows support tickets), but some advanced users have reported getting it working.
 
-For detailed information about upgrading ArchiveBox and its dependencies, see: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing
+
+</details>
 
 <br/>
 
 ## Archive Layout
 
-All of ArchiveBox's state (including the index, snapshot data, and config file) is stored in a single folder called the "ArchiveBox data folder". All `archivebox` CLI commands must be run from inside this folder, and you first create it by running `archivebox init`.
+All of ArchiveBox's state (including the SQLite DB, archived assets, config, logs, etc.) is stored in a single folder called the "ArchiveBox Data Folder".  
+Data folders can be created anywhere (`~/archivebox` or `$PWD/data` as seen in our examples), and you can create more than one for different collections.
 
-The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard `index.sqlite3` database in the root of the data folder (it can also be exported as static JSON/HTML), and the archive snapshots are organized by date-added timestamp in the `./archive/` subfolder.
+<br/>
+<details>
+<summary><i>Expand to learn more about the layout of Archivebox's data on-disk...</i></summary>
+<br/>
+
+All `archivebox` CLI commands are designed to be run from inside an ArchiveBox data folder, starting with `archivebox init` to initialize a new collection inside an empty directory.
+
+```bash
+mkdir ~/archivebox && cd ~/archivebox   # just an example, can be anywhere
+archivebox init
+```
+
+The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard `index.sqlite3` database in the root of the data folder (it can also be [exported as static JSON/HTML](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html)), and the archive snapshots are organized by date-added timestamp in the `./archive/` subfolder.
 
 <img src="https://user-images.githubusercontent.com/511499/117453293-c7b91600-af12-11eb-8a3f-aa48b0f9da3c.png" width="400px" align="right">
 
 
 ```bash
-./
+/data/
     index.sqlite3
     ArchiveBox.conf
     archive/
@@ -577,14 +660,29 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
 
 Each snapshot subfolder `./archive/<timestamp>/` includes a static `index.json` and `index.html` describing its contents, and the snapshot extractor outputs are plain files within the folder.
 
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+
+</details>
 
 <br/>
 
 ## Static Archive Exporting
 
-You can export the main index to browse it statically without needing to run a server.
+You can export the main index to browse it statically as plain HTML files in a folder (without needing to run a server).
 
-*Note about large exports: These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the `archivebox list` command to export specific Snapshots or ranges.*
+<br/>
+<details>
+<summary><i>Expand to learn how to export your ArchiveBox collection...</i></summary>
+<br/>
+
+> **Note**
+> These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the `archivebox list` command to export specific Snapshots or ranges.
 
 ```bash
 # archivebox list --help
@@ -592,12 +690,20 @@ archivebox list --html --with-headers > index.html     # export to static html t
 archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
 
-# (if using docker-compose, add the -T flag when piping)
-# docker-compose run -T archivebox list --html --filter-type=search snozzberries > index.json
+# (if using Docker Compose, add the -T flag when piping)
+# docker compose run -T archivebox list --html --filter-type=search snozzberries > index.json
 ```
 
 The paths in the static exports are relative, make sure to keep them next to your `./archive` folder when backing them up or viewing them.
 
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#publishing
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#public_index--public_snapshots--public_add_view
+
+
+</details>
 
 <br/>
 
@@ -611,10 +717,15 @@ The paths in the static exports are relative, make sure to keep them next to you
 
 ### Archiving Private Content
 
-<a id="archiving-private-urls"/>
+<a id="archiving-private-urls"></a>
 
 If you're importing pages with private content or URLs containing secret tokens you don't want public (e.g Google Docs, paywalled content, unlisted videos, etc.), **you may want to disable some of the extractor methods to avoid leaking that content to 3rd party APIs or the public**.
 
+<br/>
+<details>
+<summary><i>Click to expand...</i></summary>
+
+
 ```bash
 # don't save private content to ArchiveBox, e.g.:
 archivebox add 'https://docs.google.com/document/d/12345somePrivateDocument'
@@ -633,10 +744,27 @@ archivebox config --set SAVE_FAVICON=False          # disable favicon fetching (
 archivebox config --set CHROME_BINARY=chromium      # ensure it's using Chromium instead of Chrome
 ```
 
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#cookies_file
+
+</details>
+<br/>
+
+
 ### Security Risks of Viewing Archived JS
 
 Be aware that malicious archived JS can access the contents of other pages in your archive when viewed. Because the Web UI serves all viewed snapshots from a single domain, they share a request context and **typical CSRF/CORS/XSS/CSP protections do not work to prevent cross-site request attacks**. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) page and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for more details.
 
+<br/>
+<details>
+<summary><i>Click to expand...</i></summary>
+
+
 ```bash
 # visiting an archived page with malicious JS:
 https://127.0.0.1:8000/archive/1602401954/example.com/index.html
@@ -649,11 +777,50 @@ https://127.0.0.1:8000/archive/*
 
 The admin UI is also served from the same origin as replayed JS, so malicious pages could also potentially use your ArchiveBox login cookies to perform admin actions (e.g. adding/removing links, running extractors, etc.). We are planning to fix this security shortcoming in a future version by using separate ports/origins to serve the Admin UI and archived content (see [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239)).
 
-*Note: Only the `wget` extractor method executes archived JS when viewing snapshots, all other archive methods produce static output that does not execute JS on viewing. If you are worried about these issues ^ you should disable the wget extractor method using `archivebox config --set SAVE_WGET=False`.*
+*Note: Only the `wget` & `dom` extractor methods execute archived JS when viewing snapshots, all other archive methods produce static output that does not execute JS on viewing. If you are worried about these issues ^ you should disable these extractors using `archivebox config --set SAVE_WGET=False SAVE_DOM=False`.*
+
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/issues/239
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx (`CVE-2023-45815`)
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#publishing
+
+
+</details>
+<br/>
+
+### Working Around Sites that Block Archiving
+
+For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) actively block archiving or bots in general. There are a number of approaches to work around this.
+
+<br/>
+<details>
+<summary><i>Click to expand...</i></summary>
+<br/>
+
+- Set [`CHROME_USER_AGENT`, `WGET_USER_AGENT`, `CURL_USER_AGENT`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent) to impersonate a real browser (instead of an ArchiveBox bot)
+- Set up a logged-in browser session for archiving using [`CHROME_DATA_DIR` & `COOKIES_FILE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile)
+- Rewrite your URLs before archiving to swap in an alternative frontend thats more bot-friendly e.g.  
+  `reddit.com/some/url` -> `teddit.net/some/url`: https://github.com/mendel5/alternative-front-ends
+
+
+In the future we plan on adding support for running JS scripts during archiving to block ads, cookie popups, modals, and fix other issues. Follow here for progress: [Issue #51](https://github.com/ArchiveBox/ArchiveBox/issues/51).
+
+</details>
+<br/>
+
 
 ### Saving Multiple Snapshots of a Single URL
 
-First-class support for saving multiple snapshots of each site over time will be [added eventually](https://github.com/ArchiveBox/ArchiveBox/issues/179) (along with the ability to view diffs of the changes between runs). For now **ArchiveBox is designed to only archive each unique URL with each extractor type once**. The workaround to take multiple snapshots of the same URL is to make them slightly different by adding a hash:
+ArchiveBox appends a hash with the current date `https://example.com#2020-10-24` to differentiate when a single URL is archived multiple times.
+
+<br/>
+<details>
+<summary><i>Click to expand...</i></summary>
+<br/>
+
+Because ArchiveBox uniquely identifies snapshots by URL, it must use a workaround to take multiple snapshots of the same URL (otherwise they would show up as a single Snapshot entry). It makes the URLs of repeated snapshots unique by adding a hash with the archive date at the end:
 
 ```bash
 archivebox add 'https://example.com#2020-10-24'
@@ -661,14 +828,47 @@ archivebox add 'https://example.com#2020-10-24'
 archivebox add 'https://example.com#2020-10-25'
 ```
 
-The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI is a shortcut for this hash-date workaround.
+The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI is a shortcut for this hash-date multi-snapshotting workaround.
+
+Improved support for saving multiple snapshots of a single URL without this hash-date workaround will be [added eventually](https://github.com/ArchiveBox/ArchiveBox/issues/179) (along with the ability to view diffs of the changes between runs).
+
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/issues/179
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#explanation-of-buttons-in-the-web-ui---admin-snapshots-list
+
+
+</details>
+<br/>
 
 ### Storage Requirements
 
-Because ArchiveBox is designed to ingest a firehose of browser history and bookmark feeds to a local disk, it can be much more disk-space intensive than a centralized service like the Internet Archive or Archive.today. **ArchiveBox can use anywhere from ~1gb per 1000 articles, to ~50gb per 1000 articles**, mostly dependent on whether you're saving audio & video using `SAVE_MEDIA=True` and whether you lower `MEDIA_MAX_SIZE=750mb`.
+Because ArchiveBox is designed to ingest a large volume of URLs with multiple copies of each URL stored by different 3rd-party tools, it can be quite disk-space intensive.  
+There also also some special requirements when using filesystems like NFS/SMB/FUSE.
 
-Disk usage can be reduced by using a compressed/deduplicated filesystem like ZFS/BTRFS, or by turning off extractors methods you don't need. **Don't store large collections on older filesystems like EXT3/FAT** as they may not be able to handle more than 50k directory entries in the `archive/` folder. **Try to keep the `index.sqlite3` file on local drive (not a network mount)** or SSD for maximum performance, however the `archive/` folder can be on a network mount or spinning HDD.
+<br/>
+<details>
+<summary><i>Click to expand...</i></summary>
+<br/>
 
+**ArchiveBox can use anywhere from ~1gb per 1000 articles, to ~50gb per 1000 articles**, mostly dependent on whether you're saving audio & video using `SAVE_MEDIA=True` and whether you lower `MEDIA_MAX_SIZE=750mb`.
+
+Disk usage can be reduced by using a compressed/deduplicated filesystem like ZFS/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like [fdupes](https://github.com/adrianlopezroche/fdupes) or [rdfind](https://github.com/pauldreik/rdfind). **Don't store large collections on older filesystems like EXT3/FAT** as they may not be able to handle more than 50k directory entries in the `archive/` folder. **Try to keep the `index.sqlite3` file on local drive (not a network mount)** or SSD for maximum performance, however the `archive/` folder can be on a network mount or slower HDD.
+
+If using Docker or NFS/SMB/FUSE for the `data/archive/` folder, you may need to set [`PUID` & `PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid) and [disable `root_squash`](https://github.com/ArchiveBox/ArchiveBox/issues/1304) on your fileshare server.
+
+
+#### Learn More
+
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root
+
+
+
+</details>
 <br/>
 
 ---
@@ -678,36 +878,36 @@ Disk usage can be reduced by using a compressed/deduplicated filesystem like ZFS
 ## Screenshots
 
 <div align="center" width="80%">
-<img src="https://i.imgur.com/PVO88AZ.png" width="80%"/>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0362bcd1-1dfd-43c6-b4ec-db5e78da07b2" width="80%"/>
 <table>
 <tbody>
 <tr>
 <td>
-<img src="https://i.imgur.com/npareKG.png" alt="brew install archivebox"><br/>
-<img src="https://i.imgur.com/5vSBO2R.png" alt="archivebox version">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e9fdcb16-344e-48c8-8be0-efa48ec155d5" alt="brew install archivebox"><br/>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e235c9d8-fda9-499d-a6a5-59b0e6a0efce" alt="archivebox version">
 </td>
 <td>
-<img src="https://i.imgur.com/JXXxFzB.png" alt="archivebox init"><br/>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0e3da0c7-d2c2-4a71-b096-6caedafd6ef7" alt="archivebox init"><br/>
 </td>
 <td>
-<img src="https://i.imgur.com/wNYtV3v.jpg" alt="archivebox add">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/31c47440-ec14-4a02-99a3-aae8a9078d46" alt="archivebox add">
 </td>
 <td>
-<img src="https://i.imgur.com/uZcIOn9.png" alt="archivebox data dir">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ebcdc21d-e2af-4bf8-ad4b-bc4f3151bbef" alt="archivebox data dir">
 </td>
 </tr>
 <tr>
 <td>
-<img src="https://i.imgur.com/H08eaia.png" alt="archivebox server">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ea672e6b-4df5-49d8-b550-7f450951fd27" alt="archivebox server">
 </td>
 <td>
-<img src="https://i.imgur.com/zM4z1aU.png" alt="archivebox server add">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" alt="archivebox server add">
 </td>
 <td>
-<img src="https://i.imgur.com/p6wK6KM.png" alt="archivebox server list">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2" alt="archivebox server list">
 </td>
 <td>
-<img src="https://i.imgur.com/xHvQfon.png" alt="archivebox server detail">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2" alt="archivebox server detail">
 </td>
 </tr>
 </tbody>
@@ -720,20 +920,25 @@ Disk usage can be reduced by using a compressed/deduplicated filesystem like ZFS
 <br/>
 
 <div align="center">
-<img src="https://i.imgur.com/ZSUm9mr.png" width="100%" alt="paisley graphic">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ca85432e-a2df-40c6-968f-51a1ef99b24e" width="100%" alt="paisley graphic">
 </div>
 
 # Background & Motivation
 
 ArchiveBox aims to enable more of the internet to be saved from deterioration by empowering people to self-host their own archives. The intent is for all the web content you care about to be viewable with common software in 50 - 100 years without needing to run ArchiveBox or other specialized software to replay it.
 
+<br/>
+<details>
+<summary><i>Click to read more...</i></summary>
+<br/>
+
 Vast treasure troves of knowledge are lost every day on the internet to link rot. As a society, we have an imperative to preserve some important parts of that treasure, just like we preserve our books, paintings, and music in physical libraries long after the originals go out of print or fade into obscurity.
 
 Whether it's to resist censorship by saving articles before they get taken down or edited, or just to save a collection of early 2010's flash games you love to play, having the tools to archive internet content enables to you save the stuff you care most about before it disappears.
 
 <div align="center">
-<img src="https://i.imgur.com/bC6eZcV.png" width="40%"/><br/>
- <sup><i>Image from <a href="https://digiday.com/media/wtf-link-rot/">WTF is Link Rot?</a>...</i><br/></sup>
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/71e36bc5-1c94-44e2-92b6-405fa898c734" width="40%"/><br/>
+ <sup><i>Image from <a href="https://perma.cc/">Perma.cc</a>...</i><br/></sup>
 </div>
 
 The balance between the permanence and ephemeral nature of content on the internet is part of what makes it beautiful. I don't think everything should be preserved in an automated fashion--making all content permanent and never removable, but I do think people should be able to decide for themselves and effectively archive specific content that they care about.
@@ -741,11 +946,16 @@ The balance between the permanence and ephemeral nature of content on the intern
 Because modern websites are complicated and often rely on dynamic content,
 ArchiveBox archives the sites in **several different formats** beyond what public archiving services like Archive.org/Archive.is save. Using multiple methods and the market-dominant browser to execute JS ensures we can save even the most complex, finicky websites in at least a few high-quality, long-term data formats.
 
+</details>
+<br/>
+
 ## Comparison to Other Projects
 
-<img src="https://i.imgur.com/4nkFjdv.png" width="5%" align="right" alt="comparison"/> 
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4cac62a9-e8fb-425b-85a3-ca644aa6dd42" width="5%" align="right" alt="comparison"/> 
 
-▶ **Check out our [community page](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) for an index of web archiving initiatives and projects.**
+
+> [!TIP]
+> **Check out our [community page](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) for an index of web archiving initiatives and projects.**
 
 A variety of open and closed-source archiving projects exist, but few provide a nice UI and CLI to manage a large, high-fidelity archive collection over time.
 
@@ -761,7 +971,10 @@ By having each user store their own content locally, we can save much larger por
 
 ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
-ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. It can be as simple or advanced as you want, and is designed to do everything out-of-the-box but be tuned to suit your needs.
+<details>
+<summary><i>Click to see the <b>⭐️ officially recommended</b> alternatives to ArchiveBox...</i></summary>
+<br/>
+
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
 
@@ -771,16 +984,23 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
 
 For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
 
+ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
+
+</details>
+
+<br/>
+
 <div align="center">
 <br/>
-<img src="https://i.imgur.com/q0Oe36M.png" width="100%" alt="dependencies graphic">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/04808ac2-3133-44fd-8703-3387e06dc851" width="100%" alt="dependencies graphic">
 </div>
 
 ## Internet Archiving Ecosystem
 
 Whether you want to learn which organizations are the big players in the web archiving space, want to find a specific open-source tool for your web archiving need, or just want to see where archivists hang out online, our Community Wiki page serves as an index of the broader web archiving community. Check it out to learn about some of the coolest web archiving projects and communities on the web!
 
-<img src="https://i.imgur.com/0ZOmOvN.png" width="14%" align="right"/>
+
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/78d8a725-97f4-47f5-b983-1f62843ddc51" width="14%" align="right"/>
 
 - [Community Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
   - [The Master Lists](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#the-master-lists)  
@@ -792,23 +1012,24 @@ Whether you want to learn which organizations are the big players in the web arc
   - [Communities](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#communities)  
     _A collection of the most active internet archiving communities and initiatives._
 - Check out the ArchiveBox [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap) and [Changelog](https://github.com/ArchiveBox/ArchiveBox/wiki/Changelog)
-- Learn why archiving the internet is important by reading the "[On the Importance of Web Archiving](https://parameters.ssrc.org/2018/09/on-the-importance-of-web-archiving/)" blog post.
+- Learn why archiving the internet is important by reading the "[On the Importance of Web Archiving](https://items.ssrc.org/parameters/on-the-importance-of-web-archiving/)" blog post.
 - Reach out to me for questions and comments via [@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) or [@theSquashSH](https://twitter.com/thesquashSH) on Twitter
 
+
 <br/>
 
 **Need help building a custom archiving solution?**
 
-> ✨ **[Hire the team that helps build Archivebox](https://monadical.com) to work on your project.** ([@MonadicalSAS](https://twitter.com/MonadicalSAS))
+> ✨ **[Hire the team that built Archivebox](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102) to work on your project.** ([@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp))
 
-<sup>(They also do general software consulting across many industries)</sup>
+<sup>(We also offer general software consulting across many industries)</sup>
 
 <br/>
 
 ---
 
 <div align="center">
-<img src="https://i.imgur.com/SMkGW0L.png" width="100%" alt="documentation graphic">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/897f7a88-1265-4aab-b80c-b1640afaad1f" width="100%" alt="documentation graphic">
 </div>
 
 # Documentation
@@ -824,27 +1045,32 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 - [Quickstart](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart)
 - [Install](https://github.com/ArchiveBox/ArchiveBox/wiki/Install)
 - [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker)
-
-## Reference
-
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 - [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+
+## Advanced
+
+- [Troubleshooting](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 - [Publishing Your Archive](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive)
 - [Chromium Install](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install)
+- [Cookies & Sessions Setup](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile)
 - [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview)
-- [Troubleshooting](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - [Upgrading or Merging Archives](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives)
+
+## Developers
+
+- [Developer Documentation](https://github.com/ArchiveBox/ArchiveBox#archivebox-development)
 - [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
 - [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)
 
 ## More Info
 
-- [Tickets](https://github.com/ArchiveBox/ArchiveBox/issues)
+- [Bug Tracker](https://github.com/ArchiveBox/ArchiveBox/issues)
 - [Roadmap](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap)
-- [Changelog](https://github.com/ArchiveBox/ArchiveBox/wiki/Changelog)
+- [Changelog](https://github.com/ArchiveBox/ArchiveBox/releases)
 - [Donations](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
 - [Background & Motivation](https://github.com/ArchiveBox/ArchiveBox#background--motivation)
 - [Web Archiving Community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
@@ -854,7 +1080,7 @@ You can also access the docs locally by looking in the [`ArchiveBox/docs/`](http
 ---
 
 <div align="center">
-<img src="https://i.imgur.com/EGWjbD4.png" width="100%" alt="development">
+<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e895e79f-5c7d-429b-ad8a-7df2cc183ca3" width="100%" alt="development">
 </div>
 
 # ArchiveBox Development
@@ -863,7 +1089,7 @@ All contributions to ArchiveBox are welcomed! Check our [issues](https://github.
 
 For low hanging fruit / easy first tickets, see: <a href="https://github.com/ArchiveBox/ArchiveBox/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22help+wanted%22">ArchiveBox/Issues `#good first ticket` `#help wanted`</a>.
 
-**Python API Documentation:** https://docs.archivebox.io/en/master/archivebox.html#module-archivebox.main
+**Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main
 
 ### Setup the dev environment
 
@@ -913,6 +1139,7 @@ docker run -it -p 8000:8000 \
     archivebox server 0.0.0.0:8000 --debug --reload
 
 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
+# When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
 ```
 
 </details>
@@ -940,12 +1167,33 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 <details><summary><i>Click to expand...</i></summary>
 
+##### Use a Pre-Built Image
+
+If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
+
 ```bash
-# docker:
+docker pull archivebox/archivebox:dev
+docker run archivebox/archivebox:dev version
+# verify the BUILD_TIME and COMMIT_HASH in the output are recent
+```
+
+##### Build Branch from Source
+  
+You can also build and run any branch yourself from source, for example to build & use `dev` locally:
+
+```bash
+# docker-compose.yml:
+services:
+    archivebox:
+        image: archivebox/archivebox:dev
+        build: 'https://github.com/ArchiveBox/ArchiveBox.git#dev'
+        ...
+
+# or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
 docker run -it -v $PWD:/data archivebox:dev init --setup
 
-# bare metal:
+# or with pip:
 pip install 'git+https://github.com/pirate/ArchiveBox@dev'
 npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
 archivebox init --setup
@@ -980,6 +1228,7 @@ archivebox init --setup
 <details><summary><i>Click to expand...</i></summary>
 
 Make sure to run this whenever you change things in `models.py`.
+
 ```bash
 cd archivebox/
 ./manage.py makemigrations
@@ -988,6 +1237,7 @@ cd path/to/test/data/
 archivebox shell
 archivebox manage dbshell
 ```
+
 (uses `pytest -s`)  
 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
 
@@ -995,7 +1245,9 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 #### Contributing a new extractor
 
-<details><summary><i>Click to expand...</i></summary><br/><br/>
+<details><summary><i>Click to expand...</i></summary>
+
+<br/><br/>
 
 ArchiveBox [`extractors`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/media.py) are external binaries or Python/Node scripts that ArchiveBox runs to archive content on a page.
 
@@ -1071,7 +1323,9 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 - Releases: [Github.com/ArchiveBox/ArchiveBox/releases](https://github.com/ArchiveBox/ArchiveBox/releases)
 - Wiki: [Github.com/ArchiveBox/ArchiveBox/wiki](https://github.com/ArchiveBox/ArchiveBox/wiki)
 - Issues: [Github.com/ArchiveBox/ArchiveBox/issues](https://github.com/ArchiveBox/ArchiveBox/issues)
-- Forum: [Github.com/ArchiveBox/ArchiveBox/discussions](https://github.com/ArchiveBox/ArchiveBox/discussions)
+- Discussions: [Github.com/ArchiveBox/ArchiveBox/discussions](https://github.com/ArchiveBox/ArchiveBox/discussions)
+- Community Chat: [Zulip Chat (preferred)](https://zulip.archivebox.io) or [Matrix Chat (old)](https://app.element.io/#/room/#archivebox:matrix.org)
+- Social Media: [Twitter](https://twitter.com/ArchiveBoxApp), [LinkedIn](https://www.linkedin.com/company/archivebox/), [YouTube](https://www.youtube.com/@ArchiveBoxApp), [Alternative.to](https://alternativeto.net/software/archivebox/about/), [Reddit](https://www.reddit.com/r/ArchiveBox/)
 - Donations: [Github.com/ArchiveBox/ArchiveBox/wiki/Donations](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
 
 ---
@@ -1081,7 +1335,7 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <img src="https://raw.githubusercontent.com/Monadical-SAS/redux-time/HEAD/examples/static/jeremy.jpg" height="40px"/>
 <br/>
 <i><sub>
-This project is maintained mostly in <a href="https://nicksweeting.com/blog#About">my spare time</a> with the help from generous <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">contributors</a> and <a href="https://monadical.com">Monadical</a> (✨  <a href="https://monadical.com">hire them</a> for dev work!).
+This project is maintained mostly in <a href="https://docs.sweeting.me/s/blog#About">my spare time</a> with the help from generous <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">contributors</a> and <a href="https://monadical.com">Monadical Consulting</a>.
 </sub>
 </i>
 <br/><br/>
@@ -1093,6 +1347,8 @@ This project is maintained mostly in <a href="https://nicksweeting.com/blog#Abou
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Donate_to_support_development-via_Patreon-%23DD5D76.svg?style=flat"/></a>
 <br/>
 
+<b><sub>(网站存档 / 爬虫)</sub></b>
+
 <a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>
 

SECURITY.md

@@ -0,0 +1,34 @@
+# Security Policy
+
+---
+
+## Security Information
+
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+
+- https://github.com/ArchiveBox/ArchiveBox#archive-layout
+- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
+
+---
+
+## Reporting a Vulnerability
+
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
+
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
+
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
+
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
+
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

archivebox/cli/archivebox_schedule.py

@@ -39,6 +39,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=None,
         help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
     )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
     parser.add_argument(
         '--depth', # '-d',
         type=int,
@@ -97,6 +103,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         run_all=command.run_all,
         quiet=command.quiet,
         every=command.every,
+        tag=command.tag,
         depth=command.depth,
         overwrite=command.overwrite,
         update=command.update,

archivebox/config.py

@@ -30,6 +30,7 @@ import inspect
 import getpass
 import platform
 import shutil
+import requests
 import django
 from sqlite3 import dbapi2 as sqlite3
 
@@ -40,6 +41,7 @@ from typing import Optional, Type, Tuple, Dict, Union, List
 from subprocess import run, PIPE, DEVNULL
 from configparser import ConfigParser
 from collections import defaultdict
+import importlib.metadata
 
 from .config_stubs import (
     SimpleConfigValueDict,
@@ -50,17 +52,6 @@ from .config_stubs import (
 )
 
 
-### Pre-Fetch Minimal System Config
-
-SYSTEM_USER = getpass.getuser() or os.getlogin()
-
-try:
-    import pwd
-    SYSTEM_USER = pwd.getpwuid(os.geteuid()).pw_name or SYSTEM_USER
-except ModuleNotFoundError:
-    # pwd is only needed for some linux systems, doesn't exist on windows
-    pass
-
 ############################### Config Schema ##################################
 
 CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
@@ -69,9 +60,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'USE_COLOR':                {'type': bool,  'default': lambda c: c['IS_TTY']},
         'SHOW_PROGRESS':            {'type': bool,  'default': lambda c: (c['IS_TTY'] and platform.system() != 'Darwin')},  # progress bars are buggy on mac, disable for now
         'IN_DOCKER':                {'type': bool,  'default': False},
+        'IN_QEMU':                  {'type': bool,  'default': False},
         'PUID':                     {'type': int,   'default': os.getuid()},
         'PGID':                     {'type': int,   'default': os.getgid()},
-        # TODO: 'SHOW_HINTS':       {'type:  bool,  'default': True},
     },
 
     'GENERAL_CONFIG': {
@@ -82,8 +73,13 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'MEDIA_TIMEOUT':            {'type': int,   'default': 3600},
         'OUTPUT_PERMISSIONS':       {'type': str,   'default': '644'},
         'RESTRICT_FILE_NAMES':      {'type': str,   'default': 'windows'},
-        'URL_BLACKLIST':            {'type': str,   'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$'},  # to avoid downloading code assets as their own pages
-        'URL_WHITELIST':            {'type': str,   'default': None},
+
+        'URL_DENYLIST':             {'type': str,   'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', 'aliases': ('URL_BLACKLIST',)},  # to avoid downloading code assets as their own pages
+        'URL_ALLOWLIST':            {'type': str,   'default': None, 'aliases': ('URL_WHITELIST',)},
+
+        'ADMIN_USERNAME':           {'type': str,   'default': None},
+        'ADMIN_PASSWORD':           {'type': str,   'default': None},
+
         'ENFORCE_ATOMIC_WRITES':    {'type': bool,  'default': True},
         'TAG_SEPARATOR_PATTERN':    {'type': str,   'default': r'[,]'},
     },
@@ -100,12 +96,22 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SNAPSHOTS_PER_PAGE':        {'type': int,   'default': 40},
         'CUSTOM_TEMPLATES_DIR':      {'type': str,   'default': None},
         'TIME_ZONE':                 {'type': str,   'default': 'UTC'},
-        'TIMEZONE':                 {'type': str,   'default': 'UTC'},
+        'TIMEZONE':                  {'type': str,   'default': 'UTC'},
         'REVERSE_PROXY_USER_HEADER': {'type': str,   'default': 'Remote-User'},
         'REVERSE_PROXY_WHITELIST':   {'type': str,   'default': ''},
         'LOGOUT_REDIRECT_URL':       {'type': str,   'default': '/'},
-        'PREVIEW_ORIGINALS':        {'type': bool,  'default': True},
-        'LOGOUT_REDIRECT_URL':   {'type': str,   'default': '/'},
+        'PREVIEW_ORIGINALS':         {'type': bool,  'default': True},
+
+        'LDAP':                      {'type': bool,  'default': False},
+        'LDAP_SERVER_URI':           {'type': str,   'default': None},
+        'LDAP_BIND_DN':              {'type': str,   'default': None},
+        'LDAP_BIND_PASSWORD':        {'type': str,   'default': None},
+        'LDAP_USER_BASE':            {'type': str,   'default': None},
+        'LDAP_USER_FILTER':          {'type': str,   'default': None},
+        'LDAP_USERNAME_ATTR':        {'type': str,   'default': None},
+        'LDAP_FIRSTNAME_ATTR':       {'type': str,   'default': None},
+        'LDAP_LASTNAME_ATTR':        {'type': str,   'default': None},
+        'LDAP_EMAIL_ATTR':           {'type': str,   'default': None},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -116,6 +122,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SAVE_SINGLEFILE':          {'type': bool,  'default': True, 'aliases': ('FETCH_SINGLEFILE',)},
         'SAVE_READABILITY':         {'type': bool,  'default': True, 'aliases': ('FETCH_READABILITY',)},
         'SAVE_MERCURY':             {'type': bool,  'default': True, 'aliases': ('FETCH_MERCURY',)},
+        'SAVE_HTMLTOTEXT':          {'type': bool,  'default': True, 'aliases': ('FETCH_HTMLTOTEXT',)},
         'SAVE_PDF':                 {'type': bool,  'default': True, 'aliases': ('FETCH_PDF',)},
         'SAVE_SCREENSHOT':          {'type': bool,  'default': True, 'aliases': ('FETCH_SCREENSHOT',)},
         'SAVE_DOM':                 {'type': bool,  'default': True, 'aliases': ('FETCH_DOM',)},
@@ -124,6 +131,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SAVE_GIT':                 {'type': bool,  'default': True, 'aliases': ('FETCH_GIT',)},
         'SAVE_MEDIA':               {'type': bool,  'default': True, 'aliases': ('FETCH_MEDIA',)},
         'SAVE_ARCHIVE_DOT_ORG':     {'type': bool,  'default': True, 'aliases': ('SUBMIT_ARCHIVE_DOT_ORG',)},
+        'SAVE_ALLOWLIST':           {'type': dict,  'default': {},},
+        'SAVE_DENYLIST':            {'type': dict,  'default': {},},
     },
 
     'ARCHIVE_METHOD_OPTIONS': {
@@ -132,9 +141,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
         'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/605.1.15 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/605.1.15 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/605.1.15 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
@@ -149,10 +158,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--write-thumbnail',
                                                                 '--no-call-home',
                                                                 '--write-sub',
-                                                                '--all-subs',
-                                                                # There are too many of these and youtube
-                                                                # throttles you with HTTP error 429
-                                                                #'--write-auto-subs',
+                                                                '--write-auto-subs',
                                                                 '--convert-subs=srt',
                                                                 '--yes-playlist',
                                                                 '--continue',
@@ -165,7 +171,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--ignore-errors',
                                                                 '--geo-bypass',
                                                                 '--add-metadata',
-                                                                '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']),
+                                                                '--format=(bv*+ba/b)[filesize<={}][filesize_approx<=?{}]/(bv*+ba/b)'.format(c['MEDIA_MAX_SIZE'], c['MEDIA_MAX_SIZE']),
                                                                 ]},
 
 
@@ -183,7 +189,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--compressed'
                                                                ]},
         'GIT_ARGS':                 {'type': list,  'default': ['--recursive']},
-        'SINGLEFILE_ARGS':          {'type': list,  'default' : None}
+        'SINGLEFILE_ARGS':          {'type': list,  'default' : None},
+        'FAVICON_PROVIDER':         {'type': str,   'default': 'https://www.google.com/s2/favicons?domain={}'},
     },
 
     'SEARCH_BACKEND_CONFIG' : {
@@ -193,10 +200,16 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SEARCH_BACKEND_HOST_NAME': {'type': str,   'default': 'localhost'},
         'SEARCH_BACKEND_PORT':      {'type': int,   'default': 1491},
         'SEARCH_BACKEND_PASSWORD':  {'type': str,   'default': 'SecretPassword'},
+        'SEARCH_PROCESS_HTML':      {'type': bool,  'default': True},
         # SONIC
         'SONIC_COLLECTION':         {'type': str,   'default': 'archivebox'},
         'SONIC_BUCKET':             {'type': str,   'default': 'snapshots'},
         'SEARCH_BACKEND_TIMEOUT':   {'type': int,   'default': 90},
+        # SQLite3 FTS5
+        'FTS_SEPARATE_DATABASE':    {'type': bool,  'default': True},
+        'FTS_TOKENIZERS':           {'type': str,   'default': 'porter unicode61 remove_diacritics 2'},
+        # Default from https://www.sqlite.org/limits.html#max_length
+        'FTS_SQLITE_MAX_LENGTH':    {'type': int,   'default': int(1e9)},
     },
 
     'DEPENDENCY_CONFIG': {
@@ -213,18 +226,19 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
 
         'CURL_BINARY':              {'type': str,   'default': 'curl'},
         'GIT_BINARY':               {'type': str,   'default': 'git'},
-        'WGET_BINARY':              {'type': str,   'default': 'wget'},
+        'WGET_BINARY':              {'type': str,   'default': 'wget'},     # also can accept wget2
         'SINGLEFILE_BINARY':        {'type': str,   'default': lambda c: bin_path('single-file')},
         'READABILITY_BINARY':       {'type': str,   'default': lambda c: bin_path('readability-extractor')},
-        'MERCURY_BINARY':           {'type': str,   'default': lambda c: bin_path('mercury-parser')},
-        #'YOUTUBEDL_BINARY':         {'type': str,   'default': 'youtube-dl'},
-        'YOUTUBEDL_BINARY':         {'type': str,   'default': 'yt-dlp'},
+        'MERCURY_BINARY':           {'type': str,   'default': lambda c: bin_path('postlight-parser')},
+        'YOUTUBEDL_BINARY':         {'type': str,   'default': 'yt-dlp'},   # also can accept youtube-dl
         'NODE_BINARY':              {'type': str,   'default': 'node'},
         'RIPGREP_BINARY':           {'type': str,   'default': 'rg'},
         'CHROME_BINARY':            {'type': str,   'default': None},
 
         'POCKET_CONSUMER_KEY':      {'type': str,   'default': None},
         'POCKET_ACCESS_TOKENS':     {'type': dict,  'default': {}},
+
+        'READWISE_READER_TOKENS':   {'type': dict,  'default': {}},
     },
 }
 
@@ -327,6 +341,7 @@ ALLOWED_IN_OUTPUT_DIR = {
     'yarn.lock',
     'static',
     'sonic',
+    'search.sqlite3',
     ARCHIVE_DIR_NAME,
     SOURCES_DIR_NAME,
     LOGS_DIR_NAME,
@@ -342,21 +357,126 @@ ALLOWED_IN_OUTPUT_DIR = {
     'static_index.json',
 }
 
-def get_version(config):
-    return json.loads((Path(config['PACKAGE_DIR']) / 'package.json').read_text(encoding='utf-8').strip())['version']
 
-def get_commit_hash(config):
+ALLOWDENYLIST_REGEX_FLAGS: int = re.IGNORECASE | re.UNICODE | re.MULTILINE
+
+
+############################## Version Config ##################################
+
+def get_system_user():
+    SYSTEM_USER = getpass.getuser() or os.getlogin()
+    try:
+        import pwd
+        return pwd.getpwuid(os.geteuid()).pw_name or SYSTEM_USER
+    except KeyError:
+        # Process' UID might not map to a user in cases such as running the Docker image
+        # (where `archivebox` is 999) as a different UID.
+        pass
+    except ModuleNotFoundError:
+        # pwd doesn't exist on windows
+        pass
+    except Exception:
+        # this should never happen, uncomment to debug
+        # raise
+        pass
+
+    return SYSTEM_USER
+
+def get_version(config):
+    try:
+        return importlib.metadata.version(__package__ or 'archivebox')
+    except importlib.metadata.PackageNotFoundError:
+        try:
+            pyproject_config = (config['PACKAGE_DIR'] / 'pyproject.toml').read_text()
+            for line in pyproject_config:
+                if line.startswith('version = '):
+                    return line.split(' = ', 1)[-1].strip('"')
+        except FileNotFoundError:
+            # building docs, pyproject.toml is not available
+            return 'dev'
+
+    raise Exception('Failed to detect installed archivebox version!')
+
+def get_commit_hash(config) -> Optional[str]:
+    try:
+        git_dir = config['PACKAGE_DIR'] / '../.git'
+        ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
+        commit_hash = git_dir.joinpath(ref).read_text().strip()
+        return commit_hash
+    except Exception:
+        pass
+
     try:
         return list((config['PACKAGE_DIR'] / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
     except Exception:
+        pass
+    
+    return None
+
+def get_build_time(config) -> str:
+    if config['IN_DOCKER']:
+        docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
+        return docker_build_end_time
+
+    src_last_modified_unix_timestamp = (config['PACKAGE_DIR'] / 'config.py').stat().st_mtime
+    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
+
+def get_versions_available_on_github(config):
+    """
+    returns a dictionary containing the ArchiveBox GitHub release info for
+    the recommended upgrade version and the currently installed version
+    """
+    
+    # we only want to perform the (relatively expensive) check for new versions
+    # when its most relevant, e.g. when the user runs a long-running command
+    subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
+    long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
+    if subcommand_run_by_user not in long_running_commands:
         return None
+    
+    github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
+    response = requests.get(github_releases_api)
+    if response.status_code != 200:
+        stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
+        return None
+    all_releases = response.json()
+
+    installed_version = parse_version_string(config['VERSION'])
+
+    # find current version or nearest older version (to link to)
+    current_version = None
+    for idx, release in enumerate(all_releases):
+        release_version = parse_version_string(release['tag_name'])
+        if release_version <= installed_version:
+            current_version = release
+            break
+
+    current_version = current_version or all_releases[-1]
+    
+    # recommended version is whatever comes after current_version in the release list
+    # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
+    try:
+        recommended_version = all_releases[idx+1]
+    except IndexError:
+        recommended_version = None
+
+    return {'recommended_version': recommended_version, 'current_version': current_version}
+
+def can_upgrade(config):
+    if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
+        recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
+        current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
+        return recommended_version > current_version
+    return False
+
 
 ############################## Derived Config ##################################
 
-
+# These are derived/computed values calculated *after* all user-provided config values are ingested
+# they appear in `archivebox config` output and are intended to be read-only for the user
 DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'TERM_WIDTH':               {'default': lambda c: lambda: shutil.get_terminal_size((100, 10)).columns},
-    'USER':                     {'default': lambda c: SYSTEM_USER},
+    'USER':                     {'default': lambda c: get_system_user()},
     'ANSI':                     {'default': lambda c: DEFAULT_CLI_COLORS if c['USE_COLOR'] else {k: '' for k in DEFAULT_CLI_COLORS.keys()}},
 
     'PACKAGE_DIR':              {'default': lambda c: Path(__file__).resolve().parent},
@@ -370,14 +490,19 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CONFIG_FILE':              {'default': lambda c: Path(c['CONFIG_FILE']).resolve() if c['CONFIG_FILE'] else c['OUTPUT_DIR'] / CONFIG_FILENAME},
     'COOKIES_FILE':             {'default': lambda c: c['COOKIES_FILE'] and Path(c['COOKIES_FILE']).resolve()},
     'CHROME_USER_DATA_DIR':     {'default': lambda c: find_chrome_data_dir() if c['CHROME_USER_DATA_DIR'] is None else (Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None)},   # None means unset, so we autodetect it with find_chrome_Data_dir(), but emptystring '' means user manually set it to '', and we should store it as None
-    'URL_BLACKLIST_PTN':        {'default': lambda c: c['URL_BLACKLIST'] and re.compile(c['URL_BLACKLIST'] or '', re.IGNORECASE | re.UNICODE | re.MULTILINE)},
-    'URL_WHITELIST_PTN':        {'default': lambda c: c['URL_WHITELIST'] and re.compile(c['URL_WHITELIST'] or '', re.IGNORECASE | re.UNICODE | re.MULTILINE)},
-    'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},
+    'URL_DENYLIST_PTN':         {'default': lambda c: c['URL_DENYLIST'] and re.compile(c['URL_DENYLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
+    'URL_ALLOWLIST_PTN':        {'default': lambda c: c['URL_ALLOWLIST'] and re.compile(c['URL_ALLOWLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
+    'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},  # exec is always needed to list directories
 
     'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0] or bin_path('archivebox')},
-    'VERSION':                  {'default': lambda c: get_version(c)},
-    'COMMIT_HASH':              {'default': lambda c: get_commit_hash(c)},
+
+    'VERSION':                  {'default': lambda c: get_version(c).split('+', 1)[0]},     # remove +editable from user-displayed version string
+    'COMMIT_HASH':              {'default': lambda c: get_commit_hash(c)},                  # short git commit hash of codebase HEAD commit
+    'BUILD_TIME':               {'default': lambda c: get_build_time(c)},                   # docker build completed time or python src last modified time
     
+    'VERSIONS_AVAILABLE':       {'default': lambda c: get_versions_available_on_github(c)},
+    'CAN_UPGRADE':              {'default': lambda c: can_upgrade(c)},
+
     'PYTHON_BINARY':            {'default': lambda c: sys.executable},
     'PYTHON_ENCODING':          {'default': lambda c: sys.stdout.encoding.upper()},
     'PYTHON_VERSION':           {'default': lambda c: '{}.{}.{}'.format(*sys.version_info[:3])},
@@ -387,7 +512,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     
     'SQLITE_BINARY':            {'default': lambda c: inspect.getfile(sqlite3)},
     'SQLITE_VERSION':           {'default': lambda c: sqlite3.version},
-    #'SQLITE_JOURNAL_MODE':      {'default': lambda c: 'wal'},         # set at runtime below, interesting but unused for now
+    #'SQLITE_JOURNAL_MODE':      {'default': lambda c: 'wal'},         # set at runtime below, interesting if changed later but unused for now because its always expected to be wal
     #'SQLITE_OPTIONS':           {'default': lambda c: ['JSON1']},     # set at runtime below
 
     'USE_CURL':                 {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])},
@@ -415,7 +540,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'READABILITY_VERSION':      {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None},
 
     'USE_MERCURY':              {'default': lambda c: c['USE_MERCURY'] and c['SAVE_MERCURY']},
-    'MERCURY_VERSION':          {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None},  # mercury is unversioned
+    'MERCURY_VERSION':          {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None},  # mercury doesnt expose version info until this is merged https://github.com/postlight/parser/pull/750
 
     'USE_GIT':                  {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']},
     'GIT_VERSION':              {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None},
@@ -429,6 +554,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] or find_chrome_binary()},
     'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and c['CHROME_BINARY'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
     'CHROME_VERSION':           {'default': lambda c: bin_version(c['CHROME_BINARY']) if c['USE_CHROME'] else None},
+    'CHROME_USER_AGENT':        {'default': lambda c: c['CHROME_USER_AGENT'].format(**c)},
 
     'SAVE_PDF':                 {'default': lambda c: c['USE_CHROME'] and c['SAVE_PDF']},
     'SAVE_SCREENSHOT':          {'default': lambda c: c['USE_CHROME'] and c['SAVE_SCREENSHOT']},
@@ -445,10 +571,11 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'EXTERNAL_LOCATIONS':       {'default': lambda c: get_external_locations(c)},
     'DATA_LOCATIONS':           {'default': lambda c: get_data_locations(c)},
     'CHROME_OPTIONS':           {'default': lambda c: get_chrome_info(c)},
+    'SAVE_ALLOWLIST_PTN':       {'default': lambda c: c['SAVE_ALLOWLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_ALLOWLIST'].items()}},
+    'SAVE_DENYLIST_PTN':        {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
 }
 
 
-
 ################################### Helpers ####################################
 
 
@@ -461,47 +588,60 @@ def load_config_val(key: str,
                     config_file_vars: Optional[Dict[str, str]]=None) -> ConfigValue:
     """parse bool, int, and str key=value pairs from env"""
 
+    assert isinstance(config, dict)
 
+    is_read_only = type is None
+    if is_read_only:
+        if callable(default):
+            return default(config)
+        return default
+
+    # get value from environment variables or config files
     config_keys_to_check = (key, *(aliases or ()))
+    val = None
     for key in config_keys_to_check:
         if env_vars:
             val = env_vars.get(key)
             if val:
                 break
+
         if config_file_vars:
             val = config_file_vars.get(key)
             if val:
                 break
 
-    if type is None or val is None:
+    is_unset = val is None
+    if is_unset:
         if callable(default):
-            assert isinstance(config, dict)
             return default(config)
-
         return default
 
-    elif type is bool:
-        if val.lower() in ('true', 'yes', '1'):
+    # calculate value based on expected type
+    BOOL_TRUEIES = ('true', 'yes', '1')
+    BOOL_FALSEIES = ('false', 'no', '0')
+
+    if type is bool:
+        if val.lower() in BOOL_TRUEIES:
             return True
-        elif val.lower() in ('false', 'no', '0'):
+        elif val.lower() in BOOL_FALSEIES:
             return False
         else:
             raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)')
 
     elif type is str:
-        if val.lower() in ('true', 'false', 'yes', 'no', '1', '0'):
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a string)')
+        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
+            raise ValueError(f'Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)')
         return val.strip()
 
     elif type is int:
-        if not val.isdigit():
+        if not val.strip().isdigit():
             raise ValueError(f'Invalid configuration option {key}={val} (expected an integer)')
-        return int(val)
+        return int(val.strip())
 
     elif type is list or type is dict:
         return json.loads(val)
 
-    raise Exception('Config values can only be str, bool, int or json')
+    raise Exception('Config values can only be str, bool, int, or json')
 
 
 def load_config_file(out_dir: str=None) -> Optional[Dict[str, str]]:
@@ -643,9 +783,11 @@ def load_config(defaults: ConfigDefaultDict,
 
     return extended_config
 
-# def write_config(config: ConfigDict):
 
-#     with open(os.path.join(config['OUTPUT_DIR'], CONFIG_FILENAME), 'w+') as f:
+def parse_version_string(version: str) -> Tuple[int, int, int]:
+    """parses a version tag string formatted like 'vx.x.x' into (major, minor, patch) ints"""
+    base = version.split('+')[0].split('v')[-1] # remove 'v' prefix and '+editable' suffix
+    return tuple(int(part) for part in base.split('.'))[:3]
 
 
 # Logging Helpers
@@ -689,7 +831,8 @@ def bin_version(binary: Optional[str]) -> Optional[str]:
         return None
 
     try:
-        version_str = run([abspath, "--version"], stdout=PIPE, env={'LANG': 'C'}).stdout.strip().decode()
+        bin_env = os.environ | {'LANG': 'C'}
+        version_str = run([abspath, "--version"], stdout=PIPE, env=bin_env).stdout.strip().decode()
         if not version_str:
             version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
         # take first 3 columns of first line of version info
@@ -733,6 +876,7 @@ def find_chrome_binary() -> Optional[str]:
     # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
     # make sure data dir finding precedence order always matches binary finding order
     default_executable_paths = (
+        # '~/Library/Caches/ms-playwright/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium',
         'chromium-browser',
         'chromium',
         '/Applications/Chromium.app/Contents/MacOS/Chromium',
@@ -1023,9 +1167,9 @@ globals().update(CONFIG)
 
 
 # Set timezone to UTC and umask to OUTPUT_PERMISSIONS
-assert TIMEZONE == 'UTC', 'The server timezone should always be set to UTC'  # we may allow this to change later
-os.environ["TZ"] = TIMEZONE
-os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8))  # noqa: F821
+assert TIMEZONE == 'UTC', 'The server timezone should always be set to UTC'  # noqa: F821
+os.environ["TZ"] = TIMEZONE                                                  # noqa: F821
+os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8))                        # noqa: F821
 
 # add ./node_modules/.bin to $PATH so we can use node scripts in extractors
 NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))
@@ -1042,7 +1186,6 @@ sys.path.append(NODE_BIN_PATH)
 # disable stderr "you really shouldnt disable ssl" warnings with library config
 if not CONFIG['CHECK_SSL_VALIDITY']:
     import urllib3
-    import requests
     requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -1059,14 +1202,25 @@ if not CONFIG['CHECK_SSL_VALIDITY']:
 
 def check_system_config(config: ConfigDict=CONFIG) -> None:
     ### Check system environment
-    if config['USER'] == 'root':
+    if config['USER'] == 'root' or str(config['PUID']) == "0":
         stderr('[!] ArchiveBox should never be run as root!', color='red')
         stderr('    For more information, see the security overview documentation:')
         stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root')
+        
+        if config['IN_DOCKER']:
+            attempted_command = ' '.join(sys.argv[:3])
+            stderr('')
+            stderr('    {lightred}Hint{reset}: When using Docker, you must run commands with {green}docker run{reset} instead of {lightyellow}docker exec{reset}, e.g.:'.format(**config['ANSI']))
+            stderr(f'        docker compose run archivebox {attempted_command}')
+            stderr(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}')
+            stderr('        or:')
+            stderr(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"')
+            stderr(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"')
+        
         raise SystemExit(2)
 
     ### Check Python environment
-    if sys.version_info[:3] < (3, 6, 0):
+    if sys.version_info[:3] < (3, 7, 0):
         stderr(f'[X] Python version is not new enough: {config["PYTHON_VERSION"]} (>3.6 is required)', color='red')
         stderr('    See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.')
         raise SystemExit(2)
@@ -1142,7 +1296,7 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None:
 
     if config['USE_YOUTUBEDL'] and config['MEDIA_TIMEOUT'] < 20:
         stderr(f'[!] Warning: MEDIA_TIMEOUT is set too low! (currently set to MEDIA_TIMEOUT={config["MEDIA_TIMEOUT"]} seconds)', color='red')
-        stderr('    Youtube-dl will fail to archive all media if set to less than ~20 seconds.')
+        stderr('    youtube-dl/yt-dlp will fail to archive any media if set to less than ~20 seconds.')
         stderr('    (Setting it somewhere over 60 seconds is recommended)')
         stderr()
         stderr('    If you want to disable media archiving entirely, set SAVE_MEDIA=False instead:')
@@ -1230,8 +1384,7 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
         with open(settings.ERROR_LOG, "a", encoding='utf-8') as f:
             command = ' '.join(sys.argv)
             ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
-            f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
-
+            f.write(f"\n> {command}; TS={ts} VERSION={config['VERSION']} IN_DOCKER={config['IN_DOCKER']} IS_TTY={config['IS_TTY']}\n")
 
         if check_db:
             # Enable WAL mode in sqlite3

archivebox/config_stubs.py

@@ -41,7 +41,7 @@ class ConfigDict(BaseConfig, total=False):
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
     RESTRICT_FILE_NAMES: str
-    URL_BLACKLIST: str
+    URL_DENYLIST: str
 
     SECRET_KEY: Optional[str]
     BIND_ADDR: str

archivebox/core/admin.py

@@ -48,9 +48,11 @@ class TagInline(admin.TabularInline):
 from django.contrib.admin.helpers import ActionForm
 from django.contrib.admin.widgets import AutocompleteSelectMultiple
 
+# WIP: broken by Django 3.1.2 -> 4.0 migration
 class AutocompleteTags:
     model = Tag
     search_fields = ['name']
+    name = 'tags'
 
 class AutocompleteTagsAdminStub:
     name = 'admin'
@@ -60,6 +62,7 @@ class SnapshotActionForm(ActionForm):
     tags = forms.ModelMultipleChoiceField(
         queryset=Tag.objects.all(),
         required=False,
+        # WIP: broken by Django 3.1.2 -> 4.0 migration
         widget=AutocompleteSelectMultiple(
             AutocompleteTags(),
             AutocompleteTagsAdminStub(),

archivebox/core/apps.py

@@ -3,4 +3,5 @@ from django.apps import AppConfig
 
 class CoreConfig(AppConfig):
     name = 'core'
+    # WIP: broken by Django 3.1.2 -> 4.0 migration
     default_auto_field = 'django.db.models.UUIDField'

archivebox/core/forms.py

@@ -41,7 +41,7 @@ class AddLinkForm(forms.Form):
     #     label="Exclude patterns",
     #     min_length='1',
     #     required=False,
-    #     initial=URL_BLACKLIST,
+    #     initial=URL_DENYLIST,
     # )
     # timeout = forms.IntegerField(
     #     initial=TIMEOUT,

archivebox/core/migrations/0022_auto_20231023_2008.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.14 on 2023-10-23 20:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0021_auto_20220914_0934'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('htmltotext', 'htmltotext'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+        ),
+    ]

archivebox/core/settings.py

@@ -20,6 +20,17 @@ from ..config import (
     OUTPUT_DIR,
     LOGS_DIR,
     TIMEZONE,
+
+    LDAP,
+    LDAP_SERVER_URI,
+    LDAP_BIND_DN,
+    LDAP_BIND_PASSWORD,
+    LDAP_USER_BASE,
+    LDAP_USER_FILTER,
+    LDAP_USERNAME_ATTR,
+    LDAP_FIRSTNAME_ATTR,
+    LDAP_LASTNAME_ATTR,
+    LDAP_EMAIL_ATTR,
 )
 
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
@@ -55,6 +66,12 @@ INSTALLED_APPS = [
 ]
 
 
+# For usage with https://www.jetadmin.io/integrations/django
+# INSTALLED_APPS += ['jet_django']
+# JET_PROJECT = 'archivebox'
+# JET_TOKEN = 'some-api-token-here'
+
+
 MIDDLEWARE = [
     'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
@@ -67,11 +84,59 @@ MIDDLEWARE = [
     'core.middleware.CacheControlMiddleware',
 ]
 
+################################################################################
+### Authentication Settings
+################################################################################
+
 AUTHENTICATION_BACKENDS = [
     'django.contrib.auth.backends.RemoteUserBackend',
     'django.contrib.auth.backends.ModelBackend',
 ]
 
+if LDAP:
+    try:
+        import ldap
+        from django_auth_ldap.config import LDAPSearch
+
+        global AUTH_LDAP_SERVER_URI
+        global AUTH_LDAP_BIND_DN
+        global AUTH_LDAP_BIND_PASSWORD
+        global AUTH_LDAP_USER_SEARCH
+        global AUTH_LDAP_USER_ATTR_MAP
+
+        AUTH_LDAP_SERVER_URI = LDAP_SERVER_URI
+        AUTH_LDAP_BIND_DN = LDAP_BIND_DN
+        AUTH_LDAP_BIND_PASSWORD = LDAP_BIND_PASSWORD
+
+        assert AUTH_LDAP_SERVER_URI and LDAP_USERNAME_ATTR and LDAP_USER_FILTER, 'LDAP_* config options must all be set if LDAP=True'
+
+        AUTH_LDAP_USER_SEARCH = LDAPSearch(
+            LDAP_USER_BASE,
+            ldap.SCOPE_SUBTREE,
+            '(&(' + LDAP_USERNAME_ATTR + '=%(user)s)' + LDAP_USER_FILTER + ')',
+        )
+
+        AUTH_LDAP_USER_ATTR_MAP = {
+            'username': LDAP_USERNAME_ATTR,
+            'first_name': LDAP_FIRSTNAME_ATTR,
+            'last_name': LDAP_LASTNAME_ATTR,
+            'email': LDAP_EMAIL_ATTR,
+        }
+
+        AUTHENTICATION_BACKENDS = [
+            'django.contrib.auth.backends.ModelBackend',
+            'django_auth_ldap.backend.LDAPBackend',
+        ]
+    except ModuleNotFoundError:
+        sys.stderr.write('[X] Error: Found LDAP=True config but LDAP packages not installed. You may need to run: pip install archivebox[ldap]\n\n')
+        # dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
+        # sys.exit(1)
+
+
+################################################################################
+### Debug Settings
+################################################################################
+
 # only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode)
 DEBUG_TOOLBAR = DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv)
 if DEBUG_TOOLBAR:
@@ -204,6 +269,8 @@ AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
 ]
 
+# WIP: broken by Django 3.1.2 -> 4.0 migration
+DEFAULT_AUTO_FIELD = 'django.db.models.UUIDField'
 
 ################################################################################
 ### Shell Settings
@@ -267,8 +334,8 @@ class NoisyRequestsFilter(logging.Filter):
 if LOGS_DIR.exists():
     ERROR_LOG = (LOGS_DIR / 'errors.log')
 else:
-    # meh too many edge cases here around creating log dir w/ correct permissions
-    # cant be bothered, just trash the log and let them figure it out via stdout/stderr
+    # historically too many edge cases here around creating log dir w/ correct permissions early on
+    # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr
     ERROR_LOG = tempfile.NamedTemporaryFile().name
 
 LOGGING = {

archivebox/core/urls.py

@@ -8,6 +8,10 @@ from django.views.generic.base import RedirectView
 
 from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
 
+# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
+# from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
+# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
+
 
 # print('DEBUG', settings.DEBUG)
 
@@ -31,8 +35,14 @@ urlpatterns = [
 
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', admin.site.urls),
+    
+    # do not add extra_context like this as not all admin views (e.g. ModelAdmin.autocomplete_view accept extra kwargs)
+    # path('admin/', admin.site.urls, {'extra_context': GLOBAL_CONTEXT}),
 
     path('health/', HealthCheckView.as_view(), name='healthcheck'),
+    path('error/', lambda _: 1/0),
+
+    # path('jet_api/', include('jet_django.urls')),  Enable to use https://www.jetadmin.io/integrations/django
 
     path('index.html', RedirectView.as_view(url='/')),
     path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),

archivebox/core/views.py

@@ -23,6 +23,7 @@ from ..config import (
     PUBLIC_SNAPSHOTS,
     PUBLIC_ADD_VIEW,
     VERSION,
+    COMMIT_HASH,
     FOOTER_INFO,
     SNAPSHOTS_PER_PAGE,
 )
@@ -217,6 +218,7 @@ class PublicIndexView(ListView):
         return {
             **super().get_context_data(**kwargs),
             'VERSION': VERSION,
+            'COMMIT_HASH': COMMIT_HASH,
             'FOOTER_INFO': FOOTER_INFO,
         }
 

archivebox/extractors/__init__.py

@@ -4,12 +4,16 @@ import os
 import sys
 from pathlib import Path
 
-from typing import Optional, List, Iterable, Union
+from typing import Callable, Optional, List, Iterable, Union
 from datetime import datetime, timezone
 from django.db.models import QuerySet
 
+from ..config import (
+    SAVE_ALLOWLIST_PTN,
+    SAVE_DENYLIST_PTN,
+)
 from ..core.settings import ERROR_LOG
-from ..index.schema import Link
+from ..index.schema import ArchiveResult, Link
 from ..index.sql import write_link_to_sql_index
 from ..index import (
     load_link_details,
@@ -33,6 +37,7 @@ from .wget import should_save_wget, save_wget
 from .singlefile import should_save_singlefile, save_singlefile
 from .readability import should_save_readability, save_readability
 from .mercury import should_save_mercury, save_mercury
+from .htmltotext import should_save_htmltotext, save_htmltotext
 from .pdf import should_save_pdf, save_pdf
 from .screenshot import should_save_screenshot, save_screenshot
 from .dom import should_save_dom, save_dom
@@ -42,7 +47,11 @@ from .archive_org import should_save_archive_dot_org, save_archive_dot_org
 from .headers import should_save_headers, save_headers
 
 
-def get_default_archive_methods():
+ShouldSaveFunction = Callable[[Link, Optional[Path], Optional[bool]], bool]
+SaveFunction = Callable[[Link, Optional[Path], int], ArchiveResult]
+ArchiveMethodEntry = tuple[str, ShouldSaveFunction, SaveFunction]
+
+def get_default_archive_methods() -> List[ArchiveMethodEntry]:
     return [
         ('favicon', should_save_favicon, save_favicon),
         ('headers', should_save_headers, save_headers),
@@ -51,22 +60,49 @@ def get_default_archive_methods():
         ('screenshot', should_save_screenshot, save_screenshot),
         ('dom', should_save_dom, save_dom),
         ('wget', should_save_wget, save_wget),
-        ('title', should_save_title, save_title),                   # keep title and readability below wget and singlefile, as it depends on them
+        # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
+        ('title', should_save_title, save_title),
         ('readability', should_save_readability, save_readability),
         ('mercury', should_save_mercury, save_mercury),
+        ('htmltotext', should_save_htmltotext, save_htmltotext),
         ('git', should_save_git, save_git),
         ('media', should_save_media, save_media),
         ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
-ARCHIVE_METHODS_INDEXING_PRECEDENCE = [('readability', 1), ('singlefile', 2), ('dom', 3), ('wget', 4)]
+ARCHIVE_METHODS_INDEXING_PRECEDENCE = [
+    ('readability', 1),
+    ('mercury', 2),
+    ('htmltotext', 3),
+    ('singlefile', 4),
+    ('dom', 5),
+    ('wget', 6)
+]
+
 
 @enforce_types
-def ignore_methods(to_ignore: List[str]):
+def get_archive_methods_for_link(link: Link) -> Iterable[ArchiveMethodEntry]:
+    DEFAULT_METHODS = get_default_archive_methods()
+    allowed_methods = {
+        m for pat, methods in
+        SAVE_ALLOWLIST_PTN.items()
+        if pat.search(link.url)
+        for m in methods
+    } or { m[0] for m in DEFAULT_METHODS }
+    denied_methods = {
+        m for pat, methods in
+        SAVE_DENYLIST_PTN.items()
+        if pat.search(link.url)
+        for m in methods
+    }
+    allowed_methods -= denied_methods
+
+    return (m for m in DEFAULT_METHODS if m[0] in allowed_methods)
+
+@enforce_types
+def ignore_methods(to_ignore: List[str]) -> Iterable[str]:
     ARCHIVE_METHODS = get_default_archive_methods()
-    methods = filter(lambda x: x[0] not in to_ignore, ARCHIVE_METHODS)
-    methods = map(lambda x: x[0], methods)
-    return list(methods)
+    return [x[0] for x in ARCHIVE_METHODS if x[0] not in to_ignore]
 
 @enforce_types
 def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Link:
@@ -79,11 +115,11 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
     except Snapshot.DoesNotExist:
         snapshot = write_link_to_sql_index(link)
 
-    ARCHIVE_METHODS = get_default_archive_methods()
+    active_methods = get_archive_methods_for_link(link)
     
     if methods:
-        ARCHIVE_METHODS = [
-            method for method in ARCHIVE_METHODS
+        active_methods = [
+            method for method in active_methods
             if method[0] in methods
         ]
 
@@ -100,7 +136,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
         start_ts = datetime.now(timezone.utc)
 
-        for method_name, should_run, method_function in ARCHIVE_METHODS:
+        for method_name, should_run, method_function in active_methods:
             try:
                 if method_name not in link.history:
                     link.history[method_name] = []
@@ -148,7 +184,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                         link.url,
                         command,
                         ts
-                    ) + "\n"))
+                    ) + "\n" + str(e) + "\n"))
                     #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
 
         # print('    ', stats)
@@ -162,7 +198,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
 
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
 
-        log_link_archiving_finished(link, link.link_dir, is_new, stats, start_ts)
+        log_link_archiving_finished(link, out_dir, is_new, stats, start_ts)
 
     except KeyboardInterrupt:
         try:

archivebox/extractors/dom.py

@@ -9,6 +9,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -57,6 +58,7 @@ def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     except Exception as err:
         status = 'failed'
         output = err
+        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/favicon.py

@@ -10,6 +10,7 @@ from ..util import enforce_types, domain
 from ..config import (
     TIMEOUT,
     SAVE_FAVICON,
+    FAVICON_PROVIDER,
     CURL_BINARY,
     CURL_ARGS,
     CURL_VERSION,
@@ -40,7 +41,7 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         '--output', str(output),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
+        FAVICON_PROVIDER.format(domain(link.url)),
     ]
     status = 'failed'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/htmltotext.py

@@ -0,0 +1,154 @@
+__package__ = 'archivebox.extractors'
+
+from html.parser import HTMLParser
+import io
+from pathlib import Path
+from typing import Optional
+
+from ..config import (
+    SAVE_HTMLTOTEXT,
+    TIMEOUT,
+    VERSION,
+)
+from ..index.schema import Link, ArchiveResult, ArchiveError
+from ..logging_util import TimedProgress
+from ..system import atomic_write
+from ..util import (
+    enforce_types,
+    is_static_file,
+)
+from .title import get_html
+
+class HTMLTextExtractor(HTMLParser):
+    TEXT_ATTRS = [
+        "alt", "cite", "href", "label",
+        "list", "placeholder", "title", "value"
+    ]
+    NOTEXT_TAGS = ["script", "style", "template"]
+    NOTEXT_HREF = ["data:", "javascript:", "#"]
+
+    def __init__(self):
+        super().__init__()
+
+        self.output = io.StringIO()
+        self._tag_stack = []
+
+    def _is_text_attr(self, name, value):
+        if not isinstance(value, str):
+            return False
+        if name == "href" and any(map(lambda p: value.startswith(p), self.NOTEXT_HREF)):
+            return False
+
+        if name in self.TEXT_ATTRS:
+            return True
+
+        return False
+
+    def _parent_tag(self):
+        try:
+            return self._tag_stack[-1]
+        except IndexError:
+            return None
+
+    def _in_notext_tag(self):
+        return any([t in self._tag_stack for t in self.NOTEXT_TAGS])
+
+    def handle_starttag(self, tag, attrs):
+        self._tag_stack.append(tag)
+
+        # Don't write out attribute values if any ancestor
+        # is in NOTEXT_TAGS
+        if self._in_notext_tag():
+            return
+
+        for name, value in attrs:
+            if self._is_text_attr(name, value):
+                self.output.write(f"({value.strip()}) ")
+
+    def handle_endtag(self, tag):
+        orig_stack = self._tag_stack.copy()
+        try:
+            # Keep popping tags until we find the nearest
+            # ancestor matching this end tag
+            while tag != self._tag_stack.pop():
+                pass
+            # Write a space after every tag, to ensure that tokens
+            # in tag text aren't concatenated. This may result in
+            # excess spaces, which should be ignored by search tokenizers.
+            if not self._in_notext_tag() and tag not in self.NOTEXT_TAGS:
+                self.output.write(" ")
+        except IndexError:
+            # Got to the top of the stack, but somehow missed
+            # this end tag -- maybe malformed markup -- restore the
+            # stack
+            self._tag_stack = orig_stack
+
+    def handle_data(self, data):
+        # Don't output text data if any ancestor is in NOTEXT_TAGS
+        if self._in_notext_tag():
+            return
+
+        data = data.lstrip()
+        len_before_rstrip = len(data)
+        data = data.rstrip()
+        spaces_rstripped = len_before_rstrip - len(data)
+        if data:
+            self.output.write(data)
+            if spaces_rstripped:
+                # Add back a single space if 1 or more
+                # whitespace characters were stripped
+                self.output.write(' ')
+
+    def __str__(self):
+        return self.output.getvalue()
+
+
+@enforce_types
+def should_save_htmltotext(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
+        return False
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'htmltotext.txt').exists():
+        return False
+
+    return SAVE_HTMLTOTEXT
+
+
+@enforce_types
+def save_htmltotext(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+    """extract search-indexing-friendly text from an HTML document"""
+
+    out_dir = Path(out_dir or link.link_dir)
+    output = "htmltotext.txt"
+
+    timer = TimedProgress(timeout, prefix='      ')
+    extracted_text = None
+    try:
+        extractor = HTMLTextExtractor()
+        document = get_html(link, out_dir)
+
+        if not document:
+            raise ArchiveError('htmltotext could not find HTML to parse for article text')
+
+        extractor.feed(document)
+        extractor.close()
+        extracted_text = str(extractor)
+
+        atomic_write(str(out_dir / output), extracted_text)
+    except (Exception, OSError) as err:
+        status = 'failed'
+        output = err
+        cmd = ['(internal) archivebox.extractors.htmltotext', './{singlefile,dom}.html']
+    finally:
+        timer.end()
+
+    return ArchiveResult(
+        cmd=cmd,
+        pwd=str(out_dir),
+        cmd_version=VERSION,
+        output=output,
+        status=status,
+        index_texts=[extracted_text] if extracted_text else [],
+        **timer.stats,  
+    )

archivebox/extractors/pdf.py

@@ -9,6 +9,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -54,6 +55,7 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     except Exception as err:
         status = 'failed'
         output = err
+        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/readability.py

@@ -67,13 +67,12 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
             temp_doc.name,
             link.url,
         ]
-
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:
             result_json = json.loads(result.stdout)
-            assert result_json and 'content' in result_json
+            assert result_json and 'content' in result_json, 'Readability output is not valid JSON'
         except json.JSONDecodeError:
-            raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)
+            raise ArchiveError('Readability was not able to archive the page (invalid JSON)', result.stdout + result.stderr)
 
         output_folder.mkdir(exist_ok=True)
         readability_content = result_json.pop("textContent") 
@@ -81,11 +80,9 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         atomic_write(str(output_folder / "content.txt"), readability_content)
         atomic_write(str(output_folder / "article.json"), result_json)
 
-        # parse out number of files downloaded from last line of stderr:
-        #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
         output_tail = [
             line.strip()
-            for line in (result.stdout + result.stderr).decode().rsplit('\n', 3)[-3:]
+            for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
             if line.strip()
         ]
         hints = (
@@ -95,11 +92,13 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
 
         # Check for common failure cases
         if (result.returncode > 0):
-            raise ArchiveError('Readability was not able to archive the page', hints)
+            raise ArchiveError(f'Readability was not able to archive the page (status={result.returncode})', hints)
     except (Exception, OSError) as err:
         status = 'failed'
         output = err
-        cmd = [cmd[0], './{singlefile,dom}.html']
+
+        # prefer Chrome dom output to singlefile because singlefile often contains huge url(data:image/...base64) strings that make the html too long to parse with readability
+        cmd = [cmd[0], './{dom,singlefile}.html']
     finally:
         timer.end()
 

archivebox/extractors/screenshot.py

@@ -9,6 +9,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -54,6 +55,7 @@ def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
     except Exception as err:
         status = 'failed'
         output = err
+        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/title.py

@@ -26,7 +26,7 @@ from ..logging_util import TimedProgress
 
 HTML_TITLE_REGEX = re.compile(
     r'<title.*?>'                      # start matching text after <title> tag
-    r'(.[^<>]+)',                      # get everything up to these symbols
+    r'([^<>]+)',                      # get everything up to these symbols
     re.IGNORECASE | re.MULTILINE | re.DOTALL | re.UNICODE,
 )
 
@@ -66,7 +66,9 @@ def get_html(link: Link, path: Path, timeout: int=TIMEOUT) -> str:
     """
     canonical = link.canonical_outputs()
     abs_path = path.absolute()
-    sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
+
+    # prefer chrome-generated DOM dump to singlefile as singlefile output often includes HUGE url(data:image/...base64) strings that crash parsers
+    sources = [canonical["dom_path"], canonical["singlefile_path"], canonical["wget_path"]]
     document = None
     for source in sources:
         try:

archivebox/index/__init__.py

@@ -22,8 +22,8 @@ from ..config import (
     JSON_INDEX_FILENAME,
     OUTPUT_DIR,
     TIMEOUT,
-    URL_BLACKLIST_PTN,
-    URL_WHITELIST_PTN,
+    URL_DENYLIST_PTN,
+    URL_ALLOWLIST_PTN,
     stderr,
     OUTPUT_PERMISSIONS
 )
@@ -142,9 +142,9 @@ def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
             continue
         if scheme(link.url) not in ('http', 'https', 'ftp'):
             continue
-        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(link.url):
+        if URL_DENYLIST_PTN and URL_DENYLIST_PTN.search(link.url):
             continue
-        if URL_WHITELIST_PTN and (not URL_WHITELIST_PTN.search(link.url)):
+        if URL_ALLOWLIST_PTN and (not URL_ALLOWLIST_PTN.search(link.url)):
             continue
 
         yield link

archivebox/index/html.py

@@ -143,7 +143,7 @@ def snapshot_icons(snapshot) -> str:
             "mercury": "🅼",
             "warc": "📦"
         }
-        exclude = ["favicon", "title", "headers", "archive_org"]
+        exclude = ["favicon", "title", "headers", "htmltotext", "archive_org"]
         # Missing specific entry for WARC
 
         extractor_outputs = defaultdict(lambda: None)
@@ -177,7 +177,7 @@ def snapshot_icons(snapshot) -> str:
                 # The check for archive_org is different, so it has to be handled separately
 
                 # get from db (faster)
-                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                exists = extractor in extractor_outputs and extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
                 # get from filesystem (slower)
                 # target_path = Path(path) / "archive.org.txt"
                 # exists = target_path.exists()

archivebox/index/schema.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from ..system import get_dir_size
 from ..util import ts_to_date_str, parse_date
-from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
+from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME, FAVICON_PROVIDER
 
 class ArchiveError(Exception):
     def __init__(self, message, hints=None):
@@ -423,12 +423,13 @@ class Link:
         canonical = {
             'index_path': 'index.html',
             'favicon_path': 'favicon.ico',
-            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
+            'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
             'wget_path': wget_output_path(self),
             'warc_path': 'warc/',
             'singlefile_path': 'singlefile.html',
             'readability_path': 'readability/content.html',
             'mercury_path': 'mercury/content.html',
+            'htmltotext_path': 'htmltotext.txt',
             'pdf_path': 'output.pdf',
             'screenshot_path': 'screenshot.png',
             'dom_path': 'output.html',
@@ -452,6 +453,7 @@ class Link:
                 'singlefile_path': static_path,
                 'readability_path': static_path,
                 'mercury_path': static_path,
+                'htmltotext_path': static_path,
             })
         return canonical
 

archivebox/index/sql.py

@@ -109,11 +109,13 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
         snap = Snapshot.objects.get(url=link.url)
     except Snapshot.DoesNotExist:
         snap = write_link_to_sql_index(link)
+
     snap.title = link.title
 
-    tag_list = list(dict.fromkeys(
-        tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')
-    ))
+    tag_list = list(
+        {tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')}
+        | set(snap.tags.values_list('name', flat=True))
+    )
 
     snap.save()
     snap.save_tags(tag_list)

archivebox/logging_util.py

@@ -393,7 +393,11 @@ def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats
     else:
         _LAST_RUN_STATS.succeeded += 1
 
-    size = get_dir_size(link_dir)
+    try:
+        size = get_dir_size(link_dir)
+    except FileNotFoundError:
+        size = (0, None, '0')
+
     end_ts = datetime.now(timezone.utc)
     duration = str(end_ts - start_ts).split('.')[0]
     print('        {black}{} files ({}) in {}s {reset}'.format(size[2], printable_filesize(size[0]), duration, **ANSI))
@@ -409,7 +413,7 @@ def log_archive_method_finished(result: "ArchiveResult"):
     """
     # Prettify CMD string and make it safe to copy-paste by quoting arguments
     quoted_cmd = ' '.join(
-        '"{}"'.format(arg) if ' ' in arg else arg
+        '"{}"'.format(arg) if (' ' in arg) or (':' in arg) else arg
         for arg in result.cmd
     )
 
@@ -441,15 +445,21 @@ def log_archive_method_finished(result: "ArchiveResult"):
 
             hints = (
                 '    {}{}{}'.format(ANSI['lightyellow'], line.strip(), ANSI['reset'])
-                for line in hints[:5] if line.strip()
+                for line in list(hints)[:5] if line.strip()
             )
 
+        docker_hints = ()
+        if IN_DOCKER:
+            docker_hints = (
+                '  docker run -it -v $PWD/data:/data archivebox/archivebox /bin/bash',
+            )
 
         # Collect and prefix output lines with indentation
         output_lines = [
             *hint_header,
             *hints,
             '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']),
+            *docker_hints,
             *(['    cd {};'.format(result.pwd)] if result.pwd else []),
             '    {}'.format(quoted_cmd),
         ]
@@ -517,8 +527,8 @@ def log_shell_welcome_msg():
     from .cli import list_subcommands
 
     print('{green}# ArchiveBox Imports{reset}'.format(**ANSI))
-    print('{green}from core.models import Snapshot, User{reset}'.format(**ANSI))
-    print('{green}from archivebox import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
+    print('{green}from archivebox.core.models import Snapshot, ArchiveResult, Tag, User{reset}'.format(**ANSI))
+    print('{green}from archivebox.cli import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
     print()
     print('[i] Welcome to the ArchiveBox Shell!')
     print('    https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage')
@@ -533,11 +543,27 @@ def log_shell_welcome_msg():
 ### Helpers
 
 @enforce_types
-def pretty_path(path: Union[Path, str]) -> str:
+def pretty_path(path: Union[Path, str], pwd: Union[Path, str]=OUTPUT_DIR) -> str:
     """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
-    pwd = Path('.').resolve()
-    # parent = os.path.abspath(os.path.join(pwd, os.path.pardir))
-    return str(path).replace(str(pwd) + '/', './')
+    pwd = str(Path(pwd))  # .resolve()
+    path = str(path)
+
+    if not path:
+        return path
+
+    # replace long absolute paths with ./ relative ones to save on terminal output width
+    if path.startswith(pwd) and (pwd != '/'):
+        path = path.replace(pwd, '.', 1)
+    
+    # quote paths containing spaces
+    if ' ' in path:
+        path = f'"{path}"'
+
+    # if path is just a plain dot, replace it back with the absolute path for clarity
+    if path == '.':
+        path = pwd
+
+    return path
 
 
 @enforce_types
@@ -578,6 +604,7 @@ def printable_folder_status(name: str, folder: Dict) -> str:
     else:
         color, symbol, note, num_files = 'lightyellow', '-', 'disabled', '-'
 
+
     if folder['path']:
         if Path(folder['path']).exists():
             num_files = (
@@ -592,13 +619,7 @@ def printable_folder_status(name: str, folder: Dict) -> str:
         # add symbol @ next to filecount if path is a remote filesystem mount
         num_files = f'{num_files} @' if num_files else '@'
 
-    path = str(folder['path']).replace(str(OUTPUT_DIR), '.') if folder['path'] else ''
-    if path and ' ' in path:
-        path = f'"{path}"'
-
-    # if path is just a plain dot, replace it back with the full path for clarity
-    if path == '.':
-        path = str(OUTPUT_DIR)
+    path = pretty_path(folder['path'])
 
     return ' '.join((
         ANSI[color],
@@ -629,9 +650,7 @@ def printable_dependency_version(name: str, dependency: Dict) -> str:
     else:
         color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
 
-    path = str(dependency["path"]).replace(str(OUTPUT_DIR), '.') if dependency["path"] else ''
-    if path and ' ' in path:
-        path = f'"{path}"'
+    path = pretty_path(dependency['path'])
 
     return ' '.join((
         ANSI[color],

archivebox/main.py

@@ -71,6 +71,7 @@ from .config import (
     IS_TTY,
     DEBUG,
     IN_DOCKER,
+    IN_QEMU,
     PUID,
     PGID,
     USER,
@@ -92,11 +93,16 @@ from .config import (
     SQL_INDEX_FILENAME,
     ALLOWED_IN_OUTPUT_DIR,
     SEARCH_BACKEND_ENGINE,
+    LDAP,
+    get_version,
     check_dependencies,
     check_data_folder,
     write_config_file,
     VERSION,
+    VERSIONS_AVAILABLE,
+    CAN_UPGRADE,
     COMMIT_HASH,
+    BUILD_TIME,
     CODE_LOCATIONS,
     EXTERNAL_LOCATIONS,
     DATA_LOCATIONS,
@@ -112,6 +118,8 @@ from .config import (
     load_all_config,
     CONFIG,
     USER_CONFIG,
+    ADMIN_USERNAME,
+    ADMIN_PASSWORD,
     get_real_name,
     setup_django,
 )
@@ -214,30 +222,40 @@ def version(quiet: bool=False,
     print(VERSION)
     
     if not quiet:
-        # 0.6.3
-        # ArchiveBox v0.6.3 Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY)
-        # DEBUG=False IN_DOCKER=True IS_TTY=True TZ=UTC FS_ATOMIC=True FS_REMOTE=False FS_PERMS=644 501:20 SEARCH_BACKEND=ripgrep
+        # 0.7.1
+        # ArchiveBox v0.7.1+editable COMMIT_HASH=951bba5 BUILD_TIME=2023-12-17 16:46:05 1702860365
+        # IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-14.2-arm64-arm-64bit PYTHON=Cpython
+        # FS_ATOMIC=True FS_REMOTE=False FS_USER=501:20 FS_PERMS=644
+        # DEBUG=False IS_TTY=True TZ=UTC SEARCH_BACKEND=ripgrep LDAP=False
         
         p = platform.uname()
         print(
-            'ArchiveBox v{}'.format(VERSION),
-            *((COMMIT_HASH[:7],) if COMMIT_HASH else ()),
-            sys.implementation.name.title(),
-            p.system,
-            platform.platform(),
-            p.machine,
+            'ArchiveBox v{}'.format(get_version(CONFIG)),
+            *((f'COMMIT_HASH={COMMIT_HASH[:7]}',) if COMMIT_HASH else ()),
+            f'BUILD_TIME={BUILD_TIME}',
+        )
+        print(
+            f'IN_DOCKER={IN_DOCKER}',
+            f'IN_QEMU={IN_QEMU}',
+            f'ARCH={p.machine}',
+            f'OS={p.system}',
+            f'PLATFORM={platform.platform()}',
+            f'PYTHON={sys.implementation.name.title()}',
         )
         OUTPUT_IS_REMOTE_FS = DATA_LOCATIONS['OUTPUT_DIR']['is_mount'] or DATA_LOCATIONS['ARCHIVE_DIR']['is_mount']
         print(
-            f'DEBUG={DEBUG}',
-            f'IN_DOCKER={IN_DOCKER}',
-            f'IS_TTY={IS_TTY}',
-            f'TZ={TIMEZONE}',
-            #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
             f'FS_ATOMIC={ENFORCE_ATOMIC_WRITES}',
             f'FS_REMOTE={OUTPUT_IS_REMOTE_FS}',
-            f'FS_PERMS={OUTPUT_PERMISSIONS} {PUID}:{PGID}',
+            f'FS_USER={PUID}:{PGID}',
+            f'FS_PERMS={OUTPUT_PERMISSIONS}',
+        )
+        print(
+            f'DEBUG={DEBUG}',
+            f'IS_TTY={IS_TTY}',
+            f'TZ={TIMEZONE}',
             f'SEARCH_BACKEND={SEARCH_BACKEND_ENGINE}',
+            f'LDAP={LDAP}',
+            #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
         )
         print()
 
@@ -251,22 +269,22 @@ def version(quiet: bool=False,
         
         print()
         print('{white}[i] Source-code locations:{reset}'.format(**ANSI))
-        for name, folder in CODE_LOCATIONS.items():
-            print(printable_folder_status(name, folder))
+        for name, path in CODE_LOCATIONS.items():
+            print(printable_folder_status(name, path))
 
         print()
         print('{white}[i] Secrets locations:{reset}'.format(**ANSI))
-        for name, folder in EXTERNAL_LOCATIONS.items():
-            print(printable_folder_status(name, folder))
+        for name, path in EXTERNAL_LOCATIONS.items():
+            print(printable_folder_status(name, path))
 
         print()
         if DATA_LOCATIONS['OUTPUT_DIR']['is_valid']:
             print('{white}[i] Data locations:{reset}'.format(**ANSI))
-            for name, folder in DATA_LOCATIONS.items():
-                print(printable_folder_status(name, folder))
+            for name, path in DATA_LOCATIONS.items():
+                print(printable_folder_status(name, path))
         else:
             print()
-            print('{white}[i] Data locations:{reset}'.format(**ANSI))
+            print('{white}[i] Data locations:{reset} (not in a data directory)'.format(**ANSI))
 
         print()
         check_dependencies()
@@ -419,14 +437,16 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
         write_main_index(list(pending_links.values()), out_dir=out_dir)
 
     print('\n{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
+
+    from django.contrib.auth.models import User
+
+    if (ADMIN_USERNAME and ADMIN_PASSWORD) and not User.objects.filter(username=ADMIN_USERNAME).exists():
+        print('{green}[+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.{reset}'.format(**ANSI))
+        User.objects.create_superuser(username=ADMIN_USERNAME, password=ADMIN_PASSWORD)
+
     if existing_index:
         print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**ANSI))
     else:
-        # TODO: allow creating new supersuer via env vars on first init
-        # if config.HTTP_USER and config.HTTP_PASS:
-        #     from django.contrib.auth.models import User
-        #     User.objects.create_superuser(HTTP_USER, '', HTTP_PASS)
-
         print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links) + len(pending_links), **ANSI))
 
     json_index = out_dir / JSON_INDEX_FILENAME
@@ -584,7 +604,7 @@ def add(urls: Union[str, List[str]],
         out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Add a new URL or list of URLs to your archive"""
 
-    from core.models import Tag
+    from core.models import Snapshot, Tag
 
     assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)'
 
@@ -628,6 +648,19 @@ def add(urls: Union[str, List[str]],
     write_main_index(links=new_links, out_dir=out_dir)
     all_links = load_main_index(out_dir=out_dir)
 
+    tags = [
+        Tag.objects.get_or_create(name=name.strip())[0]
+        for name in tag.split(',')
+        if name.strip()
+    ]
+    if tags:
+        for link in imported_links:
+            snapshot = Snapshot.objects.get(url=link.url)
+            snapshot.tags.add(*tags)
+            snapshot.tags_str(nocache=True)
+            snapshot.save()
+        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
+
     if index_only:
         # mock archive all the links using the fake index_only extractor method in order to update their state
         if overwrite:
@@ -659,21 +692,8 @@ def add(urls: Union[str, List[str]],
             stderr(f'[*] [{ts}] Archiving {len(new_links)}/{len(all_links)} URLs from added set...', color='green')
             archive_links(new_links, overwrite=False, **archive_kwargs)
 
-
-    # add any tags to imported links
-    tags = [
-        Tag.objects.get_or_create(name=name.strip())[0]
-        for name in tag.split(',')
-        if name.strip()
-    ]
-    if tags:
-        for link in imported_links:
-            snapshot = link.as_snapshot()
-            snapshot.tags.add(*tags)
-            snapshot.tags_str(nocache=True)
-            snapshot.save()
-        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
-
+    if CAN_UPGRADE:
+        hint(f"There's a new version of ArchiveBox available! Your current version is {VERSION}. You can upgrade to {VERSIONS_AVAILABLE['recommended_version']['tag_name']} ({VERSIONS_AVAILABLE['recommended_version']['html_url']}). For more on how to upgrade: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives\n")
 
     return all_links
 
@@ -971,33 +991,36 @@ def setup(out_dir: Path=OUTPUT_DIR) -> None:
             stderr(f'[X] Failed to install python packages: {e}', color='red')
             raise SystemExit(1)
 
-    stderr('\n    Installing CHROME_BINARY automatically using playwright...')
-    if CHROME_VERSION:
-        print(f'{CHROME_VERSION} is already installed', CHROME_BINARY)
+    if platform.machine() == 'armv7l':
+        stderr('\n    Skip the automatic installation of CHROME_BINARY because playwright is not available on armv7.')
     else:
-        try:
-            run_shell([
-                PYTHON_BINARY, '-m', 'pip',
-                'install',
-                '--upgrade',
-                '--no-cache-dir',
-                '--no-warn-script-location',
-                'playwright',
-            ], capture_output=False, cwd=out_dir)
-            run_shell([PYTHON_BINARY, '-m', 'playwright', 'install', 'chromium'], capture_output=False, cwd=out_dir)
-            proc = run_shell([PYTHON_BINARY, '-c', 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)'], capture_output=True, text=True, cwd=out_dir)
-            NEW_CHROME_BINARY = proc.stdout.decode().strip() if isinstance(proc.stdout, bytes) else proc.stdout.strip()
-            assert NEW_CHROME_BINARY and len(NEW_CHROME_BINARY), 'CHROME_BINARY must contain a path'
-            config(f'CHROME_BINARY={NEW_CHROME_BINARY}', set=True, out_dir=out_dir)
-        except BaseException as e:                                              # lgtm [py/catch-base-exception]
-            stderr(f'[X] Failed to install chromium using playwright: {e.__class__.__name__} {e}', color='red')
-            raise SystemExit(1)
+        stderr('\n    Installing CHROME_BINARY automatically using playwright...')
+        if CHROME_VERSION:
+            print(f'{CHROME_VERSION} is already installed', CHROME_BINARY)
+        else:
+            try:
+                run_shell([
+                    PYTHON_BINARY, '-m', 'pip',
+                    'install',
+                    '--upgrade',
+                    '--no-cache-dir',
+                    '--no-warn-script-location',
+                    'playwright',
+                ], capture_output=False, cwd=out_dir)
+                run_shell([PYTHON_BINARY, '-m', 'playwright', 'install', 'chromium'], capture_output=False, cwd=out_dir)
+                proc = run_shell([PYTHON_BINARY, '-c', 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)'], capture_output=True, text=True, cwd=out_dir)
+                NEW_CHROME_BINARY = proc.stdout.decode().strip() if isinstance(proc.stdout, bytes) else proc.stdout.strip()
+                assert NEW_CHROME_BINARY and len(NEW_CHROME_BINARY), 'CHROME_BINARY must contain a path'
+                config(f'CHROME_BINARY={NEW_CHROME_BINARY}', set=True, out_dir=out_dir)
+            except BaseException as e:                                              # lgtm [py/catch-base-exception]
+                stderr(f'[X] Failed to install chromium using playwright: {e.__class__.__name__} {e}', color='red')
+                raise SystemExit(1)
 
     stderr('\n    Installing SINGLEFILE_BINARY, READABILITY_BINARY, MERCURY_BINARY automatically using npm...')
     if not NODE_VERSION:
-        stderr('[X] You must first install node using your system package manager', color='red')
+        stderr('[X] You must first install node & npm using your system package manager', color='red')
         hint([
-            'curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -',
+            'https://github.com/nodesource/distributions#table-of-contents',
             'or to disable all node-based modules run: archivebox config --set USE_NODE=False',
         ])
         raise SystemExit(1)
@@ -1145,6 +1168,7 @@ def schedule(add: bool=False,
              run_all: bool=False,
              quiet: bool=False,
              every: Optional[str]=None,
+             tag: str='',
              depth: int=0,
              overwrite: bool=False,
              update: bool=not ONLY_NEW,
@@ -1178,6 +1202,7 @@ def schedule(add: bool=False,
                 'add',
                 *(['--overwrite'] if overwrite else []),
                 *(['--update'] if update else []),
+                *([f'--tag={tag}'] if tag else []),
                 f'--depth={depth}',
                 f'"{import_path}"',
             ] if import_path else ['update']),
@@ -1260,6 +1285,9 @@ def schedule(add: bool=False,
                 print('\n{green}[√] Stopped.{reset}'.format(**ANSI))
                 raise SystemExit(1)
 
+    if CAN_UPGRADE:
+        hint(f"There's a new version of ArchiveBox available! Your current version is {VERSION}. You can upgrade to {VERSIONS_AVAILABLE['recommended_version']['tag_name']} ({VERSIONS_AVAILABLE['recommended_version']['html_url']}). For more on how to upgrade: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives\n")
+
     
 @enforce_types
 def server(runserver_args: Optional[List[str]]=None,

archivebox/package.json

@@ -1 +0,0 @@
-../package.json

archivebox/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "archivebox",
+  "version": "0.7.2",
+  "description": "ArchiveBox: The self-hosted internet archive",
+  "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
+  "repository": "github:ArchiveBox/ArchiveBox",
+  "license": "MIT",
+  "dependencies": {
+    "@postlight/parser": "^2.2.3",
+    "readability-extractor": "github:ArchiveBox/readability-extractor",
+    "single-file-cli": "^1.1.46"
+  }
+}

archivebox/parsers/__init__.py

@@ -34,6 +34,7 @@ from ..index.schema import Link
 from ..logging_util import TimedProgress, log_source_saved
 
 from . import pocket_api
+from . import readwise_reader_api
 from . import wallabag_atom
 from . import pocket_html
 from . import pinboard_rss
@@ -51,6 +52,7 @@ from . import url_list
 PARSERS = {
     # Specialized parsers
     pocket_api.KEY:     (pocket_api.NAME,       pocket_api.PARSER),
+    readwise_reader_api.KEY: (readwise_reader_api.NAME, readwise_reader_api.PARSER),
     wallabag_atom.KEY:  (wallabag_atom.NAME,    wallabag_atom.PARSER),
     pocket_html.KEY:    (pocket_html.NAME,      pocket_html.PARSER),
     pinboard_rss.KEY:   (pinboard_rss.NAME,     pinboard_rss.PARSER),
@@ -233,6 +235,10 @@ _test_url_strs = {
     'https://example.com/?what=1#how-about-this=1&2%20baf': 1,
     'https://example.com?what=1#how-about-this=1&2%20baf': 1,
     '<test>http://example7.com</test>': 1,
+    'https://<test>': 0,
+    'https://[test]': 0,
+    'http://"test"': 0,
+    'http://\'test\'': 0,
     '[https://example8.com/what/is/this.php?what=1]': 1,
     '[and http://example9.com?what=1&other=3#and-thing=2]': 1,
     '<what>https://example10.com#and-thing=2 "</about>': 1,

archivebox/parsers/generic_json.py

@@ -17,7 +17,10 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)"""
 
     json_file.seek(0)
-    links = json.load(json_file)
+
+    # sometimes the first line is a comment or filepath, so we get everything after the first {
+    json_file_json_str = '{' + json_file.read().split('{', 1)[-1]
+    links = json.loads(json_file_json_str)
     json_date = lambda s: datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')
 
     for link in links:

archivebox/parsers/readwise_reader_api.py

@@ -0,0 +1,123 @@
+__package__ = "archivebox.parsers"
+
+
+import re
+import requests
+from datetime import datetime
+
+from typing import IO, Iterable, Optional
+from configparser import ConfigParser
+
+from pathlib import Path
+
+from ..index.schema import Link
+from ..util import enforce_types
+from ..system import atomic_write
+from ..config import (
+    SOURCES_DIR,
+    READWISE_READER_TOKENS,
+)
+
+
+API_DB_PATH = Path(SOURCES_DIR) / "readwise_reader_api.db"
+
+
+class ReadwiseReaderAPI:
+    cursor: Optional[str]
+
+    def __init__(self, api_token, cursor=None) -> None:
+        self.api_token = api_token
+        self.cursor = cursor
+
+    def get_archive(self):
+        response = requests.get(
+            url="https://readwise.io/api/v3/list/",
+            headers={"Authorization": f"Token {self.api_token}"},
+            params={
+                "location": "archive",
+                "pageCursor": self.cursor,
+            }
+        )
+        response.raise_for_status()
+        return response
+
+def get_readwise_reader_articles(api: ReadwiseReaderAPI):
+    response = api.get_archive()
+    body = response.json()
+    articles = body["results"]
+
+    yield from articles
+
+
+    if body['nextPageCursor']:
+        api.cursor = body["nextPageCursor"]
+        yield from get_readwise_reader_articles(api)
+
+
+def link_from_article(article: dict, sources: list):
+    url: str = article['source_url']
+    title = article["title"] or url
+    timestamp = datetime.fromisoformat(article['updated_at']).timestamp()
+
+    return Link(
+        url=url,
+        timestamp=str(timestamp),
+        title=title,
+        tags="",
+        sources=sources,
+    )
+
+
+def write_cursor(username: str, since: str):
+    if not API_DB_PATH.exists():
+        atomic_write(API_DB_PATH, "")
+
+    since_file = ConfigParser()
+    since_file.optionxform = str
+    since_file.read(API_DB_PATH)
+
+    since_file[username] = {"since": since}
+
+    with open(API_DB_PATH, "w+") as new:
+        since_file.write(new)
+
+
+def read_cursor(username: str) -> Optional[str]:
+    if not API_DB_PATH.exists():
+        atomic_write(API_DB_PATH, "")
+
+    config_file = ConfigParser()
+    config_file.optionxform = str
+    config_file.read(API_DB_PATH)
+
+    return config_file.get(username, "since", fallback=None)
+
+
+
+
+@enforce_types
+def should_parse_as_readwise_reader_api(text: str) -> bool:
+    return text.startswith("readwise-reader://")
+
+
+@enforce_types
+def parse_readwise_reader_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
+    """Parse bookmarks from the Readwise Reader API"""
+
+    input_buffer.seek(0)
+    pattern = re.compile(r"^readwise-reader:\/\/(\w+)")
+    for line in input_buffer:
+        if should_parse_as_readwise_reader_api(line):
+            username = pattern.search(line).group(1)
+            api = ReadwiseReaderAPI(READWISE_READER_TOKENS[username], cursor=read_cursor(username))
+
+            for article in get_readwise_reader_articles(api):
+                yield link_from_article(article, sources=[line])
+
+            if api.cursor:
+                write_cursor(username, api.cursor)
+
+
+KEY = "readwise_reader_api"
+NAME = "Readwise Reader API"
+PARSER = parse_readwise_reader_api_export

archivebox/search/backends/sqlite.py

@@ -0,0 +1,195 @@
+import codecs
+from typing import List, Generator
+import sqlite3
+
+from archivebox.util import enforce_types
+from archivebox.config import (
+    FTS_SEPARATE_DATABASE,
+    FTS_TOKENIZERS,
+    FTS_SQLITE_MAX_LENGTH
+)
+
+FTS_TABLE = "snapshot_fts"
+FTS_ID_TABLE = "snapshot_id_fts"
+FTS_COLUMN = "texts"
+
+if FTS_SEPARATE_DATABASE:
+    database = sqlite3.connect("search.sqlite3")
+    # Make get_connection callable, because `django.db.connection.cursor()`
+    # has to be called to get a context manager, but sqlite3.Connection
+    # is a context manager without being called.
+    def get_connection():
+        return database
+    SQLITE_BIND = "?"
+else:
+    from django.db import connection as database  # type: ignore[no-redef, assignment]
+    get_connection = database.cursor
+    SQLITE_BIND = "%s"
+
+# Only Python >= 3.11 supports sqlite3.Connection.getlimit(),
+# so fall back to the default if the API to get the real value isn't present
+try:
+    limit_id = sqlite3.SQLITE_LIMIT_LENGTH
+    try:
+        with database.temporary_connection() as cursor:  # type: ignore[attr-defined]
+            SQLITE_LIMIT_LENGTH = cursor.connection.getlimit(limit_id)
+    except AttributeError:
+        SQLITE_LIMIT_LENGTH = database.getlimit(limit_id)
+except AttributeError:
+    SQLITE_LIMIT_LENGTH = FTS_SQLITE_MAX_LENGTH
+
+
+def _escape_sqlite3(value: str, *, quote: str, errors='strict') -> str:
+    assert isinstance(quote, str), "quote is not a str"
+    assert len(quote) == 1, "quote must be a single character"
+
+    encodable = value.encode('utf-8', errors).decode('utf-8')
+
+    nul_index = encodable.find("\x00")
+    if nul_index >= 0:
+        error = UnicodeEncodeError("NUL-terminated utf-8", encodable,
+                                   nul_index, nul_index + 1, "NUL not allowed")
+        error_handler = codecs.lookup_error(errors)
+        replacement, _ = error_handler(error)
+        assert isinstance(replacement, str), "handling a UnicodeEncodeError should return a str replacement"
+        encodable = encodable.replace("\x00", replacement)
+
+    return quote + encodable.replace(quote, quote * 2) + quote
+
+def _escape_sqlite3_value(value: str, errors='strict') -> str:
+    return _escape_sqlite3(value, quote="'", errors=errors)
+
+def _escape_sqlite3_identifier(value: str) -> str:
+    return _escape_sqlite3(value, quote='"', errors='strict')
+
+@enforce_types
+def _create_tables():
+    table = _escape_sqlite3_identifier(FTS_TABLE)
+    # Escape as value, because fts5() expects
+    # string literal column names
+    column = _escape_sqlite3_value(FTS_COLUMN)
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+    tokenizers = _escape_sqlite3_value(FTS_TOKENIZERS)
+    trigger_name = _escape_sqlite3_identifier(f"{FTS_ID_TABLE}_ad")
+
+    with get_connection() as cursor:
+        # Create a contentless-delete FTS5 table that indexes
+        # but does not store the texts of snapshots
+        try:
+            cursor.execute(
+                f"CREATE VIRTUAL TABLE {table}"
+                f" USING fts5({column},"
+                f" tokenize={tokenizers},"
+                " content='', contentless_delete=1);"
+                )
+        except Exception as e:
+            msg = str(e)
+            if 'unrecognized option: "contentlessdelete"' in msg:
+                sqlite_version = getattr(sqlite3, "sqlite_version", "Unknown")
+                raise RuntimeError(
+                    "SQLite full-text search requires SQLite >= 3.43.0;"
+                    f" the running version is {sqlite_version}"
+                ) from e
+            else:
+                raise
+        # Create a one-to-one mapping between ArchiveBox snapshot_id
+        # and FTS5 rowid, because the column type of rowid can't be
+        # customized.
+        cursor.execute(
+            f"CREATE TABLE {id_table}("
+            " rowid INTEGER PRIMARY KEY AUTOINCREMENT,"
+            " snapshot_id char(32) NOT NULL UNIQUE"
+            ");"
+            )
+        # Create a trigger to delete items from the FTS5 index when
+        # the snapshot_id is deleted from the mapping, to maintain
+        # consistency and make the `flush()` query simpler.
+        cursor.execute(
+            f"CREATE TRIGGER {trigger_name}"
+            f" AFTER DELETE ON {id_table} BEGIN"
+            f" DELETE FROM {table} WHERE rowid=old.rowid;"
+            " END;"
+            )
+
+def _handle_query_exception(exc: Exception):
+    message = str(exc)
+    if message.startswith("no such table:"):
+        raise RuntimeError(
+            "SQLite full-text search index has not yet"
+            " been created; run `archivebox update --index-only`."
+        )
+    else:
+        raise exc
+
+@enforce_types
+def index(snapshot_id: str, texts: List[str]):
+    text = ' '.join(texts)[:SQLITE_LIMIT_LENGTH]
+
+    table = _escape_sqlite3_identifier(FTS_TABLE)
+    column = _escape_sqlite3_identifier(FTS_COLUMN)
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+
+    with get_connection() as cursor:
+        retries = 2
+        while retries > 0:
+            retries -= 1
+            try:
+                # If there is already an FTS index rowid to snapshot_id mapping,
+                # then don't insert a new one, silently ignoring the operation.
+                # {id_table}.rowid is AUTOINCREMENT, so will generate an unused
+                # rowid for the index if it is an unindexed snapshot_id.
+                cursor.execute(
+                    f"INSERT OR IGNORE INTO {id_table}(snapshot_id) VALUES({SQLITE_BIND})",
+                    [snapshot_id])
+                # Fetch the FTS index rowid for the given snapshot_id
+                id_res = cursor.execute(
+                    f"SELECT rowid FROM {id_table} WHERE snapshot_id = {SQLITE_BIND}",
+                    [snapshot_id])
+                rowid = id_res.fetchone()[0]
+                # (Re-)index the content
+                cursor.execute(
+                    "INSERT OR REPLACE INTO"
+                    f" {table}(rowid, {column}) VALUES ({SQLITE_BIND}, {SQLITE_BIND})",
+                    [rowid, text])
+                # All statements succeeded; return
+                return
+            except Exception as e:
+                if str(e).startswith("no such table:") and retries > 0:
+                    _create_tables()
+                else:
+                    raise
+
+    raise RuntimeError("Failed to create tables for SQLite FTS5 search")
+
+@enforce_types
+def search(text: str) -> List[str]:
+    table = _escape_sqlite3_identifier(FTS_TABLE)
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+
+    with get_connection() as cursor:
+        try:
+            res = cursor.execute(
+                f"SELECT snapshot_id FROM {table}"
+                f" INNER JOIN {id_table}"
+                f" ON {id_table}.rowid = {table}.rowid"
+                f" WHERE {table} MATCH {SQLITE_BIND}",
+                [text])
+        except Exception as e:
+            _handle_query_exception(e)
+
+        snap_ids = [row[0] for row in res.fetchall()]
+    return snap_ids
+
+@enforce_types
+def flush(snapshot_ids: Generator[str, None, None]):
+    snapshot_ids = list(snapshot_ids)  # type: ignore[assignment]
+
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+
+    with get_connection() as cursor:
+        try:
+            cursor.executemany(
+                f"DELETE FROM {id_table} WHERE snapshot_id={SQLITE_BIND}",
+                [snapshot_ids])
+        except Exception as e:
+            _handle_query_exception(e)

archivebox/static

@@ -0,0 +1 @@
+templates/static

archivebox/system.py

@@ -185,17 +185,19 @@ def dedupe_cron_jobs(cron: CronTab) -> CronTab:
 
 
 class suppress_output(object):
-    '''
+    """
     A context manager for doing a "deep suppression" of stdout and stderr in 
     Python, i.e. will suppress all print, even if the print originates in a 
     compiled C/Fortran sub-function.
-       This will not suppress raised exceptions, since exceptions are printed
+    
+    This will not suppress raised exceptions, since exceptions are printed
     to stderr just before a script exits, and after the context manager has
     exited (at least, I think that is why it lets exceptions through).      
 
     with suppress_stdout_stderr():
         rogue_function()
-    '''
+    """
+    
     def __init__(self, stdout=True, stderr=True):
         # Open a pair of null files
         # Save the actual stdout (1) and stderr (2) file descriptors.

archivebox/templates/admin/base.html

@@ -12,7 +12,26 @@
         {% endblock %}
         
         <link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
-        {% block extrastyle %}{% endblock %}
+        {% block extrastyle %}
+        <style>
+            #upgrade-banner {
+                position: fixed;
+                right: 20px;
+                bottom: 20px;
+                background-color: #f8f8f8;
+                color: #333333;
+                border: 2px solid #772948;
+                padding: 10px 20px;
+                z-index: 1000;
+                text-align: center;
+            }
+            #dismiss-btn {
+                background: #aa1e55;
+                color: white;
+                cursor: pointer;
+            }
+        </style>
+        {% endblock %}
         
         {% if LANGUAGE_BIDI %}
             <link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">
@@ -122,6 +141,43 @@
             {% block footer %}<div id="footer"></div>{% endblock %}
         </div>
 
+        {% if user.is_authenticated and user.is_superuser and CAN_UPGRADE %}
+            <script>
+                if (!localStorage.getItem("bannerDismissed")) {
+                    const upgradeVersionTag = "{{VERSIONS_AVAILABLE.recommended_version.tag_name}}"
+                    const upgradeVersionURL = "{{VERSIONS_AVAILABLE.recommended_version.html_url}}"
+                    const currentVersionTag = "{{VERSION}}"
+                    const currentVersionURL = "{{VERSIONS_AVAILABLE.recommended_version.html_url}}"
+
+                    createBanner(currentVersionTag, currentVersionURL, upgradeVersionTag, upgradeVersionURL)
+                }
+
+                function createBanner(currentVersionTag, currentVersionURL, upgradeVersionTag, upgradeVersionURL) {
+                    const banner = document.createElement('div')
+                    banner.setAttribute('id', 'upgrade-banner');
+                    banner.innerHTML = `
+                        <p>There's a new version of ArchiveBox available!</p>
+                        Your version: <a href=${currentVersionURL}>${currentVersionTag}</a> | New version: <a href=${upgradeVersionURL}>${upgradeVersionTag}</a>
+                        <p>
+                        <a href=https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives>Upgrade Instructions</a> | <a href=https://github.com/ArchiveBox/ArchiveBox/releases>Changelog</a> | <a href=https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap>Roadmap</a> 
+                        </p>
+                        <button id="dismiss-btn">Dismiss</button>
+                    `
+                    document.body.appendChild(banner);
+                    const dismissButton = document.querySelector("#dismiss-btn")
+                    if (dismissButton) {
+                        dismissButton.addEventListener("click", dismissBanner)
+                    }
+                }
+
+                function dismissBanner() {
+                    const banner = document.getElementById("upgrade-banner")
+                    banner.style.display = "none"
+                    localStorage.setItem("bannerDismissed", "true")
+                }
+            </script>
+        {% endif %}
+        
         <script>
             $ = django.jQuery;
             $.fn.reverse = [].reverse;

archivebox/templates/admin/private_index.html

@@ -1,62 +1,3 @@
-{% extends "base.html" %}
-{% load static %}
-
-{% block body %}
-    <div id="toolbar">
-        <form id="changelist-search" action="{% url 'public-index' %}" method="get">
-            <div>
-                <label for="searchbar"><img src="/static/admin/img/search.svg" alt="Search"></label>
-                <input type="text" size="40" name="q" value="" id="searchbar" autofocus placeholder="Title, URL, tags, timestamp, or content...".>
-                <input type="submit" value="Search" style="height: 36px; padding-top: 6px; margin: 8px"/>
-                <input type="button"
-                       value="♺"
-                       title="Refresh..."
-                       onclick="location.href='{% url 'public-index' %}'"
-                       style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
-                </input>
-            </div>
-        </form>
-    </div>
-    <table id="table-bookmarks">
-        <thead>
-            <tr>
-                <th style="width: 100px;">Bookmarked</th>
-                <th style="width: 26vw;">Snapshot ({{object_list|length}})</th>
-                <th style="width: 140px">Files</th>
-                <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
-            </tr>
-        </thead>
-            <tbody>
-                {% for link in object_list %}
-                    {% include 'main_index_row.html' with link=link  %}
-                {% endfor %}
-            </tbody>
-        </table>
-        <center>
-            <span class="step-links">
-                {% if page_obj.has_previous %}
-                    <a href="{% url 'public-index' %}?page=1">&laquo; first</a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
-                {% endif %}
-        
-                <span class="current">
-                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
-                </span>
-        
-                {% if page_obj.has_next %}
-                    <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
-                {% endif %}
-            </span>
-    
-            {% if page_obj.has_next %}
-                <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
-                <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
-            {% endif %}
-        </span>
-        <br>
-    </center>
-{% endblock %}
 {% extends "admin/base_site.html" %}
 {% load i18n admin_urls static admin_list %}
 {% load core_tags %}

archivebox/templates/core/add.html

@@ -33,7 +33,7 @@
                     <br/>
                     <div class="loader"></div>
                     <br/>
-                    Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for progress...
+                    Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for detailed progress...
                 </center>
             </div>
             <form id="add-form" method="POST" class="p-form">{% csrf_token %}
@@ -46,19 +46,22 @@
             </form>
             <br/><br/><br/>
             <center id="delay-warning" style="display: none">
-                <small>(it's safe to leave this page, adding will continue in the background)</small>
+                <small>(you will be redirected to your <a href="/">Snapshot list</a> momentarily, its safe to close this page at any time)</small>
             </center>
             {% if absolute_add_path %}
-            <center id="bookmarklet">
+            <!-- <center id="bookmarklet">
               <p>Bookmark this link to quickly add to your archive:
                 <a href="javascript:void(window.open('{{ absolute_add_path }}?url='+encodeURIComponent(document.location.href)));">Add to ArchiveBox</a></p>
-            </center>
+            </center> -->
             {% endif %}
             <script>
                 document.getElementById('add-form').addEventListener('submit', function(event) {
                     document.getElementById('in-progress').style.display = 'block'
                     document.getElementById('add-form').style.display = 'none'
                     document.getElementById('delay-warning').style.display = 'block'
+                    setTimeout(function() {
+                        window.location = '/'
+                    }, 2000)
                     return true
                 })
             </script>

archivebox/templates/core/base.html

@@ -38,8 +38,11 @@
                     <br />
                     <center>
                         <small>
-                            Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a> version
-                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{VERSION}}" title="Releases">v{{VERSION}}</a>.
+                            Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
+                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{VERSION}}" title="Releases">v{{VERSION}}</a>
+                            {% if COMMIT_HASH %}
+                                (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{{COMMIT_HASH}}"><code>{{COMMIT_HASH|truncatechars:9}}</code></a>)
+                            {% endif %}.
                             <br/><br/>
                             {{FOOTER_INFO}}
                         </small>

archivebox/util.py

@@ -17,6 +17,8 @@ from requests.exceptions import RequestException, ReadTimeout
 
 from .vendor.base32_crockford import encode as base32_encode                            # type: ignore
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
+from os.path import lexists
+from os import remove as remove_file
 
 try:
     import chardet
@@ -59,7 +61,7 @@ URL_REGEX = re.compile(
     r'(?=('
     r'http[s]?://'                    # start matching from allowed schemes
     r'(?:[a-zA-Z]|[0-9]'              # followed by allowed alphanum characters
-    r'|[$-_@.&+]|[!*\(\),]'           #    or allowed symbols
+    r'|[-_$@.&+!*\(\),]'           #    or allowed symbols (keep hyphen first to match literal hyphen)
     r'|(?:%[0-9a-fA-F][0-9a-fA-F]))'  #    or allowed unicode bytes
     r'[^\]\[\(\)<>"\'\s]+'          # stop parsing at these symbols
     r'))',
@@ -229,7 +231,8 @@ def chrome_args(**options) -> List[str]:
     cmd_args = [options['CHROME_BINARY']]
 
     if options['CHROME_HEADLESS']:
-        if int(CHROME_VERSION.split()[1].split('.')[0]) >= 111:
+        chrome_major_version = int(re.search(r'\s(\d+)\.\d', CHROME_VERSION)[1])
+        if chrome_major_version >= 111:
             cmd_args += ("--headless=new",)
         else:
             cmd_args += ('--headless',)
@@ -271,6 +274,16 @@ def chrome_args(**options) -> List[str]:
     
     return cmd_args
 
+def chrome_cleanup():
+    """
+    Cleans up any state or runtime files that chrome leaves behind when killed by
+    a timeout or other error
+    """
+
+    from .config import IN_DOCKER
+    
+    if IN_DOCKER and lexists("/home/archivebox/.config/chromium/SingletonLock"):
+        remove_file("/home/archivebox/.config/chromium/SingletonLock")
 
 def ansi_to_html(text):
     """

archivebox/vendor/package-lock.json

@@ -0,0 +1 @@
+../../package-lock.json

archivebox/vendor/package.json

@@ -0,0 +1 @@
+../../package.json

bin/build_brew.sh

@@ -23,7 +23,24 @@ fi
 
 cd "$REPO_DIR/brew_dist"
 # make sure archivebox.rb is up-to-date with the dependencies
+git pull
+git status | grep 'up to date'
 
-echo "[+] Building Homebrew bottle"
-brew install --build-bottle ./archivebox.rb
+echo
+echo "[+] Uninstalling any exisitng archivebox versions..."
+brew uninstall archivebox || true
+brew untap archivebox/archivebox || true
+
+# echo "[*] Running Formula linters and test build..."
+# brew test-bot --tap=ArchiveBox/homebrew-archivebox archivebox/archivebox/archivebox || true
+# brew uninstall archivebox || true
+# brew untap archivebox/archivebox || true
+
+echo
+echo "[+] Installing and building hombrew bottle from https://Github.com/ArchiveBox/homebrew-archivebox#main"
+brew tap archivebox/archivebox
+brew install --build-bottle archivebox
 brew bottle archivebox
+
+echo
+echo "[√] Finished. Make sure to commit the outputted .tar.gz and bottle files!"

bin/build_dev.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# ./bin/build_docker.sh dev 'linux/arm/v7'
+
+### Bash Environment Setup
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+# set -o xtrace
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+IFS=$'\n'
+
+REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
+cd "$REPO_DIR"
+which docker > /dev/null || exit 1
+
+
+TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
+VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
+SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+REQUIRED_PLATFORMS="${2:-"linux/arm64,linux/amd64,linux/arm/v7"}"
+
+echo "[+] Building Docker image: tag=$TAG_NAME version=$SHORT_VERSION arch=$REQUIRED_PLATFORMS"
+
+
+echo "[+] Building archivebox:$VERSION docker image..."
+# docker builder prune
+docker build . --no-cache -t archivebox-dev --load
+
+# docker buildx build --platform "$REQUIRED_PLATFORMS" --load . \
+#                -t archivebox \
+#                -t archivebox:$TAG_NAME \
+#                -t archivebox:$VERSION \
+#                -t archivebox:$SHORT_VERSION

bin/build_docker.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# ./bin/build_docker.sh dev 'linux/arm/v7'
 
 ### Bash Environment Setup
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
@@ -12,18 +13,25 @@ IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
 cd "$REPO_DIR"
+
 which docker > /dev/null || exit 1
+which jq > /dev/null || exit 1
+# which pdm > /dev/null || exit 1
 
+SUPPORTED_PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
 
-TAG_NAME="dev"
+TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
 VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
 SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
-REQUIRED_PLATFORMS=('linux/arm64','linux/amd64','linux/arm/v8','linux/arm/v7')
+GIT_SHA=sha-"$(git rev-parse --short HEAD)"
+SELECTED_PLATFORMS="${2:-$SUPPORTED_PLATFORMS}"
+
+echo "[+] Building Docker image: tag=$TAG_NAME version=$SHORT_VERSION arch=$SELECTED_PLATFORMS"
 
 function check_platforms() {
     INSTALLED_PLATFORMS="$(docker buildx inspect | grep 'Platforms:' )"
 
-    for REQUIRED_PLATFORM in ${REQUIRED_PLATFORMS//,/$IFS}; do
+    for REQUIRED_PLATFORM in ${SELECTED_PLATFORMS//,/$IFS}; do
         echo "[+] Checking for: $REQUIRED_PLATFORM..."
         if ! (echo "$INSTALLED_PLATFORMS" | grep -q "$REQUIRED_PLATFORM"); then
             return 1
@@ -33,50 +41,64 @@ function check_platforms() {
     return 0
 }
 
+function remove_builder() {
+    # remove existing xbuilder
+    docker buildx stop xbuilder || true
+    docker buildx rm xbuilder || true
+}
+
 function create_builder() {
-    echo "[+] Creating new xbuilder for: $REQUIRED_PLATFORMS"
+    docker buildx use xbuilder && return 0
+    echo "[+] Creating new xbuilder for: $SELECTED_PLATFORMS"
     echo
+    docker pull 'moby/buildkit:buildx-stable-1'
 
     # Switch to buildx builder if already present / previously created
-    docker buildx create --name xbuilder --driver docker-container --bootstrap --use --platform "$REQUIRED_PLATFORMS" || true
+    docker buildx create --name xbuilder --driver docker-container --bootstrap --use --platform "$SELECTED_PLATFORMS" || true
     docker buildx inspect --bootstrap || true
-
-    echo
 }
 
 function recreate_builder() {
     # Install QEMU binaries for cross-platform building if not installed
     docker run --privileged --rm 'tonistiigi/binfmt' --install all
 
-    # remove existing xbuilder
-    docker buildx stop xbuilder || true
-    docker buildx rm xbuilder || true
-
-    # Create Docker builder for cross-platform building
-    docker buildx use xbuilder && return 0
-
+    remove_builder
     create_builder
 }
 
-
 # Check if docker is ready for cross-plaform builds, if not, recreate builder
-docker buildx use xbuilder || create_builder
+docker buildx use xbuilder 2>&1 >/dev/null || create_builder
 check_platforms || (recreate_builder && check_platforms) || exit 1
 
 
+# Build python package lists
+echo "[+] Generating requirements.txt and pdm.lock from pyproject.toml..."
+pdm lock --group=':all' --strategy="inherit_metadata" --production
+pdm export --group=':all' --production --without-hashes -o requirements.txt
+
+
 echo "[+] Building archivebox:$VERSION docker image..."
-#docker build . \
-docker buildx build --platform "$REQUIRED_PLATFORMS" --push . \
-               -t archivebox \
-               -t archivebox:$TAG_NAME \
-               -t archivebox:$VERSION \
-               -t archivebox:$SHORT_VERSION \
-               -t docker.io/nikisweeting/archivebox:$TAG_NAME \
-               -t docker.io/nikisweeting/archivebox:$VERSION \
-               -t docker.io/nikisweeting/archivebox:$SHORT_VERSION \
-               -t docker.io/archivebox/archivebox:$TAG_NAME \
-               -t docker.io/archivebox/archivebox:$VERSION \
-               -t docker.io/archivebox/archivebox:$SHORT_VERSION \
-               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$TAG_NAME \
-               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$VERSION \
-               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$SHORT_VERSION
+# docker builder prune
+# docker build . --no-cache -t archivebox-dev \
+# replace --load with --push to deploy
+docker buildx build --platform "$SELECTED_PLATFORMS" --load . \
+               -t archivebox/archivebox \
+               -t archivebox/archivebox:$TAG_NAME \
+               -t archivebox/archivebox:$VERSION \
+               -t archivebox/archivebox:$SHORT_VERSION \
+               -t archivebox/archivebox:$GIT_SHA \
+               -t archivebox/archivebox:latest \
+               -t archivebox/archivebox:stable \
+               -t nikisweeting/archivebox \
+               -t nikisweeting/archivebox:$TAG_NAME \
+               -t nikisweeting/archivebox:$VERSION \
+               -t nikisweeting/archivebox:$SHORT_VERSION \
+               -t nikisweeting/archivebox:$GIT_SHA \
+               -t nikisweeting/archivebox:latest \
+               -t nikisweeting/archivebox:stable \
+               -t ghcr.io/archivebox/archivebox/archivebox:$TAG_NAME \
+               -t ghcr.io/archivebox/archivebox/archivebox:$VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$SHORT_VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$GIT_SHA \
+               -t ghcr.io/archivebox/archivebox/archivebox:latest \
+               -t ghcr.io/archivebox/archivebox/archivebox:stable

bin/build_pip.sh

@@ -15,17 +15,25 @@ REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && p
 if [[ -f "$REPO_DIR/.venv/bin/activate" ]]; then
     source "$REPO_DIR/.venv/bin/activate"
 else
-    echo "[!] Warning: No virtualenv presesnt in $REPO_DIR.venv"
+    echo "[!] Warning: No virtualenv presesnt in $REPO_DIR/.venv, creating one now..."
+    python3 -m venv --system-site-packages --symlinks $REPO_DIR/.venv
 fi
 cd "$REPO_DIR"
 
-
 echo "[*] Cleaning up build dirs"
 cd "$REPO_DIR"
 rm -Rf build dist
 
 echo "[+] Building sdist, bdist_wheel, and egg_info"
-python3 setup.py \
-    sdist --dist-dir=./pip_dist \
-    bdist_wheel --dist-dir=./pip_dist \
-    egg_info --egg-base=./pip_dist
+rm -f archivebox/package.json
+cp package.json archivebox/package.json
+
+pdm self update
+pdm install
+pdm build
+pdm export --without-hashes -o ./pip_dist/requirements.txt
+
+cp dist/* ./pip_dist/
+
+echo
+echo "[√] Finished. Don't forget to commit the new sdist and wheel files in ./pip_dist/"

bin/docker_entrypoint.sh

@@ -1,45 +1,157 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
-DATA_DIR="${DATA_DIR:-/data}"
-ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
+# This Docker ENTRYPOINT script is called by `docker run archivebox ...` or `docker compose run archivebox ...`.
+# It takes a CMD as $* shell arguments and runs it following these setup steps:
 
+# - Set the archivebox user to use the correct PUID & PGID
+#     1. highest precedence is for valid PUID and PGID env vars passsed in explicitly
+#     2. fall back to DETECTED_PUID of files found within existing data dir
+#     3. fall back to DEFAULT_PUID if no data dir or its owned by root
+# - Create a new /data dir if necessary and set the correct ownership on it
+# - Create a new /browsers dir if necessary and set the correct ownership on it
+# - Check whether we're running inside QEMU emulation and show a warning if so.
+# - Check that enough free space is available on / and /data
+# - Drop down to archivebox user permisisons and execute passed CMD command.
 
-# Set the archivebox user UID & GID
-if [[ -n "$PUID" && "$PUID" != 0 ]]; then
-    usermod -u "$PUID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
-fi
-if [[ -n "$PGID" && "$PGID" != 0 ]]; then
-    groupmod -g "$PGID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+# Bash Environment Setup
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+# set -o xtrace
+# set -o nounset
+set -o errexit
+set -o errtrace
+set -o pipefail
+# IFS=$'\n'
+
+# Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime)
+export DATA_DIR="${DATA_DIR:-/data}"
+export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
+
+# Global default PUID and PGID if data dir is empty and no intended PUID+PGID is set manually by user
+export DEFAULT_PUID=911
+export DEFAULT_PGID=911
+
+# If user tires to set PUID and PGID to root values manually, catch and reject because root is not allowed
+if [[ "$PUID" == "0" ]]; then
+    echo -e "\n[X] Error: Got PUID=$PUID and PGID=$PGID but ArchiveBox is not allowed to be run as root, please change or unset PUID & PGID and try again." > /dev/stderr
+    echo -e "    Hint: some NFS/SMB/FUSE/etc. filesystems force-remap/ignore all permissions," > /dev/stderr
+        echo -e "          leave PUID/PGID unset, or use values the filesystem prefers (defaults to $DEFAULT_PUID:$DEFAULT_PGID)" > /dev/stderr
+        echo -e "    https://linux.die.net/man/8/mount.cifs#:~:text=does%20not%20provide%20unix%20ownership" > /dev/stderr
+    exit 3
 fi
 
+# If data directory already exists, autodetect detect owner by looking at files within
+export DETECTED_PUID="$(stat -c '%u' "$DATA_DIR/logs/errors.log" 2>/dev/null || echo "$DEFAULT_PUID")"
+export DETECTED_PGID="$(stat -c '%g' "$DATA_DIR/logs/errors.log" 2>/dev/null || echo "$DEFAULT_PGID")"
 
-# Set the permissions of the data dir to match the archivebox user
+# If data directory exists but is owned by root, use defaults instead of root because root is not allowed
+[[ "$DETECTED_PUID" == "0" ]] && export DETECTED_PUID="$DEFAULT_PUID"
+
+# Set archivebox user and group ids to desired PUID/PGID
+usermod -o -u "${PUID:-$DETECTED_PUID}" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+groupmod -o -g "${PGID:-$DETECTED_PGID}" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+
+# re-set PUID and PGID to values reported by system instead of values we tried to set,
+# in case wonky filesystems or Docker setups try to play UID/GID remapping tricks on us
+export PUID="$(id -u archivebox)"
+export PGID="$(id -g archivebox)"
+
+# Check the permissions of the data dir (or create if it doesn't exist)
 if [[ -d "$DATA_DIR/archive" ]]; then
-    # check data directory permissions
-    if [[ ! "$(stat -c %u $DATA_DIR/archive)" = "$(id -u archivebox)" ]]; then
-        echo "Change in ownership detected, please be patient while we chown existing files"
-        echo "This could take some time..."
-        chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER -R "$DATA_DIR"
+    if touch "$DATA_DIR/archive/.permissions_test_safe_to_delete" 2>/dev/null; then
+        # It's fine, we are able to write to the data directory (as root inside the container)
+        rm -f "$DATA_DIR/archive/.permissions_test_safe_to_delete"
+        # echo "[√] Permissions are correct"
+    else
+     # the only time this fails is if the host filesystem doesn't allow us to write as root (e.g. some NFS mapall/maproot problems, connection issues, drive dissapeared, etc.)
+        echo -e "\n[X] Error: archivebox user (PUID=$PUID) is not able to write to your ./data dir (currently owned by $(stat -c '%u' "$DATA_DIR"):$(stat -c '%g' "$DATA_DIR")." >&2
+        echo -e "    Change ./data to be owned by PUID=$PUID PGID=$PGID on the host and retry:" > /dev/stderr
+        echo -e "       \$ chown -R $PUID:$PGID ./data\n" > /dev/stderr
+        echo -e "    Configure the PUID & PGID environment variables to change the desired owner:" > /dev/stderr
+        echo -e "       https://docs.linuxserver.io/general/understanding-puid-and-pgid\n" > /dev/stderr
+        echo -e "    Hint: some NFS/SMB/FUSE/etc. filesystems force-remap/ignore all permissions," > /dev/stderr
+        echo -e "          leave PUID/PGID unset, or use values the filesystem prefers (defaults to $DEFAULT_PUID:$DEFAULT_PGID)" > /dev/stderr
+        echo -e "    https://linux.die.net/man/8/mount.cifs#:~:text=does%20not%20provide%20unix%20ownership" > /dev/stderr
+        exit 3
     fi
 else
     # create data directory
     mkdir -p "$DATA_DIR/logs"
-    chown -R $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 fi
-chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 
+# force set the ownership of the data dir contents to the archivebox user and group
+# this is needed because Docker Desktop often does not map user permissions from the host properly
+chown $PUID:$PGID "$DATA_DIR"
+chown $PUID:$PGID "$DATA_DIR"/*
+
+# also chown BROWSERS_DIR because otherwise 'archivebox setup' wont be able to install chrome at runtime
+export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-/browsers}"
+mkdir -p "$PLAYWRIGHT_BROWSERS_PATH/permissions_test_safe_to_delete"
+chown $PUID:$PGID "$PLAYWRIGHT_BROWSERS_PATH"
+chown $PUID:$PGID "$PLAYWRIGHT_BROWSERS_PATH"/*
+rm -Rf "$PLAYWRIGHT_BROWSERS_PATH/permissions_test_safe_to_delete"
+
+
+# (this check is written in blood in 2023, QEMU silently breaks things in ways that are not obvious)
+export IN_QEMU="$(pmap 1 | grep qemu >/dev/null && echo 'True' || echo 'False')"
+if [[ "$IN_QEMU" == "True" ]]; then
+    echo -e "\n[!] Warning: Running $(uname -m) docker image using QEMU emulation, some things will break!" > /dev/stderr
+    echo -e "    chromium (screenshot, pdf, dom), singlefile, and any dependencies that rely on inotify will not run in QEMU." > /dev/stderr
+    echo -e "    See here for more info: https://github.com/microsoft/playwright/issues/17395#issuecomment-1250830493\n" > /dev/stderr
+fi
+
+# check disk space free on / and /data, warn on <500Mb free, error on <100Mb free
+export ROOT_USAGE="$(df --output=pcent,avail / | tail -n 1 | xargs)"
+export ROOT_USED_PCT="${ROOT_USAGE%%%*}"
+export ROOT_AVAIL_KB="$(echo "$ROOT_USAGE" | awk '{print $2}')"
+if [[ "$ROOT_AVAIL_KB" -lt 100000 ]]; then
+    echo -e "\n[!] Warning: Docker root filesystem is completely out of space! (${ROOT_USED_PCT}% used on /)" > /dev/stderr
+    echo -e "    you need to free up at least 100Mb in your Docker VM to continue:" > /dev/stderr
+    echo -e "    \$ docker system prune\n" > /dev/stderr
+    df -kh / > /dev/stderr
+    exit 3
+elif [[ "$ROOT_USED_PCT" -ge 99 ]] || [[ "$ROOT_AVAIL_KB" -lt 500000 ]]; then
+    echo -e "\n[!] Warning: Docker root filesystem is running out of space! (${ROOT_USED_PCT}% used on /)" > /dev/stderr
+    echo -e "    you may need to free up space in your Docker VM soon:" > /dev/stderr
+    echo -e "    \$ docker system prune\n" > /dev/stderr
+    df -kh / > /dev/stderr
+fi
+
+export DATA_USAGE="$(df --output=pcent,avail /data | tail -n 1 | xargs)"
+export DATA_USED_PCT="${DATA_USAGE%%%*}"
+export DATA_AVAIL_KB="$(echo "$DATA_USAGE" | awk '{print $2}')"
+if [[ "$DATA_AVAIL_KB" -lt 100000 ]]; then
+    echo -e "\n[!] Warning: Docker data volume is completely out of space! (${DATA_USED_PCT}% used on /data)" > /dev/stderr
+    echo -e "    you need to free up at least 100Mb on the drive holding your data directory" > /dev/stderr
+    echo -e "    \$ ncdu -x data\n" > /dev/stderr
+    df -kh /data > /dev/stderr
+    sleep 5
+elif [[ "$DATA_USED_PCT" -ge 99 ]] || [[ "$ROOT_AVAIL_KB" -lt 500000 ]]; then
+    echo -e "\n[!] Warning: Docker data volume is running out of space! (${DATA_USED_PCT}% used on /data)" > /dev/stderr
+    echo -e "    you may need to free up space on the drive holding your data directory soon" > /dev/stderr
+    echo -e "    \$ ncdu -x data\n" > /dev/stderr
+    df -kh /data > /dev/stderr
+fi
+
+
+export ARCHIVEBOX_BIN_PATH="$(which archivebox)"
 
 # Drop permissions to run commands as the archivebox user
-if [[ "$1" == /* || "$1" == "echo" || "$1" == "archivebox" ]]; then
-    # arg 1 is a binary, execute it verbatim
-    # e.g. "archivebox init"
-    #      "/bin/bash"
-    #      "echo"
-    exec gosu "$ARCHIVEBOX_USER" bash -c "$*"
+if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" == "cat" || "$1" == "whoami" || "$1" == "archivebox" ]]; then
+    # handle "docker run archivebox /bin/somecommand --with=some args" by passing args directly to bash -c
+    # e.g. "docker run archivebox archivebox init:
+    #      "docker run archivebox /venv/bin/ipython3"
+    #      "docker run archivebox /bin/bash -c '...'"
+    #      "docker run archivebox cat /VERSION.txt"
+    exec gosu "$PUID" /bin/bash -c "exec $(printf ' %q' "$@")"
+    # printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113
+    # gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
+    # outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command
 else
-    # no command given, assume args were meant to be passed to archivebox cmd
-    # e.g. "add https://example.com"
-    #      "manage createsupseruser"
-    #      "server 0.0.0.0:8000"
-    exec gosu "$ARCHIVEBOX_USER" bash -c "archivebox $*"
+    # handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand
+    # e.g. "docker run archivebox help"
+    #      "docker run archivebox add --depth=1 https://example.com"
+    #      "docker run archivebox manage createsupseruser"
+    #      "docker run archivebox server 0.0.0.0:8000"
+    exec gosu "$PUID" "$ARCHIVEBOX_BIN_PATH" "$@"
 fi

bin/export_browser_history.sh

@@ -44,7 +44,24 @@ if [[ "$1" == "--firefox" ]]; then
     fi
     
     sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', last_visit_date, 'description', title, 'href', url)) || \"]\" FROM moz_places;" > "$OUTPUT_DIR/firefox_history.json"
-    sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', b.dateAdded, 'description', b.title, 'href', f.url)) || \"]\" FROM moz_bookmarks AS b JOIN moz_places AS f ON f.id = b.fk" > "$OUTPUT_DIR/firefox_bookmarks.json"
+
+    sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "
+    with recursive tags AS (
+          select id, title, '' AS tags
+          FROM moz_bookmarks
+          where parent == 0
+        UNION ALL
+          select c.id, p.title, c.title || ',' || tags AS tags
+          from moz_bookmarks AS c
+          JOIN tags AS p
+          ON c.parent = p.id
+        )
+
+        SELECT '[' || group_concat(json_object('timestamp', b.dateAdded, 'description', b.title, 'href', f.url, 'tags', tags.tags)) || ']'
+        FROM moz_bookmarks AS b
+        JOIN moz_places AS f ON f.id = b.fk
+        JOIN tags ON tags.id = b.parent
+        WHERE f.url LIKE '%://%';" > "$OUTPUT_DIR/firefox_bookmarks.json"
     
     rm "$OUTPUT_DIR"/firefox_history.db.*
     echo "Firefox history exported to:"

bin/release_brew.sh

@@ -11,9 +11,20 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
-SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+
+
+CURRENT_PLAFORM="$(uname)"
+REQUIRED_PLATFORM="Darwin"
+if [[ "$CURRENT_PLAFORM" != "$REQUIRED_PLATFORM" ]]; then
+   echo "[!] Skipping the Homebrew package build on $CURRENT_PLAFORM (it can only be run on $REQUIRED_PLATFORM)."
+   exit 0
+fi
+
 cd "$REPO_DIR"
 
-# TODO
-exit 0
+./bin/build_brew.sh
+
+git add '*.bottle.tar.gz'
+git commit -m "add new release bottle"
+git pull
+git push

bin/release_docker.sh

@@ -11,15 +11,44 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
-SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
 cd "$REPO_DIR"
 
+SUPPORTED_PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
+
+TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
+VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
+SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+GIT_SHA=sha-"$(git rev-parse --short HEAD)"
+SELECTED_PLATFORMS="${2:-$SUPPORTED_PLATFORMS}"
+
+
+# echo "[*] Logging in to Docker Hub & Github Container Registry"
+# docker login --username=nikisweeting
+# docker login ghcr.io --username=pirate
+
+# echo "[^] Building docker image"
+# ./bin/build_docker.sh "$TAG_NAME" "$SELECTED_PLATFORMS"
 
 echo "[^] Uploading docker image"
-# docker login --username=nikisweeting
-# docker login docker.pkg.github.com --username=pirate
-docker push archivebox/archivebox:$VERSION archivebox/archivebox:$SHORT_VERSION archivebox/archivebox:latest
-docker push docker.io/nikisweeting/archivebox
-docker push docker.io/archivebox/archivebox
-docker push docker.pkg.github.com/archivebox/archivebox/archivebox
+docker buildx build --platform "$SELECTED_PLATFORMS" --push . \
+               -t archivebox/archivebox \
+               -t archivebox/archivebox:$TAG_NAME \
+               -t archivebox/archivebox:$VERSION \
+               -t archivebox/archivebox:$SHORT_VERSION \
+               -t archivebox/archivebox:$GIT_SHA \
+               -t archivebox/archivebox:latest \
+               -t archivebox/archivebox:stable \
+               -t nikisweeting/archivebox \
+               -t nikisweeting/archivebox:$TAG_NAME \
+               -t nikisweeting/archivebox:$VERSION \
+               -t nikisweeting/archivebox:$SHORT_VERSION \
+               -t nikisweeting/archivebox:$GIT_SHA \
+               -t nikisweeting/archivebox:latest \
+               -t nikisweeting/archivebox:stable \
+               -t ghcr.io/archivebox/archivebox/archivebox:$TAG_NAME \
+               -t ghcr.io/archivebox/archivebox/archivebox:$VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$SHORT_VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$GIT_SHA \
+               -t ghcr.io/archivebox/archivebox/archivebox:latest \
+               -t ghcr.io/archivebox/archivebox/archivebox:stable
+

bin/release_pip.sh

@@ -11,17 +11,11 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
 cd "$REPO_DIR"
 source "$REPO_DIR/.venv/bin/activate"
 
+echo "[^] Publishing to Test PyPI..."
+pdm publish --repository testpypi
 
-# apt install python3 python3-all python3-dev
-# pip install '.[dev]'
-
-
-echo "[^] Uploading to test.pypi.org"
-python3 -m twine upload --repository testpypi pip_dist/archivebox-${VERSION}*.{whl,tar.gz}
-
-echo "[^] Uploading to pypi.org"
-python3 -m twine upload --repository pypi pip_dist/archivebox-${VERSION}*.{whl,tar.gz}
+echo "[^] Publishing to PyPI..."
+pdm publish --no-build

docker-compose.yml

@@ -1,77 +1,110 @@
 # Usage:
-#     docker-compose run archivebox init --setup
-#     docker-compose up
-#     echo "https://example.com" | docker-compose run archivebox archivebox add
-#     docker-compose run archivebox add --depth=1 https://example.com/some/feed.rss
-#     docker-compose run archivebox config --set PUBLIC_INDEX=True
-#     docker-compose run archivebox help
+#     docker compose run archivebox init --setup
+#     docker compose up
+#     echo "https://example.com" | docker compose run archivebox archivebox add
+#     docker compose run archivebox add --depth=1 https://example.com/some/feed.rss
+#     docker compose run archivebox config --set MEDIA_MAX_SIZE=750m
+#     docker compose run archivebox help
 # Documentation:
 #     https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker-compose
 
-version: '2.4'   # '3.9' or greater also works
-
 services:
     archivebox:
-        # build: .                              # for developers working on archivebox
-        image: ${DOCKER_IMAGE:-archivebox/archivebox:master}
+        #image: ${DOCKER_IMAGE:-archivebox/archivebox:dev}
+        image: archivebox/archivebox:stable
         command: server --quick-init 0.0.0.0:8000
         ports:
             - 8000:8000
-        environment:
-            - ALLOWED_HOSTS=*                   # add any config options you want as env vars
-            - MEDIA_MAX_SIZE=750m
-            # - SEARCH_BACKEND_ENGINE=sonic     # uncomment these if you enable sonic below
-            # - SEARCH_BACKEND_HOST_NAME=sonic
-            # - SEARCH_BACKEND_PASSWORD=SecretPassword
-        # dns:                                  # uncomment to use pihole below for ad/tracker blocking during archiving
-            # - pihole
         volumes:
             - ./data:/data
-            # - ./archivebox:/app/archivebox    # for developers working on archivebox
+            # - ./etc/crontabs:/var/spool/cron/crontabs  # uncomment this and archivebox_scheduler below to set up automatic recurring archive jobs
+            # - ./archivebox:/app/archivebox             # uncomment this to mount the ArchiveBox source code at runtime (for developers working on archivebox)
+        # build: .                                       # uncomment this to build the image from source code at buildtime (for developers working on archivebox)
+        environment:
+            - ALLOWED_HOSTS=*                   # restrict this to only accept incoming traffic via specific domain name
+            # - PUBLIC_INDEX=True               # set to False to prevent anonymous users from viewing snapshot list
+            # - PUBLIC_SNAPSHOTS=True           # set to False to prevent anonymous users from viewing snapshot content
+            # - PUBLIC_ADD_VIEW=False           # set to True to allow anonymous users to submit new URLs to archive
+            # - ADMIN_USERNAME=admin            # create an admin user on first run with the given user/pass combo
+            # - ADMIN_PASSWORD=SomeSecretPassword
+            # - PUID=911                        # set to your host user's UID & GID if you encounter permissions issues
+            # - PGID=911
+            # - SEARCH_BACKEND_ENGINE=sonic     # uncomment these and sonic container below for better full-text search
+            # - SEARCH_BACKEND_HOST_NAME=sonic
+            # - SEARCH_BACKEND_PASSWORD=SomeSecretPassword
+            # - MEDIA_MAX_SIZE=750m             # increase this filesize limit to allow archiving larger audio/video files
+            # - TIMEOUT=60                      # increase this number to 120+ seconds if you see many slow downloads timing out
+            # - CHECK_SSL_VALIDITY=True         # set to False to disable strict SSL checking (allows saving URLs w/ broken certs)
+            # - SAVE_ARCHIVE_DOT_ORG=True       # set to False to disable submitting all URLs to Archive.org when archiving
+            # ...
+            # add further configuration options from archivebox/config.py as needed (to apply them only to this container)
+            # or set using `docker compose run archivebox config --set SOME_KEY=someval` (to persist config across all containers)
+
+        # For ad-blocking during archiving, uncomment this section and pihole service section below 
+        # networks:
+        #   - dns
+        # dns:
+        #   - 172.20.0.53
 
 
-    ### Optional Addons: tweak these examples as needed for your specific use case
+    ######## Optional Addons: tweak examples below as needed for your specific use case ########
 
-    ### Example: To run the Sonic full-text search backend, first download the config file to sonic.cfg
-    # curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/etc/sonic.cfg
-    # after starting, backfill any existing Snapshots into the index: docker-compose run archivebox update --index-only
+    ### Example: To run the Sonic full-text search backend, uncomment this block and the sonic config above
 
     # sonic:
-    #    image: valeriansaliou/sonic:v1.3.1
+    #    image: archivebox/sonic:latest
     #    expose:
     #        - 1491
     #    environment:
-    #        - SEARCH_BACKEND_PASSWORD=SecretPassword
+    #        - SEARCH_BACKEND_PASSWORD=SomeSecretPassword
     #    volumes:
     #        - ./sonic.cfg:/etc/sonic.cfg:ro
     #        - ./data/sonic:/var/lib/sonic/store
-    
-    
+
+
     ### Example: To run pihole in order to block ad/tracker requests during archiving,
     # uncomment this block and set up pihole using its admin interface
 
     # pihole:
     #   image: pihole/pihole:latest
     #   ports:
-    #     - 80:80       # uncomment to access the admin HTTP interface on http://localhost:80
+    #     - 127.0.0.1:8090:80       # uncomment to access the admin HTTP interface on http://localhost:8090
     #   environment:
-    #     WEBPASSWORD: 'set a secure password here or it will be random'
+    #     - WEBPASSWORD=SET_THIS_TO_SOME_SECRET_PASSWORD_FOR_ADMIN_DASHBOARD
+    #     - DNSMASQ_LISTENING=all
+    #   dns:
+    #     - 127.0.0.1
+    #     - 1.1.1.1
+    #   networks:
+    #     dns:
+    #       ipv4_address: 172.20.0.53
     #   volumes:
-    #     - ./data/pihole:/etc/pihole
-    #     - ./data/dnsmasq:/etc/dnsmasq.d
+    #     - ./etc/pihole:/etc/pihole
+    #     - ./etc/dnsmasq:/etc/dnsmasq.d
 
 
-    ### Example: Run scheduled imports in a docker instead of using cron on the
-    #     host machine, add tasks and see more info with archivebox schedule --help
+    ### Example: Enable ability to run regularly scheduled archiving tasks by uncommenting this container
+    #   $ docker compose run archivebox schedule --every=day --depth=1 'https://example.com/some/rss/feed.xml'
+    # then restart the scheduler container to apply the changes to the schedule
+    #   $ docker compose restart archivebox_scheduler
 
-    # scheduler:
-    #    image: archivebox/archivebox:latest
-    #    command: schedule --foreground --every=day --depth=1 'https://getpocket.com/users/USERNAME/feed/all'
+    # archivebox_scheduler:
+    #    image: ${DOCKER_IMAGE:-archivebox/archivebox:dev}
+    #    command: schedule --foreground
     #    environment:
-    #        - USE_COLOR=True
-    #        - SHOW_PROGRESS=False
+    #        - MEDIA_MAX_SIZE=750m               # increase this number to allow archiving larger audio/video files
+    #        # - TIMEOUT=60                      # increase if you see timeouts often during archiving / on slow networks
+    #        # - ONLY_NEW=True                   # set to False to retry previously failed URLs when re-adding instead of skipping them
+    #        # - CHECK_SSL_VALIDITY=True         # set to False to allow saving URLs w/ broken SSL certs
+    #        # - SAVE_ARCHIVE_DOT_ORG=True       # set to False to disable submitting URLs to Archive.org when archiving
+    #        # - PUID=502                        # set to your host user's UID & GID if you encounter permissions issues
+    #        # - PGID=20
     #    volumes:
     #        - ./data:/data
+    #        - ./etc/crontabs:/var/spool/cron/crontabs
+    #    # cpus: 2                               # uncomment / edit these values to limit container resource consumption
+    #    # mem_limit: 2048m
+    #    # shm_size: 1024m
 
 
     ### Example: Put Nginx in front of the ArchiveBox server for SSL termination
@@ -82,14 +115,14 @@ services:
     #         - 443:443
     #         - 80:80
     #     volumes:
-    #         - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf
+    #         - ./etc/nginx.conf:/etc/nginx/nginx.conf
     #         - ./data:/var/www
 
 
     ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel
 
     # wireguard:
-    #   image: linuxserver/wireguard
+    #   image: linuxserver/wireguard:latest
     #   network_mode: 'service:archivebox'
     #   cap_add:
     #     - NET_ADMIN
@@ -114,3 +147,13 @@ services:
     #     volumes:
     #         - ./data:/archivebox
     #         - ./data/wayback:/webarchive
+
+
+networks:
+
+    # network needed for pihole container to offer :53 dns resolving on fixed ip for archivebox container
+    dns:
+        ipam:
+            driver: default
+            config:
+                - subnet: 172.20.0.0/24

etc/crontabs/archivebox

@@ -0,0 +1,8 @@
+# DO NOT EDIT THIS FILE - edit the master and reinstall.
+# (/tmp/tmpe3dawo9u installed on Tue Jun 13 23:21:48 2023)
+# (Cron version -- $Id: crontab.c,v 2.13 1994/01/17 03:20:37 vixie Exp $)
+
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "https://example.com/3" >> /data/logs/schedule.log 2>&1 # archivebox_schedule
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "https://example.com/2" >> /data/logs/schedule.log 2>&1 # archivebox_schedule
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "https://example.com" >> /data/logs/schedule.log 2>&1 # archivebox_schedule
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "update" >> /data/logs/schedule.log 2>&1 # archivebox_schedule

etc/nginx.conf

@@ -34,12 +34,14 @@ http {
     server {
         listen                      80 default_server;
         server_name                 _;
-
-        root                        /var/www;
+        
         index                       index.html;
         autoindex                   on;
-
         try_files                   $uri $uri/ $uri.html =404;
+
+        location /archive {
+            root                    /var/www/archive;
+        }
     }
 }
 

package.json

@@ -1,13 +1,13 @@
 {
   "name": "archivebox",
-  "version": "0.6.3",
+  "version": "0.7.3",
   "description": "ArchiveBox: The self-hosted internet archive",
   "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {
-    "@postlight/mercury-parser": "git+https://github.com/postlight/mercury-parser.git",
-    "readability-extractor": "git+https://github.com/ArchiveBox/readability-extractor.git",
-    "single-file": "git+https://github.com/gildas-lormeau/SingleFile.git"
+    "@postlight/parser": "^2.2.3",
+    "readability-extractor": "github:ArchiveBox/readability-extractor",
+    "single-file-cli": "^1.1.54"
   }
 }

pdm.lock

@@ -0,0 +1,664 @@
+# This file is @generated by PDM.
+# It is not intended for manual editing.
+
+[metadata]
+groups = ["default", "ldap", "sonic"]
+strategy = ["cross_platform", "inherit_metadata"]
+lock_version = "4.5.0"
+content_hash = "sha256:69e9db2cbb19d8e69e92e9821841ebb67d8d1e37aade4f4e6bf8e1e01b5bc079"
+
+[[metadata.targets]]
+requires_python = ">=3.9,<3.12"
+
+[[package]]
+name = "asgiref"
+version = "3.8.1"
+requires_python = ">=3.8"
+summary = "ASGI specs, helper code, and adapters"
+groups = ["default", "ldap"]
+dependencies = [
+    "typing-extensions>=4; python_version < \"3.11\"",
+]
+files = [
+    {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"},
+    {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"},
+]
+
+[[package]]
+name = "asttokens"
+version = "3.0.0"
+requires_python = ">=3.8"
+summary = "Annotate AST trees with source code positions"
+groups = ["default"]
+files = [
+    {file = "asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2"},
+    {file = "asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7"},
+]
+
+[[package]]
+name = "certifi"
+version = "2024.12.14"
+requires_python = ">=3.6"
+summary = "Python package for providing Mozilla's CA Bundle."
+groups = ["default"]
+files = [
+    {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"},
+    {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"},
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.4.0"
+requires_python = ">=3.7.0"
+summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+groups = ["default"]
+files = [
+    {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"},
+    {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"},
+    {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"},
+    {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"},
+    {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"},
+    {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"},
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+summary = "Cross-platform colored terminal text."
+groups = ["default"]
+marker = "sys_platform == \"win32\""
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+
+[[package]]
+name = "croniter"
+version = "5.0.1"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.6"
+summary = "croniter provides iteration for datetime object with cron like format"
+groups = ["default"]
+dependencies = [
+    "python-dateutil",
+    "pytz>2021.1",
+]
+files = [
+    {file = "croniter-5.0.1-py2.py3-none-any.whl", hash = "sha256:eb28439742291f6c10b181df1a5ecf421208b1fc62ef44501daec1780a0b09e9"},
+    {file = "croniter-5.0.1.tar.gz", hash = "sha256:7d9b1ef25b10eece48fdf29d8ac52f9b6252abff983ac614ade4f3276294019e"},
+]
+
+[[package]]
+name = "dateparser"
+version = "1.2.0"
+requires_python = ">=3.7"
+summary = "Date parsing library designed to parse dates from HTML pages"
+groups = ["default"]
+dependencies = [
+    "python-dateutil",
+    "pytz",
+    "regex!=2019.02.19,!=2021.8.27",
+    "tzlocal",
+]
+files = [
+    {file = "dateparser-1.2.0-py2.py3-none-any.whl", hash = "sha256:0b21ad96534e562920a0083e97fd45fa959882d4162acc358705144520a35830"},
+    {file = "dateparser-1.2.0.tar.gz", hash = "sha256:7975b43a4222283e0ae15be7b4999d08c9a70e2d378ac87385b1ccf2cffbbb30"},
+]
+
+[[package]]
+name = "decorator"
+version = "5.1.1"
+requires_python = ">=3.5"
+summary = "Decorators for Humans"
+groups = ["default"]
+files = [
+    {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
+    {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
+]
+
+[[package]]
+name = "django"
+version = "3.1.14"
+requires_python = ">=3.6"
+summary = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design."
+groups = ["default", "ldap"]
+dependencies = [
+    "asgiref<4,>=3.2.10",
+    "pytz",
+    "sqlparse>=0.2.2",
+]
+files = [
+    {file = "Django-3.1.14-py3-none-any.whl", hash = "sha256:0fabc786489af16ad87a8c170ba9d42bfd23f7b699bd5ef05675864e8d012859"},
+    {file = "Django-3.1.14.tar.gz", hash = "sha256:72a4a5a136a214c39cf016ccdd6b69e2aa08c7479c66d93f3a9b5e4bb9d8a347"},
+]
+
+[[package]]
+name = "django-auth-ldap"
+version = "4.1.0"
+requires_python = ">=3.7"
+summary = "Django LDAP authentication backend."
+groups = ["ldap"]
+dependencies = [
+    "Django>=2.2",
+    "python-ldap>=3.1",
+]
+files = [
+    {file = "django-auth-ldap-4.1.0.tar.gz", hash = "sha256:77f749d3b17807ce8eb56a9c9c8e5746ff316567f81d5ba613495d9c7495a949"},
+    {file = "django_auth_ldap-4.1.0-py3-none-any.whl", hash = "sha256:68870e7921e84b1a9867e268a9c8a3e573e8a0d95ea08bcf31be178f5826ff36"},
+]
+
+[[package]]
+name = "django-extensions"
+version = "3.1.5"
+requires_python = ">=3.6"
+summary = "Extensions for Django"
+groups = ["default"]
+dependencies = [
+    "Django>=2.2",
+]
+files = [
+    {file = "django-extensions-3.1.5.tar.gz", hash = "sha256:28e1e1bf49f0e00307ba574d645b0af3564c981a6dfc87209d48cb98f77d0b1a"},
+    {file = "django_extensions-3.1.5-py3-none-any.whl", hash = "sha256:9238b9e016bb0009d621e05cf56ea8ce5cce9b32e91ad2026996a7377ca28069"},
+]
+
+[[package]]
+name = "exceptiongroup"
+version = "1.2.2"
+requires_python = ">=3.7"
+summary = "Backport of PEP 654 (exception groups)"
+groups = ["default"]
+marker = "python_version < \"3.11\""
+files = [
+    {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
+    {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
+]
+
+[[package]]
+name = "executing"
+version = "2.1.0"
+requires_python = ">=3.8"
+summary = "Get the currently executing AST node of a frame, and other information"
+groups = ["default"]
+files = [
+    {file = "executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf"},
+    {file = "executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"},
+]
+
+[[package]]
+name = "idna"
+version = "3.10"
+requires_python = ">=3.6"
+summary = "Internationalized Domain Names in Applications (IDNA)"
+groups = ["default"]
+files = [
+    {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"},
+    {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"},
+]
+
+[[package]]
+name = "ipython"
+version = "8.18.1"
+requires_python = ">=3.9"
+summary = "IPython: Productive Interactive Computing"
+groups = ["default"]
+dependencies = [
+    "colorama; sys_platform == \"win32\"",
+    "decorator",
+    "exceptiongroup; python_version < \"3.11\"",
+    "jedi>=0.16",
+    "matplotlib-inline",
+    "pexpect>4.3; sys_platform != \"win32\"",
+    "prompt-toolkit<3.1.0,>=3.0.41",
+    "pygments>=2.4.0",
+    "stack-data",
+    "traitlets>=5",
+    "typing-extensions; python_version < \"3.10\"",
+]
+files = [
+    {file = "ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"},
+    {file = "ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27"},
+]
+
+[[package]]
+name = "jedi"
+version = "0.19.2"
+requires_python = ">=3.6"
+summary = "An autocompletion tool for Python that can be used for text editors."
+groups = ["default"]
+dependencies = [
+    "parso<0.9.0,>=0.8.4",
+]
+files = [
+    {file = "jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9"},
+    {file = "jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0"},
+]
+
+[[package]]
+name = "matplotlib-inline"
+version = "0.1.7"
+requires_python = ">=3.8"
+summary = "Inline Matplotlib backend for Jupyter"
+groups = ["default"]
+dependencies = [
+    "traitlets",
+]
+files = [
+    {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"},
+    {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"},
+]
+
+[[package]]
+name = "mypy-extensions"
+version = "1.0.0"
+requires_python = ">=3.5"
+summary = "Type system extensions for programs checked with the mypy type checker."
+groups = ["default"]
+files = [
+    {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
+    {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
+]
+
+[[package]]
+name = "parso"
+version = "0.8.4"
+requires_python = ">=3.6"
+summary = "A Python Parser"
+groups = ["default"]
+files = [
+    {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"},
+    {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"},
+]
+
+[[package]]
+name = "pexpect"
+version = "4.9.0"
+summary = "Pexpect allows easy control of interactive console applications."
+groups = ["default"]
+marker = "sys_platform != \"win32\""
+dependencies = [
+    "ptyprocess>=0.5",
+]
+files = [
+    {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"},
+    {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"},
+]
+
+[[package]]
+name = "prompt-toolkit"
+version = "3.0.48"
+requires_python = ">=3.7.0"
+summary = "Library for building powerful interactive command lines in Python"
+groups = ["default"]
+dependencies = [
+    "wcwidth",
+]
+files = [
+    {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"},
+    {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"},
+]
+
+[[package]]
+name = "ptyprocess"
+version = "0.7.0"
+summary = "Run a subprocess in a pseudo terminal"
+groups = ["default"]
+marker = "sys_platform != \"win32\""
+files = [
+    {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
+    {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"},
+]
+
+[[package]]
+name = "pure-eval"
+version = "0.2.3"
+summary = "Safely evaluate AST nodes without side effects"
+groups = ["default"]
+files = [
+    {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"},
+    {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"},
+]
+
+[[package]]
+name = "pyasn1"
+version = "0.6.1"
+requires_python = ">=3.8"
+summary = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
+groups = ["ldap"]
+files = [
+    {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"},
+    {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"},
+]
+
+[[package]]
+name = "pyasn1-modules"
+version = "0.4.1"
+requires_python = ">=3.8"
+summary = "A collection of ASN.1-based protocols modules"
+groups = ["ldap"]
+dependencies = [
+    "pyasn1<0.7.0,>=0.4.6",
+]
+files = [
+    {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"},
+    {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"},
+]
+
+[[package]]
+name = "pygments"
+version = "2.18.0"
+requires_python = ">=3.8"
+summary = "Pygments is a syntax highlighting package written in Python."
+groups = ["default"]
+files = [
+    {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"},
+    {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"},
+]
+
+[[package]]
+name = "python-crontab"
+version = "3.2.0"
+summary = "Python Crontab API"
+groups = ["default"]
+dependencies = [
+    "python-dateutil",
+]
+files = [
+    {file = "python_crontab-3.2.0-py3-none-any.whl", hash = "sha256:82cb9b6a312d41ff66fd3caf3eed7115c28c195bfb50711bc2b4b9592feb9fe5"},
+    {file = "python_crontab-3.2.0.tar.gz", hash = "sha256:40067d1dd39ade3460b2ad8557c7651514cd3851deffff61c5c60e1227c5c36b"},
+]
+
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+summary = "Extensions to the standard Python datetime module"
+groups = ["default"]
+dependencies = [
+    "six>=1.5",
+]
+files = [
+    {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
+    {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
+]
+
+[[package]]
+name = "python-ldap"
+version = "3.4.4"
+requires_python = ">=3.6"
+summary = "Python modules for implementing LDAP clients"
+groups = ["ldap"]
+dependencies = [
+    "pyasn1-modules>=0.1.5",
+    "pyasn1>=0.3.7",
+]
+files = [
+    {file = "python-ldap-3.4.4.tar.gz", hash = "sha256:7edb0accec4e037797705f3a05cbf36a9fde50d08c8f67f2aef99a2628fab828"},
+]
+
+[[package]]
+name = "pytz"
+version = "2024.2"
+summary = "World timezone definitions, modern and historical"
+groups = ["default", "ldap"]
+files = [
+    {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"},
+    {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"},
+]
+
+[[package]]
+name = "regex"
+version = "2024.11.6"
+requires_python = ">=3.8"
+summary = "Alternative regular expression module, to replace re."
+groups = ["default"]
+files = [
+    {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"},
+    {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"},
+    {file = "regex-2024.11.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164d8b7b3b4bcb2068b97428060b2a53be050085ef94eca7f240e7947f1b080e"},
+    {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3660c82f209655a06b587d55e723f0b813d3a7db2e32e5e7dc64ac2a9e86fde"},
+    {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d22326fcdef5e08c154280b71163ced384b428343ae16a5ab2b3354aed12436e"},
+    {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1ac758ef6aebfc8943560194e9fd0fa18bcb34d89fd8bd2af18183afd8da3a2"},
+    {file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997d6a487ff00807ba810e0f8332c18b4eb8d29463cfb7c820dc4b6e7562d0cf"},
+    {file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02a02d2bb04fec86ad61f3ea7f49c015a0681bf76abb9857f945d26159d2968c"},
+    {file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f02f93b92358ee3f78660e43b4b0091229260c5d5c408d17d60bf26b6c900e86"},
+    {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06eb1be98df10e81ebaded73fcd51989dcf534e3c753466e4b60c4697a003b67"},
+    {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:040df6fe1a5504eb0f04f048e6d09cd7c7110fef851d7c567a6b6e09942feb7d"},
+    {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabbfc59f2c6edba2a6622c647b716e34e8e3867e0ab975412c5c2f79b82da2"},
+    {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8447d2d39b5abe381419319f942de20b7ecd60ce86f16a23b0698f22e1b70008"},
+    {file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da8f5fc57d1933de22a9e23eec290a0d8a5927a5370d24bda9a6abe50683fe62"},
+    {file = "regex-2024.11.6-cp310-cp310-win32.whl", hash = "sha256:b489578720afb782f6ccf2840920f3a32e31ba28a4b162e13900c3e6bd3f930e"},
+    {file = "regex-2024.11.6-cp310-cp310-win_amd64.whl", hash = "sha256:5071b2093e793357c9d8b2929dfc13ac5f0a6c650559503bb81189d0a3814519"},
+    {file = "regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638"},
+    {file = "regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7"},
+    {file = "regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20"},
+    {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114"},
+    {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3"},
+    {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f"},
+    {file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0"},
+    {file = "regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55"},
+    {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89"},
+    {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d"},
+    {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34"},
+    {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d"},
+    {file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45"},
+    {file = "regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9"},
+    {file = "regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60"},
+    {file = "regex-2024.11.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5704e174f8ccab2026bd2f1ab6c510345ae8eac818b613d7d73e785f1310f839"},
+    {file = "regex-2024.11.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:220902c3c5cc6af55d4fe19ead504de80eb91f786dc102fbd74894b1551f095e"},
+    {file = "regex-2024.11.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7e351589da0850c125f1600a4c4ba3c722efefe16b297de54300f08d734fbf"},
+    {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5056b185ca113c88e18223183aa1a50e66507769c9640a6ff75859619d73957b"},
+    {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e34b51b650b23ed3354b5a07aab37034d9f923db2a40519139af34f485f77d0"},
+    {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5670bce7b200273eee1840ef307bfa07cda90b38ae56e9a6ebcc9f50da9c469b"},
+    {file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08986dce1339bc932923e7d1232ce9881499a0e02925f7402fb7c982515419ef"},
+    {file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93c0b12d3d3bc25af4ebbf38f9ee780a487e8bf6954c115b9f015822d3bb8e48"},
+    {file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:764e71f22ab3b305e7f4c21f1a97e1526a25ebdd22513e251cf376760213da13"},
+    {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f056bf21105c2515c32372bbc057f43eb02aae2fda61052e2f7622c801f0b4e2"},
+    {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:69ab78f848845569401469da20df3e081e6b5a11cb086de3eed1d48f5ed57c95"},
+    {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:86fddba590aad9208e2fa8b43b4c098bb0ec74f15718bb6a704e3c63e2cef3e9"},
+    {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:684d7a212682996d21ca12ef3c17353c021fe9de6049e19ac8481ec35574a70f"},
+    {file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a03e02f48cd1abbd9f3b7e3586d97c8f7a9721c436f51a5245b3b9483044480b"},
+    {file = "regex-2024.11.6-cp39-cp39-win32.whl", hash = "sha256:41758407fc32d5c3c5de163888068cfee69cb4c2be844e7ac517a52770f9af57"},
+    {file = "regex-2024.11.6-cp39-cp39-win_amd64.whl", hash = "sha256:b2837718570f95dd41675328e111345f9b7095d821bac435aac173ac80b19983"},
+    {file = "regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519"},
+]
+
+[[package]]
+name = "requests"
+version = "2.32.3"
+requires_python = ">=3.8"
+summary = "Python HTTP for Humans."
+groups = ["default"]
+dependencies = [
+    "certifi>=2017.4.17",
+    "charset-normalizer<4,>=2",
+    "idna<4,>=2.5",
+    "urllib3<3,>=1.21.1",
+]
+files = [
+    {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
+    {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
+]
+
+[[package]]
+name = "six"
+version = "1.17.0"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+summary = "Python 2 and 3 compatibility utilities"
+groups = ["default"]
+files = [
+    {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
+    {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
+]
+
+[[package]]
+name = "sonic-client"
+version = "1.0.0"
+summary = "python client for sonic search backend"
+groups = ["sonic"]
+files = [
+    {file = "sonic-client-1.0.0.tar.gz", hash = "sha256:fe324c7354670488ed84847f6a6727d3cb5fb3675cb9b61396dcf5720e5aca66"},
+    {file = "sonic_client-1.0.0-py3-none-any.whl", hash = "sha256:291bf292861e97a2dd765ff0c8754ea9631383680d31a63ec3da6f5aa5f4beda"},
+]
+
+[[package]]
+name = "sqlparse"
+version = "0.5.3"
+requires_python = ">=3.8"
+summary = "A non-validating SQL parser."
+groups = ["default", "ldap"]
+files = [
+    {file = "sqlparse-0.5.3-py3-none-any.whl", hash = "sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca"},
+    {file = "sqlparse-0.5.3.tar.gz", hash = "sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272"},
+]
+
+[[package]]
+name = "stack-data"
+version = "0.6.3"
+summary = "Extract data from python stack frames and tracebacks for informative displays"
+groups = ["default"]
+dependencies = [
+    "asttokens>=2.1.0",
+    "executing>=1.2.0",
+    "pure-eval",
+]
+files = [
+    {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"},
+    {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"},
+]
+
+[[package]]
+name = "traitlets"
+version = "5.14.3"
+requires_python = ">=3.8"
+summary = "Traitlets Python configuration system"
+groups = ["default"]
+files = [
+    {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"},
+    {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"},
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.12.2"
+requires_python = ">=3.8"
+summary = "Backported and Experimental Type Hints for Python 3.8+"
+groups = ["default", "ldap"]
+marker = "python_version < \"3.11\""
+files = [
+    {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
+    {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
+]
+
+[[package]]
+name = "tzdata"
+version = "2024.2"
+requires_python = ">=2"
+summary = "Provider of IANA time zone data"
+groups = ["default"]
+marker = "platform_system == \"Windows\""
+files = [
+    {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"},
+    {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"},
+]
+
+[[package]]
+name = "tzlocal"
+version = "5.2"
+requires_python = ">=3.8"
+summary = "tzinfo object for the local timezone"
+groups = ["default"]
+dependencies = [
+    "backports-zoneinfo; python_version < \"3.9\"",
+    "tzdata; platform_system == \"Windows\"",
+]
+files = [
+    {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"},
+    {file = "tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"},
+]
+
+[[package]]
+name = "urllib3"
+version = "2.2.3"
+requires_python = ">=3.8"
+summary = "HTTP library with thread-safe connection pooling, file post, and more."
+groups = ["default"]
+files = [
+    {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"},
+    {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"},
+]
+
+[[package]]
+name = "w3lib"
+version = "2.2.1"
+requires_python = ">=3.8"
+summary = "Library of web-related functions"
+groups = ["default"]
+files = [
+    {file = "w3lib-2.2.1-py3-none-any.whl", hash = "sha256:e56d81c6a6bf507d7039e0c95745ab80abd24b465eb0f248af81e3eaa46eb510"},
+    {file = "w3lib-2.2.1.tar.gz", hash = "sha256:756ff2d94c64e41c8d7c0c59fea12a5d0bc55e33a531c7988b4a163deb9b07dd"},
+]
+
+[[package]]
+name = "wcwidth"
+version = "0.2.13"
+summary = "Measures the displayed width of unicode strings in a terminal"
+groups = ["default"]
+dependencies = [
+    "backports-functools-lru-cache>=1.2.1; python_version < \"3.2\"",
+]
+files = [
+    {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"},
+    {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"},
+]
+
+[[package]]
+name = "yt-dlp"
+version = "2024.12.13"
+requires_python = ">=3.9"
+summary = "A feature-rich command-line audio/video downloader"
+groups = ["default"]
+files = [
+    {file = "yt_dlp-2024.12.13-py3-none-any.whl", hash = "sha256:5a16b7511e8500cbb13ff0babc9c6deb1e049dc1c854a51738aad2529167fcdf"},
+    {file = "yt_dlp-2024.12.13.tar.gz", hash = "sha256:77e15afb9d460ecb7294a39bb5e39dc9f4e8a65f3a37ef4db58800b94d095511"},
+]

pyproject.toml

@@ -0,0 +1,122 @@
+[project]
+name = "archivebox"
+version = "0.7.3"
+description = "Self-hosted internet archiving solution."
+authors = [
+    {name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"},
+]
+dependencies = [
+    "croniter>=0.3.34",
+    "dateparser>=1.0.0",
+    "django-extensions>=3.0.3",
+    "django>=3.1.3,<3.2",
+    "ipython>5.0.0",
+    "mypy-extensions>=0.4.3",
+    "python-crontab>=2.5.1",
+    "requests>=2.24.0",
+    "w3lib>=1.22.0",
+    "yt-dlp>=2023.10.13",
+    # "playwright>=1.39.0; platform_machine != 'armv7l'",
+]
+requires-python = ">=3.9,<3.12"
+readme = "README.md"
+license = {text = "MIT"}
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Framework :: Django",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: End Users/Desktop",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Legal Industry",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: MIT License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
+    "Topic :: Sociology :: History",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: System :: Archiving",
+    "Topic :: System :: Archiving :: Backup",
+    "Topic :: System :: Recovery Tools",
+    "Topic :: Utilities",
+    "Typing :: Typed",
+]
+
+# pdm lock -G:all
+# pdm install -G:all
+[tool.pdm.dev-dependencies]
+dev = [
+    # build
+    "setuptools>=69.0.3",
+    "wheel",
+    "pdm",
+    "homebrew-pypi-poet>=0.10.0",
+    # docs
+    "recommonmark",
+    "sphinx",
+    "sphinx-rtd-theme",
+    # debug
+    "django-debug-toolbar",
+    "djdt_flamegraph",
+    "ipdb",
+    # test
+    "pytest",
+    # lint
+    "flake8",
+    "mypy",
+    "django-stubs",
+]
+
+[tool.pdm.scripts]
+lint = "./bin/lint.sh"
+test = "./bin/test.sh"
+# all = {composite = ["lint mypackage/", "test -v tests/"]}
+
+[project.optional-dependencies]
+sonic = [
+    # echo "deb [signed-by=/usr/share/keyrings/valeriansaliou_sonic.gpg] https://packagecloud.io/valeriansaliou/sonic/debian/ bookworm main" > /etc/apt/sources.list.d/valeriansaliou_sonic.list
+    # curl -fsSL https://packagecloud.io/valeriansaliou/sonic/gpgkey | gpg --dearmor -o /usr/share/keyrings/valeriansaliou_sonic.gpg
+    "sonic-client>=0.0.5",
+]
+ldap = [
+    # apt install libldap2-dev libsasl2-dev
+    "python-ldap>=3.4.3",
+    "django-auth-ldap>=4.1.0",
+]
+# playwright = [
+#     platform_machine isnt respected by pdm export -o requirements.txt, this breaks arm/v7
+#     "playwright>=1.39.0; platform_machine != 'armv7l'",
+# ]
+
+[project.scripts]
+archivebox = "archivebox.cli:main"
+
+[build-system]
+requires = ["pdm-backend"]
+build-backend = "pdm.backend"
+
+
+[project.urls]
+Homepage = "https://github.com/ArchiveBox/ArchiveBox"
+Source = "https://github.com/ArchiveBox/ArchiveBox"
+Documentation = "https://github.com/ArchiveBox/ArchiveBox/wiki"
+"Bug Tracker" = "https://github.com/ArchiveBox/ArchiveBox/issues"
+Changelog = "https://github.com/ArchiveBox/ArchiveBox/releases"
+Roadmap = "https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap"
+Community = "https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community"
+Demo = "https://demo.archivebox.io"
+Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
+
+

requirements.txt

@@ -0,0 +1,47 @@
+# This file is @generated by PDM.
+# Please do not edit it manually.
+
+asgiref==3.8.1
+asttokens==3.0.0
+certifi==2024.12.14
+charset-normalizer==3.4.0
+colorama==0.4.6; sys_platform == "win32"
+croniter==5.0.1
+dateparser==1.2.0
+decorator==5.1.1
+django==3.1.14
+django-auth-ldap==4.1.0
+django-extensions==3.1.5
+exceptiongroup==1.2.2; python_version < "3.11"
+executing==2.1.0
+idna==3.10
+ipython==8.18.1
+jedi==0.19.2
+matplotlib-inline==0.1.7
+mypy-extensions==1.0.0
+parso==0.8.4
+pexpect==4.9.0; sys_platform != "win32"
+prompt-toolkit==3.0.48
+ptyprocess==0.7.0; sys_platform != "win32"
+pure-eval==0.2.3
+pyasn1==0.6.1
+pyasn1-modules==0.4.1
+pygments==2.18.0
+python-crontab==3.2.0
+python-dateutil==2.9.0.post0
+python-ldap==3.4.4
+pytz==2024.2
+regex==2024.11.6
+requests==2.32.3
+six==1.17.0
+sonic-client==1.0.0
+sqlparse==0.5.3
+stack-data==0.6.3
+traitlets==5.14.3
+typing-extensions==4.12.2; python_version < "3.11"
+tzdata==2024.2; platform_system == "Windows"
+tzlocal==5.2
+urllib3==2.2.3
+w3lib==2.2.1
+wcwidth==0.2.13
+yt-dlp==2024.12.13

setup.py

@@ -1,145 +0,0 @@
-import json
-import setuptools
-from setuptools.command.test import test
-
-from pathlib import Path
-
-
-PKG_NAME = "archivebox"
-DESCRIPTION = "Self-hosted internet archiving solution."
-LICENSE = "MIT"
-AUTHOR = "Nick Sweeting"
-AUTHOR_EMAIL="git@nicksweeting.com"
-REPO_URL = "https://github.com/ArchiveBox/ArchiveBox"
-PROJECT_URLS = {
-    "Source":           f"{REPO_URL}",
-    "Documentation":    f"{REPO_URL}/wiki",
-    "Bug Tracker":      f"{REPO_URL}/issues",
-    "Changelog":        f"{REPO_URL}/releases",
-    "Roadmap":          f"{REPO_URL}/wiki/Roadmap",
-    "Community":        f"{REPO_URL}/wiki/Web-Archiving-Community",
-    "Demo":             f"https://demo.archivebox.io",
-    "Donate":           f"{REPO_URL}/wiki/Donations",
-}
-
-ROOT_DIR = Path(__file__).parent.resolve()
-PACKAGE_DIR = ROOT_DIR / PKG_NAME
-
-README = (PACKAGE_DIR / "README.md").read_text(encoding='utf-8', errors='ignore')
-VERSION = json.loads((PACKAGE_DIR / "package.json").read_text().strip())['version']
-
-PYTHON_REQUIRES = ">=3.7"
-SETUP_REQUIRES = ["wheel"]
-INSTALL_REQUIRES = [
-    # only add things here that have corresponding apt python3-packages available
-    # anything added here also needs to be added to our package dependencies in
-    # stdeb.cfg (apt), archivebox.rb (brew), Dockerfile, etc.
-    # if there is no apt python3-package equivalent, then vendor it instead in
-    # ./archivebox/vendor/
-    "requests>=2.24.0",
-    "mypy-extensions>=0.4.3",
-    "django>=3.1.3,<3.2",
-    "django-extensions>=3.0.3",
-    "dateparser>=1.0.0",
-    "youtube-dl>=2021.04.17",
-    "yt-dlp>=2021.4.11",
-    "python-crontab>=2.5.1",
-    "croniter>=0.3.34",
-    "w3lib>=1.22.0",
-    "ipython>5.0.0",
-]
-EXTRAS_REQUIRE = {
-    'sonic': [
-        "sonic-client>=0.0.5",
-    ],
-    'dev': [
-        "setuptools",
-        "twine",
-        "wheel",
-        "flake8",
-        "ipdb",
-        "mypy",
-        "django-stubs",
-        "sphinx",
-        "sphinx-rtd-theme",
-        "recommonmark",
-        "pytest",
-        "bottle",
-        "stdeb",
-        "django-debug-toolbar",
-        "djdt_flamegraph",
-    ],
-}
-
-# To see when setup.py gets called (uncomment for debugging):
-# import sys
-# print(PACKAGE_DIR, f"     (v{VERSION})")
-# print('>', sys.executable, *sys.argv)
-
-
-class DisabledTestCommand(test):
-    def run(self):
-        # setup.py test is deprecated, disable it here by force so stdeb doesnt run it
-        print()
-        print('[X] Running tests via setup.py test is deprecated.')
-        print('    Hint: Use the ./bin/test.sh script or pytest instead')
-
-
-setuptools.setup(
-    name=PKG_NAME,
-    version=VERSION,
-    license=LICENSE,
-    author=AUTHOR,
-    author_email=AUTHOR_EMAIL,
-    description=DESCRIPTION,
-    long_description=README,
-    long_description_content_type="text/markdown",
-    url=REPO_URL,
-    project_urls=PROJECT_URLS,
-    python_requires=PYTHON_REQUIRES,
-    setup_requires=SETUP_REQUIRES,
-    install_requires=INSTALL_REQUIRES,
-    extras_require=EXTRAS_REQUIRE,
-    packages=[PKG_NAME],
-    include_package_data=True,   # see MANIFEST.in
-    entry_points={
-        "console_scripts": [
-            f"{PKG_NAME} = {PKG_NAME}.cli:main",
-        ],
-    },
-    classifiers=[
-        "License :: OSI Approved :: MIT License",
-        "Natural Language :: English",
-        "Operating System :: OS Independent",
-        "Development Status :: 4 - Beta",
-
-        "Topic :: Utilities",
-        "Topic :: System :: Archiving",
-        "Topic :: System :: Archiving :: Backup",
-        "Topic :: System :: Recovery Tools",
-        "Topic :: Sociology :: History",
-        "Topic :: Internet :: WWW/HTTP",
-        "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
-        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-
-        "Intended Audience :: Developers",
-        "Intended Audience :: Education",
-        "Intended Audience :: End Users/Desktop",
-        "Intended Audience :: Information Technology",
-        "Intended Audience :: Legal Industry",
-        "Intended Audience :: System Administrators",
-        
-        "Environment :: Console",
-        "Environment :: Web Environment",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Framework :: Django",
-        "Typing :: Typed",
-    ],
-    cmdclass={
-        "test": DisabledTestCommand,
-    },
-)

stdeb.cfg

@@ -1,11 +0,0 @@
-[DEFAULT]
-Source: archivebox
-Package: archivebox
-Package3: archivebox
-Suite: focal
-Suite3: focal
-Build-Depends: debhelper, dh-python, python3-all, python3-pip, python3-setuptools, python3-wheel, python3-stdeb
-Depends3: nodejs, wget, curl, git, ffmpeg, youtube-dl, yt-dlp, python3-all, python3-pip, python3-setuptools, python3-croniter, python3-crontab, python3-dateparser, python3-django, python3-django-extensions, python3-django-jsonfield, python3-mypy-extensions, python3-requests, python3-w3lib, ripgrep
-X-Python3-Version: >= 3.7
-XS-Python-Version: >= 3.7
-Setup-Env-Vars: DEB_BUILD_OPTIONS=nocheck

tests/fixtures.py

@@ -17,6 +17,7 @@ def disable_extractors_dict():
         "USE_SINGLEFILE": "false",
         "USE_READABILITY": "false",
         "USE_MERCURY": "false",
+        "SAVE_HTMLTOTEXT": "false",
         "SAVE_PDF": "false",
         "SAVE_SCREENSHOT": "false",
         "SAVE_DOM": "false",

tests/test_extractors.py

@@ -13,12 +13,51 @@ def test_ignore_methods():
     Takes the passed method out of the default methods list and returns that value
     """
     ignored = ignore_methods(['title'])
-    assert should_save_title not in ignored
+    assert "title" not in ignored
+
+def test_save_allowdenylist_works(tmp_path, process, disable_extractors_dict):
+    allow_list = {
+        r'/static': ["headers", "singlefile"],
+        r'example\.com\.html$': ["headers"],
+    }
+    deny_list = {
+        "/static": ["singlefile"],
+    }
+    disable_extractors_dict.update({
+        "SAVE_HEADERS": "true",
+        "USE_SINGLEFILE": "true",
+        "SAVE_ALLOWLIST": pyjson.dumps(allow_list),
+        "SAVE_DENYLIST": pyjson.dumps(deny_list),
+    })
+    add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],
+                                  capture_output=True, env=disable_extractors_dict) 
+    archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
+    singlefile_file = archived_item_path / "singlefile.html"
+    assert not singlefile_file.exists()
+    headers_file = archived_item_path / "headers.json"
+    assert headers_file.exists()
+
+def test_save_denylist_works(tmp_path, process, disable_extractors_dict):
+    deny_list = {
+        "/static": ["singlefile"],
+    }
+    disable_extractors_dict.update({
+        "SAVE_HEADERS": "true",
+        "USE_SINGLEFILE": "true",
+        "SAVE_DENYLIST": pyjson.dumps(deny_list),
+    })
+    add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],
+                                  capture_output=True, env=disable_extractors_dict) 
+    archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
+    singlefile_file = archived_item_path / "singlefile.html"
+    assert not singlefile_file.exists()
+    headers_file = archived_item_path / "headers.json"
+    assert headers_file.exists()
 
 def test_singlefile_works(tmp_path, process, disable_extractors_dict):
     disable_extractors_dict.update({"USE_SINGLEFILE": "true"})
     add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],
-                                  capture_output=True, env=disable_extractors_dict) 
+                                  capture_output=True, env=disable_extractors_dict)
     archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
     output_file = archived_item_path / "singlefile.html" 
     assert output_file.exists()
@@ -39,6 +78,14 @@ def test_mercury_works(tmp_path, process, disable_extractors_dict):
     output_file = archived_item_path / "mercury" / "content.html"
     assert output_file.exists()
 
+def test_htmltotext_works(tmp_path, process, disable_extractors_dict):
+    disable_extractors_dict.update({"SAVE_HTMLTOTEXT": "true"})
+    add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],
+                                  capture_output=True, env=disable_extractors_dict)
+    archived_item_path = list(tmp_path.glob("archive/**/*"))[0]
+    output_file = archived_item_path / "htmltotext.txt"
+    assert output_file.exists()
+
 def test_readability_works_with_wget(tmp_path, process, disable_extractors_dict):
     disable_extractors_dict.update({"USE_READABILITY": "true", "USE_WGET": "true"})
     add_process = subprocess.run(['archivebox', 'add', 'http://127.0.0.1:8080/static/example.com.html'],

tests/test_oneshot.py

@@ -25,4 +25,36 @@ def test_oneshot_command_saves_page_in_right_folder(tmp_path, disable_extractors
     assert "index.json" in items
     assert not "index.sqlite3" in current_path
     assert "output.html" in items
-    
+
+def test_oneshot_command_succeeds(tmp_path, disable_extractors_dict):
+    disable_extractors_dict.update({"SAVE_DOM": "true"})
+    process = subprocess.run(
+        [
+            "archivebox",
+            "oneshot",
+            f"--out-dir={tmp_path}",
+            "--extract=title,favicon,dom",
+            "http://127.0.0.1:8080/static/example.com.html",
+        ],
+        capture_output=True,
+        env=disable_extractors_dict,
+    )
+
+    assert process.returncode == 0
+
+def test_oneshot_command_logs_archiving_finished(tmp_path, disable_extractors_dict):
+    disable_extractors_dict.update({"SAVE_DOM": "true"})
+    process = subprocess.run(
+        [
+            "archivebox",
+            "oneshot",
+            f"--out-dir={tmp_path}",
+            "--extract=title,favicon,dom",
+            "http://127.0.0.1:8080/static/example.com.html",
+        ],
+        capture_output=True,
+        env=disable_extractors_dict,
+    )
+
+    output_str = process.stdout.decode("utf-8")
+    assert "4 files" in output_str

website/README.md

@@ -0,0 +1 @@
+../README.md