fix docker release script

Co-authored-by: Nick Sweeting <git@sweeting.me>

.dockerignore

@@ -5,17 +5,27 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .github/
+.pdm-build/
+.pdm-python
+.eggs/
+.git/
+!.git/HEAD
+!.git/refs/heads/*
 
 venv/
 .venv/
+.venv-old/
 .docker-venv/
+node_modules/
 
+docs/
 build/
 dist/
-pip_dist/
-!pip_dist/archivebox.egg-info/requires.txt
 brew_dist/
+deb_dist/
+pip_dist/
 assets/
+docker/
 
 data/
 output/

.flake8

@@ -1,6 +0,0 @@
-[flake8]
-ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391
-select = F,E9,W
-max-line-length = 130
-max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv

.github/workflows/debian.yml

@@ -9,10 +9,10 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: true
           fetch-depth: 1
@@ -25,36 +25,23 @@ jobs:
             dh-python debhelper devscripts dput software-properties-common \
             python3-distutils python3-setuptools python3-wheel python3-stdeb
 
-      - name: Build Debian/Apt sdist_dsc
-        run: |
-          rm -Rf deb_dist/*
-          python3 setup.py --command-packages=stdeb.command sdist_dsc
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
 
-      - name: Build Debian/Apt bdist_deb
-        run: |
-          python3 setup.py --command-packages=stdeb.command bdist_deb
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
 
-      - name: Install archivebox from deb
-        run: |
-          cd deb_dist/
-          sudo apt-get install ./archivebox*.deb
-
-      - name: Check ArchiveBox version
-        run: |
-          # must create dir needed for snaps to run as non-root on github actions
-          sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-          mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-          archivebox init
-          archivebox config --set SAVE_READABILITY=False
-          archivebox config --set SAVE_MERCURY=False
-          archivebox config --set SAVE_SINGLEFILE=False
-          archivebox --version
-
-      - name: Add some links to test
-        run: |
-          cd "${{ github.workspace }}/data"
-          archivebox add 'https://example.com'
-          archivebox status
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
 
       # - name: Commit built package
       #   run: |

.github/workflows/docker.yml

@@ -11,27 +11,27 @@ on:
 
 env:
   DOCKER_IMAGE: archivebox-ci
-
-
+      
 jobs:
   buildx:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
       
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
@@ -40,7 +40,7 @@ jobs:
         run: echo ${{ steps.buildx.outputs.platforms }}
       
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -48,7 +48,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Docker Login
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
            username: ${{ secrets.DOCKER_USERNAME }}
@@ -56,20 +56,19 @@ jobs:
       
       - name: Collect Docker tags
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v5
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          flavor: |
-              latest=auto
           tags: |
               type=ref,event=branch
               type=semver,pattern={{version}}
               type=semver,pattern={{major}}.{{minor}}
               type=sha
+              type=raw,value=latest,enable={{is_default_branch}}
       
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
@@ -77,8 +76,18 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
           platforms: linux/amd64,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
+       
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/homebrew.yml

@@ -23,11 +23,12 @@ jobs:
           cd brew_dist/
           brew install --build-bottle ./archivebox.rb
           # brew bottle archivebox
+          archivebox version
 
       - name: Add some links to test
         run: |
           mkdir data && cd data
-          archivebox init
+          archivebox init --setup
           archivebox add 'https://example.com'
           archivebox version
           archivebox status

.github/workflows/jekyll-gh-pages.yml

@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/lint.yml

@@ -28,7 +28,8 @@ jobs:
 
       - name: Lint with flake8
         run: |
+          cd archivebox
           # one pass for show-stopper syntax errors or undefined names
-          flake8 archivebox --count --show-source --statistics
+          flake8 . --count --show-source --statistics
           # one pass for small stylistic things
-          flake8 archivebox --count --max-line-length="$MAX_LINE_LENGTH" --statistics
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics

.github/workflows/pip.yml

@@ -7,29 +7,42 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
+
+    permissions:
+      id-token: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: '3.11'
           architecture: x64
 
-      - name: Build Python Package
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
+      - uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
+
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --group :all --no-self
+
+      - name: Build package
         run: |
-          pip3 install --upgrade pip setuptools wheel
-          rm -Rf pip_dist/*.whl
-          python3 setup.py \
-            sdist --dist-dir=./pip_dist \
-            bdist_wheel --dist-dir=./pip_dist \
-            egg_info --egg-base=./pip_dist
-          pip install pip_dist/archivebox-*.whl
+          pdm build
+
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl
 
       - name: Add some links to test
         run: |
@@ -39,6 +52,15 @@ jobs:
           archivebox version
           archivebox status
 
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
+
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
+
       # - name: Commit built package
       #   run: |
       #     cd pip_dist/
@@ -54,8 +76,3 @@ jobs:
       #     branch: ${{ github.ref }}
       #     directory: pip_dist
 
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}

.github/workflows/test.yml

@@ -13,26 +13,33 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
-        python: [3.7]
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
-      - name: Set up Node JS 14.7.0
-        uses: actions/setup-node@v1
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          node-version: 14.7.0
+          node-version: 20.10.0
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
       ### Install Python & JS Dependencies
       - name: Get pip cache dir
@@ -41,7 +48,7 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-pip
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -49,11 +56,16 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
       - name: Install pip dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel pytest bottle
-          ./bin/build_pip.sh
-          python -m pip install .
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
 
       - name: Get npm cache dir
         id: npm-cache
@@ -61,7 +73,7 @@ jobs:
           echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
 
       - name: Cache npm
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-npm
         with:
           path: ${{ steps.npm-cache.outputs.dir }}
@@ -90,13 +102,13 @@ jobs:
         # TODO: remove this exception for windows once we get tests passing on that platform
         if: ${{ !contains(matrix.os, 'windows') }}
         run: |
-          python -m pytest -s --ignore=archivebox/vendor
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
 
   docker_tests:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1

.gitignore

@@ -13,6 +13,8 @@ venv/
 node_modules/
 
 # Packaging artifacts
+.pdm-python
+.pdm-build
 archivebox.egg-info
 archivebox-*.tar.gz
 build/
@@ -24,3 +26,6 @@ data1/
 data2/
 data3/
 output/
+
+# vim
+*.sw?

.readthedocs.yaml

@@ -0,0 +1,34 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements.txt
+    - requirements: docs/requirements.txt

Dockerfile

@@ -1,125 +1,297 @@
 # This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, single-file
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
 # Usage:
+#     git submodule update --init --recursive
+#     git pull --recurse-submodules
 #     docker build . -t archivebox --no-cache
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
+# Multi-arch build:
+#     docker buildx create --use
+#     docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
-FROM python:3.9-slim-buster
+
+# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
+FROM python:3.11-slim-bookworm
 
 LABEL name="archivebox" \
-    maintainer="Nick Sweeting <archivebox-docker@sweeting.me>" \
+    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
     description="All-in-one personal internet archiving container" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
     documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
 
-# System-level base config
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+######### Environment Variables #################################
+
+# Global system-level config
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
     PYTHONIOENCODING=UTF-8 \
     PYTHONUNBUFFERED=1 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error
 
-# Application-level base config
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=20
+
+# User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911
+
+# Global paths
 ENV CODE_DIR=/app \
-    VENV_PATH=/venv \
     DATA_DIR=/data \
-    NODE_DIR=/node \
-    ARCHIVEBOX_USER="archivebox"
+    GLOBAL_VENV=/venv \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
+
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
+
+# Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
 
 # Create non-privileged user for archivebox and chrome
-RUN groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
 
-# Install system dependencies
-RUN apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        apt-transport-https ca-certificates gnupg2 zlib1g-dev \
-        dumb-init gosu cron unzip curl \
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        # nano iputils-ping dnsutils htop procps jq yq
     && rm -rf /var/lib/apt/lists/*
 
-# Install apt dependencies
-RUN apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        wget curl chromium git ffmpeg youtube-dl ripgrep \
-        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-    && rm -rf /var/lib/apt/lists/*
+######### Language Environments ####################################
 
 # Install Node environment
-RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
-    && echo 'deb https://deb.nodesource.com/node_15.x buster main' >> /etc/apt/sources.list \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
     && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        nodejs \
-    # && npm install -g npm \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        nodejs libatomic1 python3-minimal \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    && ( \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Extractor Dependencies ##################################
+
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        curl wget git yt-dlp ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+        # fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+    && apt-get update -qq \
+    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
+        # install Chromium using playwright
+        pip install playwright \
+        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+        && playwright install --with-deps chromium \
+        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
+    else \
+        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.) 
+        apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+            chromium fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        && export CHROME_BINARY="$(which chromium)"; \
+    fi \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # Save version info
+    && ( \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
 
 # Install Node dependencies
-WORKDIR "$NODE_DIR"
-ENV PATH="${PATH}:$NODE_DIR/node_modules/.bin" \
-    npm_config_loglevel=error
-ADD ./package.json ./package.json
-ADD ./package-lock.json ./package-lock.json
-RUN npm ci
-
-# Install Python dependencies
 WORKDIR "$CODE_DIR"
-ENV PATH="${PATH}:$VENV_PATH/bin"
-RUN python -m venv --clear --symlinks "$VENV_PATH" \
-    && pip install --upgrade --quiet pip setuptools
-ADD "./setup.py" "$CODE_DIR/"
-ADD "./package.json" "$CODE_DIR/archivebox/"
-RUN apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential python-dev python3-dev \
-    && echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \
-    && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
-    && pip install --quiet -r /tmp/requirements.txt \
-    && apt-get purge -y build-essential python-dev python3-dev \
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
+        which node && node --version \
+        && which npm && npm version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Build Dependencies ####################################
+
+# Install ArchiveBox Python dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        build-essential \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
-# Install apt development dependencies
-# RUN apt-get install -qq \
-#     && apt-get install -qq -y --no-install-recommends \
-#         python3 python3-dev python3-pip python3-venv python3-all \
-#         dh-python debhelper devscripts dput software-properties-common \
-#         python3-distutils python3-setuptools python3-wheel python3-stdeb
-# RUN python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.extras_require["dev"]))' > /tmp/dev_requirements.txt \
-    # && pip install --quiet -r /tmp/dev_requirements.txt
+# Install ArchiveBox Python package from source
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    #     build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install ArchiveBox Python package and its dependencies
-WORKDIR "$CODE_DIR"
-ADD . "$CODE_DIR"
-RUN pip install -e .
+####################################################
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True \
-    CHROME_SANDBOX=False \
-    CHROME_BINARY="chromium" \
-    USE_SINGLEFILE=True \
-    SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \
-    USE_READABILITY=True \
-    READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor" \
-    USE_MERCURY=True \
-    MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser"
+ENV IN_DOCKER=True
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # CHROME_SANDBOX=False \
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
 
 # Print version for nice docker finish summary
-# RUN archivebox version
-RUN /app/bin/docker_entrypoint.sh archivebox version
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
 
 # Open up the interfaces to the outside world
+WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
+# Optional:
+# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+#     CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
 CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

MANIFEST.in

@@ -1,6 +0,0 @@
-graft archivebox
-global-exclude .DS_Store
-global-exclude __pycache__
-global-exclude *.pyc
-
-prune tests/

Pipfile

@@ -1,12 +0,0 @@
-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
-verify_ssl = true
-
-[packages]
-# see setup.py for package dependency list
-"e1839a8" = {path = ".", editable = true}
-
-[dev-packages]
-# see setup.py for dev package dependency list
-"e1839a8" = {path = ".", extras = ["dev"], editable = true}

SECURITY.md

@@ -0,0 +1,34 @@
+# Security Policy
+
+---
+
+## Security Information
+
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+
+- https://github.com/ArchiveBox/ArchiveBox#archive-layout
+- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
+
+---
+
+## Reporting a Vulnerability
+
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
+
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
+
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
+
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
+
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-merlot

archivebox/.flake8

@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,static,venv,.venv,.venv2,.docker-venv
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv

archivebox/cli/archivebox_add.py

@@ -30,11 +30,17 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
     )
     parser.add_argument(
-        '--update-all', #'-n',
+        '--update', #'-u',
         action='store_true',
         default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
+    parser.add_argument(
+        '--update-all', #'-n',
+        action='store_true',
+        default=False, 
+        help="Also update ALL links in index when finished adding new links",
+    )
     parser.add_argument(
         '--index-only', #'-o',
         action='store_true',
@@ -104,6 +110,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         urls=stdin_urls or urls,
         depth=command.depth,
         tag=command.tag,
+        update=command.update,
         update_all=command.update_all,
         index_only=command.index_only,
         overwrite=command.overwrite,

archivebox/cli/archivebox_schedule.py

@@ -39,6 +39,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=None,
         help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
     )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
     parser.add_argument(
         '--depth', # '-d',
         type=int,
@@ -51,6 +57,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
     )
+    parser.add_argument(
+        '--update',
+        action='store_true',
+        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
+    )
     group.add_argument(
         '--clear', # '-c'
         action='store_true',
@@ -92,8 +103,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         run_all=command.run_all,
         quiet=command.quiet,
         every=command.every,
+        tag=command.tag,
         depth=command.depth,
         overwrite=command.overwrite,
+        update=command.update,
         import_path=command.import_path,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/config.py

@@ -26,11 +26,13 @@ import io
 import re
 import sys
 import json
+import inspect
 import getpass
 import platform
 import shutil
-import sqlite3
+import requests
 import django
+from sqlite3 import dbapi2 as sqlite3
 
 from hashlib import md5
 from pathlib import Path
@@ -39,6 +41,7 @@ from typing import Optional, Type, Tuple, Dict, Union, List
 from subprocess import run, PIPE, DEVNULL
 from configparser import ConfigParser
 from collections import defaultdict
+import importlib.metadata
 
 from .config_stubs import (
     SimpleConfigValueDict,
@@ -48,6 +51,7 @@ from .config_stubs import (
     ConfigDefaultDict,
 )
 
+
 ############################### Config Schema ##################################
 
 CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
@@ -56,7 +60,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'USE_COLOR':                {'type': bool,  'default': lambda c: c['IS_TTY']},
         'SHOW_PROGRESS':            {'type': bool,  'default': lambda c: (c['IS_TTY'] and platform.system() != 'Darwin')},  # progress bars are buggy on mac, disable for now
         'IN_DOCKER':                {'type': bool,  'default': False},
-        # TODO: 'SHOW_HINTS':       {'type:  bool,  'default': True},
+        'IN_QEMU':                  {'type': bool,  'default': False},
+        'PUID':                     {'type': int,   'default': os.getuid()},
+        'PGID':                     {'type': int,   'default': os.getgid()},
     },
 
     'GENERAL_CONFIG': {
@@ -65,23 +71,47 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'ONLY_NEW':                 {'type': bool,  'default': True},
         'TIMEOUT':                  {'type': int,   'default': 60},
         'MEDIA_TIMEOUT':            {'type': int,   'default': 3600},
-        'OUTPUT_PERMISSIONS':       {'type': str,   'default': '755'},
+        'OUTPUT_PERMISSIONS':       {'type': str,   'default': '644'},
         'RESTRICT_FILE_NAMES':      {'type': str,   'default': 'windows'},
-        'URL_BLACKLIST':            {'type': str,   'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$'},  # to avoid downloading code assets as their own pages
+
+        'URL_DENYLIST':             {'type': str,   'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', 'aliases': ('URL_BLACKLIST',)},  # to avoid downloading code assets as their own pages
+        'URL_ALLOWLIST':            {'type': str,   'default': None, 'aliases': ('URL_WHITELIST',)},
+
+        'ADMIN_USERNAME':           {'type': str,   'default': None},
+        'ADMIN_PASSWORD':           {'type': str,   'default': None},
+
+        'ENFORCE_ATOMIC_WRITES':    {'type': bool,  'default': True},
+        'TAG_SEPARATOR_PATTERN':    {'type': str,   'default': r'[,]'},
     },
 
     'SERVER_CONFIG': {
-        'SECRET_KEY':               {'type': str,   'default': None},
-        'BIND_ADDR':                {'type': str,   'default': lambda c: ['127.0.0.1:8000', '0.0.0.0:8000'][c['IN_DOCKER']]},
-        'ALLOWED_HOSTS':            {'type': str,   'default': '*'},
-        'DEBUG':                    {'type': bool,  'default': False},
-        'PUBLIC_INDEX':             {'type': bool,  'default': True},
-        'PUBLIC_SNAPSHOTS':         {'type': bool,  'default': True},
-        'PUBLIC_ADD_VIEW':          {'type': bool,  'default': False},
-        'FOOTER_INFO':              {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
-        'SNAPSHOTS_PER_PAGE':       {'type': int,   'default': 40},
-        'CUSTOM_TEMPLATES_DIR':     {'type': str,   'default': None},
-        'TIME_ZONE':                {'type': str,   'default': 'UTC'},
+        'SECRET_KEY':                {'type': str,   'default': None},
+        'BIND_ADDR':                 {'type': str,   'default': lambda c: ['127.0.0.1:8000', '0.0.0.0:8000'][c['IN_DOCKER']]},
+        'ALLOWED_HOSTS':             {'type': str,   'default': '*'},
+        'DEBUG':                     {'type': bool,  'default': False},
+        'PUBLIC_INDEX':              {'type': bool,  'default': True},
+        'PUBLIC_SNAPSHOTS':          {'type': bool,  'default': True},
+        'PUBLIC_ADD_VIEW':           {'type': bool,  'default': False},
+        'FOOTER_INFO':               {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
+        'SNAPSHOTS_PER_PAGE':        {'type': int,   'default': 40},
+        'CUSTOM_TEMPLATES_DIR':      {'type': str,   'default': None},
+        'TIME_ZONE':                 {'type': str,   'default': 'UTC'},
+        'TIMEZONE':                  {'type': str,   'default': 'UTC'},
+        'REVERSE_PROXY_USER_HEADER': {'type': str,   'default': 'Remote-User'},
+        'REVERSE_PROXY_WHITELIST':   {'type': str,   'default': ''},
+        'LOGOUT_REDIRECT_URL':       {'type': str,   'default': '/'},
+        'PREVIEW_ORIGINALS':         {'type': bool,  'default': True},
+
+        'LDAP':                      {'type': bool,  'default': False},
+        'LDAP_SERVER_URI':           {'type': str,   'default': None},
+        'LDAP_BIND_DN':              {'type': str,   'default': None},
+        'LDAP_BIND_PASSWORD':        {'type': str,   'default': None},
+        'LDAP_USER_BASE':            {'type': str,   'default': None},
+        'LDAP_USER_FILTER':          {'type': str,   'default': None},
+        'LDAP_USERNAME_ATTR':        {'type': str,   'default': None},
+        'LDAP_FIRSTNAME_ATTR':       {'type': str,   'default': None},
+        'LDAP_LASTNAME_ATTR':        {'type': str,   'default': None},
+        'LDAP_EMAIL_ATTR':           {'type': str,   'default': None},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -92,6 +122,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SAVE_SINGLEFILE':          {'type': bool,  'default': True, 'aliases': ('FETCH_SINGLEFILE',)},
         'SAVE_READABILITY':         {'type': bool,  'default': True, 'aliases': ('FETCH_READABILITY',)},
         'SAVE_MERCURY':             {'type': bool,  'default': True, 'aliases': ('FETCH_MERCURY',)},
+        'SAVE_HTMLTOTEXT':          {'type': bool,  'default': True, 'aliases': ('FETCH_HTMLTOTEXT',)},
         'SAVE_PDF':                 {'type': bool,  'default': True, 'aliases': ('FETCH_PDF',)},
         'SAVE_SCREENSHOT':          {'type': bool,  'default': True, 'aliases': ('FETCH_SCREENSHOT',)},
         'SAVE_DOM':                 {'type': bool,  'default': True, 'aliases': ('FETCH_DOM',)},
@@ -100,6 +131,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SAVE_GIT':                 {'type': bool,  'default': True, 'aliases': ('FETCH_GIT',)},
         'SAVE_MEDIA':               {'type': bool,  'default': True, 'aliases': ('FETCH_MEDIA',)},
         'SAVE_ARCHIVE_DOT_ORG':     {'type': bool,  'default': True, 'aliases': ('SUBMIT_ARCHIVE_DOT_ORG',)},
+        'SAVE_ALLOWLIST':           {'type': dict,  'default': {},},
+        'SAVE_DENYLIST':            {'type': dict,  'default': {},},
     },
 
     'ARCHIVE_METHOD_OPTIONS': {
@@ -108,13 +141,14 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
         'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
 
+        'CHROME_TIMEOUT':           {'type': int,   'default': 0},
         'CHROME_HEADLESS':          {'type': bool,  'default': True},
         'CHROME_SANDBOX':           {'type': bool,  'default': lambda c: not c['IN_DOCKER']},
         'YOUTUBEDL_ARGS':           {'type': list,  'default': lambda c: [
@@ -124,17 +158,22 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--write-thumbnail',
                                                                 '--no-call-home',
                                                                 '--write-sub',
-                                                                '--all-subs',
-                                                                '--write-auto-sub',
+                                                                '--write-auto-subs',
                                                                 '--convert-subs=srt',
                                                                 '--yes-playlist',
                                                                 '--continue',
+                                                                # This flag doesn't exist in youtube-dl
+                                                                # only in yt-dlp
+                                                                '--no-abort-on-error',
+                                                                # --ignore-errors must come AFTER
+                                                                # --no-abort-on-error
+                                                                # https://github.com/yt-dlp/yt-dlp/issues/4914
                                                                 '--ignore-errors',
                                                                 '--geo-bypass',
                                                                 '--add-metadata',
-                                                                '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']),
+                                                                '--format=(bv*+ba/b)[filesize<={}][filesize_approx<=?{}]/(bv*+ba/b)'.format(c['MEDIA_MAX_SIZE'], c['MEDIA_MAX_SIZE']),
                                                                 ]},
-                                                                    
+
 
         'WGET_ARGS':                {'type': list,  'default': ['--no-verbose',
                                                                 '--adjust-extension',
@@ -150,6 +189,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--compressed'
                                                                ]},
         'GIT_ARGS':                 {'type': list,  'default': ['--recursive']},
+        'SINGLEFILE_ARGS':          {'type': list,  'default' : None},
+        'FAVICON_PROVIDER':         {'type': str,   'default': 'https://www.google.com/s2/favicons?domain={}'},
     },
 
     'SEARCH_BACKEND_CONFIG' : {
@@ -159,10 +200,16 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'SEARCH_BACKEND_HOST_NAME': {'type': str,   'default': 'localhost'},
         'SEARCH_BACKEND_PORT':      {'type': int,   'default': 1491},
         'SEARCH_BACKEND_PASSWORD':  {'type': str,   'default': 'SecretPassword'},
+        'SEARCH_PROCESS_HTML':      {'type': bool,  'default': True},
         # SONIC
         'SONIC_COLLECTION':         {'type': str,   'default': 'archivebox'},
         'SONIC_BUCKET':             {'type': str,   'default': 'snapshots'},
         'SEARCH_BACKEND_TIMEOUT':   {'type': int,   'default': 90},
+        # SQLite3 FTS5
+        'FTS_SEPARATE_DATABASE':    {'type': bool,  'default': True},
+        'FTS_TOKENIZERS':           {'type': str,   'default': 'porter unicode61 remove_diacritics 2'},
+        # Default from https://www.sqlite.org/limits.html#max_length
+        'FTS_SQLITE_MAX_LENGTH':    {'type': int,   'default': int(1e9)},
     },
 
     'DEPENDENCY_CONFIG': {
@@ -176,20 +223,22 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'USE_NODE':                 {'type': bool,  'default': True},
         'USE_YOUTUBEDL':            {'type': bool,  'default': True},
         'USE_RIPGREP':              {'type': bool,  'default': True},
-        
+
         'CURL_BINARY':              {'type': str,   'default': 'curl'},
         'GIT_BINARY':               {'type': str,   'default': 'git'},
-        'WGET_BINARY':              {'type': str,   'default': 'wget'},
+        'WGET_BINARY':              {'type': str,   'default': 'wget'},     # also can accept wget2
         'SINGLEFILE_BINARY':        {'type': str,   'default': lambda c: bin_path('single-file')},
         'READABILITY_BINARY':       {'type': str,   'default': lambda c: bin_path('readability-extractor')},
-        'MERCURY_BINARY':           {'type': str,   'default': lambda c: bin_path('mercury-parser')},
-        'YOUTUBEDL_BINARY':         {'type': str,   'default': 'youtube-dl'},
+        'MERCURY_BINARY':           {'type': str,   'default': lambda c: bin_path('postlight-parser')},
+        'YOUTUBEDL_BINARY':         {'type': str,   'default': 'yt-dlp'},   # also can accept youtube-dl
         'NODE_BINARY':              {'type': str,   'default': 'node'},
         'RIPGREP_BINARY':           {'type': str,   'default': 'rg'},
         'CHROME_BINARY':            {'type': str,   'default': None},
 
         'POCKET_CONSUMER_KEY':      {'type': str,   'default': None},
         'POCKET_ACCESS_TOKENS':     {'type': dict,  'default': {}},
+
+        'READWISE_READER_TOKENS':   {'type': dict,  'default': {}},
     },
 }
 
@@ -257,7 +306,7 @@ STATICFILE_EXTENSIONS = {
     # that can be downloaded as-is, not html pages that need to be rendered
     'gif', 'jpeg', 'jpg', 'png', 'tif', 'tiff', 'wbmp', 'ico', 'jng', 'bmp',
     'svg', 'svgz', 'webp', 'ps', 'eps', 'ai',
-    'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v', 
+    'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v',
     'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8',
     'pdf', 'txt', 'rtf', 'rtfd', 'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx',
     'atom', 'rss', 'css', 'js', 'json',
@@ -266,7 +315,7 @@ STATICFILE_EXTENSIONS = {
 
     # Less common extensions to consider adding later
     # jar, swf, bin, com, exe, dll, deb
-    # ear, hqx, eot, wmlc, kml, kmz, cco, jardiff, jnlp, run, msi, msp, msm, 
+    # ear, hqx, eot, wmlc, kml, kmz, cco, jardiff, jnlp, run, msi, msp, msm,
     # pl pm, prc pdb, rar, rpm, sea, sit, tcl tk, der, pem, crt, xpi, xspf,
     # ra, mng, asx, asf, 3gpp, 3gp, mid, midi, kar, jad, wml, htc, mml
 
@@ -274,14 +323,160 @@ STATICFILE_EXTENSIONS = {
     # html, htm, shtml, xhtml, xml, aspx, php, cgi
 }
 
+# When initializing archivebox in a new directory, we check to make sure the dir is
+# actually empty so that we dont clobber someone's home directory or desktop by accident.
+# These files are exceptions to the is_empty check when we're trying to init a new dir,
+# as they could be from a previous archivebox version, system artifacts, dependencies, etc.
+ALLOWED_IN_OUTPUT_DIR = {
+    '.gitignore',
+    'lost+found',
+    '.DS_Store',
+    '.venv',
+    'venv',
+    'virtualenv',
+    '.virtualenv',
+    'node_modules',
+    'package.json',
+    'package-lock.json',
+    'yarn.lock',
+    'static',
+    'sonic',
+    'search.sqlite3',
+    ARCHIVE_DIR_NAME,
+    SOURCES_DIR_NAME,
+    LOGS_DIR_NAME,
+    SQL_INDEX_FILENAME,
+    f'{SQL_INDEX_FILENAME}-wal',
+    f'{SQL_INDEX_FILENAME}-shm',
+    JSON_INDEX_FILENAME,
+    HTML_INDEX_FILENAME,
+    ROBOTS_TXT_FILENAME,
+    FAVICON_FILENAME,
+    CONFIG_FILENAME,
+    f'{CONFIG_FILENAME}.bak',
+    'static_index.json',
+}
+
+
+ALLOWDENYLIST_REGEX_FLAGS: int = re.IGNORECASE | re.UNICODE | re.MULTILINE
+
+
+############################## Version Config ##################################
+
+def get_system_user():
+    SYSTEM_USER = getpass.getuser() or os.getlogin()
+    try:
+        import pwd
+        return pwd.getpwuid(os.geteuid()).pw_name or SYSTEM_USER
+    except KeyError:
+        # Process' UID might not map to a user in cases such as running the Docker image
+        # (where `archivebox` is 999) as a different UID.
+        pass
+    except ModuleNotFoundError:
+        # pwd doesn't exist on windows
+        pass
+    except Exception:
+        # this should never happen, uncomment to debug
+        # raise
+        pass
+
+    return SYSTEM_USER
+
+def get_version(config):
+    try:
+        return importlib.metadata.version(__package__ or 'archivebox')
+    except importlib.metadata.PackageNotFoundError:
+        try:
+            pyproject_config = (config['PACKAGE_DIR'] / 'pyproject.toml').read_text()
+            for line in pyproject_config:
+                if line.startswith('version = '):
+                    return line.split(' = ', 1)[-1].strip('"')
+        except FileNotFoundError:
+            # building docs, pyproject.toml is not available
+            return 'dev'
+
+    raise Exception('Failed to detect installed archivebox version!')
+
+def get_commit_hash(config) -> Optional[str]:
+    try:
+        git_dir = config['PACKAGE_DIR'] / '../.git'
+        ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
+        commit_hash = git_dir.joinpath(ref).read_text().strip()
+        return commit_hash
+    except Exception:
+        pass
+
+    try:
+        return list((config['PACKAGE_DIR'] / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
+    except Exception:
+        pass
+    
+    return None
+
+def get_build_time(config) -> str:
+    if config['IN_DOCKER']:
+        docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
+        return docker_build_end_time
+
+    src_last_modified_unix_timestamp = (config['PACKAGE_DIR'] / 'config.py').stat().st_mtime
+    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
+
+def get_versions_available_on_github(config):
+    """
+    returns a dictionary containing the ArchiveBox GitHub release info for
+    the recommended upgrade version and the currently installed version
+    """
+    
+    # we only want to perform the (relatively expensive) check for new versions
+    # when its most relevant, e.g. when the user runs a long-running command
+    subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
+    long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
+    if subcommand_run_by_user not in long_running_commands:
+        return None
+    
+    github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
+    response = requests.get(github_releases_api)
+    if response.status_code != 200:
+        stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
+        return None
+    all_releases = response.json()
+
+    installed_version = parse_version_string(config['VERSION'])
+
+    # find current version or nearest older version (to link to)
+    current_version = None
+    for idx, release in enumerate(all_releases):
+        release_version = parse_version_string(release['tag_name'])
+        if release_version <= installed_version:
+            current_version = release
+            break
+
+    current_version = current_version or all_releases[-1]
+    
+    # recommended version is whatever comes after current_version in the release list
+    # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
+    try:
+        recommended_version = all_releases[idx+1]
+    except IndexError:
+        recommended_version = None
+
+    return {'recommended_version': recommended_version, 'current_version': current_version}
+
+def can_upgrade(config):
+    if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
+        recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
+        current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
+        return recommended_version > current_version
+    return False
 
 
 ############################## Derived Config ##################################
 
-
+# These are derived/computed values calculated *after* all user-provided config values are ingested
+# they appear in `archivebox config` output and are intended to be read-only for the user
 DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'TERM_WIDTH':               {'default': lambda c: lambda: shutil.get_terminal_size((100, 10)).columns},
-    'USER':                     {'default': lambda c: getpass.getuser() or os.getlogin()},
+    'USER':                     {'default': lambda c: get_system_user()},
     'ANSI':                     {'default': lambda c: DEFAULT_CLI_COLORS if c['USE_COLOR'] else {k: '' for k in DEFAULT_CLI_COLORS.keys()}},
 
     'PACKAGE_DIR':              {'default': lambda c: Path(__file__).resolve().parent},
@@ -295,17 +490,30 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CONFIG_FILE':              {'default': lambda c: Path(c['CONFIG_FILE']).resolve() if c['CONFIG_FILE'] else c['OUTPUT_DIR'] / CONFIG_FILENAME},
     'COOKIES_FILE':             {'default': lambda c: c['COOKIES_FILE'] and Path(c['COOKIES_FILE']).resolve()},
     'CHROME_USER_DATA_DIR':     {'default': lambda c: find_chrome_data_dir() if c['CHROME_USER_DATA_DIR'] is None else (Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None)},   # None means unset, so we autodetect it with find_chrome_Data_dir(), but emptystring '' means user manually set it to '', and we should store it as None
-    'URL_BLACKLIST_PTN':        {'default': lambda c: c['URL_BLACKLIST'] and re.compile(c['URL_BLACKLIST'] or '', re.IGNORECASE | re.UNICODE | re.MULTILINE)},
+    'URL_DENYLIST_PTN':         {'default': lambda c: c['URL_DENYLIST'] and re.compile(c['URL_DENYLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
+    'URL_ALLOWLIST_PTN':        {'default': lambda c: c['URL_ALLOWLIST'] and re.compile(c['URL_ALLOWLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
+    'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},  # exec is always needed to list directories
 
     'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0] or bin_path('archivebox')},
-    'VERSION':                  {'default': lambda c: json.loads((Path(c['PACKAGE_DIR']) / 'package.json').read_text(encoding='utf-8').strip())['version']},
+
+    'VERSION':                  {'default': lambda c: get_version(c).split('+', 1)[0]},     # remove +editable from user-displayed version string
+    'COMMIT_HASH':              {'default': lambda c: get_commit_hash(c)},                  # short git commit hash of codebase HEAD commit
+    'BUILD_TIME':               {'default': lambda c: get_build_time(c)},                   # docker build completed time or python src last modified time
+    
+    'VERSIONS_AVAILABLE':       {'default': lambda c: get_versions_available_on_github(c)},
+    'CAN_UPGRADE':              {'default': lambda c: can_upgrade(c)},
 
     'PYTHON_BINARY':            {'default': lambda c: sys.executable},
     'PYTHON_ENCODING':          {'default': lambda c: sys.stdout.encoding.upper()},
     'PYTHON_VERSION':           {'default': lambda c: '{}.{}.{}'.format(*sys.version_info[:3])},
 
-    'DJANGO_BINARY':            {'default': lambda c: django.__file__.replace('__init__.py', 'bin/django-admin.py')},
+    'DJANGO_BINARY':            {'default': lambda c: inspect.getfile(django)},
     'DJANGO_VERSION':           {'default': lambda c: '{}.{}.{} {} ({})'.format(*django.VERSION)},
+    
+    'SQLITE_BINARY':            {'default': lambda c: inspect.getfile(sqlite3)},
+    'SQLITE_VERSION':           {'default': lambda c: sqlite3.version},
+    #'SQLITE_JOURNAL_MODE':      {'default': lambda c: 'wal'},         # set at runtime below, interesting if changed later but unused for now because its always expected to be wal
+    #'SQLITE_OPTIONS':           {'default': lambda c: ['JSON1']},     # set at runtime below
 
     'USE_CURL':                 {'default': lambda c: c['USE_CURL'] and (c['SAVE_FAVICON'] or c['SAVE_TITLE'] or c['SAVE_ARCHIVE_DOT_ORG'])},
     'CURL_VERSION':             {'default': lambda c: bin_version(c['CURL_BINARY']) if c['USE_CURL'] else None},
@@ -326,12 +534,13 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
 
     'USE_SINGLEFILE':           {'default': lambda c: c['USE_SINGLEFILE'] and c['SAVE_SINGLEFILE']},
     'SINGLEFILE_VERSION':       {'default': lambda c: bin_version(c['SINGLEFILE_BINARY']) if c['USE_SINGLEFILE'] else None},
+    'SINGLEFILE_ARGS':          {'default': lambda c: c['SINGLEFILE_ARGS'] or []},
 
     'USE_READABILITY':          {'default': lambda c: c['USE_READABILITY'] and c['SAVE_READABILITY']},
     'READABILITY_VERSION':      {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None},
 
     'USE_MERCURY':              {'default': lambda c: c['USE_MERCURY'] and c['SAVE_MERCURY']},
-    'MERCURY_VERSION':          {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None},  # mercury is unversioned
+    'MERCURY_VERSION':          {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None},  # mercury doesnt expose version info until this is merged https://github.com/postlight/parser/pull/750
 
     'USE_GIT':                  {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']},
     'GIT_VERSION':              {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None},
@@ -345,14 +554,15 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] or find_chrome_binary()},
     'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and c['CHROME_BINARY'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
     'CHROME_VERSION':           {'default': lambda c: bin_version(c['CHROME_BINARY']) if c['USE_CHROME'] else None},
-    
+    'CHROME_USER_AGENT':        {'default': lambda c: c['CHROME_USER_AGENT'].format(**c)},
+
     'SAVE_PDF':                 {'default': lambda c: c['USE_CHROME'] and c['SAVE_PDF']},
     'SAVE_SCREENSHOT':          {'default': lambda c: c['USE_CHROME'] and c['SAVE_SCREENSHOT']},
     'SAVE_DOM':                 {'default': lambda c: c['USE_CHROME'] and c['SAVE_DOM']},
     'SAVE_SINGLEFILE':          {'default': lambda c: c['USE_CHROME'] and c['SAVE_SINGLEFILE'] and c['USE_NODE']},
     'SAVE_READABILITY':         {'default': lambda c: c['USE_READABILITY'] and c['USE_NODE']},
     'SAVE_MERCURY':             {'default': lambda c: c['USE_MERCURY'] and c['USE_NODE']},
-    
+
     'USE_NODE':                 {'default': lambda c: c['USE_NODE'] and (c['SAVE_READABILITY'] or c['SAVE_SINGLEFILE'] or c['SAVE_MERCURY'])},
     'NODE_VERSION':             {'default': lambda c: bin_version(c['NODE_BINARY']) if c['USE_NODE'] else None},
 
@@ -361,10 +571,11 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'EXTERNAL_LOCATIONS':       {'default': lambda c: get_external_locations(c)},
     'DATA_LOCATIONS':           {'default': lambda c: get_data_locations(c)},
     'CHROME_OPTIONS':           {'default': lambda c: get_chrome_info(c)},
+    'SAVE_ALLOWLIST_PTN':       {'default': lambda c: c['SAVE_ALLOWLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_ALLOWLIST'].items()}},
+    'SAVE_DENYLIST_PTN':        {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
 }
 
 
-
 ################################### Helpers ####################################
 
 
@@ -377,47 +588,60 @@ def load_config_val(key: str,
                     config_file_vars: Optional[Dict[str, str]]=None) -> ConfigValue:
     """parse bool, int, and str key=value pairs from env"""
 
+    assert isinstance(config, dict)
 
+    is_read_only = type is None
+    if is_read_only:
+        if callable(default):
+            return default(config)
+        return default
+
+    # get value from environment variables or config files
     config_keys_to_check = (key, *(aliases or ()))
+    val = None
     for key in config_keys_to_check:
         if env_vars:
             val = env_vars.get(key)
             if val:
                 break
+
         if config_file_vars:
             val = config_file_vars.get(key)
             if val:
                 break
 
-    if type is None or val is None:
+    is_unset = val is None
+    if is_unset:
         if callable(default):
-            assert isinstance(config, dict)
             return default(config)
-
         return default
 
-    elif type is bool:
-        if val.lower() in ('true', 'yes', '1'):
+    # calculate value based on expected type
+    BOOL_TRUEIES = ('true', 'yes', '1')
+    BOOL_FALSEIES = ('false', 'no', '0')
+
+    if type is bool:
+        if val.lower() in BOOL_TRUEIES:
             return True
-        elif val.lower() in ('false', 'no', '0'):
+        elif val.lower() in BOOL_FALSEIES:
             return False
         else:
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)') 
+            raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)')
 
     elif type is str:
-        if val.lower() in ('true', 'false', 'yes', 'no', '1', '0'):
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a string)')
+        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
+            raise ValueError(f'Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)')
         return val.strip()
 
     elif type is int:
-        if not val.isdigit():
+        if not val.strip().isdigit():
             raise ValueError(f'Invalid configuration option {key}={val} (expected an integer)')
-        return int(val)
+        return int(val.strip())
 
     elif type is list or type is dict:
         return json.loads(val)
 
-    raise Exception('Config values can only be str, bool, int or json')
+    raise Exception('Config values can only be str, bool, int, or json')
 
 
 def load_config_file(out_dir: str=None) -> Optional[Dict[str, str]]:
@@ -427,7 +651,7 @@ def load_config_file(out_dir: str=None) -> Optional[Dict[str, str]]:
     config_path = Path(out_dir) / CONFIG_FILENAME
     if config_path.exists():
         config_file = ConfigParser()
-        config_file.optionxform = str 
+        config_file.optionxform = str
         config_file.read(config_path)
         # flatten into one namespace
         config_file_vars = {
@@ -451,7 +675,7 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
     #
     # You can add options here manually in INI format, or automatically by running:
     #    archivebox config --set KEY=VALUE
-    # 
+    #
     # If you modify this file manually, make sure to update your archive after by running:
     #    archivebox init
     #
@@ -462,7 +686,7 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
 
     out_dir = out_dir or Path(os.getenv('OUTPUT_DIR', '.')).resolve()
     config_path = Path(out_dir) /  CONFIG_FILENAME
-    
+
     if not config_path.exists():
         atomic_write(config_path, CONFIG_HEADER)
 
@@ -500,7 +724,7 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
 
     with open(config_path, 'w+', encoding='utf-8') as new:
         config_file.write(new)
-    
+
     try:
         # validate the config by attempting to re-parse it
         CONFIG = load_all_config()
@@ -513,20 +737,20 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
 
     if Path(f'{config_path}.bak').exists():
         os.remove(f'{config_path}.bak')
-    
+
     return {
         key.upper(): CONFIG.get(key.upper())
         for key in config.keys()
     }
 
-   
+
 
 def load_config(defaults: ConfigDefaultDict,
                 config: Optional[ConfigDict]=None,
                 out_dir: Optional[str]=None,
                 env_vars: Optional[os._Environ]=None,
                 config_file_vars: Optional[Dict[str, str]]=None) -> ConfigDict:
-    
+
     env_vars = env_vars or os.environ
     config_file_vars = config_file_vars or load_config_file(out_dir=out_dir)
 
@@ -556,12 +780,14 @@ def load_config(defaults: ConfigDefaultDict,
             stderr()
             # raise
             raise SystemExit(2)
-    
+
     return extended_config
 
-# def write_config(config: ConfigDict):
 
-#     with open(os.path.join(config['OUTPUT_DIR'], CONFIG_FILENAME), 'w+') as f:
+def parse_version_string(version: str) -> Tuple[int, int, int]:
+    """parses a version tag string formatted like 'vx.x.x' into (major, minor, patch) ints"""
+    base = version.split('+')[0].split('v')[-1] # remove 'v' prefix and '+editable' suffix
+    return tuple(int(part) for part in base.split('.'))[:3]
 
 
 # Logging Helpers
@@ -605,7 +831,10 @@ def bin_version(binary: Optional[str]) -> Optional[str]:
         return None
 
     try:
-        version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
+        bin_env = os.environ | {'LANG': 'C'}
+        version_str = run([abspath, "--version"], stdout=PIPE, env=bin_env).stdout.strip().decode()
+        if not version_str:
+            version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
         # take first 3 columns of first line of version info
         return ' '.join(version_str.split('\n')[0].strip().split()[:3])
     except OSError:
@@ -639,7 +868,7 @@ def bin_hash(binary: Optional[str]) -> Optional[str]:
     with io.open(abs_path, mode='rb') as f:
         for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b''):
             file_hash.update(chunk)
-            
+
     return f'md5:{file_hash.hexdigest()}'
 
 def find_chrome_binary() -> Optional[str]:
@@ -647,6 +876,7 @@ def find_chrome_binary() -> Optional[str]:
     # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
     # make sure data dir finding precedence order always matches binary finding order
     default_executable_paths = (
+        # '~/Library/Caches/ms-playwright/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium',
         'chromium-browser',
         'chromium',
         '/Applications/Chromium.app/Contents/MacOS/Chromium',
@@ -664,7 +894,7 @@ def find_chrome_binary() -> Optional[str]:
         full_path_exists = shutil.which(name)
         if full_path_exists:
             return name
-    
+
     return None
 
 def find_chrome_data_dir() -> Optional[str]:
@@ -748,6 +978,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'path': config['OUTPUT_DIR'].resolve(),
             'enabled': True,
             'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(),
+            'is_mount': os.path.ismount(config['OUTPUT_DIR'].resolve()),
         },
         'SOURCES_DIR': {
             'path': config['SOURCES_DIR'].resolve(),
@@ -763,6 +994,7 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'path': config['ARCHIVE_DIR'].resolve(),
             'enabled': True,
             'is_valid': config['ARCHIVE_DIR'].exists(),
+            'is_mount': os.path.ismount(config['ARCHIVE_DIR'].resolve()),
         },
         'CONFIG_FILE': {
             'path': config['CONFIG_FILE'].resolve(),
@@ -773,18 +1005,12 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'path': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).resolve(),
             'enabled': True,
             'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(),
+            'is_mount': os.path.ismount((config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).resolve()),
         },
     }
 
 def get_dependency_info(config: ConfigDict) -> ConfigValue:
     return {
-        'ARCHIVEBOX_BINARY': {
-            'path': bin_path(config['ARCHIVEBOX_BINARY']),
-            'version': config['VERSION'],
-            'hash': bin_hash(config['ARCHIVEBOX_BINARY']),
-            'enabled': True,
-            'is_valid': True,
-        },
         'PYTHON_BINARY': {
             'path': bin_path(config['PYTHON_BINARY']),
             'version': config['PYTHON_VERSION'],
@@ -792,6 +1018,13 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
             'enabled': True,
             'is_valid': bool(config['PYTHON_VERSION']),
         },
+        'SQLITE_BINARY': {
+            'path': bin_path(config['SQLITE_BINARY']),
+            'version': config['SQLITE_VERSION'],
+            'hash': bin_hash(config['SQLITE_BINARY']),
+            'enabled': True,
+            'is_valid': bool(config['SQLITE_VERSION']),
+        },
         'DJANGO_BINARY': {
             'path': bin_path(config['DJANGO_BINARY']),
             'version': config['DJANGO_VERSION'],
@@ -799,6 +1032,14 @@ def get_dependency_info(config: ConfigDict) -> ConfigValue:
             'enabled': True,
             'is_valid': bool(config['DJANGO_VERSION']),
         },
+        'ARCHIVEBOX_BINARY': {
+            'path': bin_path(config['ARCHIVEBOX_BINARY']),
+            'version': config['VERSION'],
+            'hash': bin_hash(config['ARCHIVEBOX_BINARY']),
+            'enabled': True,
+            'is_valid': True,
+        },
+        
         'CURL_BINARY': {
             'path': bin_path(config['CURL_BINARY']),
             'version': config['CURL_VERSION'],
@@ -884,7 +1125,8 @@ def get_chrome_info(config: ConfigDict) -> ConfigValue:
         'TIMEOUT': config['TIMEOUT'],
         'RESOLUTION': config['RESOLUTION'],
         'CHECK_SSL_VALIDITY': config['CHECK_SSL_VALIDITY'],
-        'CHROME_BINARY': config['CHROME_BINARY'],
+        'CHROME_BINARY': bin_path(config['CHROME_BINARY']),
+        'CHROME_TIMEOUT': config['CHROME_TIMEOUT'],
         'CHROME_HEADLESS': config['CHROME_HEADLESS'],
         'CHROME_SANDBOX': config['CHROME_SANDBOX'],
         'CHROME_USER_AGENT': config['CHROME_USER_AGENT'],
@@ -925,38 +1167,69 @@ globals().update(CONFIG)
 
 
 # Set timezone to UTC and umask to OUTPUT_PERMISSIONS
-os.environ["TZ"] = 'UTC'
-os.umask(0o777 - int(OUTPUT_PERMISSIONS, base=8))  # noqa: F821
+assert TIMEZONE == 'UTC', 'The server timezone should always be set to UTC'  # noqa: F821
+os.environ["TZ"] = TIMEZONE                                                  # noqa: F821
+os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8))                        # noqa: F821
 
 # add ./node_modules/.bin to $PATH so we can use node scripts in extractors
 NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))
 sys.path.append(NODE_BIN_PATH)
 
+# OPTIONAL: also look around the host system for node modules to use
+#   avoid enabling this unless absolutely needed,
+#   having overlapping potential sources of libs is a big source of bugs/confusing to users
+# DEV_NODE_BIN_PATH = str((Path(CONFIG["PACKAGE_DIR"]).absolute() / '..' / 'node_modules' / '.bin'))
+# sys.path.append(DEV_NODE_BIN_PATH)
+# USER_NODE_BIN_PATH = str(Path('~/.node_modules/.bin').resolve())
+# sys.path.append(USER_NODE_BIN_PATH)
+
 # disable stderr "you really shouldnt disable ssl" warnings with library config
 if not CONFIG['CHECK_SSL_VALIDITY']:
     import urllib3
-    import requests
     requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
+# get SQLite database version, compile options, and runtime options
+# TODO: make this a less hacky proper assertion checker helper function in somewhere like setup_django
+#cursor = sqlite3.connect(':memory:').cursor()
+#DYNAMIC_CONFIG_SCHEMA['SQLITE_VERSION'] = lambda c: cursor.execute("SELECT sqlite_version();").fetchone()[0]
+#DYNAMIC_CONFIG_SCHEMA['SQLITE_JOURNAL_MODE'] = lambda c: cursor.execute('PRAGMA journal_mode;').fetchone()[0]
+#DYNAMIC_CONFIG_SCHEMA['SQLITE_OPTIONS'] = lambda c: [option[0] for option in cursor.execute('PRAGMA compile_options;').fetchall()]
+#cursor.close()
 
 ########################### Config Validity Checkers ###########################
 
 
 def check_system_config(config: ConfigDict=CONFIG) -> None:
     ### Check system environment
-    if config['USER'] == 'root':
+    if config['USER'] == 'root' or str(config['PUID']) == "0":
         stderr('[!] ArchiveBox should never be run as root!', color='red')
         stderr('    For more information, see the security overview documentation:')
         stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root')
+        
+        if config['IN_DOCKER']:
+            attempted_command = ' '.join(sys.argv[:3])
+            stderr('')
+            stderr('    {lightred}Hint{reset}: When using Docker, you must run commands with {green}docker run{reset} instead of {lightyellow}docker exec{reset}, e.g.:'.format(**config['ANSI']))
+            stderr(f'        docker compose run archivebox {attempted_command}')
+            stderr(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}')
+            stderr('        or:')
+            stderr(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"')
+            stderr(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"')
+        
         raise SystemExit(2)
 
     ### Check Python environment
-    if sys.version_info[:3] < (3, 6, 0):
+    if sys.version_info[:3] < (3, 7, 0):
         stderr(f'[X] Python version is not new enough: {config["PYTHON_VERSION"]} (>3.6 is required)', color='red')
         stderr('    See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.')
         raise SystemExit(2)
 
+    if int(CONFIG['DJANGO_VERSION'].split('.')[0]) < 3:
+        stderr(f'[X] Django version is not new enough: {config["DJANGO_VERSION"]} (>3.0 is required)', color='red')
+        stderr('    Upgrade django using pip or your system package manager: pip3 install --upgrade django')
+        raise SystemExit(2)
+
     if config['PYTHON_ENCODING'] not in ('UTF-8', 'UTF8'):
         stderr(f'[X] Your system is running python3 scripts with a bad locale setting: {config["PYTHON_ENCODING"]} (it should be UTF-8).', color='red')
         stderr('    To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)')
@@ -1023,12 +1296,13 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None:
 
     if config['USE_YOUTUBEDL'] and config['MEDIA_TIMEOUT'] < 20:
         stderr(f'[!] Warning: MEDIA_TIMEOUT is set too low! (currently set to MEDIA_TIMEOUT={config["MEDIA_TIMEOUT"]} seconds)', color='red')
-        stderr('    Youtube-dl will fail to archive all media if set to less than ~20 seconds.')
+        stderr('    youtube-dl/yt-dlp will fail to archive any media if set to less than ~20 seconds.')
         stderr('    (Setting it somewhere over 60 seconds is recommended)')
         stderr()
         stderr('    If you want to disable media archiving entirely, set SAVE_MEDIA=False instead:')
         stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_media')
         stderr()
+
         
 def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=CONFIG) -> None:
     output_dir = out_dir or config['OUTPUT_DIR']
@@ -1068,7 +1342,7 @@ def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CO
 
 def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, in_memory_db=False) -> None:
     check_system_config()
-    
+
     output_dir = out_dir or Path(config['OUTPUT_DIR'])
 
     assert isinstance(output_dir, Path) and isinstance(config['PACKAGE_DIR'], Path)
@@ -1103,25 +1377,30 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
             # Otherwise use default sqlite3 file-based database and initialize django
             # without running migrations automatically (user runs them manually by calling init)
             django.setup()
-            
 
         from django.conf import settings
 
         # log startup message to the error log
-        with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f:
+        with open(settings.ERROR_LOG, "a", encoding='utf-8') as f:
             command = ' '.join(sys.argv)
             ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
-            f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
-
+            f.write(f"\n> {command}; TS={ts} VERSION={config['VERSION']} IN_DOCKER={config['IN_DOCKER']} IS_TTY={config['IS_TTY']}\n")
 
         if check_db:
             # Enable WAL mode in sqlite3
             from django.db import connection
             with connection.cursor() as cursor:
+
+                # Set Journal mode to WAL to allow for multiple writers
                 current_mode = cursor.execute("PRAGMA journal_mode")
                 if current_mode != 'wal':
                     cursor.execute("PRAGMA journal_mode=wal;")
 
+                # Set max blocking delay for concurrent writes and write sync mode
+                # https://litestream.io/tips/#busy-timeout
+                cursor.execute("PRAGMA busy_timeout = 5000;")
+                cursor.execute("PRAGMA synchronous = NORMAL;")
+
             # Create cache table in DB if needed
             try:
                 from django.core.cache import cache
@@ -1129,7 +1408,6 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
             except django.db.utils.OperationalError:
                 call_command("createcachetable", verbosity=0)
 
-
             # if archivebox gets imported multiple times, we have to close
             # the sqlite3 whenever we init from scratch to avoid multiple threads
             # sharing the same connection by accident

archivebox/config_stubs.py

@@ -41,7 +41,7 @@ class ConfigDict(BaseConfig, total=False):
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
     RESTRICT_FILE_NAMES: str
-    URL_BLACKLIST: str
+    URL_DENYLIST: str
 
     SECRET_KEY: Optional[str]
     BIND_ADDR: str
@@ -74,6 +74,7 @@ class ConfigDict(BaseConfig, total=False):
     CHROME_USER_AGENT: str
     COOKIES_FILE: Union[str, Path, None]
     CHROME_USER_DATA_DIR: Union[str, Path, None]
+    CHROME_TIMEOUT: int
     CHROME_HEADLESS: bool
     CHROME_SANDBOX: bool
 
@@ -98,6 +99,7 @@ class ConfigDict(BaseConfig, total=False):
     WGET_ARGS: List[str]
     CURL_ARGS: List[str]
     GIT_ARGS: List[str]
+    TAG_SEPARATOR_PATTERN: str
 
 
 ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

archivebox/core/admin.py

@@ -48,9 +48,11 @@ class TagInline(admin.TabularInline):
 from django.contrib.admin.helpers import ActionForm
 from django.contrib.admin.widgets import AutocompleteSelectMultiple
 
+# WIP: broken by Django 3.1.2 -> 4.0 migration
 class AutocompleteTags:
     model = Tag
     search_fields = ['name']
+    name = 'tags'
 
 class AutocompleteTagsAdminStub:
     name = 'admin'
@@ -60,6 +62,7 @@ class SnapshotActionForm(ActionForm):
     tags = forms.ModelMultipleChoiceField(
         queryset=Tag.objects.all(),
         required=False,
+        # WIP: broken by Django 3.1.2 -> 4.0 migration
         widget=AutocompleteSelectMultiple(
             AutocompleteTags(),
             AutocompleteTagsAdminStub(),

archivebox/core/apps.py

@@ -3,4 +3,5 @@ from django.apps import AppConfig
 
 class CoreConfig(AppConfig):
     name = 'core'
+    # WIP: broken by Django 3.1.2 -> 4.0 migration
     default_auto_field = 'django.db.models.UUIDField'

archivebox/core/forms.py

@@ -41,7 +41,7 @@ class AddLinkForm(forms.Form):
     #     label="Exclude patterns",
     #     min_length='1',
     #     required=False,
-    #     initial=URL_BLACKLIST,
+    #     initial=URL_DENYLIST,
     # )
     # timeout = forms.IntegerField(
     #     initial=TIMEOUT,

archivebox/core/middleware.py

@@ -1,8 +1,11 @@
 __package__ = 'archivebox.core'
 
+import ipaddress
 from django.utils import timezone
+from django.contrib.auth.middleware import RemoteUserMiddleware
+from django.core.exceptions import ImproperlyConfigured
 
-from ..config import PUBLIC_SNAPSHOTS
+from ..config import PUBLIC_SNAPSHOTS, REVERSE_PROXY_USER_HEADER, REVERSE_PROXY_WHITELIST
 
 
 def detect_timezone(request, activate: bool=True):
@@ -35,3 +38,23 @@ def CacheControlMiddleware(get_response):
         return response
 
     return middleware
+
+class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
+    header = 'HTTP_{normalized}'.format(normalized=REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
+
+    def process_request(self, request):
+        if REVERSE_PROXY_WHITELIST == '':
+            return
+
+        ip = request.META.get('REMOTE_ADDR')
+
+        for cidr in REVERSE_PROXY_WHITELIST.split(','):
+            try:
+                network = ipaddress.ip_network(cidr)
+            except ValueError:
+                raise ImproperlyConfigured(
+                    "The REVERSE_PROXY_WHITELIST config paramater is in invalid format, or "
+                    "contains invalid CIDR. Correct format is a coma-separated list of IPv4/IPv6 CIDRs.")
+
+            if ipaddress.ip_address(ip) in network:
+                return super().process_request(request)

archivebox/core/migrations/0021_auto_20220914_0934.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.14 on 2022-09-14 09:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0020_auto_20210410_1031'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+        ),
+    ]

archivebox/core/migrations/0022_auto_20231023_2008.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.14 on 2023-10-23 20:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0021_auto_20220914_0934'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('htmltotext', 'htmltotext'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+        ),
+    ]

archivebox/core/settings.py

@@ -9,7 +9,7 @@ import tempfile
 from pathlib import Path
 from django.utils.crypto import get_random_string
 
-from ..config import (                                                          # noqa: F401
+from ..config import (
     DEBUG,
     SECRET_KEY,
     ALLOWED_HOSTS,
@@ -19,7 +19,18 @@ from ..config import (
     SQL_INDEX_FILENAME,
     OUTPUT_DIR,
     LOGS_DIR,
-    TIME_ZONE,
+    TIMEZONE,
+
+    LDAP,
+    LDAP_SERVER_URI,
+    LDAP_BIND_DN,
+    LDAP_BIND_PASSWORD,
+    LDAP_USER_BASE,
+    LDAP_USER_FILTER,
+    LDAP_USERNAME_ATTR,
+    LDAP_FIRSTNAME_ATTR,
+    LDAP_LASTNAME_ATTR,
+    LDAP_EMAIL_ATTR,
 )
 
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
@@ -34,7 +45,8 @@ WSGI_APPLICATION = 'core.wsgi.application'
 ROOT_URLCONF = 'core.urls'
 
 LOGIN_URL = '/accounts/login/'
-LOGOUT_REDIRECT_URL = '/'
+LOGOUT_REDIRECT_URL = os.environ.get('LOGOUT_REDIRECT_URL', '/')
+
 PASSWORD_RESET_URL = '/accounts/password_reset/'
 APPEND_SLASH = True
 
@@ -54,6 +66,12 @@ INSTALLED_APPS = [
 ]
 
 
+# For usage with https://www.jetadmin.io/integrations/django
+# INSTALLED_APPS += ['jet_django']
+# JET_PROJECT = 'archivebox'
+# JET_TOKEN = 'some-api-token-here'
+
+
 MIDDLEWARE = [
     'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
@@ -61,14 +79,64 @@ MIDDLEWARE = [
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'core.middleware.ReverseProxyAuthMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'core.middleware.CacheControlMiddleware',
 ]
 
+################################################################################
+### Authentication Settings
+################################################################################
+
 AUTHENTICATION_BACKENDS = [
+    'django.contrib.auth.backends.RemoteUserBackend',
     'django.contrib.auth.backends.ModelBackend',
 ]
 
+if LDAP:
+    try:
+        import ldap
+        from django_auth_ldap.config import LDAPSearch
+
+        global AUTH_LDAP_SERVER_URI
+        global AUTH_LDAP_BIND_DN
+        global AUTH_LDAP_BIND_PASSWORD
+        global AUTH_LDAP_USER_SEARCH
+        global AUTH_LDAP_USER_ATTR_MAP
+
+        AUTH_LDAP_SERVER_URI = LDAP_SERVER_URI
+        AUTH_LDAP_BIND_DN = LDAP_BIND_DN
+        AUTH_LDAP_BIND_PASSWORD = LDAP_BIND_PASSWORD
+
+        assert AUTH_LDAP_SERVER_URI and LDAP_USERNAME_ATTR and LDAP_USER_FILTER, 'LDAP_* config options must all be set if LDAP=True'
+
+        AUTH_LDAP_USER_SEARCH = LDAPSearch(
+            LDAP_USER_BASE,
+            ldap.SCOPE_SUBTREE,
+            '(&(' + LDAP_USERNAME_ATTR + '=%(user)s)' + LDAP_USER_FILTER + ')',
+        )
+
+        AUTH_LDAP_USER_ATTR_MAP = {
+            'username': LDAP_USERNAME_ATTR,
+            'first_name': LDAP_FIRSTNAME_ATTR,
+            'last_name': LDAP_LASTNAME_ATTR,
+            'email': LDAP_EMAIL_ATTR,
+        }
+
+        AUTHENTICATION_BACKENDS = [
+            'django.contrib.auth.backends.ModelBackend',
+            'django_auth_ldap.backend.LDAPBackend',
+        ]
+    except ModuleNotFoundError:
+        sys.stderr.write('[X] Error: Found LDAP=True config but LDAP packages not installed. You may need to run: pip install archivebox[ldap]\n\n')
+        # dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
+        # sys.exit(1)
+
+
+################################################################################
+### Debug Settings
+################################################################################
+
 # only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode)
 DEBUG_TOOLBAR = DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv)
 if DEBUG_TOOLBAR:
@@ -154,7 +222,7 @@ DATABASES = {
             'timeout': 60,
             'check_same_thread': False,
         },
-        'TIME_ZONE': 'UTC',
+        'TIME_ZONE': TIMEZONE,
         # DB setup is sometimes modified at runtime by setup_django() in config.py
     }
 }
@@ -201,6 +269,8 @@ AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
 ]
 
+# WIP: broken by Django 3.1.2 -> 4.0 migration
+DEFAULT_AUTO_FIELD = 'django.db.models.UUIDField'
 
 ################################################################################
 ### Shell Settings
@@ -224,6 +294,8 @@ USE_L10N = True
 USE_TZ = True
 DATETIME_FORMAT = 'Y-m-d g:iA'
 SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
+TIME_ZONE = TIMEZONE        # django convention is TIME_ZONE, archivebox config uses TIMEZONE, they are equivalent
+
 
 from django.conf.locale.en import formats as en_formats
 
@@ -262,8 +334,8 @@ class NoisyRequestsFilter(logging.Filter):
 if LOGS_DIR.exists():
     ERROR_LOG = (LOGS_DIR / 'errors.log')
 else:
-    # meh too many edge cases here around creating log dir w/ correct permissions
-    # cant be bothered, just trash the log and let them figure it out via stdout/stderr
+    # historically too many edge cases here around creating log dir w/ correct permissions early on
+    # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr
     ERROR_LOG = tempfile.NamedTemporaryFile().name
 
 LOGGING = {

archivebox/core/urls.py

@@ -6,7 +6,11 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.conf import settings
 from django.views.generic.base import RedirectView
 
-from core.views import HomepageView, SnapshotView, PublicIndexView, AddView
+from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
+
+# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
+# from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
+# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
 
 
 # print('DEBUG', settings.DEBUG)
@@ -24,7 +28,7 @@ urlpatterns = [
 
     path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
     path('add/', AddView.as_view(), name='add'),
-    
+
     path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
     path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
 
@@ -32,6 +36,14 @@ urlpatterns = [
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', admin.site.urls),
     
+    # do not add extra_context like this as not all admin views (e.g. ModelAdmin.autocomplete_view accept extra kwargs)
+    # path('admin/', admin.site.urls, {'extra_context': GLOBAL_CONTEXT}),
+
+    path('health/', HealthCheckView.as_view(), name='healthcheck'),
+    path('error/', lambda _: 1/0),
+
+    # path('jet_api/', include('jet_django.urls')),  Enable to use https://www.jetadmin.io/integrations/django
+
     path('index.html', RedirectView.as_view(url='/')),
     path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),
     path('', HomepageView.as_view(), name='Home'),

archivebox/core/views.py

@@ -11,6 +11,8 @@ from django.views.generic.list import ListView
 from django.views.generic import FormView
 from django.db.models import Q
 from django.contrib.auth.mixins import UserPassesTestMixin
+from django.views.decorators.csrf import csrf_exempt
+from django.utils.decorators import method_decorator
 
 from core.models import Snapshot
 from core.forms import AddLinkForm
@@ -21,6 +23,7 @@ from ..config import (
     PUBLIC_SNAPSHOTS,
     PUBLIC_ADD_VIEW,
     VERSION,
+    COMMIT_HASH,
     FOOTER_INFO,
     SNAPSHOTS_PER_PAGE,
 )
@@ -36,7 +39,7 @@ class HomepageView(View):
 
         if PUBLIC_INDEX:
             return redirect('/public')
-        
+
         return redirect(f'/admin/login/?next={request.path}')
 
 
@@ -203,7 +206,7 @@ class SnapshotView(View):
                 content_type="text/html",
                 status=404,
             )
-        
+
 
 class PublicIndexView(ListView):
     template_name = 'public_index.html'
@@ -215,10 +218,11 @@ class PublicIndexView(ListView):
         return {
             **super().get_context_data(**kwargs),
             'VERSION': VERSION,
+            'COMMIT_HASH': COMMIT_HASH,
             'FOOTER_INFO': FOOTER_INFO,
         }
 
-    def get_queryset(self, **kwargs): 
+    def get_queryset(self, **kwargs):
         qs = super().get_queryset(**kwargs)
         query = self.request.GET.get('q')
         if query and query.strip():
@@ -236,7 +240,7 @@ class PublicIndexView(ListView):
         else:
             return redirect(f'/admin/login/?next={self.request.path}')
 
-
+@method_decorator(csrf_exempt, name='dispatch')
 class AddView(UserPassesTestMixin, FormView):
     template_name = "add.html"
     form_class = AddLinkForm
@@ -247,7 +251,7 @@ class AddView(UserPassesTestMixin, FormView):
             url = self.request.GET.get('url', None)
             if url:
                 return {'url': url if '://' in url else f'https://{url}'}
-        
+
         return super().get_initial()
 
     def test_func(self):
@@ -293,3 +297,18 @@ class AddView(UserPassesTestMixin, FormView):
             "form": AddLinkForm()
         })
         return render(template_name=self.template_name, request=self.request, context=context)
+
+
+class HealthCheckView(View):
+    """
+    A Django view that renders plain text "OK" for service discovery tools
+    """
+    def get(self, request):
+        """
+        Handle a GET request
+        """
+        return HttpResponse(
+            'OK',
+            content_type='text/plain',
+            status=200
+        )

archivebox/extractors/__init__.py

@@ -1,13 +1,19 @@
 __package__ = 'archivebox.extractors'
 
 import os
+import sys
 from pathlib import Path
 
-from typing import Optional, List, Iterable, Union
+from typing import Callable, Optional, List, Iterable, Union
 from datetime import datetime, timezone
 from django.db.models import QuerySet
 
-from ..index.schema import Link
+from ..config import (
+    SAVE_ALLOWLIST_PTN,
+    SAVE_DENYLIST_PTN,
+)
+from ..core.settings import ERROR_LOG
+from ..index.schema import ArchiveResult, Link
 from ..index.sql import write_link_to_sql_index
 from ..index import (
     load_link_details,
@@ -31,6 +37,7 @@ from .wget import should_save_wget, save_wget
 from .singlefile import should_save_singlefile, save_singlefile
 from .readability import should_save_readability, save_readability
 from .mercury import should_save_mercury, save_mercury
+from .htmltotext import should_save_htmltotext, save_htmltotext
 from .pdf import should_save_pdf, save_pdf
 from .screenshot import should_save_screenshot, save_screenshot
 from .dom import should_save_dom, save_dom
@@ -40,9 +47,12 @@ from .archive_org import should_save_archive_dot_org, save_archive_dot_org
 from .headers import should_save_headers, save_headers
 
 
-def get_default_archive_methods():
+ShouldSaveFunction = Callable[[Link, Optional[Path], Optional[bool]], bool]
+SaveFunction = Callable[[Link, Optional[Path], int], ArchiveResult]
+ArchiveMethodEntry = tuple[str, ShouldSaveFunction, SaveFunction]
+
+def get_default_archive_methods() -> List[ArchiveMethodEntry]:
     return [
-        ('title', should_save_title, save_title),
         ('favicon', should_save_favicon, save_favicon),
         ('headers', should_save_headers, save_headers),
         ('singlefile', should_save_singlefile, save_singlefile),
@@ -50,21 +60,49 @@ def get_default_archive_methods():
         ('screenshot', should_save_screenshot, save_screenshot),
         ('dom', should_save_dom, save_dom),
         ('wget', should_save_wget, save_wget),
-        ('readability', should_save_readability, save_readability),  # keep readability below wget and singlefile, as it depends on them
+        # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
+        ('title', should_save_title, save_title),
+        ('readability', should_save_readability, save_readability),
         ('mercury', should_save_mercury, save_mercury),
+        ('htmltotext', should_save_htmltotext, save_htmltotext),
         ('git', should_save_git, save_git),
         ('media', should_save_media, save_media),
         ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
-ARCHIVE_METHODS_INDEXING_PRECEDENCE = [('readability', 1), ('singlefile', 2), ('dom', 3), ('wget', 4)]
+ARCHIVE_METHODS_INDEXING_PRECEDENCE = [
+    ('readability', 1),
+    ('mercury', 2),
+    ('htmltotext', 3),
+    ('singlefile', 4),
+    ('dom', 5),
+    ('wget', 6)
+]
+
 
 @enforce_types
-def ignore_methods(to_ignore: List[str]):
+def get_archive_methods_for_link(link: Link) -> Iterable[ArchiveMethodEntry]:
+    DEFAULT_METHODS = get_default_archive_methods()
+    allowed_methods = {
+        m for pat, methods in
+        SAVE_ALLOWLIST_PTN.items()
+        if pat.search(link.url)
+        for m in methods
+    } or { m[0] for m in DEFAULT_METHODS }
+    denied_methods = {
+        m for pat, methods in
+        SAVE_DENYLIST_PTN.items()
+        if pat.search(link.url)
+        for m in methods
+    }
+    allowed_methods -= denied_methods
+
+    return (m for m in DEFAULT_METHODS if m[0] in allowed_methods)
+
+@enforce_types
+def ignore_methods(to_ignore: List[str]) -> Iterable[str]:
     ARCHIVE_METHODS = get_default_archive_methods()
-    methods = filter(lambda x: x[0] not in to_ignore, ARCHIVE_METHODS)
-    methods = map(lambda x: x[0], methods)
-    return list(methods)
+    return [x[0] for x in ARCHIVE_METHODS if x[0] not in to_ignore]
 
 @enforce_types
 def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Link:
@@ -77,11 +115,11 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
     except Snapshot.DoesNotExist:
         snapshot = write_link_to_sql_index(link)
 
-    ARCHIVE_METHODS = get_default_archive_methods()
+    active_methods = get_archive_methods_for_link(link)
     
     if methods:
-        ARCHIVE_METHODS = [
-            method for method in ARCHIVE_METHODS
+        active_methods = [
+            method for method in active_methods
             if method[0] in methods
         ]
 
@@ -98,7 +136,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
         start_ts = datetime.now(timezone.utc)
 
-        for method_name, should_run, method_function in ARCHIVE_METHODS:
+        for method_name, should_run, method_function in active_methods:
             try:
                 if method_name not in link.history:
                     link.history[method_name] = []
@@ -127,10 +165,27 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
                     stats['skipped'] += 1
             except Exception as e:
+                # Disabled until https://github.com/ArchiveBox/ArchiveBox/issues/984
+                # and https://github.com/ArchiveBox/ArchiveBox/issues/1014
+                # are fixed.
+                """
                 raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
                     method_name,
                     link.url,
                 )) from e
+                """
+                # Instead, use the kludgy workaround from
+                # https://github.com/ArchiveBox/ArchiveBox/issues/984#issuecomment-1150541627
+                with open(ERROR_LOG, "a", encoding='utf-8') as f:
+                    command = ' '.join(sys.argv)
+                    ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
+                    f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={})) command={}; ts={}'.format(
+                        method_name,
+                        link.url,
+                        command,
+                        ts
+                    ) + "\n" + str(e) + "\n"))
+                    #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
 
         # print('    ', stats)
 
@@ -143,7 +198,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
 
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
 
-        log_link_archiving_finished(link, link.link_dir, is_new, stats, start_ts)
+        log_link_archiving_finished(link, out_dir, is_new, stats, start_ts)
 
     except KeyboardInterrupt:
         try:

archivebox/extractors/dom.py

@@ -9,6 +9,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -26,7 +27,8 @@ def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
 
     out_dir = out_dir or Path(link.link_dir)
     if not overwrite and (out_dir / 'output.html').exists():
-        return False
+        if (out_dir / 'output.html').stat().st_size > 1:
+            return False
 
     return SAVE_DOM
 
@@ -38,7 +40,7 @@ def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     output: ArchiveOutput = 'output.html'
     output_path = out_dir / output
     cmd = [
-        *chrome_args(TIMEOUT=timeout),
+        *chrome_args(),
         '--dump-dom',
         link.url
     ]
@@ -56,6 +58,7 @@ def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     except Exception as err:
         status = 'failed'
         output = err
+        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/favicon.py

@@ -10,6 +10,7 @@ from ..util import enforce_types, domain
 from ..config import (
     TIMEOUT,
     SAVE_FAVICON,
+    FAVICON_PROVIDER,
     CURL_BINARY,
     CURL_ARGS,
     CURL_VERSION,
@@ -40,7 +41,7 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         '--output', str(output),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
+        FAVICON_PROVIDER.format(domain(link.url)),
     ]
     status = 'failed'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/htmltotext.py

@@ -0,0 +1,154 @@
+__package__ = 'archivebox.extractors'
+
+from html.parser import HTMLParser
+import io
+from pathlib import Path
+from typing import Optional
+
+from ..config import (
+    SAVE_HTMLTOTEXT,
+    TIMEOUT,
+    VERSION,
+)
+from ..index.schema import Link, ArchiveResult, ArchiveError
+from ..logging_util import TimedProgress
+from ..system import atomic_write
+from ..util import (
+    enforce_types,
+    is_static_file,
+)
+from .title import get_html
+
+class HTMLTextExtractor(HTMLParser):
+    TEXT_ATTRS = [
+        "alt", "cite", "href", "label",
+        "list", "placeholder", "title", "value"
+    ]
+    NOTEXT_TAGS = ["script", "style", "template"]
+    NOTEXT_HREF = ["data:", "javascript:", "#"]
+
+    def __init__(self):
+        super().__init__()
+
+        self.output = io.StringIO()
+        self._tag_stack = []
+
+    def _is_text_attr(self, name, value):
+        if not isinstance(value, str):
+            return False
+        if name == "href" and any(map(lambda p: value.startswith(p), self.NOTEXT_HREF)):
+            return False
+
+        if name in self.TEXT_ATTRS:
+            return True
+
+        return False
+
+    def _parent_tag(self):
+        try:
+            return self._tag_stack[-1]
+        except IndexError:
+            return None
+
+    def _in_notext_tag(self):
+        return any([t in self._tag_stack for t in self.NOTEXT_TAGS])
+
+    def handle_starttag(self, tag, attrs):
+        self._tag_stack.append(tag)
+
+        # Don't write out attribute values if any ancestor
+        # is in NOTEXT_TAGS
+        if self._in_notext_tag():
+            return
+
+        for name, value in attrs:
+            if self._is_text_attr(name, value):
+                self.output.write(f"({value.strip()}) ")
+
+    def handle_endtag(self, tag):
+        orig_stack = self._tag_stack.copy()
+        try:
+            # Keep popping tags until we find the nearest
+            # ancestor matching this end tag
+            while tag != self._tag_stack.pop():
+                pass
+            # Write a space after every tag, to ensure that tokens
+            # in tag text aren't concatenated. This may result in
+            # excess spaces, which should be ignored by search tokenizers.
+            if not self._in_notext_tag() and tag not in self.NOTEXT_TAGS:
+                self.output.write(" ")
+        except IndexError:
+            # Got to the top of the stack, but somehow missed
+            # this end tag -- maybe malformed markup -- restore the
+            # stack
+            self._tag_stack = orig_stack
+
+    def handle_data(self, data):
+        # Don't output text data if any ancestor is in NOTEXT_TAGS
+        if self._in_notext_tag():
+            return
+
+        data = data.lstrip()
+        len_before_rstrip = len(data)
+        data = data.rstrip()
+        spaces_rstripped = len_before_rstrip - len(data)
+        if data:
+            self.output.write(data)
+            if spaces_rstripped:
+                # Add back a single space if 1 or more
+                # whitespace characters were stripped
+                self.output.write(' ')
+
+    def __str__(self):
+        return self.output.getvalue()
+
+
+@enforce_types
+def should_save_htmltotext(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
+        return False
+
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'htmltotext.txt').exists():
+        return False
+
+    return SAVE_HTMLTOTEXT
+
+
+@enforce_types
+def save_htmltotext(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+    """extract search-indexing-friendly text from an HTML document"""
+
+    out_dir = Path(out_dir or link.link_dir)
+    output = "htmltotext.txt"
+
+    timer = TimedProgress(timeout, prefix='      ')
+    extracted_text = None
+    try:
+        extractor = HTMLTextExtractor()
+        document = get_html(link, out_dir)
+
+        if not document:
+            raise ArchiveError('htmltotext could not find HTML to parse for article text')
+
+        extractor.feed(document)
+        extractor.close()
+        extracted_text = str(extractor)
+
+        atomic_write(str(out_dir / output), extracted_text)
+    except (Exception, OSError) as err:
+        status = 'failed'
+        output = err
+        cmd = ['(internal) archivebox.extractors.htmltotext', './{singlefile,dom}.html']
+    finally:
+        timer.end()
+
+    return ArchiveResult(
+        cmd=cmd,
+        pwd=str(out_dir),
+        cmd_version=VERSION,
+        output=output,
+        status=status,
+        index_texts=[extracted_text] if extracted_text else [],
+        **timer.stats,  
+    )

archivebox/extractors/media.py

@@ -33,7 +33,7 @@ def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optio
 
 @enforce_types
 def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
-    """Download playlists or individual video, audio, and subtitles using youtube-dl"""
+    """Download playlists or individual video, audio, and subtitles using youtube-dl or yt-dlp"""
 
     out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'media'
@@ -43,6 +43,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
         YOUTUBEDL_BINARY,
         *YOUTUBEDL_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
+        # TODO: add --cookies-from-browser={CHROME_USER_DATA_DIR}
         link.url,
     ]
     status = 'succeeded'
@@ -60,7 +61,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
                 pass
             else:
                 hints = (
-                    'Got youtube-dl response code: {}.'.format(result.returncode),
+                    'Got youtube-dl (or yt-dlp) response code: {}.'.format(result.returncode),
                     *result.stderr.decode().split('\n'),
                 )
                 raise ArchiveError('Failed to save media', hints)
@@ -71,8 +72,18 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
         timer.end()
 
     # add video description and subtitles to full-text index
+    # Let's try a few different 
     index_texts = [
-        text_file.read_text(encoding='utf-8').strip()
+        # errors:
+        # * 'strict' to raise a ValueError exception if there is an
+        #   encoding error. The default value of None has the same effect.
+        # * 'ignore' ignores errors. Note that ignoring encoding errors
+        #   can lead to data loss.
+        # * 'xmlcharrefreplace' is only supported when writing to a
+        #   file. Characters not supported by the encoding are replaced with
+        #   the appropriate XML character reference &#nnn;.
+        # There are a few more options described in https://docs.python.org/3/library/functions.html#open
+        text_file.read_text(encoding='utf-8', errors='xmlcharrefreplace').strip()
         for text_file in (
             *output_path.glob('*.description'),
             *output_path.glob('*.srt'),

archivebox/extractors/pdf.py

@@ -9,6 +9,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -37,7 +38,7 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'output.pdf'
     cmd = [
-        *chrome_args(TIMEOUT=timeout),
+        *chrome_args(),
         '--print-to-pdf',
         link.url,
     ]
@@ -54,6 +55,7 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     except Exception as err:
         status = 'failed'
         output = err
+        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/readability.py

@@ -10,9 +10,7 @@ from ..index.schema import Link, ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
-    download_url,
     is_static_file,
-    
 )
 from ..config import (
     TIMEOUT,
@@ -22,28 +20,8 @@ from ..config import (
     READABILITY_VERSION,
 )
 from ..logging_util import TimedProgress
+from .title import get_html
 
-@enforce_types
-def get_html(link: Link, path: Path) -> str:
-    """
-    Try to find wget, singlefile and then dom files.
-    If none is found, download the url again.
-    """
-    canonical = link.canonical_outputs()
-    abs_path = path.absolute()
-    sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
-    document = None
-    for source in sources:
-        try:
-            with open(abs_path / source, "r", encoding="utf-8") as f:
-                document = f.read()
-                break
-        except (FileNotFoundError, TypeError):
-            continue
-    if document is None:
-        return download_url(link.url)
-    else:
-        return document
 
 @enforce_types
 def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
@@ -87,14 +65,14 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         cmd = [
             DEPENDENCIES['READABILITY_BINARY']['path'],
             temp_doc.name,
+            link.url,
         ]
-
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:
             result_json = json.loads(result.stdout)
-            assert result_json and 'content' in result_json
+            assert result_json and 'content' in result_json, 'Readability output is not valid JSON'
         except json.JSONDecodeError:
-            raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)
+            raise ArchiveError('Readability was not able to archive the page (invalid JSON)', result.stdout + result.stderr)
 
         output_folder.mkdir(exist_ok=True)
         readability_content = result_json.pop("textContent") 
@@ -102,11 +80,9 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         atomic_write(str(output_folder / "content.txt"), readability_content)
         atomic_write(str(output_folder / "article.json"), result_json)
 
-        # parse out number of files downloaded from last line of stderr:
-        #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
         output_tail = [
             line.strip()
-            for line in (result.stdout + result.stderr).decode().rsplit('\n', 3)[-3:]
+            for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
             if line.strip()
         ]
         hints = (
@@ -116,11 +92,13 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
 
         # Check for common failure cases
         if (result.returncode > 0):
-            raise ArchiveError('Readability was not able to archive the page', hints)
+            raise ArchiveError(f'Readability was not able to archive the page (status={result.returncode})', hints)
     except (Exception, OSError) as err:
         status = 'failed'
         output = err
-        cmd = [cmd[0], './{singlefile,dom}.html']
+
+        # prefer Chrome dom output to singlefile because singlefile often contains huge url(data:image/...base64) strings that make the html too long to parse with readability
+        cmd = [cmd[0], './{dom,singlefile}.html']
     finally:
         timer.end()
 

archivebox/extractors/screenshot.py

@@ -9,6 +9,7 @@ from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
+    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -37,7 +38,7 @@ def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
     out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'screenshot.png'
     cmd = [
-        *chrome_args(TIMEOUT=timeout),
+        *chrome_args(),
         '--screenshot',
         link.url,
     ]
@@ -54,6 +55,7 @@ def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
     except Exception as err:
         status = 'failed'
         output = err
+        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/singlefile.py

@@ -17,6 +17,7 @@ from ..config import (
     SAVE_SINGLEFILE,
     DEPENDENCIES,
     SINGLEFILE_VERSION,
+    SINGLEFILE_ARGS,
     CHROME_BINARY,
 )
 from ..logging_util import TimedProgress
@@ -41,14 +42,35 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
     out_dir = out_dir or Path(link.link_dir)
     output = "singlefile.html"
 
-    browser_args = chrome_args(TIMEOUT=0)
+    browser_args = chrome_args(CHROME_TIMEOUT=0)
 
     # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
     browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:]))
-    cmd = [
-        DEPENDENCIES['SINGLEFILE_BINARY']['path'],
+    options = [
+        *SINGLEFILE_ARGS,
         '--browser-executable-path={}'.format(CHROME_BINARY),
         browser_args,
+    ]
+
+    # Deduplicate options (single-file doesn't like when you use the same option two times)
+    #
+    # NOTE: Options names that come first clobber conflicting names that come later
+    # My logic is SINGLEFILE_ARGS is the option that affects the singlefile command with most 
+    # specificity, therefore the user sets it with a lot intent, therefore it should take precedence 
+    # kind of like the ergonomic principle of lexical scope in programming languages.
+    seen_option_names = []
+    def test_seen(argument):
+        option_name = argument.split("=")[0]
+        if option_name in seen_option_names:
+            return False
+        else:
+            seen_option_names.append(option_name)
+            return True
+    deduped_options = list(filter(test_seen, options))
+
+    cmd = [
+        DEPENDENCIES['SINGLEFILE_BINARY']['path'],
+        *deduped_options,
         link.url,
         output,
     ]

archivebox/extractors/title.py

@@ -26,7 +26,7 @@ from ..logging_util import TimedProgress
 
 HTML_TITLE_REGEX = re.compile(
     r'<title.*?>'                      # start matching text after <title> tag
-    r'(.[^<>]+)',                      # get everything up to these symbols
+    r'([^<>]+)',                      # get everything up to these symbols
     re.IGNORECASE | re.MULTILINE | re.DOTALL | re.UNICODE,
 )
 
@@ -58,6 +58,29 @@ class TitleParser(HTMLParser):
         if tag.lower() == "title":
             self.inside_title_tag = False
 
+@enforce_types
+def get_html(link: Link, path: Path, timeout: int=TIMEOUT) -> str:
+    """
+    Try to find wget, singlefile and then dom files.
+    If none is found, download the url again.
+    """
+    canonical = link.canonical_outputs()
+    abs_path = path.absolute()
+
+    # prefer chrome-generated DOM dump to singlefile as singlefile output often includes HUGE url(data:image/...base64) strings that crash parsers
+    sources = [canonical["dom_path"], canonical["singlefile_path"], canonical["wget_path"]]
+    document = None
+    for source in sources:
+        try:
+            with open(abs_path / source, "r", encoding="utf-8") as f:
+                document = f.read()
+                break
+        except (FileNotFoundError, TypeError):
+            continue
+    if document is None:
+        return download_url(link.url, timeout=timeout)
+    else:
+        return document
 
 @enforce_types
 def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
@@ -90,7 +113,7 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        html = download_url(link.url, timeout=timeout)
+        html = get_html(link, out_dir, timeout=timeout)
         try:
             # try using relatively strict html parser first
             parser = TitleParser()

archivebox/index/__init__.py

@@ -22,7 +22,8 @@ from ..config import (
     JSON_INDEX_FILENAME,
     OUTPUT_DIR,
     TIMEOUT,
-    URL_BLACKLIST_PTN,
+    URL_DENYLIST_PTN,
+    URL_ALLOWLIST_PTN,
     stderr,
     OUTPUT_PERMISSIONS
 )
@@ -141,7 +142,9 @@ def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
             continue
         if scheme(link.url) not in ('http', 'https', 'ftp'):
             continue
-        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(link.url):
+        if URL_DENYLIST_PTN and URL_DENYLIST_PTN.search(link.url):
+            continue
+        if URL_ALLOWLIST_PTN and (not URL_ALLOWLIST_PTN.search(link.url)):
             continue
 
         yield link

archivebox/index/html.py

@@ -24,6 +24,7 @@ from ..config import (
     FOOTER_INFO,
     HTML_INDEX_FILENAME,
     SAVE_ARCHIVE_DOT_ORG,
+    PREVIEW_ORIGINALS,
 )
 
 MAIN_INDEX_TEMPLATE = 'static_index.html'
@@ -105,6 +106,7 @@ def link_details_template(link: Link) -> str:
         'status_color': 'success' if link.is_archived else 'danger',
         'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
         'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
+        'PREVIEW_ORIGINALS': PREVIEW_ORIGINALS,
     })
 
 @enforce_types
@@ -141,7 +143,7 @@ def snapshot_icons(snapshot) -> str:
             "mercury": "🅼",
             "warc": "📦"
         }
-        exclude = ["favicon", "title", "headers", "archive_org"]
+        exclude = ["favicon", "title", "headers", "htmltotext", "archive_org"]
         # Missing specific entry for WARC
 
         extractor_outputs = defaultdict(lambda: None)
@@ -175,7 +177,7 @@ def snapshot_icons(snapshot) -> str:
                 # The check for archive_org is different, so it has to be handled separately
 
                 # get from db (faster)
-                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                exists = extractor in extractor_outputs and extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
                 # get from filesystem (slower)
                 # target_path = Path(path) / "archive.org.txt"
                 # exists = target_path.exists()

archivebox/index/json.py

@@ -60,7 +60,18 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     index_path = Path(out_dir) / JSON_INDEX_FILENAME
     if index_path.exists():
         with open(index_path, 'r', encoding='utf-8') as f:
-            links = pyjson.load(f)['links']
+            try:
+                links = pyjson.load(f)['links']
+                if links:
+                    Link.from_json(links[0])
+            except Exception as err:
+                print("    {lightyellow}! Found an index.json in the project root but couldn't load links from it: {} {}".format(
+                    err.__class__.__name__,
+                    err,
+                    **ANSI,
+                ))
+                return ()
+
             for link_json in links:
                 try:
                     yield Link.from_json(link_json)

archivebox/index/schema.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from ..system import get_dir_size
 from ..util import ts_to_date_str, parse_date
-from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
+from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME, FAVICON_PROVIDER
 
 class ArchiveError(Exception):
     def __init__(self, message, hints=None):
@@ -423,12 +423,13 @@ class Link:
         canonical = {
             'index_path': 'index.html',
             'favicon_path': 'favicon.ico',
-            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
+            'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
             'wget_path': wget_output_path(self),
             'warc_path': 'warc/',
             'singlefile_path': 'singlefile.html',
             'readability_path': 'readability/content.html',
             'mercury_path': 'mercury/content.html',
+            'htmltotext_path': 'htmltotext.txt',
             'pdf_path': 'output.pdf',
             'screenshot_path': 'screenshot.png',
             'dom_path': 'output.html',
@@ -452,6 +453,7 @@ class Link:
                 'singlefile_path': static_path,
                 'readability_path': static_path,
                 'mercury_path': static_path,
+                'htmltotext_path': static_path,
             })
         return canonical
 

archivebox/index/sql.py

@@ -1,5 +1,7 @@
 __package__ = 'archivebox.index'
 
+import re
+
 from io import StringIO
 from pathlib import Path
 from typing import List, Tuple, Iterator
@@ -8,7 +10,10 @@ from django.db import transaction
 
 from .schema import Link
 from ..util import enforce_types, parse_date
-from ..config import OUTPUT_DIR
+from ..config import (
+    OUTPUT_DIR,
+    TAG_SEPARATOR_PATTERN,
+)
 
 
 ### Main Links Index
@@ -33,9 +38,11 @@ def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir:
 def write_link_to_sql_index(link: Link):
     from core.models import Snapshot, ArchiveResult
     info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}
-    tags = info.pop("tags")
-    if tags is None:
-        tags = []
+
+    tag_list = list(dict.fromkeys(
+        tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')
+    ))
+    info.pop('tags')
 
     try:
         info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp
@@ -44,7 +51,7 @@ def write_link_to_sql_index(link: Link):
             info["timestamp"] = str(float(info["timestamp"]) + 1.0)
 
         snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info)
-    snapshot.save_tags(tags)
+    snapshot.save_tags(tag_list)
 
     for extractor, entries in link.history.items():
         for entry in entries:
@@ -102,12 +109,13 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
         snap = Snapshot.objects.get(url=link.url)
     except Snapshot.DoesNotExist:
         snap = write_link_to_sql_index(link)
+
     snap.title = link.title
 
-    tag_set = (
-        set(tag.strip() for tag in (link.tags or '').split(','))
+    tag_list = list(
+        {tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')}
+        | set(snap.tags.values_list('name', flat=True))
     )
-    tag_list = list(tag_set) or []
 
     snap.save()
     snap.save_tags(tag_list)

archivebox/logging_util.py

@@ -108,12 +108,12 @@ def reject_stdin(caller: str, stdin: Optional[IO]=sys.stdin) -> None:
     if not stdin.isatty():
         # stderr('READING STDIN TO REJECT...')
         stdin_raw_text = stdin.read()
-        if stdin_raw_text:
+        if stdin_raw_text.strip():
             # stderr('GOT STDIN!', len(stdin_str))
-            stderr(f'[X] The "{caller}" command does not accept stdin.', color='red')
+            stderr(f'[!] The "{caller}" command does not accept stdin (ignoring).', color='red')
             stderr(f'    Run archivebox "{caller} --help" to see usage and examples.')
             stderr()
-            raise SystemExit(1)
+            # raise SystemExit(1)
     return None
 
 
@@ -393,7 +393,11 @@ def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats
     else:
         _LAST_RUN_STATS.succeeded += 1
 
-    size = get_dir_size(link_dir)
+    try:
+        size = get_dir_size(link_dir)
+    except FileNotFoundError:
+        size = (0, None, '0')
+
     end_ts = datetime.now(timezone.utc)
     duration = str(end_ts - start_ts).split('.')[0]
     print('        {black}{} files ({}) in {}s {reset}'.format(size[2], printable_filesize(size[0]), duration, **ANSI))
@@ -409,7 +413,7 @@ def log_archive_method_finished(result: "ArchiveResult"):
     """
     # Prettify CMD string and make it safe to copy-paste by quoting arguments
     quoted_cmd = ' '.join(
-        '"{}"'.format(arg) if ' ' in arg else arg
+        '"{}"'.format(arg) if (' ' in arg) or (':' in arg) else arg
         for arg in result.cmd
     )
 
@@ -432,18 +436,30 @@ def log_archive_method_finished(result: "ArchiveResult"):
         # Prettify error output hints string and limit to five lines
         hints = getattr(result.output, 'hints', None) or ()
         if hints:
-            hints = hints if isinstance(hints, (list, tuple)) else hints.split('\n')
+            if isinstance(hints, (list, tuple, type(_ for _ in ()))):
+                hints = [hint.decode() for hint in hints if isinstance(hint, bytes)]
+            else:
+                if isinstance(hints, bytes):
+                    hints = hints.decode()
+                hints = hints.split('\n')
+
             hints = (
                 '    {}{}{}'.format(ANSI['lightyellow'], line.strip(), ANSI['reset'])
-                for line in hints[:5] if line.strip()
+                for line in list(hints)[:5] if line.strip()
             )
 
+        docker_hints = ()
+        if IN_DOCKER:
+            docker_hints = (
+                '  docker run -it -v $PWD/data:/data archivebox/archivebox /bin/bash',
+            )
 
         # Collect and prefix output lines with indentation
         output_lines = [
             *hint_header,
             *hints,
             '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']),
+            *docker_hints,
             *(['    cd {};'.format(result.pwd)] if result.pwd else []),
             '    {}'.format(quoted_cmd),
         ]
@@ -511,8 +527,8 @@ def log_shell_welcome_msg():
     from .cli import list_subcommands
 
     print('{green}# ArchiveBox Imports{reset}'.format(**ANSI))
-    print('{green}from core.models import Snapshot, User{reset}'.format(**ANSI))
-    print('{green}from archivebox import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
+    print('{green}from archivebox.core.models import Snapshot, ArchiveResult, Tag, User{reset}'.format(**ANSI))
+    print('{green}from archivebox.cli import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
     print()
     print('[i] Welcome to the ArchiveBox Shell!')
     print('    https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage')
@@ -527,11 +543,27 @@ def log_shell_welcome_msg():
 ### Helpers
 
 @enforce_types
-def pretty_path(path: Union[Path, str]) -> str:
+def pretty_path(path: Union[Path, str], pwd: Union[Path, str]=OUTPUT_DIR) -> str:
     """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
-    pwd = Path('.').resolve()
-    # parent = os.path.abspath(os.path.join(pwd, os.path.pardir))
-    return str(path).replace(str(pwd) + '/', './')
+    pwd = str(Path(pwd))  # .resolve()
+    path = str(path)
+
+    if not path:
+        return path
+
+    # replace long absolute paths with ./ relative ones to save on terminal output width
+    if path.startswith(pwd) and (pwd != '/'):
+        path = path.replace(pwd, '.', 1)
+    
+    # quote paths containing spaces
+    if ' ' in path:
+        path = f'"{path}"'
+
+    # if path is just a plain dot, replace it back with the absolute path for clarity
+    if path == '.':
+        path = pwd
+
+    return path
 
 
 @enforce_types
@@ -566,12 +598,13 @@ def printable_config(config: ConfigDict, prefix: str='') -> str:
 def printable_folder_status(name: str, folder: Dict) -> str:
     if folder['enabled']:
         if folder['is_valid']:
-            color, symbol, note = 'green', '√', 'valid'
+            color, symbol, note, num_files = 'green', '√', 'valid', ''
         else:
             color, symbol, note, num_files = 'red', 'X', 'invalid', '?'
     else:
         color, symbol, note, num_files = 'lightyellow', '-', 'disabled', '-'
 
+
     if folder['path']:
         if Path(folder['path']).exists():
             num_files = (
@@ -581,14 +614,12 @@ def printable_folder_status(name: str, folder: Dict) -> str:
             )
         else:
             num_files = 'missing'
+        
+    if folder.get('is_mount'):
+        # add symbol @ next to filecount if path is a remote filesystem mount
+        num_files = f'{num_files} @' if num_files else '@'
 
-    path = str(folder['path']).replace(str(OUTPUT_DIR), '.') if folder['path'] else ''
-    if path and ' ' in path:
-        path = f'"{path}"'
-
-    # if path is just a plain dot, replace it back with the full path for clarity
-    if path == '.':
-        path = str(OUTPUT_DIR)
+    path = pretty_path(folder['path'])
 
     return ' '.join((
         ANSI[color],
@@ -619,9 +650,7 @@ def printable_dependency_version(name: str, dependency: Dict) -> str:
     else:
         color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
 
-    path = str(dependency["path"]).replace(str(OUTPUT_DIR), '.') if dependency["path"] else ''
-    if path and ' ' in path:
-        path = f'"{path}"'
+    path = pretty_path(dependency['path'])
 
     return ' '.join((
         ANSI[color],

archivebox/main.py

@@ -4,8 +4,9 @@ import os
 import sys
 import shutil
 import platform
+from django.utils import timezone
 from pathlib import Path
-from datetime import date
+from datetime import date, datetime
 
 from typing import Dict, List, Optional, Iterable, IO, Union
 from crontab import CronTab, CronSlices
@@ -70,7 +71,13 @@ from .config import (
     IS_TTY,
     DEBUG,
     IN_DOCKER,
+    IN_QEMU,
+    PUID,
+    PGID,
     USER,
+    TIMEZONE,
+    ENFORCE_ATOMIC_WRITES,
+    OUTPUT_PERMISSIONS,
     PYTHON_BINARY,
     ARCHIVEBOX_BINARY,
     ONLY_NEW,
@@ -80,25 +87,26 @@ from .config import (
     LOGS_DIR,
     PACKAGE_DIR,
     CONFIG_FILE,
-    CONFIG_FILENAME,
     ARCHIVE_DIR_NAME,
-    SOURCES_DIR_NAME,
-    LOGS_DIR_NAME,
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,
     SQL_INDEX_FILENAME,
-    ROBOTS_TXT_FILENAME,
-    FAVICON_FILENAME,
+    ALLOWED_IN_OUTPUT_DIR,
     SEARCH_BACKEND_ENGINE,
+    LDAP,
+    get_version,
     check_dependencies,
     check_data_folder,
     write_config_file,
     VERSION,
+    VERSIONS_AVAILABLE,
+    CAN_UPGRADE,
+    COMMIT_HASH,
+    BUILD_TIME,
     CODE_LOCATIONS,
     EXTERNAL_LOCATIONS,
     DATA_LOCATIONS,
     DEPENDENCIES,
-    USE_CHROME,
     CHROME_BINARY,
     CHROME_VERSION,
     YOUTUBEDL_BINARY,
@@ -106,12 +114,12 @@ from .config import (
     SINGLEFILE_VERSION,
     READABILITY_VERSION,
     MERCURY_VERSION,
-    USE_YOUTUBEDL,
-    USE_NODE,
     NODE_VERSION,
     load_all_config,
     CONFIG,
     USER_CONFIG,
+    ADMIN_USERNAME,
+    ADMIN_PASSWORD,
     get_real_name,
     setup_django,
 )
@@ -133,33 +141,7 @@ from .logging_util import (
 
 from .search import flush_search_index, index_links
 
-ALLOWED_IN_OUTPUT_DIR = {
-    '.gitignore',
-    'lost+found',
-    '.DS_Store',
-    '.venv',
-    'venv',
-    'virtualenv',
-    '.virtualenv',
-    'node_modules',
-    'package.json',
-    'package-lock.json',
-    'yarn.lock',
-    'static',
-    'sonic',
-    ARCHIVE_DIR_NAME,
-    SOURCES_DIR_NAME,
-    LOGS_DIR_NAME,
-    SQL_INDEX_FILENAME,
-    f'{SQL_INDEX_FILENAME}-wal',
-    f'{SQL_INDEX_FILENAME}-shm',
-    JSON_INDEX_FILENAME,
-    HTML_INDEX_FILENAME,
-    ROBOTS_TXT_FILENAME,
-    FAVICON_FILENAME,
-    CONFIG_FILENAME,
-    f'{CONFIG_FILENAME}.bak',
-}
+
 
 @enforce_types
 def help(out_dir: Path=OUTPUT_DIR) -> None:
@@ -236,51 +218,73 @@ def help(out_dir: Path=OUTPUT_DIR) -> None:
 def version(quiet: bool=False,
             out_dir: Path=OUTPUT_DIR) -> None:
     """Print the ArchiveBox version and dependency information"""
-
-    if quiet:
-        print(VERSION)
-    else:
-        # ArchiveBox v0.5.6
-        # Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY)
-        print('ArchiveBox v{}'.format(VERSION))
+    
+    print(VERSION)
+    
+    if not quiet:
+        # 0.7.1
+        # ArchiveBox v0.7.1+editable COMMIT_HASH=951bba5 BUILD_TIME=2023-12-17 16:46:05 1702860365
+        # IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-14.2-arm64-arm-64bit PYTHON=Cpython
+        # FS_ATOMIC=True FS_REMOTE=False FS_USER=501:20 FS_PERMS=644
+        # DEBUG=False IS_TTY=True TZ=UTC SEARCH_BACKEND=ripgrep LDAP=False
+        
         p = platform.uname()
         print(
-            sys.implementation.name.title(),
-            p.system,
-            platform.platform(),
-            p.machine,
+            'ArchiveBox v{}'.format(get_version(CONFIG)),
+            *((f'COMMIT_HASH={COMMIT_HASH[:7]}',) if COMMIT_HASH else ()),
+            f'BUILD_TIME={BUILD_TIME}',
         )
         print(
             f'IN_DOCKER={IN_DOCKER}',
+            f'IN_QEMU={IN_QEMU}',
+            f'ARCH={p.machine}',
+            f'OS={p.system}',
+            f'PLATFORM={platform.platform()}',
+            f'PYTHON={sys.implementation.name.title()}',
+        )
+        OUTPUT_IS_REMOTE_FS = DATA_LOCATIONS['OUTPUT_DIR']['is_mount'] or DATA_LOCATIONS['ARCHIVE_DIR']['is_mount']
+        print(
+            f'FS_ATOMIC={ENFORCE_ATOMIC_WRITES}',
+            f'FS_REMOTE={OUTPUT_IS_REMOTE_FS}',
+            f'FS_USER={PUID}:{PGID}',
+            f'FS_PERMS={OUTPUT_PERMISSIONS}',
+        )
+        print(
             f'DEBUG={DEBUG}',
             f'IS_TTY={IS_TTY}',
-            f'TZ={os.environ.get("TZ", "UTC")}',
-            f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}',
+            f'TZ={TIMEZONE}',
+            f'SEARCH_BACKEND={SEARCH_BACKEND_ENGINE}',
+            f'LDAP={LDAP}',
+            #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
         )
         print()
 
         print('{white}[i] Dependency versions:{reset}'.format(**ANSI))
         for name, dependency in DEPENDENCIES.items():
             print(printable_dependency_version(name, dependency))
+            
+            # add a newline between core dependencies and extractor dependencies for easier reading
+            if name == 'ARCHIVEBOX_BINARY':
+                print()
         
         print()
         print('{white}[i] Source-code locations:{reset}'.format(**ANSI))
-        for name, folder in CODE_LOCATIONS.items():
-            print(printable_folder_status(name, folder))
+        for name, path in CODE_LOCATIONS.items():
+            print(printable_folder_status(name, path))
 
         print()
         print('{white}[i] Secrets locations:{reset}'.format(**ANSI))
-        for name, folder in EXTERNAL_LOCATIONS.items():
-            print(printable_folder_status(name, folder))
+        for name, path in EXTERNAL_LOCATIONS.items():
+            print(printable_folder_status(name, path))
 
         print()
         if DATA_LOCATIONS['OUTPUT_DIR']['is_valid']:
             print('{white}[i] Data locations:{reset}'.format(**ANSI))
-            for name, folder in DATA_LOCATIONS.items():
-                print(printable_folder_status(name, folder))
+            for name, path in DATA_LOCATIONS.items():
+                print(printable_folder_status(name, path))
         else:
             print()
-            print('{white}[i] Data locations:{reset}'.format(**ANSI))
+            print('{white}[i] Data locations:{reset} (not in a data directory)'.format(**ANSI))
 
         print()
         check_dependencies()
@@ -311,7 +315,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
 
     if (out_dir / JSON_INDEX_FILENAME).exists():
         stderr("[!] This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.", color="lightyellow")
-        stderr("    You can run `archivebox list --json --with-headers > index.json` to manually generate it.", color="lightyellow")
+        stderr("    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.", color="lightyellow")
 
     existing_index = (out_dir / SQL_INDEX_FILENAME).exists()
 
@@ -433,6 +437,13 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
         write_main_index(list(pending_links.values()), out_dir=out_dir)
 
     print('\n{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
+
+    from django.contrib.auth.models import User
+
+    if (ADMIN_USERNAME and ADMIN_PASSWORD) and not User.objects.filter(username=ADMIN_USERNAME).exists():
+        print('{green}[+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.{reset}'.format(**ANSI))
+        User.objects.create_superuser(username=ADMIN_USERNAME, password=ADMIN_PASSWORD)
+
     if existing_index:
         print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**ANSI))
     else:
@@ -455,7 +466,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
         print('        archivebox server  # then visit http://127.0.0.1:8000')
         print()
         print('    To add new links, you can run:')
-        print("        archivebox add ~/some/path/or/url/to/list_of_links.txt")
+        print("        archivebox add < ~/some/path/to/list_of_links.txt")
         print()
         print('    For more usage and examples, run:')
         print('        archivebox help')
@@ -582,7 +593,8 @@ def oneshot(url: str, extractors: str="", out_dir: Path=OUTPUT_DIR):
 def add(urls: Union[str, List[str]],
         tag: str='',
         depth: int=0,
-        update_all: bool=not ONLY_NEW,
+        update: bool=not ONLY_NEW,
+        update_all: bool=False,
         index_only: bool=False,
         overwrite: bool=False,
         # duplicate: bool=False,  # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically
@@ -592,7 +604,7 @@ def add(urls: Union[str, List[str]],
         out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Add a new URL or list of URLs to your archive"""
 
-    from core.models import Tag
+    from core.models import Snapshot, Tag
 
     assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)'
 
@@ -615,6 +627,7 @@ def add(urls: Union[str, List[str]],
         # save verbatim args to sources
         write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir)
     
+
     new_links += parse_links_from_source(write_ahead_log, root_url=None, parser=parser)
 
     # If we're going one level deeper, download each link and look for more links
@@ -622,8 +635,11 @@ def add(urls: Union[str, List[str]],
     if new_links and depth == 1:
         log_crawl_started(new_links)
         for new_link in new_links:
-            downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir)
-            new_links_depth += parse_links_from_source(downloaded_file, root_url=new_link.url)
+            try:
+                downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir)
+                new_links_depth += parse_links_from_source(downloaded_file, root_url=new_link.url)
+            except Exception as err:
+                stderr('[!] Failed to get contents of URL {new_link.url}', err, color='red')
 
     imported_links = list({link.url: link for link in (new_links + new_links_depth)}.values())
     
@@ -632,6 +648,19 @@ def add(urls: Union[str, List[str]],
     write_main_index(links=new_links, out_dir=out_dir)
     all_links = load_main_index(out_dir=out_dir)
 
+    tags = [
+        Tag.objects.get_or_create(name=name.strip())[0]
+        for name in tag.split(',')
+        if name.strip()
+    ]
+    if tags:
+        for link in imported_links:
+            snapshot = Snapshot.objects.get(url=link.url)
+            snapshot.tags.add(*tags)
+            snapshot.tags_str(nocache=True)
+            snapshot.save()
+        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
+
     if index_only:
         # mock archive all the links using the fake index_only extractor method in order to update their state
         if overwrite:
@@ -646,28 +675,25 @@ def add(urls: Union[str, List[str]],
         if extractors:
             archive_kwargs["methods"] = extractors
 
-        if update_all:
+        stderr()
+
+        ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+
+        if update:
+            stderr(f'[*] [{ts}] Archiving + updating {len(imported_links)}/{len(all_links)}', len(imported_links), 'URLs from added set...', color='green')
+            archive_links(imported_links, overwrite=overwrite, **archive_kwargs)
+        elif update_all:
+            stderr(f'[*] [{ts}] Archiving + updating {len(all_links)}/{len(all_links)}', len(all_links), 'URLs from entire library...', color='green')
             archive_links(all_links, overwrite=overwrite, **archive_kwargs)
         elif overwrite:
+            stderr(f'[*] [{ts}] Archiving + overwriting {len(imported_links)}/{len(all_links)}', len(imported_links), 'URLs from added set...', color='green')
             archive_links(imported_links, overwrite=True, **archive_kwargs)
         elif new_links:
+            stderr(f'[*] [{ts}] Archiving {len(new_links)}/{len(all_links)} URLs from added set...', color='green')
             archive_links(new_links, overwrite=False, **archive_kwargs)
 
-
-    # add any tags to imported links
-    tags = [
-        Tag.objects.get_or_create(name=name.strip())[0]
-        for name in tag.split(',')
-        if name.strip()
-    ]
-    if tags:
-        for link in imported_links:
-            snapshot = link.as_snapshot()
-            snapshot.tags.add(*tags)
-            snapshot.tags_str(nocache=True)
-            snapshot.save()
-        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
-
+    if CAN_UPGRADE:
+        hint(f"There's a new version of ArchiveBox available! Your current version is {VERSION}. You can upgrade to {VERSIONS_AVAILABLE['recommended_version']['tag_name']} ({VERSIONS_AVAILABLE['recommended_version']['html_url']}). For more on how to upgrade: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives\n")
 
     return all_links
 
@@ -940,34 +966,35 @@ def setup(out_dir: Path=OUTPUT_DIR) -> None:
     stderr('\n[+] Installing enabled ArchiveBox dependencies automatically...', color='green')
 
     stderr('\n    Installing YOUTUBEDL_BINARY automatically using pip...')
-    if USE_YOUTUBEDL:
-        if YOUTUBEDL_VERSION:
-            print(f'{YOUTUBEDL_VERSION} is already installed', YOUTUBEDL_BINARY)
-        else:
-            try:
-                run_shell([
-                    PYTHON_BINARY, '-m', 'pip',
-                    'install',
-                    '--upgrade',
-                    '--no-cache-dir',
-                    '--no-warn-script-location',
-                    'youtube_dl',
-                ], capture_output=False, cwd=out_dir)
-                pkg_path = run_shell([
-                    PYTHON_BINARY, '-m', 'pip',
-                    'show',
-                    'youtube_dl',
-                ], capture_output=True, text=True, cwd=out_dir).stdout.split('Location: ')[-1].split('\n', 1)[0]
-                NEW_YOUTUBEDL_BINARY = Path(pkg_path) / 'youtube_dl' / '__main__.py'
-                os.chmod(NEW_YOUTUBEDL_BINARY, 0o777)
-                assert NEW_YOUTUBEDL_BINARY.exists(), f'youtube_dl must exist inside {pkg_path}'
-                config(f'YOUTUBEDL_BINARY={NEW_YOUTUBEDL_BINARY}', set=True, out_dir=out_dir)
-            except BaseException as e:
-                stderr(f'[X] Failed to install python packages: {e}', color='red')
-                raise SystemExit(1)
+    if YOUTUBEDL_VERSION:
+        print(f'{YOUTUBEDL_VERSION} is already installed', YOUTUBEDL_BINARY)
+    else:
+        try:
+            run_shell([
+                PYTHON_BINARY, '-m', 'pip',
+                'install',
+                '--upgrade',
+                '--no-cache-dir',
+                '--no-warn-script-location',
+                'youtube_dl',
+            ], capture_output=False, cwd=out_dir)
+            pkg_path = run_shell([
+                PYTHON_BINARY, '-m', 'pip',
+                'show',
+                'youtube_dl',
+            ], capture_output=True, text=True, cwd=out_dir).stdout.decode().split('Location: ')[-1].split('\n', 1)[0]
+            NEW_YOUTUBEDL_BINARY = Path(pkg_path) / 'youtube_dl' / '__main__.py'
+            os.chmod(NEW_YOUTUBEDL_BINARY, 0o777)
+            assert NEW_YOUTUBEDL_BINARY.exists(), f'youtube_dl must exist inside {pkg_path}'
+            config(f'YOUTUBEDL_BINARY={NEW_YOUTUBEDL_BINARY}', set=True, out_dir=out_dir)
+        except BaseException as e:                                              # lgtm [py/catch-base-exception]
+            stderr(f'[X] Failed to install python packages: {e}', color='red')
+            raise SystemExit(1)
 
-    stderr('\n    Installing CHROME_BINARY automatically using playwright...')
-    if USE_CHROME:
+    if platform.machine() == 'armv7l':
+        stderr('\n    Skip the automatic installation of CHROME_BINARY because playwright is not available on armv7.')
+    else:
+        stderr('\n    Installing CHROME_BINARY automatically using playwright...')
         if CHROME_VERSION:
             print(f'{CHROME_VERSION} is already installed', CHROME_BINARY)
         else:
@@ -982,59 +1009,60 @@ def setup(out_dir: Path=OUTPUT_DIR) -> None:
                 ], capture_output=False, cwd=out_dir)
                 run_shell([PYTHON_BINARY, '-m', 'playwright', 'install', 'chromium'], capture_output=False, cwd=out_dir)
                 proc = run_shell([PYTHON_BINARY, '-c', 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)'], capture_output=True, text=True, cwd=out_dir)
-                NEW_CHROME_BINARY = proc.stdout.strip()
+                NEW_CHROME_BINARY = proc.stdout.decode().strip() if isinstance(proc.stdout, bytes) else proc.stdout.strip()
                 assert NEW_CHROME_BINARY and len(NEW_CHROME_BINARY), 'CHROME_BINARY must contain a path'
                 config(f'CHROME_BINARY={NEW_CHROME_BINARY}', set=True, out_dir=out_dir)
-            except BaseException as e:
+            except BaseException as e:                                              # lgtm [py/catch-base-exception]
                 stderr(f'[X] Failed to install chromium using playwright: {e.__class__.__name__} {e}', color='red')
                 raise SystemExit(1)
 
     stderr('\n    Installing SINGLEFILE_BINARY, READABILITY_BINARY, MERCURY_BINARY automatically using npm...')
-    if USE_NODE:
-        if not NODE_VERSION:
-            stderr('[X] You must first install node using your system package manager', color='red')
-            hint([
-                'curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -',
-                'or to disable all node-based modules run: archivebox config --set USE_NODE=False',
-            ])
+    if not NODE_VERSION:
+        stderr('[X] You must first install node & npm using your system package manager', color='red')
+        hint([
+            'https://github.com/nodesource/distributions#table-of-contents',
+            'or to disable all node-based modules run: archivebox config --set USE_NODE=False',
+        ])
+        raise SystemExit(1)
+
+    if all((SINGLEFILE_VERSION, READABILITY_VERSION, MERCURY_VERSION)):
+        print('SINGLEFILE_BINARY, READABILITY_BINARY, and MERCURURY_BINARY are already installed')
+    else:
+        try:
+            # clear out old npm package locations
+            paths = (
+                out_dir / 'package.json',
+                out_dir / 'package_lock.json',
+                out_dir / 'node_modules',
+            )
+            for path in paths:
+                if path.is_dir():
+                    shutil.rmtree(path, ignore_errors=True)
+                elif path.is_file():
+                    os.remove(path)
+
+            shutil.copyfile(PACKAGE_DIR / 'package.json', out_dir / 'package.json')   # copy the js requirements list from the source install into the data dir
+            # lets blindly assume that calling out to npm via shell works reliably cross-platform 🤡 (until proven otherwise via support tickets)
+            run_shell([
+                'npm',
+                'install',
+                '--prefix', str(out_dir),        # force it to put the node_modules dir in this folder
+                '--force',                       # overwrite any existing node_modules
+                '--no-save',                     # don't bother saving updating the package.json or package-lock.json file
+                '--no-audit',                    # don't bother checking for newer versions with security vuln fixes
+                '--no-fund',                     # hide "please fund our project" messages
+                '--loglevel', 'error',           # only show erros (hide warn/info/debug) during installation
+                # these args are written in blood, change with caution
+            ], capture_output=False, cwd=out_dir)
+            os.remove(out_dir / 'package.json')
+        except BaseException as e:                                              # lgtm [py/catch-base-exception]
+            stderr(f'[X] Failed to install npm packages: {e}', color='red')
+            hint(f'Try deleting {out_dir}/node_modules and running it again')
             raise SystemExit(1)
 
-        if all((SINGLEFILE_VERSION, READABILITY_VERSION, MERCURY_VERSION)):
-            print('SINGLEFILE_BINARY, READABILITY_BINARY, and MERCURURY_BINARY are already installed')
-        else:
-            try:
-                # clear out old npm package locations
-                paths = (
-                    out_dir / 'package.json',
-                    out_dir / 'package_lock.json',
-                    out_dir / 'node_modules',
-                )
-                for path in paths:
-                    if path.is_dir():
-                        shutil.rmtree(path, ignore_errors=True)
-                    elif path.is_file():
-                        os.remove(path)
-
-                shutil.copyfile(PACKAGE_DIR / 'package.json', out_dir / 'package.json')
-                run_shell([
-                    'npm',
-                    'install',
-                    '--prefix', str(out_dir),
-                    '--force',
-                    '--no-save',
-                    '--no-audit',
-                    '--no-fund',
-                    '--loglevel', 'error',
-                ], capture_output=False, cwd=out_dir)
-                os.remove(out_dir / 'package.json')
-            except BaseException as e:
-                stderr(f'[X] Failed to install npm packages: {e}', color='red')
-                hint(f'Try deleting {out_dir}/node_modules and running it again')
-                raise SystemExit(1)
-
     stderr('\n[√] Set up ArchiveBox and its dependencies successfully.', color='green')
     
-    run_shell([ARCHIVEBOX_BINARY, '--version'], capture_output=False, cwd=out_dir)
+    run_shell([PYTHON_BINARY, ARCHIVEBOX_BINARY, '--version'], capture_output=False, cwd=out_dir)
 
 @enforce_types
 def config(config_options_str: Optional[str]=None,
@@ -1140,8 +1168,10 @@ def schedule(add: bool=False,
              run_all: bool=False,
              quiet: bool=False,
              every: Optional[str]=None,
+             tag: str='',
              depth: int=0,
              overwrite: bool=False,
+             update: bool=not ONLY_NEW,
              import_path: Optional[str]=None,
              out_dir: Path=OUTPUT_DIR):
     """Set ArchiveBox to regularly import URLs at specific times using cron"""
@@ -1171,6 +1201,8 @@ def schedule(add: bool=False,
             *([
                 'add',
                 *(['--overwrite'] if overwrite else []),
+                *(['--update'] if update else []),
+                *([f'--tag={tag}'] if tag else []),
                 f'--depth={depth}',
                 f'"{import_path}"',
             ] if import_path else ['update']),
@@ -1253,6 +1285,9 @@ def schedule(add: bool=False,
                 print('\n{green}[√] Stopped.{reset}'.format(**ANSI))
                 raise SystemExit(1)
 
+    if CAN_UPGRADE:
+        hint(f"There's a new version of ArchiveBox available! Your current version is {VERSION}. You can upgrade to {VERSIONS_AVAILABLE['recommended_version']['tag_name']} ({VERSIONS_AVAILABLE['recommended_version']['html_url']}). For more on how to upgrade: https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives\n")
+
     
 @enforce_types
 def server(runserver_args: Optional[List[str]]=None,

archivebox/package.json

@@ -1 +0,0 @@
-../package.json

archivebox/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "archivebox",
+  "version": "0.7.2",
+  "description": "ArchiveBox: The self-hosted internet archive",
+  "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
+  "repository": "github:ArchiveBox/ArchiveBox",
+  "license": "MIT",
+  "dependencies": {
+    "@postlight/parser": "^2.2.3",
+    "readability-extractor": "github:ArchiveBox/readability-extractor",
+    "single-file-cli": "^1.1.46"
+  }
+}

archivebox/parsers/__init__.py

@@ -34,6 +34,7 @@ from ..index.schema import Link
 from ..logging_util import TimedProgress, log_source_saved
 
 from . import pocket_api
+from . import readwise_reader_api
 from . import wallabag_atom
 from . import pocket_html
 from . import pinboard_rss
@@ -51,6 +52,7 @@ from . import url_list
 PARSERS = {
     # Specialized parsers
     pocket_api.KEY:     (pocket_api.NAME,       pocket_api.PARSER),
+    readwise_reader_api.KEY: (readwise_reader_api.NAME, readwise_reader_api.PARSER),
     wallabag_atom.KEY:  (wallabag_atom.NAME,    wallabag_atom.PARSER),
     pocket_html.KEY:    (pocket_html.NAME,      pocket_html.PARSER),
     pinboard_rss.KEY:   (pinboard_rss.NAME,     pinboard_rss.PARSER),
@@ -149,7 +151,17 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None,
 def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str:
     ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
     source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts))
-    atomic_write(source_path, raw_text)
+
+    referenced_texts = ''
+
+    for entry in raw_text.split():
+        try:
+            if Path(entry).exists():
+                referenced_texts += Path(entry).read_text()
+        except Exception as err:
+            print(err)
+
+    atomic_write(source_path, raw_text + '\n' + referenced_texts)
     log_source_saved(source_file=source_path)
     return source_path
 
@@ -176,7 +188,7 @@ def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{ba
                 ANSI['reset'],
             ))
             print('    ', e)
-            raise SystemExit(1)
+            raise e
 
     else:
         # Source is a path to a local file on the filesystem
@@ -223,6 +235,10 @@ _test_url_strs = {
     'https://example.com/?what=1#how-about-this=1&2%20baf': 1,
     'https://example.com?what=1#how-about-this=1&2%20baf': 1,
     '<test>http://example7.com</test>': 1,
+    'https://<test>': 0,
+    'https://[test]': 0,
+    'http://"test"': 0,
+    'http://\'test\'': 0,
     '[https://example8.com/what/is/this.php?what=1]': 1,
     '[and http://example9.com?what=1&other=3#and-thing=2]': 1,
     '<what>https://example10.com#and-thing=2 "</about>': 1,

archivebox/parsers/generic_json.py

@@ -17,7 +17,10 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)"""
 
     json_file.seek(0)
-    links = json.load(json_file)
+
+    # sometimes the first line is a comment or filepath, so we get everything after the first {
+    json_file_json_str = '{' + json_file.read().split('{', 1)[-1]
+    links = json.loads(json_file_json_str)
     json_date = lambda s: datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')
 
     for link in links:

archivebox/parsers/pinboard_rss.py

@@ -21,13 +21,18 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     root = ElementTree.parse(rss_file).getroot()
     items = root.findall("{http://purl.org/rss/1.0/}item")
     for item in items:
-        find = lambda p: item.find(p).text.strip() if item.find(p) else None    # type: ignore
+        find = lambda p: item.find(p).text.strip() if item.find(p) is not None else None    # type: ignore
 
         url = find("{http://purl.org/rss/1.0/}link")
         tags = find("{http://purl.org/dc/elements/1.1/}subject")
         title = find("{http://purl.org/rss/1.0/}title")
         ts_str = find("{http://purl.org/dc/elements/1.1/}date")
         
+        if url is None:
+            # Yielding a Link with no URL will
+            # crash on a URL validation assertion
+            continue
+
         # Pinboard includes a colon in its date stamp timezone offsets, which
         # Python can't parse. Remove it:
         if ts_str and ts_str[-3:-2] == ":":

archivebox/parsers/pocket_api.py

@@ -47,11 +47,11 @@ def get_pocket_articles(api: Pocket, since=None, page=0):
 
 
 def link_from_article(article: dict, sources: list):
-    url: str = article['resolved_url'] or article['given_url']
+    url: str = article.get('resolved_url') or article['given_url']
     broken_protocol = _BROKEN_PROTOCOL_RE.match(url)
     if broken_protocol:
         url = url.replace(f'{broken_protocol.group(1)}:/', f'{broken_protocol.group(1)}://')
-    title = article['resolved_title'] or article['given_title'] or url
+    title = article.get('resolved_title') or article.get('given_title') or url
 
     return Link(
         url=url,

archivebox/parsers/readwise_reader_api.py

@@ -0,0 +1,123 @@
+__package__ = "archivebox.parsers"
+
+
+import re
+import requests
+from datetime import datetime
+
+from typing import IO, Iterable, Optional
+from configparser import ConfigParser
+
+from pathlib import Path
+
+from ..index.schema import Link
+from ..util import enforce_types
+from ..system import atomic_write
+from ..config import (
+    SOURCES_DIR,
+    READWISE_READER_TOKENS,
+)
+
+
+API_DB_PATH = Path(SOURCES_DIR) / "readwise_reader_api.db"
+
+
+class ReadwiseReaderAPI:
+    cursor: Optional[str]
+
+    def __init__(self, api_token, cursor=None) -> None:
+        self.api_token = api_token
+        self.cursor = cursor
+
+    def get_archive(self):
+        response = requests.get(
+            url="https://readwise.io/api/v3/list/",
+            headers={"Authorization": f"Token {self.api_token}"},
+            params={
+                "location": "archive",
+                "pageCursor": self.cursor,
+            }
+        )
+        response.raise_for_status()
+        return response
+
+def get_readwise_reader_articles(api: ReadwiseReaderAPI):
+    response = api.get_archive()
+    body = response.json()
+    articles = body["results"]
+
+    yield from articles
+
+
+    if body['nextPageCursor']:
+        api.cursor = body["nextPageCursor"]
+        yield from get_readwise_reader_articles(api)
+
+
+def link_from_article(article: dict, sources: list):
+    url: str = article['source_url']
+    title = article["title"] or url
+    timestamp = datetime.fromisoformat(article['updated_at']).timestamp()
+
+    return Link(
+        url=url,
+        timestamp=str(timestamp),
+        title=title,
+        tags="",
+        sources=sources,
+    )
+
+
+def write_cursor(username: str, since: str):
+    if not API_DB_PATH.exists():
+        atomic_write(API_DB_PATH, "")
+
+    since_file = ConfigParser()
+    since_file.optionxform = str
+    since_file.read(API_DB_PATH)
+
+    since_file[username] = {"since": since}
+
+    with open(API_DB_PATH, "w+") as new:
+        since_file.write(new)
+
+
+def read_cursor(username: str) -> Optional[str]:
+    if not API_DB_PATH.exists():
+        atomic_write(API_DB_PATH, "")
+
+    config_file = ConfigParser()
+    config_file.optionxform = str
+    config_file.read(API_DB_PATH)
+
+    return config_file.get(username, "since", fallback=None)
+
+
+
+
+@enforce_types
+def should_parse_as_readwise_reader_api(text: str) -> bool:
+    return text.startswith("readwise-reader://")
+
+
+@enforce_types
+def parse_readwise_reader_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
+    """Parse bookmarks from the Readwise Reader API"""
+
+    input_buffer.seek(0)
+    pattern = re.compile(r"^readwise-reader:\/\/(\w+)")
+    for line in input_buffer:
+        if should_parse_as_readwise_reader_api(line):
+            username = pattern.search(line).group(1)
+            api = ReadwiseReaderAPI(READWISE_READER_TOKENS[username], cursor=read_cursor(username))
+
+            for article in get_readwise_reader_articles(api):
+                yield link_from_article(article, sources=[line])
+
+            if api.cursor:
+                write_cursor(username, api.cursor)
+
+
+KEY = "readwise_reader_api"
+NAME = "Readwise Reader API"
+PARSER = parse_readwise_reader_api_export

archivebox/parsers/wallabag_atom.py

@@ -34,13 +34,19 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
 
         trailing_removed = entry.split('</entry>', 1)[0]
         leading_removed = trailing_removed.strip()
-        rows = leading_removed.split('\n')
+        splits_fixed = leading_removed.replace('"\n              href="', '" href="')
+        rows = splits_fixed.split('\n')
 
-        def get_row(key):
-            return [r.strip() for r in rows if r.strip().startswith('<{}'.format(key))][0]
+        def get_row(prefix):
+            return [
+                row.strip()
+                for row in rows
+                if row.strip().startswith('<{}'.format(prefix))
+            ][0]
 
         title = str_between(get_row('title'), '<title><![CDATA[', ']]></title>').strip()
-        url = str_between(get_row('link rel="via"'), '<link rel="via">', '</link>')
+        url_inside_link = str_between(get_row('link rel="via"'), '<link rel="via">', '</link>')
+        url_inside_attr = str_between(get_row('link rel="via"'), 'href="', '"/>')
         ts_str = str_between(get_row('published'), '<published>', '</published>')
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
         try:
@@ -49,7 +55,7 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags = None
 
         yield Link(
-            url=htmldecode(url),
+            url=htmldecode(url_inside_attr or url_inside_link),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
             tags=tags or '',

archivebox/search/backends/sqlite.py

@@ -0,0 +1,195 @@
+import codecs
+from typing import List, Generator
+import sqlite3
+
+from archivebox.util import enforce_types
+from archivebox.config import (
+    FTS_SEPARATE_DATABASE,
+    FTS_TOKENIZERS,
+    FTS_SQLITE_MAX_LENGTH
+)
+
+FTS_TABLE = "snapshot_fts"
+FTS_ID_TABLE = "snapshot_id_fts"
+FTS_COLUMN = "texts"
+
+if FTS_SEPARATE_DATABASE:
+    database = sqlite3.connect("search.sqlite3")
+    # Make get_connection callable, because `django.db.connection.cursor()`
+    # has to be called to get a context manager, but sqlite3.Connection
+    # is a context manager without being called.
+    def get_connection():
+        return database
+    SQLITE_BIND = "?"
+else:
+    from django.db import connection as database  # type: ignore[no-redef, assignment]
+    get_connection = database.cursor
+    SQLITE_BIND = "%s"
+
+# Only Python >= 3.11 supports sqlite3.Connection.getlimit(),
+# so fall back to the default if the API to get the real value isn't present
+try:
+    limit_id = sqlite3.SQLITE_LIMIT_LENGTH
+    try:
+        with database.temporary_connection() as cursor:  # type: ignore[attr-defined]
+            SQLITE_LIMIT_LENGTH = cursor.connection.getlimit(limit_id)
+    except AttributeError:
+        SQLITE_LIMIT_LENGTH = database.getlimit(limit_id)
+except AttributeError:
+    SQLITE_LIMIT_LENGTH = FTS_SQLITE_MAX_LENGTH
+
+
+def _escape_sqlite3(value: str, *, quote: str, errors='strict') -> str:
+    assert isinstance(quote, str), "quote is not a str"
+    assert len(quote) == 1, "quote must be a single character"
+
+    encodable = value.encode('utf-8', errors).decode('utf-8')
+
+    nul_index = encodable.find("\x00")
+    if nul_index >= 0:
+        error = UnicodeEncodeError("NUL-terminated utf-8", encodable,
+                                   nul_index, nul_index + 1, "NUL not allowed")
+        error_handler = codecs.lookup_error(errors)
+        replacement, _ = error_handler(error)
+        assert isinstance(replacement, str), "handling a UnicodeEncodeError should return a str replacement"
+        encodable = encodable.replace("\x00", replacement)
+
+    return quote + encodable.replace(quote, quote * 2) + quote
+
+def _escape_sqlite3_value(value: str, errors='strict') -> str:
+    return _escape_sqlite3(value, quote="'", errors=errors)
+
+def _escape_sqlite3_identifier(value: str) -> str:
+    return _escape_sqlite3(value, quote='"', errors='strict')
+
+@enforce_types
+def _create_tables():
+    table = _escape_sqlite3_identifier(FTS_TABLE)
+    # Escape as value, because fts5() expects
+    # string literal column names
+    column = _escape_sqlite3_value(FTS_COLUMN)
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+    tokenizers = _escape_sqlite3_value(FTS_TOKENIZERS)
+    trigger_name = _escape_sqlite3_identifier(f"{FTS_ID_TABLE}_ad")
+
+    with get_connection() as cursor:
+        # Create a contentless-delete FTS5 table that indexes
+        # but does not store the texts of snapshots
+        try:
+            cursor.execute(
+                f"CREATE VIRTUAL TABLE {table}"
+                f" USING fts5({column},"
+                f" tokenize={tokenizers},"
+                " content='', contentless_delete=1);"
+                )
+        except Exception as e:
+            msg = str(e)
+            if 'unrecognized option: "contentlessdelete"' in msg:
+                sqlite_version = getattr(sqlite3, "sqlite_version", "Unknown")
+                raise RuntimeError(
+                    "SQLite full-text search requires SQLite >= 3.43.0;"
+                    f" the running version is {sqlite_version}"
+                ) from e
+            else:
+                raise
+        # Create a one-to-one mapping between ArchiveBox snapshot_id
+        # and FTS5 rowid, because the column type of rowid can't be
+        # customized.
+        cursor.execute(
+            f"CREATE TABLE {id_table}("
+            " rowid INTEGER PRIMARY KEY AUTOINCREMENT,"
+            " snapshot_id char(32) NOT NULL UNIQUE"
+            ");"
+            )
+        # Create a trigger to delete items from the FTS5 index when
+        # the snapshot_id is deleted from the mapping, to maintain
+        # consistency and make the `flush()` query simpler.
+        cursor.execute(
+            f"CREATE TRIGGER {trigger_name}"
+            f" AFTER DELETE ON {id_table} BEGIN"
+            f" DELETE FROM {table} WHERE rowid=old.rowid;"
+            " END;"
+            )
+
+def _handle_query_exception(exc: Exception):
+    message = str(exc)
+    if message.startswith("no such table:"):
+        raise RuntimeError(
+            "SQLite full-text search index has not yet"
+            " been created; run `archivebox update --index-only`."
+        )
+    else:
+        raise exc
+
+@enforce_types
+def index(snapshot_id: str, texts: List[str]):
+    text = ' '.join(texts)[:SQLITE_LIMIT_LENGTH]
+
+    table = _escape_sqlite3_identifier(FTS_TABLE)
+    column = _escape_sqlite3_identifier(FTS_COLUMN)
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+
+    with get_connection() as cursor:
+        retries = 2
+        while retries > 0:
+            retries -= 1
+            try:
+                # If there is already an FTS index rowid to snapshot_id mapping,
+                # then don't insert a new one, silently ignoring the operation.
+                # {id_table}.rowid is AUTOINCREMENT, so will generate an unused
+                # rowid for the index if it is an unindexed snapshot_id.
+                cursor.execute(
+                    f"INSERT OR IGNORE INTO {id_table}(snapshot_id) VALUES({SQLITE_BIND})",
+                    [snapshot_id])
+                # Fetch the FTS index rowid for the given snapshot_id
+                id_res = cursor.execute(
+                    f"SELECT rowid FROM {id_table} WHERE snapshot_id = {SQLITE_BIND}",
+                    [snapshot_id])
+                rowid = id_res.fetchone()[0]
+                # (Re-)index the content
+                cursor.execute(
+                    "INSERT OR REPLACE INTO"
+                    f" {table}(rowid, {column}) VALUES ({SQLITE_BIND}, {SQLITE_BIND})",
+                    [rowid, text])
+                # All statements succeeded; return
+                return
+            except Exception as e:
+                if str(e).startswith("no such table:") and retries > 0:
+                    _create_tables()
+                else:
+                    raise
+
+    raise RuntimeError("Failed to create tables for SQLite FTS5 search")
+
+@enforce_types
+def search(text: str) -> List[str]:
+    table = _escape_sqlite3_identifier(FTS_TABLE)
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+
+    with get_connection() as cursor:
+        try:
+            res = cursor.execute(
+                f"SELECT snapshot_id FROM {table}"
+                f" INNER JOIN {id_table}"
+                f" ON {id_table}.rowid = {table}.rowid"
+                f" WHERE {table} MATCH {SQLITE_BIND}",
+                [text])
+        except Exception as e:
+            _handle_query_exception(e)
+
+        snap_ids = [row[0] for row in res.fetchall()]
+    return snap_ids
+
+@enforce_types
+def flush(snapshot_ids: Generator[str, None, None]):
+    snapshot_ids = list(snapshot_ids)  # type: ignore[assignment]
+
+    id_table = _escape_sqlite3_identifier(FTS_ID_TABLE)
+
+    with get_connection() as cursor:
+        try:
+            cursor.executemany(
+                f"DELETE FROM {id_table} WHERE snapshot_id={SQLITE_BIND}",
+                [snapshot_ids])
+        except Exception as e:
+            _handle_query_exception(e)

archivebox/static

@@ -0,0 +1 @@
+templates/static

archivebox/system.py

@@ -14,11 +14,11 @@ from crontab import CronTab
 from .vendor.atomicwrites import atomic_write as lib_atomic_write
 
 from .util import enforce_types, ExtendedEncoder
-from .config import OUTPUT_PERMISSIONS
+from .config import PYTHON_BINARY, OUTPUT_PERMISSIONS, DIR_OUTPUT_PERMISSIONS, ENFORCE_ATOMIC_WRITES
 
 
 
-def run(*args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs):
+def run(cmd, *args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs):
     """Patched of subprocess.run to kill forked child subprocesses and fix blocking io making timeout=innefective
         Mostly copied from https://github.com/python/cpython/blob/master/Lib/subprocess.py
     """
@@ -37,7 +37,10 @@ def run(*args, input=None, capture_output=True, timeout=None, check=False, text=
 
     pgid = None
     try:
-        with Popen(*args, start_new_session=start_new_session, **kwargs) as process:
+        if isinstance(cmd, (list, tuple)) and cmd[0].endswith('.py'):
+            cmd = (PYTHON_BINARY, *cmd)
+
+        with Popen(cmd, *args, start_new_session=start_new_session, **kwargs) as process:
             pgid = os.getpgid(process.pid)
             try:
                 stdout, stderr = process.communicate(input, timeout=timeout)
@@ -89,14 +92,24 @@ def atomic_write(path: Union[Path, str], contents: Union[dict, str, bytes], over
             elif isinstance(contents, (bytes, str)):
                 f.write(contents)
     except OSError as e:
-        print(f"[X] OSError: Failed to write {path} with fcntl.F_FULLFSYNC. ({e})")
-        print("    You can store the archive/ subfolder on a hard drive or network share that doesn't support support syncronous writes,")
-        print("    but the main folder containing the index.sqlite3 and ArchiveBox.conf files must be on a filesystem that supports FSYNC.")
-        raise SystemExit(1)
+        if ENFORCE_ATOMIC_WRITES:
+            print(f"[X] OSError: Failed to write {path} with fcntl.F_FULLFSYNC. ({e})")
+            print("    You can store the archive/ subfolder on a hard drive or network share that doesn't support support syncronous writes,")
+            print("    but the main folder containing the index.sqlite3 and ArchiveBox.conf files must be on a filesystem that supports FSYNC.")
+            raise SystemExit(1)
+
+        # retry the write without forcing FSYNC (aka atomic mode)
+        with open(path, mode=mode, encoding=encoding) as f:
+            if isinstance(contents, dict):
+                dump(contents, f, indent=4, sort_keys=True, cls=ExtendedEncoder)
+            elif isinstance(contents, (bytes, str)):
+                f.write(contents)
+
+    # set file permissions
     os.chmod(path, int(OUTPUT_PERMISSIONS, base=8))
 
 @enforce_types
-def chmod_file(path: str, cwd: str='.', permissions: str=OUTPUT_PERMISSIONS) -> None:
+def chmod_file(path: str, cwd: str='.') -> None:
     """chmod -R <permissions> <cwd>/<path>"""
 
     root = Path(cwd) / path
@@ -104,10 +117,15 @@ def chmod_file(path: str, cwd: str='.', permissions: str=OUTPUT_PERMISSIONS) ->
         raise Exception('Failed to chmod: {} does not exist (did the previous step fail?)'.format(path))
 
     if not root.is_dir():
+        # path is just a plain file
         os.chmod(root, int(OUTPUT_PERMISSIONS, base=8))
     else:
         for subpath in Path(path).glob('**/*'):
-            os.chmod(subpath, int(OUTPUT_PERMISSIONS, base=8))
+            if subpath.is_dir():
+                # directories need execute permissions to be able to list contents
+                os.chmod(subpath, int(DIR_OUTPUT_PERMISSIONS, base=8))
+            else:
+                os.chmod(subpath, int(OUTPUT_PERMISSIONS, base=8))
 
 
 @enforce_types
@@ -167,17 +185,19 @@ def dedupe_cron_jobs(cron: CronTab) -> CronTab:
 
 
 class suppress_output(object):
-    '''
+    """
     A context manager for doing a "deep suppression" of stdout and stderr in 
     Python, i.e. will suppress all print, even if the print originates in a 
     compiled C/Fortran sub-function.
-       This will not suppress raised exceptions, since exceptions are printed
+    
+    This will not suppress raised exceptions, since exceptions are printed
     to stderr just before a script exits, and after the context manager has
     exited (at least, I think that is why it lets exceptions through).      
 
     with suppress_stdout_stderr():
         rogue_function()
-    '''
+    """
+    
     def __init__(self, stdout=True, stderr=True):
         # Open a pair of null files
         # Save the actual stdout (1) and stderr (2) file descriptors.

archivebox/templates/admin/base.html

@@ -12,7 +12,26 @@
         {% endblock %}
         
         <link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
-        {% block extrastyle %}{% endblock %}
+        {% block extrastyle %}
+        <style>
+            #upgrade-banner {
+                position: fixed;
+                right: 20px;
+                bottom: 20px;
+                background-color: #f8f8f8;
+                color: #333333;
+                border: 2px solid #772948;
+                padding: 10px 20px;
+                z-index: 1000;
+                text-align: center;
+            }
+            #dismiss-btn {
+                background: #aa1e55;
+                color: white;
+                cursor: pointer;
+            }
+        </style>
+        {% endblock %}
         
         {% if LANGUAGE_BIDI %}
             <link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">
@@ -122,6 +141,43 @@
             {% block footer %}<div id="footer"></div>{% endblock %}
         </div>
 
+        {% if user.is_authenticated and user.is_superuser and CAN_UPGRADE %}
+            <script>
+                if (!localStorage.getItem("bannerDismissed")) {
+                    const upgradeVersionTag = "{{VERSIONS_AVAILABLE.recommended_version.tag_name}}"
+                    const upgradeVersionURL = "{{VERSIONS_AVAILABLE.recommended_version.html_url}}"
+                    const currentVersionTag = "{{VERSION}}"
+                    const currentVersionURL = "{{VERSIONS_AVAILABLE.recommended_version.html_url}}"
+
+                    createBanner(currentVersionTag, currentVersionURL, upgradeVersionTag, upgradeVersionURL)
+                }
+
+                function createBanner(currentVersionTag, currentVersionURL, upgradeVersionTag, upgradeVersionURL) {
+                    const banner = document.createElement('div')
+                    banner.setAttribute('id', 'upgrade-banner');
+                    banner.innerHTML = `
+                        <p>There's a new version of ArchiveBox available!</p>
+                        Your version: <a href=${currentVersionURL}>${currentVersionTag}</a> | New version: <a href=${upgradeVersionURL}>${upgradeVersionTag}</a>
+                        <p>
+                        <a href=https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives>Upgrade Instructions</a> | <a href=https://github.com/ArchiveBox/ArchiveBox/releases>Changelog</a> | <a href=https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap>Roadmap</a> 
+                        </p>
+                        <button id="dismiss-btn">Dismiss</button>
+                    `
+                    document.body.appendChild(banner);
+                    const dismissButton = document.querySelector("#dismiss-btn")
+                    if (dismissButton) {
+                        dismissButton.addEventListener("click", dismissBanner)
+                    }
+                }
+
+                function dismissBanner() {
+                    const banner = document.getElementById("upgrade-banner")
+                    banner.style.display = "none"
+                    localStorage.setItem("bannerDismissed", "true")
+                }
+            </script>
+        {% endif %}
+        
         <script>
             $ = django.jQuery;
             $.fn.reverse = [].reverse;
@@ -197,7 +253,7 @@
 
                             // select the action button from the dropdown
                             container.find('select[name=action]')
-                                .find('op:selected').removeAttr('selected').end()
+                                .find('[selected]').removeAttr('selected').end()
                                 .find('[value=' + action_type + ']').attr('selected', 'selected').click()
                             
                             // click submit & replace the archivebox logo with a spinner

archivebox/templates/admin/private_index.html

@@ -1,62 +1,3 @@
-{% extends "base.html" %}
-{% load static %}
-
-{% block body %}
-    <div id="toolbar">
-        <form id="changelist-search" action="{% url 'public-index' %}" method="get">
-            <div>
-                <label for="searchbar"><img src="/static/admin/img/search.svg" alt="Search"></label>
-                <input type="text" size="40" name="q" value="" id="searchbar" autofocus placeholder="Title, URL, tags, timestamp, or content...".>
-                <input type="submit" value="Search" style="height: 36px; padding-top: 6px; margin: 8px"/>
-                <input type="button"
-                       value="♺"
-                       title="Refresh..."
-                       onclick="location.href='{% url 'public-index' %}'"
-                       style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
-                </input>
-            </div>
-        </form>
-    </div>
-    <table id="table-bookmarks">
-        <thead>
-            <tr>
-                <th style="width: 100px;">Bookmarked</th>
-                <th style="width: 26vw;">Snapshot ({{object_list|length}})</th>
-                <th style="width: 140px">Files</th>
-                <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
-            </tr>
-        </thead>
-            <tbody>
-                {% for link in object_list %}
-                    {% include 'main_index_row.html' with link=link  %}
-                {% endfor %}
-            </tbody>
-        </table>
-        <center>
-            <span class="step-links">
-                {% if page_obj.has_previous %}
-                    <a href="{% url 'public-index' %}?page=1">&laquo; first</a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
-                {% endif %}
-        
-                <span class="current">
-                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
-                </span>
-        
-                {% if page_obj.has_next %}
-                    <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
-                {% endif %}
-            </span>
-    
-            {% if page_obj.has_next %}
-                <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
-                <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
-            {% endif %}
-        </span>
-        <br>
-    </center>
-{% endblock %}
 {% extends "admin/base_site.html" %}
 {% load i18n admin_urls static admin_list %}
 {% load core_tags %}

archivebox/templates/admin/snapshots_grid.html

@@ -150,7 +150,6 @@
             <a href="{% url 'admin:core_snapshot_change' obj.id %}">
               <span class="timestamp">{{obj.added}}</span>
             </a>
-            &nbsp; &nbsp;
             <label>
               <span class="num_outputs">📄 &nbsp; {{obj.num_outputs}}</span> &nbsp; &nbsp;
               <span>🗄&nbsp; {{ obj.archive_size | file_size }}</span>

archivebox/templates/core/add.html

@@ -28,6 +28,14 @@
                 <a href="/add" id="submit">&nbsp; Add more URLs ➕</a>
             </center>
         {% else %}
+            <div id="in-progress" style="display: none;">
+                <center><h3>Adding URLs to index and running archive methods...</h3>
+                    <br/>
+                    <div class="loader"></div>
+                    <br/>
+                    Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for detailed progress...
+                </center>
+            </div>
             <form id="add-form" method="POST" class="p-form">{% csrf_token %}
                 <h1>Add new URLs to your archive</h1>
                 <br/>
@@ -38,20 +46,22 @@
             </form>
             <br/><br/><br/>
             <center id="delay-warning" style="display: none">
-                <small>(it's safe to leave this page, adding will continue in the background)</small>
+                <small>(you will be redirected to your <a href="/">Snapshot list</a> momentarily, its safe to close this page at any time)</small>
             </center>
             {% if absolute_add_path %}
-            <center id="bookmarklet">
+            <!-- <center id="bookmarklet">
               <p>Bookmark this link to quickly add to your archive:
-                <a href="javascript:void(window.open('{{ absolute_add_path }}?url='+document.location.href));">Add to ArchiveBox</a></p>
-            </center>
+                <a href="javascript:void(window.open('{{ absolute_add_path }}?url='+encodeURIComponent(document.location.href)));">Add to ArchiveBox</a></p>
+            </center> -->
             {% endif %}
             <script>
                 document.getElementById('add-form').addEventListener('submit', function(event) {
+                    document.getElementById('in-progress').style.display = 'block'
+                    document.getElementById('add-form').style.display = 'none'
+                    document.getElementById('delay-warning').style.display = 'block'
                     setTimeout(function() {
-                        document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for progress...</center>'
-                        document.getElementById('delay-warning').style.display = 'block'
-                    }, 200)
+                        window.location = '/'
+                    }, 2000)
                     return true
                 })
             </script>

archivebox/templates/core/base.html

@@ -38,8 +38,11 @@
                     <br />
                     <center>
                         <small>
-                            Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a> version
-                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{VERSION}}" title="Releases">v{{VERSION}}</a>.
+                            Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
+                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{VERSION}}" title="Releases">v{{VERSION}}</a>
+                            {% if COMMIT_HASH %}
+                                (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{{COMMIT_HASH}}"><code>{{COMMIT_HASH|truncatechars:9}}</code></a>)
+                            {% endif %}.
                             <br/><br/>
                             {{FOOTER_INFO}}
                         </small>

archivebox/templates/core/index_row.html

@@ -36,7 +36,7 @@
             {% endif %}
         </span>
     </td>
-    <td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; title="{{link.url}}">
+    <td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;" title="{{link.url}}">
         <a href="{{link.url}}">
             {{link.url}}
         </a>

archivebox/templates/core/snapshot.html

@@ -414,17 +414,21 @@
                         </div>
                     </div>
                     {% endif %}
+                    {% if PREVIEW_ORIGINALS %}
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
+                            <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy" referrerpolicy="no-referrer"></iframe>
                             <div class="card-body">
-                                <a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener">
+                                <a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener" referrerpolicy="no-referrer">
                                     <p class="card-text"><code>🌐 {{domain}}</code></p>
                                 </a>
-                                <a href="{{url}}" target="preview" id="original-btn"><h4 class="card-title">Original</h4></a>
+                                <a href="{{url}}" target="preview" id="original-btn" referrerpolicy="no-referrer">
+                                    <h4 class="card-title">Original</h4>
+                                </a>
                           </div>
                         </div>
                     </div>
+                    {% endif %}
                     <div class="col-lg-2">
                         <div class="card">
                             <iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>

archivebox/util.py

@@ -17,6 +17,8 @@ from requests.exceptions import RequestException, ReadTimeout
 
 from .vendor.base32_crockford import encode as base32_encode                            # type: ignore
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
+from os.path import lexists
+from os import remove as remove_file
 
 try:
     import chardet
@@ -59,7 +61,7 @@ URL_REGEX = re.compile(
     r'(?=('
     r'http[s]?://'                    # start matching from allowed schemes
     r'(?:[a-zA-Z]|[0-9]'              # followed by allowed alphanum characters
-    r'|[$-_@.&+]|[!*\(\),]'           #    or allowed symbols
+    r'|[-_$@.&+!*\(\),]'           #    or allowed symbols (keep hyphen first to match literal hyphen)
     r'|(?:%[0-9a-fA-F][0-9a-fA-F]))'  #    or allowed unicode bytes
     r'[^\]\[\(\)<>"\'\s]+'          # stop parsing at these symbols
     r'))',
@@ -219,7 +221,7 @@ def get_headers(url: str, timeout: int=None) -> str:
 def chrome_args(**options) -> List[str]:
     """helper to build up a chrome shell command with arguments"""
 
-    from .config import CHROME_OPTIONS
+    from .config import CHROME_OPTIONS, CHROME_VERSION
 
     options = {**CHROME_OPTIONS, **options}
 
@@ -229,19 +231,29 @@ def chrome_args(**options) -> List[str]:
     cmd_args = [options['CHROME_BINARY']]
 
     if options['CHROME_HEADLESS']:
-        cmd_args += ('--headless',)
-    
+        chrome_major_version = int(re.search(r'\s(\d+)\.\d', CHROME_VERSION)[1])
+        if chrome_major_version >= 111:
+            cmd_args += ("--headless=new",)
+        else:
+            cmd_args += ('--headless',)
+
     if not options['CHROME_SANDBOX']:
         # assume this means we are running inside a docker container
-        # in docker, GPU support is limited, sandboxing is unecessary, 
+        # in docker, GPU support is limited, sandboxing is unecessary,
         # and SHM is limited to 64MB by default (which is too low to be usable).
         cmd_args += (
-            '--no-sandbox',
-            '--disable-gpu',
-            '--disable-dev-shm-usage',
-            '--disable-software-rasterizer',
-            '--run-all-compositor-stages-before-draw',
-            '--hide-scrollbars',
+            "--no-sandbox",
+            "--no-zygote",
+            "--disable-dev-shm-usage",
+            "--disable-software-rasterizer",
+            "--run-all-compositor-stages-before-draw",
+            "--hide-scrollbars",
+            "--window-size=1440,2000",
+            "--autoplay-policy=no-user-gesture-required",
+            "--no-first-run",
+            "--use-fake-ui-for-media-stream",
+            "--use-fake-device-for-media-stream",
+            "--disable-sync",
         )
 
 
@@ -254,14 +266,24 @@ def chrome_args(**options) -> List[str]:
     if options['RESOLUTION']:
         cmd_args += ('--window-size={}'.format(options['RESOLUTION']),)
 
-    if options['TIMEOUT']:
-        cmd_args += ('--timeout={}'.format(options['TIMEOUT'] * 1000),)
+    if options['CHROME_TIMEOUT']:
+       cmd_args += ('--timeout={}'.format(options['CHROME_TIMEOUT'] * 1000),)
 
     if options['CHROME_USER_DATA_DIR']:
         cmd_args.append('--user-data-dir={}'.format(options['CHROME_USER_DATA_DIR']))
     
     return cmd_args
 
+def chrome_cleanup():
+    """
+    Cleans up any state or runtime files that chrome leaves behind when killed by
+    a timeout or other error
+    """
+
+    from .config import IN_DOCKER
+    
+    if IN_DOCKER and lexists("/home/archivebox/.config/chromium/SingletonLock"):
+        remove_file("/home/archivebox/.config/chromium/SingletonLock")
 
 def ansi_to_html(text):
     """

archivebox/vendor/package-lock.json

@@ -0,0 +1 @@
+../../package-lock.json

archivebox/vendor/package.json

@@ -0,0 +1 @@
+../../package.json

bin/archive

@@ -1,11 +1,13 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# DEPRECATED: this helper exists for backwards compatibility with <v0.4 only
+# Do not use this to call archivebox, instead use the archivebox binary directly.
 
 if python3 -m django --version >/dev/null 2>&1; then
     python3 -m archivebox "$*"
 else
-    echo '[X] ArchiveBox must be installed before using:'
-    echo "    pip install archivebox"
+    echo '[X] ArchiveBox not found, is it installed and present in your $PATH?'
+    echo '    pip3 install archivebox'
     echo
-    echo "Hint: Did you forget to activate a virtuenv or set your $$PATH?"
+    echo 'Hint: Did you forget to activate a virtualenv?'
     exit 2
 fi

bin/build_brew.sh

@@ -23,7 +23,24 @@ fi
 
 cd "$REPO_DIR/brew_dist"
 # make sure archivebox.rb is up-to-date with the dependencies
+git pull
+git status | grep 'up to date'
 
-echo "[+] Building Homebrew bottle"
-brew install --build-bottle ./archivebox.rb
+echo
+echo "[+] Uninstalling any exisitng archivebox versions..."
+brew uninstall archivebox || true
+brew untap archivebox/archivebox || true
+
+# echo "[*] Running Formula linters and test build..."
+# brew test-bot --tap=ArchiveBox/homebrew-archivebox archivebox/archivebox/archivebox || true
+# brew uninstall archivebox || true
+# brew untap archivebox/archivebox || true
+
+echo
+echo "[+] Installing and building hombrew bottle from https://Github.com/ArchiveBox/homebrew-archivebox#main"
+brew tap archivebox/archivebox
+brew install --build-bottle archivebox
 brew bottle archivebox
+
+echo
+echo "[√] Finished. Make sure to commit the outputted .tar.gz and bottle files!"

bin/build_dev.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# ./bin/build_docker.sh dev 'linux/arm/v7'
+
+### Bash Environment Setup
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+# set -o xtrace
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+IFS=$'\n'
+
+REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
+cd "$REPO_DIR"
+which docker > /dev/null || exit 1
+
+
+TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
+VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
+SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+REQUIRED_PLATFORMS="${2:-"linux/arm64,linux/amd64,linux/arm/v7"}"
+
+echo "[+] Building Docker image: tag=$TAG_NAME version=$SHORT_VERSION arch=$REQUIRED_PLATFORMS"
+
+
+echo "[+] Building archivebox:$VERSION docker image..."
+# docker builder prune
+docker build . --no-cache -t archivebox-dev --load
+
+# docker buildx build --platform "$REQUIRED_PLATFORMS" --load . \
+#                -t archivebox \
+#                -t archivebox:$TAG_NAME \
+#                -t archivebox:$VERSION \
+#                -t archivebox:$SHORT_VERSION

bin/build_docker.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# ./bin/build_docker.sh dev 'linux/arm/v7'
 
 ### Bash Environment Setup
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
@@ -11,23 +12,93 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
-SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
 cd "$REPO_DIR"
 
-which docker > /dev/null
+which docker > /dev/null || exit 1
+which jq > /dev/null || exit 1
+# which pdm > /dev/null || exit 1
+
+SUPPORTED_PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
+
+TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
+VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
+SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+GIT_SHA=sha-"$(git rev-parse --short HEAD)"
+SELECTED_PLATFORMS="${2:-$SUPPORTED_PLATFORMS}"
+
+echo "[+] Building Docker image: tag=$TAG_NAME version=$SHORT_VERSION arch=$SELECTED_PLATFORMS"
+
+function check_platforms() {
+    INSTALLED_PLATFORMS="$(docker buildx inspect | grep 'Platforms:' )"
+
+    for REQUIRED_PLATFORM in ${SELECTED_PLATFORMS//,/$IFS}; do
+        echo "[+] Checking for: $REQUIRED_PLATFORM..."
+        if ! (echo "$INSTALLED_PLATFORMS" | grep -q "$REQUIRED_PLATFORM"); then
+            return 1
+        fi
+    done
+    echo
+    return 0
+}
+
+function remove_builder() {
+    # remove existing xbuilder
+    docker buildx stop xbuilder || true
+    docker buildx rm xbuilder || true
+}
+
+function create_builder() {
+    docker buildx use xbuilder && return 0
+    echo "[+] Creating new xbuilder for: $SELECTED_PLATFORMS"
+    echo
+    docker pull 'moby/buildkit:buildx-stable-1'
+
+    # Switch to buildx builder if already present / previously created
+    docker buildx create --name xbuilder --driver docker-container --bootstrap --use --platform "$SELECTED_PLATFORMS" || true
+    docker buildx inspect --bootstrap || true
+}
+
+function recreate_builder() {
+    # Install QEMU binaries for cross-platform building if not installed
+    docker run --privileged --rm 'tonistiigi/binfmt' --install all
+
+    remove_builder
+    create_builder
+}
+
+# Check if docker is ready for cross-plaform builds, if not, recreate builder
+docker buildx use xbuilder 2>&1 >/dev/null || create_builder
+check_platforms || (recreate_builder && check_platforms) || exit 1
+
+
+# Build python package lists
+echo "[+] Generating requirements.txt and pdm.lock from pyproject.toml..."
+pdm lock --group=':all' --strategy="inherit_metadata" --production
+pdm export --group=':all' --production --without-hashes -o requirements.txt
+
 
 echo "[+] Building archivebox:$VERSION docker image..."
-docker build . -t archivebox \
-               -t archivebox:latest \
-               -t archivebox:$VERSION \
-               -t archivebox:$SHORT_VERSION \
-               -t docker.io/nikisweeting/archivebox:latest \
-               -t docker.io/nikisweeting/archivebox:$VERSION \
-               -t docker.io/nikisweeting/archivebox:$SHORT_VERSION \
-               -t docker.io/archivebox/archivebox:latest \
-               -t docker.io/archivebox/archivebox:$VERSION \
-               -t docker.io/archivebox/archivebox:$SHORT_VERSION \
-               -t docker.pkg.github.com/archivebox/archivebox/archivebox:latest \
-               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$VERSION \
-               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$SHORT_VERSION
+# docker builder prune
+# docker build . --no-cache -t archivebox-dev \
+# replace --load with --push to deploy
+docker buildx build --platform "$SELECTED_PLATFORMS" --load . \
+               -t archivebox/archivebox \
+               -t archivebox/archivebox:$TAG_NAME \
+               -t archivebox/archivebox:$VERSION \
+               -t archivebox/archivebox:$SHORT_VERSION \
+               -t archivebox/archivebox:$GIT_SHA \
+               -t archivebox/archivebox:latest \
+               -t archivebox/archivebox:stable \
+               -t nikisweeting/archivebox \
+               -t nikisweeting/archivebox:$TAG_NAME \
+               -t nikisweeting/archivebox:$VERSION \
+               -t nikisweeting/archivebox:$SHORT_VERSION \
+               -t nikisweeting/archivebox:$GIT_SHA \
+               -t nikisweeting/archivebox:latest \
+               -t nikisweeting/archivebox:stable \
+               -t ghcr.io/archivebox/archivebox/archivebox:$TAG_NAME \
+               -t ghcr.io/archivebox/archivebox/archivebox:$VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$SHORT_VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$GIT_SHA \
+               -t ghcr.io/archivebox/archivebox/archivebox:latest \
+               -t ghcr.io/archivebox/archivebox/archivebox:stable

bin/build_pip.sh

@@ -15,17 +15,25 @@ REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && p
 if [[ -f "$REPO_DIR/.venv/bin/activate" ]]; then
     source "$REPO_DIR/.venv/bin/activate"
 else
-    echo "[!] Warning: No virtualenv presesnt in $REPO_DIR.venv"
+    echo "[!] Warning: No virtualenv presesnt in $REPO_DIR/.venv, creating one now..."
+    python3 -m venv --system-site-packages --symlinks $REPO_DIR/.venv
 fi
 cd "$REPO_DIR"
 
-
 echo "[*] Cleaning up build dirs"
 cd "$REPO_DIR"
 rm -Rf build dist
 
 echo "[+] Building sdist, bdist_wheel, and egg_info"
-python3 setup.py \
-    sdist --dist-dir=./pip_dist \
-    bdist_wheel --dist-dir=./pip_dist \
-    egg_info --egg-base=./pip_dist
+rm -f archivebox/package.json
+cp package.json archivebox/package.json
+
+pdm self update
+pdm install
+pdm build
+pdm export --without-hashes -o ./pip_dist/requirements.txt
+
+cp dist/* ./pip_dist/
+
+echo
+echo "[√] Finished. Don't forget to commit the new sdist and wheel files in ./pip_dist/"

bin/docker_entrypoint.sh

@@ -1,45 +1,157 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
-DATA_DIR="${DATA_DIR:-/data}"
-ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
+# This Docker ENTRYPOINT script is called by `docker run archivebox ...` or `docker compose run archivebox ...`.
+# It takes a CMD as $* shell arguments and runs it following these setup steps:
 
+# - Set the archivebox user to use the correct PUID & PGID
+#     1. highest precedence is for valid PUID and PGID env vars passsed in explicitly
+#     2. fall back to DETECTED_PUID of files found within existing data dir
+#     3. fall back to DEFAULT_PUID if no data dir or its owned by root
+# - Create a new /data dir if necessary and set the correct ownership on it
+# - Create a new /browsers dir if necessary and set the correct ownership on it
+# - Check whether we're running inside QEMU emulation and show a warning if so.
+# - Check that enough free space is available on / and /data
+# - Drop down to archivebox user permisisons and execute passed CMD command.
 
-# Set the archivebox user UID & GID
-if [[ -n "$PUID" && "$PUID" != 0 ]]; then
-    usermod -u "$PUID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
-fi
-if [[ -n "$PGID" && "$PGID" != 0 ]]; then
-    groupmod -g "$PGID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+# Bash Environment Setup
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
+# set -o xtrace
+# set -o nounset
+set -o errexit
+set -o errtrace
+set -o pipefail
+# IFS=$'\n'
+
+# Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime)
+export DATA_DIR="${DATA_DIR:-/data}"
+export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
+
+# Global default PUID and PGID if data dir is empty and no intended PUID+PGID is set manually by user
+export DEFAULT_PUID=911
+export DEFAULT_PGID=911
+
+# If user tires to set PUID and PGID to root values manually, catch and reject because root is not allowed
+if [[ "$PUID" == "0" ]]; then
+    echo -e "\n[X] Error: Got PUID=$PUID and PGID=$PGID but ArchiveBox is not allowed to be run as root, please change or unset PUID & PGID and try again." > /dev/stderr
+    echo -e "    Hint: some NFS/SMB/FUSE/etc. filesystems force-remap/ignore all permissions," > /dev/stderr
+        echo -e "          leave PUID/PGID unset, or use values the filesystem prefers (defaults to $DEFAULT_PUID:$DEFAULT_PGID)" > /dev/stderr
+        echo -e "    https://linux.die.net/man/8/mount.cifs#:~:text=does%20not%20provide%20unix%20ownership" > /dev/stderr
+    exit 3
 fi
 
+# If data directory already exists, autodetect detect owner by looking at files within
+export DETECTED_PUID="$(stat -c '%u' "$DATA_DIR/logs/errors.log" 2>/dev/null || echo "$DEFAULT_PUID")"
+export DETECTED_PGID="$(stat -c '%g' "$DATA_DIR/logs/errors.log" 2>/dev/null || echo "$DEFAULT_PGID")"
 
-# Set the permissions of the data dir to match the archivebox user
+# If data directory exists but is owned by root, use defaults instead of root because root is not allowed
+[[ "$DETECTED_PUID" == "0" ]] && export DETECTED_PUID="$DEFAULT_PUID"
+
+# Set archivebox user and group ids to desired PUID/PGID
+usermod -o -u "${PUID:-$DETECTED_PUID}" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+groupmod -o -g "${PGID:-$DETECTED_PGID}" "$ARCHIVEBOX_USER" > /dev/null 2>&1
+
+# re-set PUID and PGID to values reported by system instead of values we tried to set,
+# in case wonky filesystems or Docker setups try to play UID/GID remapping tricks on us
+export PUID="$(id -u archivebox)"
+export PGID="$(id -g archivebox)"
+
+# Check the permissions of the data dir (or create if it doesn't exist)
 if [[ -d "$DATA_DIR/archive" ]]; then
-    # check data directory permissions
-    if [[ ! "$(stat -c %u $DATA_DIR/archive)" = "$(id -u archivebox)" ]]; then
-        echo "Change in ownership detected, please be patient while we chown existing files"
-        echo "This could take some time..."
-        chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER -R "$DATA_DIR"
+    if touch "$DATA_DIR/archive/.permissions_test_safe_to_delete" 2>/dev/null; then
+        # It's fine, we are able to write to the data directory (as root inside the container)
+        rm -f "$DATA_DIR/archive/.permissions_test_safe_to_delete"
+        # echo "[√] Permissions are correct"
+    else
+     # the only time this fails is if the host filesystem doesn't allow us to write as root (e.g. some NFS mapall/maproot problems, connection issues, drive dissapeared, etc.)
+        echo -e "\n[X] Error: archivebox user (PUID=$PUID) is not able to write to your ./data dir (currently owned by $(stat -c '%u' "$DATA_DIR"):$(stat -c '%g' "$DATA_DIR")." >&2
+        echo -e "    Change ./data to be owned by PUID=$PUID PGID=$PGID on the host and retry:" > /dev/stderr
+        echo -e "       \$ chown -R $PUID:$PGID ./data\n" > /dev/stderr
+        echo -e "    Configure the PUID & PGID environment variables to change the desired owner:" > /dev/stderr
+        echo -e "       https://docs.linuxserver.io/general/understanding-puid-and-pgid\n" > /dev/stderr
+        echo -e "    Hint: some NFS/SMB/FUSE/etc. filesystems force-remap/ignore all permissions," > /dev/stderr
+        echo -e "          leave PUID/PGID unset, or use values the filesystem prefers (defaults to $DEFAULT_PUID:$DEFAULT_PGID)" > /dev/stderr
+        echo -e "    https://linux.die.net/man/8/mount.cifs#:~:text=does%20not%20provide%20unix%20ownership" > /dev/stderr
+        exit 3
     fi
 else
     # create data directory
     mkdir -p "$DATA_DIR/logs"
-    chown -R $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 fi
-chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 
+# force set the ownership of the data dir contents to the archivebox user and group
+# this is needed because Docker Desktop often does not map user permissions from the host properly
+chown $PUID:$PGID "$DATA_DIR"
+chown $PUID:$PGID "$DATA_DIR"/*
+
+# also chown BROWSERS_DIR because otherwise 'archivebox setup' wont be able to install chrome at runtime
+export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-/browsers}"
+mkdir -p "$PLAYWRIGHT_BROWSERS_PATH/permissions_test_safe_to_delete"
+chown $PUID:$PGID "$PLAYWRIGHT_BROWSERS_PATH"
+chown $PUID:$PGID "$PLAYWRIGHT_BROWSERS_PATH"/*
+rm -Rf "$PLAYWRIGHT_BROWSERS_PATH/permissions_test_safe_to_delete"
+
+
+# (this check is written in blood in 2023, QEMU silently breaks things in ways that are not obvious)
+export IN_QEMU="$(pmap 1 | grep qemu >/dev/null && echo 'True' || echo 'False')"
+if [[ "$IN_QEMU" == "True" ]]; then
+    echo -e "\n[!] Warning: Running $(uname -m) docker image using QEMU emulation, some things will break!" > /dev/stderr
+    echo -e "    chromium (screenshot, pdf, dom), singlefile, and any dependencies that rely on inotify will not run in QEMU." > /dev/stderr
+    echo -e "    See here for more info: https://github.com/microsoft/playwright/issues/17395#issuecomment-1250830493\n" > /dev/stderr
+fi
+
+# check disk space free on / and /data, warn on <500Mb free, error on <100Mb free
+export ROOT_USAGE="$(df --output=pcent,avail / | tail -n 1 | xargs)"
+export ROOT_USED_PCT="${ROOT_USAGE%%%*}"
+export ROOT_AVAIL_KB="$(echo "$ROOT_USAGE" | awk '{print $2}')"
+if [[ "$ROOT_AVAIL_KB" -lt 100000 ]]; then
+    echo -e "\n[!] Warning: Docker root filesystem is completely out of space! (${ROOT_USED_PCT}% used on /)" > /dev/stderr
+    echo -e "    you need to free up at least 100Mb in your Docker VM to continue:" > /dev/stderr
+    echo -e "    \$ docker system prune\n" > /dev/stderr
+    df -kh / > /dev/stderr
+    exit 3
+elif [[ "$ROOT_USED_PCT" -ge 99 ]] || [[ "$ROOT_AVAIL_KB" -lt 500000 ]]; then
+    echo -e "\n[!] Warning: Docker root filesystem is running out of space! (${ROOT_USED_PCT}% used on /)" > /dev/stderr
+    echo -e "    you may need to free up space in your Docker VM soon:" > /dev/stderr
+    echo -e "    \$ docker system prune\n" > /dev/stderr
+    df -kh / > /dev/stderr
+fi
+
+export DATA_USAGE="$(df --output=pcent,avail /data | tail -n 1 | xargs)"
+export DATA_USED_PCT="${DATA_USAGE%%%*}"
+export DATA_AVAIL_KB="$(echo "$DATA_USAGE" | awk '{print $2}')"
+if [[ "$DATA_AVAIL_KB" -lt 100000 ]]; then
+    echo -e "\n[!] Warning: Docker data volume is completely out of space! (${DATA_USED_PCT}% used on /data)" > /dev/stderr
+    echo -e "    you need to free up at least 100Mb on the drive holding your data directory" > /dev/stderr
+    echo -e "    \$ ncdu -x data\n" > /dev/stderr
+    df -kh /data > /dev/stderr
+    sleep 5
+elif [[ "$DATA_USED_PCT" -ge 99 ]] || [[ "$ROOT_AVAIL_KB" -lt 500000 ]]; then
+    echo -e "\n[!] Warning: Docker data volume is running out of space! (${DATA_USED_PCT}% used on /data)" > /dev/stderr
+    echo -e "    you may need to free up space on the drive holding your data directory soon" > /dev/stderr
+    echo -e "    \$ ncdu -x data\n" > /dev/stderr
+    df -kh /data > /dev/stderr
+fi
+
+
+export ARCHIVEBOX_BIN_PATH="$(which archivebox)"
 
 # Drop permissions to run commands as the archivebox user
-if [[ "$1" == /* || "$1" == "echo" || "$1" == "archivebox" ]]; then
-    # arg 1 is a binary, execute it verbatim
-    # e.g. "archivebox init"
-    #      "/bin/bash"
-    #      "echo"
-    exec gosu "$ARCHIVEBOX_USER" bash -c "$*"
+if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" == "cat" || "$1" == "whoami" || "$1" == "archivebox" ]]; then
+    # handle "docker run archivebox /bin/somecommand --with=some args" by passing args directly to bash -c
+    # e.g. "docker run archivebox archivebox init:
+    #      "docker run archivebox /venv/bin/ipython3"
+    #      "docker run archivebox /bin/bash -c '...'"
+    #      "docker run archivebox cat /VERSION.txt"
+    exec gosu "$PUID" /bin/bash -c "exec $(printf ' %q' "$@")"
+    # printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113
+    # gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
+    # outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command
 else
-    # no command given, assume args were meant to be passed to archivebox cmd
-    # e.g. "add https://example.com"
-    #      "manage createsupseruser"
-    #      "server 0.0.0.0:8000"
-    exec gosu "$ARCHIVEBOX_USER" bash -c "archivebox $*"
+    # handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand
+    # e.g. "docker run archivebox help"
+    #      "docker run archivebox add --depth=1 https://example.com"
+    #      "docker run archivebox manage createsupseruser"
+    #      "docker run archivebox server 0.0.0.0:8000"
+    exec gosu "$PUID" "$ARCHIVEBOX_BIN_PATH" "$@"
 fi

bin/export_browser_history.sh

@@ -1,4 +1,15 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# Helper script to export browser history and bookmarks to a format ArchiveBox can ingest.
+# Usage:
+#    curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/export_browser_history.sh'
+#    bash export_browser_history.sh --chrome
+#    bash export_browser_history.sh --firefox
+#    bash export_browser_history.sh --safari
+#    ls
+#        chrome_history.json
+#        firefox_history.json
+#        firefox_bookmarks.json
+#        safari_history.json
 
 OUTPUT_DIR="$(pwd)"
 
@@ -16,9 +27,9 @@ if [[ "$1" == "--chrome" ]]; then
     sqlite3 "$OUTPUT_DIR/chrome_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', last_visit_time, 'description', title, 'href', url)) || \"]\" FROM urls;" > "$OUTPUT_DIR/chrome_history.json"
     jq < "$(dirname "${2:-$default}")"/Bookmarks '.roots.other.children[] | {href: .url, description: .name, timestamp: .date_added}' > "$OUTPUT_DIR/chrome_bookmarks.json"
     
-    rm "$DATA_DIR"/output/sources/chrome_history.db.*
+    rm "$OUTPUT_DIR"/chrome_history.db.*
     echo "Chrome history exported to:"
-    echo "    output/sources/chrome_history.json"
+    echo "    $OUTPUT_DIR/chrome_history.json"
 fi
 
 if [[ "$1" == "--firefox" ]]; then
@@ -33,12 +44,29 @@ if [[ "$1" == "--firefox" ]]; then
     fi
     
     sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', last_visit_date, 'description', title, 'href', url)) || \"]\" FROM moz_places;" > "$OUTPUT_DIR/firefox_history.json"
-    sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "SELECT \"[\" || group_concat(json_object('timestamp', b.dateAdded, 'description', b.title, 'href', f.url)) || \"]\" FROM moz_bookmarks AS b JOIN moz_places AS f ON f.id = b.fk" > "$OUTPUT_DIR/firefox_bookmarks.json"
+
+    sqlite3 "$OUTPUT_DIR/firefox_history.db.tmp" "
+    with recursive tags AS (
+          select id, title, '' AS tags
+          FROM moz_bookmarks
+          where parent == 0
+        UNION ALL
+          select c.id, p.title, c.title || ',' || tags AS tags
+          from moz_bookmarks AS c
+          JOIN tags AS p
+          ON c.parent = p.id
+        )
+
+        SELECT '[' || group_concat(json_object('timestamp', b.dateAdded, 'description', b.title, 'href', f.url, 'tags', tags.tags)) || ']'
+        FROM moz_bookmarks AS b
+        JOIN moz_places AS f ON f.id = b.fk
+        JOIN tags ON tags.id = b.parent
+        WHERE f.url LIKE '%://%';" > "$OUTPUT_DIR/firefox_bookmarks.json"
     
-    rm "$DATA_DIR"/output/sources/firefox_history.db.*
+    rm "$OUTPUT_DIR"/firefox_history.db.*
     echo "Firefox history exported to:"
-    echo "    output/sources/firefox_history.json"
-    echo "    output/sources/firefox_bookmarks.json"
+    echo "    $OUTPUT_DIR/firefox_history.json"
+    echo "    $OUTPUT_DIR/firefox_bookmarks.json"
 fi
 
 if [[ "$1" == "--safari" ]]; then
@@ -54,7 +82,7 @@ if [[ "$1" == "--safari" ]]; then
     
     sqlite3 "$OUTPUT_DIR/safari_history.db.tmp" "select url from history_items" > "$OUTPUT_DIR/safari_history.json"
     
-    rm "$DATA_DIR"/output/sources/safari_history.db.*
+    rm "$OUTPUT_DIR"/safari_history.db.*
     echo "Safari history exported to:"
-    echo "    output/sources/safari_history.json"
+    echo "    $OUTPUT_DIR/safari_history.json"
 fi

bin/lint.sh

@@ -15,7 +15,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
 source "$DIR/.venv/bin/activate"
 
 echo "[*] Running flake8..."
-flake8 archivebox && echo "√ No errors found."
+cd archivebox
+flake8 . && echo "√ No errors found."
 
 echo
 

bin/release_brew.sh

@@ -11,9 +11,20 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
-SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+
+
+CURRENT_PLAFORM="$(uname)"
+REQUIRED_PLATFORM="Darwin"
+if [[ "$CURRENT_PLAFORM" != "$REQUIRED_PLATFORM" ]]; then
+   echo "[!] Skipping the Homebrew package build on $CURRENT_PLAFORM (it can only be run on $REQUIRED_PLATFORM)."
+   exit 0
+fi
+
 cd "$REPO_DIR"
 
-# TODO
-exit 0
+./bin/build_brew.sh
+
+git add '*.bottle.tar.gz'
+git commit -m "add new release bottle"
+git pull
+git push

bin/release_docker.sh

@@ -11,15 +11,44 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
-SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
 cd "$REPO_DIR"
 
+SUPPORTED_PLATFORMS="linux/amd64,linux/arm64,linux/arm/v7"
+
+TAG_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}"
+VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
+SHORT_VERSION="$(echo "$VERSION" | perl -pe 's/(\d+)\.(\d+)\.(\d+)/$1.$2/g')"
+GIT_SHA=sha-"$(git rev-parse --short HEAD)"
+SELECTED_PLATFORMS="${2:-$SUPPORTED_PLATFORMS}"
+
+
+# echo "[*] Logging in to Docker Hub & Github Container Registry"
+# docker login --username=nikisweeting
+# docker login ghcr.io --username=pirate
+
+# echo "[^] Building docker image"
+# ./bin/build_docker.sh "$TAG_NAME" "$SELECTED_PLATFORMS"
 
 echo "[^] Uploading docker image"
-# docker login --username=nikisweeting
-# docker login docker.pkg.github.com --username=pirate
-docker push archivebox/archivebox:$VERSION archivebox/archivebox:$SHORT_VERSION archivebox/archivebox:latest
-docker push docker.io/nikisweeting/archivebox
-docker push docker.io/archivebox/archivebox
-docker push docker.pkg.github.com/archivebox/archivebox/archivebox
+docker buildx build --platform "$SELECTED_PLATFORMS" --push . \
+               -t archivebox/archivebox \
+               -t archivebox/archivebox:$TAG_NAME \
+               -t archivebox/archivebox:$VERSION \
+               -t archivebox/archivebox:$SHORT_VERSION \
+               -t archivebox/archivebox:$GIT_SHA \
+               -t archivebox/archivebox:latest \
+               -t archivebox/archivebox:stable \
+               -t nikisweeting/archivebox \
+               -t nikisweeting/archivebox:$TAG_NAME \
+               -t nikisweeting/archivebox:$VERSION \
+               -t nikisweeting/archivebox:$SHORT_VERSION \
+               -t nikisweeting/archivebox:$GIT_SHA \
+               -t nikisweeting/archivebox:latest \
+               -t nikisweeting/archivebox:stable \
+               -t ghcr.io/archivebox/archivebox/archivebox:$TAG_NAME \
+               -t ghcr.io/archivebox/archivebox/archivebox:$VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$SHORT_VERSION \
+               -t ghcr.io/archivebox/archivebox/archivebox:$GIT_SHA \
+               -t ghcr.io/archivebox/archivebox/archivebox:latest \
+               -t ghcr.io/archivebox/archivebox/archivebox:stable
+

bin/release_pip.sh

@@ -11,17 +11,11 @@ set -o pipefail
 IFS=$'\n'
 
 REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
 cd "$REPO_DIR"
 source "$REPO_DIR/.venv/bin/activate"
 
+echo "[^] Publishing to Test PyPI..."
+pdm publish --repository testpypi
 
-# apt install python3 python3-all python3-dev
-# pip install '.[dev]'
-
-
-echo "[^] Uploading to test.pypi.org"
-python3 -m twine upload --repository testpypi pip_dist/archivebox-${VERSION}*.{whl,tar.gz}
-
-echo "[^] Uploading to pypi.org"
-python3 -m twine upload --repository pypi pip_dist/archivebox-${VERSION}*.{whl,tar.gz}
+echo "[^] Publishing to PyPI..."
+pdm publish --no-build

bin/setup.sh

@@ -1,120 +1,200 @@
-#!/usr/bin/env bash
-# ArchiveBox Setup Script
-# https://github.com/ArchiveBox/ArchiveBox
+#!/usr/bin/env sh
+# ArchiveBox Setup Script: https://github.com/ArchiveBox/ArchiveBox
+# Supported Platforms: Ubuntu/Debian/FreeBSD/macOS
+# Usage:
+#    curl -sSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/setup.sh' | sh
+
+clear
+
+if [ $(id -u) -eq 0 ]; then
+    echo ""
+    echo "[X] You cannot run this script as root. You must run it as a non-root user with sudo ability."
+    echo "    Create a new non-privileged user 'archivebox' if necessary."
+    echo "      adduser archivebox && usermod -a archivebox -G sudo && su archivebox"
+    echo "    https://www.digitalocean.com/community/tutorials/how-to-create-a-new-sudo-enabled-user-on-ubuntu-20-04-quickstart"
+    echo "    https://www.vultr.com/docs/create-a-sudo-user-on-freebsd"
+    echo "    Then re-run this script as the non-root user."
+    echo ""
+    exit 2
+fi
+
+if (which docker-compose > /dev/null && docker pull archivebox/archivebox:latest); then
+    echo "[+] Initializing an ArchiveBox data folder at ~/archivebox/data using Docker Compose..."
+    mkdir -p ~/archivebox
+    cd ~/archivebox
+    mkdir -p data
+    if [ -f "./index.sqlite3" ]; then
+        mv ~/archivebox/* ~/archivebox/data/
+    fi
+    curl -O 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/docker-compose.yml'
+    docker-compose run --rm archivebox init --setup
+    echo
+    echo "[+] Starting ArchiveBox server using: docker-compose up -d..."
+    docker-compose up -d
+    sleep 7
+    open http://127.0.0.1:8000 || true
+    echo
+    echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized in ~/archivebox/data. Usage:"
+    echo "    cd ~/archivebox"
+    echo "    docker-compose ps"
+    echo "    docker-compose down"
+    echo "    docker-compose pull"
+    echo "    docker-compose up"
+    echo "    docker-compose run archivebox manage createsuperuser"
+    echo "    docker-compose run archivebox add 'https://example.com'"
+    echo "    docker-compose run archivebox list"
+    echo "    docker-compose run archivebox help"
+    exit 0
+elif (which docker > /dev/null && docker pull archivebox/archivebox:latest); then
+    echo "[+] Initializing an ArchiveBox data folder at ~/archivebox using Docker..."
+    mkdir -p ~/archivebox
+    cd ~/archivebox
+    if [ -f "./data/index.sqlite3" ]; then
+        cd ./data
+    fi
+    docker run -v "$PWD":/data -it --rm archivebox/archivebox:latest init --setup
+    echo
+    echo "[+] Starting ArchiveBox server using: docker run -d archivebox/archivebox..."
+    docker run -v "$PWD":/data -it -d -p 8000:8000 --name=archivebox archivebox/archivebox:latest
+    sleep 7
+    open http://127.0.0.1:8000 || true
+    echo
+    echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized in ~/archivebox. Usage:"
+    echo "    cd ~/archivebox"
+    echo "    docker ps --filter name=archivebox"
+    echo "    docker kill archivebox"
+    echo "    docker pull archivebox/archivebox"
+    echo "    docker run -v $PWD:/data -d -p 8000:8000 --name=archivebox archivebox/archivebox"
+    echo "    docker run -v $PWD:/data -it archivebox/archivebox manage createsuperuser"
+    echo "    docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'"
+    echo "    docker run -v $PWD:/data -it archivebox/archivebox list"
+    echo "    docker run -v $PWD:/data -it archivebox/archivebox help"
+    exit 0
+fi
+
+echo ""
+echo "[!] It's highly recommended to use ArchiveBox with Docker, but Docker wasn't found."
+echo ""
+echo "    ⚠️ If you want to use Docker, press [Ctrl-C] to cancel now. ⚠️"
+echo "        Get Docker: https://docs.docker.com/get-docker/"
+echo "        After you've installed Docker, run this script again."
+echo ""
+echo "Otherwise, install will continue with apt/brew/pip in 12s... (press [Ctrl+C] to cancel)"
+echo ""
+sleep 12 || exit 1
+echo "Proceeding with system package manager..."
+echo ""
 
 echo "[i] ArchiveBox Setup Script 📦"
 echo ""
-echo "    This is a helper script which installs the ArchiveBox dependencies on your system using homebrew/aptitude."
-echo "    You may be prompted for a password in order to install the following:"
+echo "    This is a helper script which installs the ArchiveBox dependencies on your system using brew/apt/pip3."
+echo "    You may be prompted for a sudo password in order to install the following:"
 echo ""
-echo "        - python3, python3-pip, python3-distutils"
-echo "        - curl"
-echo "        - wget"
-echo "        - git"
-echo "        - youtube-dl"
-echo "        - chromium-browser  (skip this if Chrome/Chromium is already installed)"
-echo "        - nodejs            (used for singlefile, readability, mercury, and more)"
+echo "        - archivebox"
+echo "        - python3, pip, nodejs, npm            (languages used by ArchiveBox, and its extractor modules)"
+echo "        - curl, wget, git, youtube-dl, yt-dlp  (used for extracting title, favicon, git, media, and more)"
+echo "        - chromium                             (skips this if any Chrome/Chromium version is already installed)"
 echo ""
-echo "    If you'd rather install these manually, you can find documentation here:"
+echo "    If you'd rather install these manually as-needed, you can find detailed documentation here:"
 echo "        https://github.com/ArchiveBox/ArchiveBox/wiki/Install"
 echo ""
-read -p "Press [enter] to continue with the automatic install, or Ctrl+C to cancel..." REPLY
+echo "Continuing in 12s... (press [Ctrl+C] to cancel)"
+echo ""
+sleep 12 || exit 1
+echo "Proceeding to install dependencies..."
 echo ""
 
 # On Linux:
 if which apt-get > /dev/null; then
-    echo "[+] Adding ArchiveBox apt repo to sources..."
-    sudo apt install software-properties-common
-    sudo add-apt-repository -u ppa:archivebox/archivebox
-    echo "[+] Installing python3, wget, curl..."
-    sudo apt install -y git python3 python3-pip python3-distutils wget curl youtube-dl nodejs npm ripgrep
-    # sudo apt install archivebox
-
-    if which google-chrome; then
-        echo "[i] You already have google-chrome installed, if you would like to download chromium instead (they work pretty much the same), follow the Manual Setup instructions"
-        google-chrome --version
-    elif which chromium-browser; then
-        echo "[i] chromium-browser already installed, using existing installation."
-        chromium-browser --version
-    elif which chromium; then
-        echo "[i] chromium already installed, using existing installation."
-        chromium --version
-    else
-        echo "[+] Installing chromium..."
-        sudo apt install chromium || sudo apt install chromium-browser
+    echo "[+] Adding ArchiveBox apt repo and signing key to sources..."
+    if ! (sudo apt install -y software-properties-common && sudo add-apt-repository -u ppa:archivebox/archivebox); then
+        echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
+        echo "deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee -a /etc/apt/sources.list.d/archivebox.list
+        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
+        sudo apt-get update -qq
     fi
-
+    echo
+    echo "[+] Installing ArchiveBox system dependencies using apt..."
+    sudo apt-get install -y git python3 python3-pip python3-distutils wget curl youtube-dl yt-dlp ffmpeg git nodejs npm ripgrep
+    sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true
+    sudo apt-get install -y archivebox
+    sudo apt-get --only-upgrade install -y archivebox
+    echo ""
+    echo "[+] Installing ArchiveBox python dependencies using pip3..."
+    sudo python3 -m pip install --upgrade --ignore-installed archivebox
 # On Mac:
-elif which brew > /dev/null; then   # 🐍 eye of newt
-    echo "[+] Installing python3, wget, curl  (ignore 'already installed' warnings)..."
-    brew install git wget curl youtube-dl ripgrep node
-    if which python3; then
-        if python3 -c 'import sys; raise SystemExit(sys.version_info < (3,5,0))'; then
-            echo "[√] Using existing $(which python3)..."
-        else
-            echo "[+] Installing python3..."
-            brew install python3
-        fi
-    else
-        echo "[+] Installing python3..."
-        brew install python3
-    fi
-
-    if ls /Applications/Google\ Chrome*.app > /dev/null; then
-        echo "[√] Using existing /Applications/Google Chrome.app"
-    elif ls /Applications/Chromium.app; then
-        echo "[√] Using existing /Applications/Chromium.app"
-    elif which chromium-browser; then
-        echo "[√] Using existing $(which chromium-browser)"
-    elif which chromium; then
-        echo "[√] Using existing $(which chromium)"
-    else
-        echo "[+] Installing chromium..."
-        brew cask install chromium
-    fi
+elif which brew > /dev/null; then
+    echo "[+] Installing ArchiveBox system dependencies using brew..."
+    brew tap archivebox/archivebox
+    brew update
+    brew install --fetch-HEAD -f archivebox
+    echo ""
+    echo "[+] Installing ArchiveBox python dependencies using pip3..."
+    python3 -m pip install --upgrade --ignore-installed archivebox
+elif which pkg > /dev/null; then
+    echo "[+] Installing ArchiveBox system dependencies using pkg and pip (python3.9)..."
+    sudo pkg install -y python3 py39-pip py39-sqlite3 npm wget curl youtube_dl ffmpeg git ripgrep
+    sudo pkg install -y chromium
+    echo ""
+    echo "[+] Installing ArchiveBox python dependencies using pip..."
+    # don't use sudo here so that pip installs in $HOME/.local instead of into /usr/local
+    python3 -m pip install --upgrade --ignore-installed archivebox
 else
-    echo "[X] Could not find aptitude or homebrew! ‼️"
+    echo "[!] Warning: Could not find aptitude/homebrew/pkg! May not be able to install all dependencies automatically."
     echo ""
     echo "    If you're on macOS, make sure you have homebrew installed:     https://brew.sh/"
-    echo "    If you're on Ubuntu/Debian, make sure you have apt installed:  https://help.ubuntu.com/lts/serverguide/apt.html"
-    echo "    (those are the only currently supported systems for the automatic setup script)"
+    echo "    If you're on Linux, only Ubuntu/Debian/BSD systems are officially supported with this script."
+    echo "    If you're on Windows, this script is not officially supported (Docker is recommeded instead)."
     echo ""
-    echo "See the README.md for Manual Setup & Troubleshooting instructions."
+    echo "See the README.md for Manual Setup & Troubleshooting instructions if you you're unable to run ArchiveBox after this script completes."
+fi
+
+echo ""
+
+if ! (python3 --version && python3 -m pip --version && python3 -m django --version); then
+    echo "[X] Python 3 pip was not found on your system!"
+    echo "    You must first install Python >= 3.7 (and pip3):"
+    echo "      https://www.python.org/downloads/"
+    echo "      https://wiki.python.org/moin/BeginnersGuide/Download"
+    echo "    After installing, run this script again."
     exit 1
 fi
 
-npm i -g npm
-pip3 install --upgrade pip setuptools
+if ! (python3 -m django --version && python3 -m archivebox version --quiet); then
+    echo "[X] Django and ArchiveBox were not found after installing!"
+    echo "    Check to see if a previous step failed."
+    echo ""
+    exit 1
+fi
 
-pip3 install --upgrade archivebox
-npm install -g 'git+https://github.com/ArchiveBox/ArchiveBox.git' 
+# echo ""
+# echo "[+] Upgrading npm and pip..."
+# sudo npm i -g npm || true
+# sudo python3 -m pip install --upgrade pip setuptools || true
 
-# Check:
-echo ""
-echo "[*] Checking installed versions:"
-echo "---------------------------------------------------"
-which python3 &&
-python3 --version | head -n 1 &&
-echo "" &&
-which git &&
-git --version | head -n 1 &&
-echo "" &&
-which wget &&
-wget --version | head -n 1 &&
-echo "" &&
-which curl &&
-curl --version | head -n 1 &&
-echo "" &&
-which youtube-dl &&
-youtube-dl --version | head -n 1 &&
-echo "---------------------------------------------------" &&
-archivebox version &&
-echo "[√] All dependencies installed. ✅" &&
-exit 0
+echo
+echo "[+] Initializing ArchiveBox data folder at ~/archivebox..."
+mkdir -p ~/archivebox
+cd ~/archivebox
+if [ -f "./data/index.sqlite3" ]; then
+    cd ./data
+fi
+: | python3 -m archivebox init --setup || true   # pipe in empty command to make sure stdin is closed
 
-echo "---------------------------------------------------"
-echo "[X] Failed to install some dependencies! ‼️"
-echo "    - Try the Manual Setup instructions in the README.md"
-echo "    - Try the Troubleshooting: Dependencies instructions in the README.md"
-echo "    - Open an issue on github to get help: https://github.com/ArchiveBox/ArchiveBox/issues"
-exit 1
+echo
+echo "[+] Starting ArchiveBox server using: nohup archivebox server &..."
+nohup python3 -m archivebox server 0.0.0.0:8000 > ./logs/server.log 2>&1 &
+sleep 7
+which open > /dev/null && open http://127.0.0.1:8000 || true
+
+echo
+echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized in ~/archivebox. Usage:"
+echo "    cd ~/archivebox"
+echo "    ps aux | grep archivebox"
+echo "    pkill -f archivebox"
+echo "    python3 -m pip install --upgrade archivebox"
+echo "    archivebox server --quick-init 0.0.0.0:8000"
+echo "    archivebox manage createsuperuser"
+echo "    archivebox add 'https://example.com'"
+echo "    archivebox list"
+echo "    archivebox help"

bin/test.sh

@@ -14,4 +14,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
 
 source "$DIR/.venv/bin/activate"
 
-pytest -s --basetemp=tests/out --ignore=archivebox/vendor
+pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist

docker-compose.yml

@@ -1,72 +1,128 @@
 # Usage:
-#     docker-compose run archivebox init --setup
-#     docker-compose up
-#     echo "https://example.com" | docker-compose run archivebox archivebox add
-#     docker-compose run archivebox add --depth=1 https://example.com/some/feed.rss
-#     docker-compose run archivebox config --set PUBLIC_INDEX=True
-#     docker-compose run archivebox help
+#     docker compose run archivebox init --setup
+#     docker compose up
+#     echo "https://example.com" | docker compose run archivebox archivebox add
+#     docker compose run archivebox add --depth=1 https://example.com/some/feed.rss
+#     docker compose run archivebox config --set MEDIA_MAX_SIZE=750m
+#     docker compose run archivebox help
 # Documentation:
 #     https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker-compose
 
-version: '2.4'
-
 services:
     archivebox:
-        # build: .                              # for developers working on archivebox
-        image: ${DOCKER_IMAGE:-archivebox/archivebox:latest}
+        #image: ${DOCKER_IMAGE:-archivebox/archivebox:dev}
+        image: archivebox/archivebox:stable
         command: server --quick-init 0.0.0.0:8000
         ports:
             - 8000:8000
-        environment:
-            - ALLOWED_HOSTS=*                   # add any config options you want as env vars
-            - MEDIA_MAX_SIZE=750m
-            # - SEARCH_BACKEND_ENGINE=sonic     # uncomment these if you enable sonic below
-            # - SEARCH_BACKEND_HOST_NAME=sonic
-            # - SEARCH_BACKEND_PASSWORD=SecretPassword
         volumes:
             - ./data:/data
-            # - ./archivebox:/app/archivebox    # for developers working on archivebox
+            # - ./etc/crontabs:/var/spool/cron/crontabs  # uncomment this and archivebox_scheduler below to set up automatic recurring archive jobs
+            # - ./archivebox:/app/archivebox             # uncomment this to mount the ArchiveBox source code at runtime (for developers working on archivebox)
+        # build: .                                       # uncomment this to build the image from source code at buildtime (for developers working on archivebox)
+        environment:
+            - ALLOWED_HOSTS=*                   # restrict this to only accept incoming traffic via specific domain name
+            # - PUBLIC_INDEX=True               # set to False to prevent anonymous users from viewing snapshot list
+            # - PUBLIC_SNAPSHOTS=True           # set to False to prevent anonymous users from viewing snapshot content
+            # - PUBLIC_ADD_VIEW=False           # set to True to allow anonymous users to submit new URLs to archive
+            # - ADMIN_USERNAME=admin            # create an admin user on first run with the given user/pass combo
+            # - ADMIN_PASSWORD=SomeSecretPassword
+            # - PUID=911                        # set to your host user's UID & GID if you encounter permissions issues
+            # - PGID=911
+            # - SEARCH_BACKEND_ENGINE=sonic     # uncomment these and sonic container below for better full-text search
+            # - SEARCH_BACKEND_HOST_NAME=sonic
+            # - SEARCH_BACKEND_PASSWORD=SomeSecretPassword
+            # - MEDIA_MAX_SIZE=750m             # increase this filesize limit to allow archiving larger audio/video files
+            # - TIMEOUT=60                      # increase this number to 120+ seconds if you see many slow downloads timing out
+            # - CHECK_SSL_VALIDITY=True         # set to False to disable strict SSL checking (allows saving URLs w/ broken certs)
+            # - SAVE_ARCHIVE_DOT_ORG=True       # set to False to disable submitting all URLs to Archive.org when archiving
+            # ...
+            # add further configuration options from archivebox/config.py as needed (to apply them only to this container)
+            # or set using `docker compose run archivebox config --set SOME_KEY=someval` (to persist config across all containers)
+
+        # For ad-blocking during archiving, uncomment this section and pihole service section below 
+        # networks:
+        #   - dns
+        # dns:
+        #   - 172.20.0.53
+
+
+    ######## Optional Addons: tweak examples below as needed for your specific use case ########
+
+    ### Example: To run the Sonic full-text search backend, uncomment this block and the sonic config above
 
-    # To run the Sonic full-text search backend, first download the config file to sonic.cfg
-    # curl -O https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/master/etc/sonic.cfg
-    # after starting, backfill any existing Snapshots into the index: docker-compose run archivebox update --index-only
     # sonic:
-    #    image: valeriansaliou/sonic:v1.3.0
+    #    image: archivebox/sonic:latest
     #    expose:
     #        - 1491
     #    environment:
-    #        - SEARCH_BACKEND_PASSWORD=SecretPassword
+    #        - SEARCH_BACKEND_PASSWORD=SomeSecretPassword
     #    volumes:
     #        - ./sonic.cfg:/etc/sonic.cfg:ro
     #        - ./data/sonic:/var/lib/sonic/store
 
 
-    ### Optional Addons: tweak these examples as needed for your specific use case
+    ### Example: To run pihole in order to block ad/tracker requests during archiving,
+    # uncomment this block and set up pihole using its admin interface
 
-    # Example: Run scheduled imports in a docker instead of using cron on the
-    # host machine, add tasks and see more info with archivebox schedule --help
-    # scheduler:
-    #    image: archivebox/archivebox:latest
-    #    command: schedule --foreground --every=day --depth=1 'https://getpocket.com/users/USERNAME/feed/all'
+    # pihole:
+    #   image: pihole/pihole:latest
+    #   ports:
+    #     - 127.0.0.1:8090:80       # uncomment to access the admin HTTP interface on http://localhost:8090
+    #   environment:
+    #     - WEBPASSWORD=SET_THIS_TO_SOME_SECRET_PASSWORD_FOR_ADMIN_DASHBOARD
+    #     - DNSMASQ_LISTENING=all
+    #   dns:
+    #     - 127.0.0.1
+    #     - 1.1.1.1
+    #   networks:
+    #     dns:
+    #       ipv4_address: 172.20.0.53
+    #   volumes:
+    #     - ./etc/pihole:/etc/pihole
+    #     - ./etc/dnsmasq:/etc/dnsmasq.d
+
+
+    ### Example: Enable ability to run regularly scheduled archiving tasks by uncommenting this container
+    #   $ docker compose run archivebox schedule --every=day --depth=1 'https://example.com/some/rss/feed.xml'
+    # then restart the scheduler container to apply the changes to the schedule
+    #   $ docker compose restart archivebox_scheduler
+
+    # archivebox_scheduler:
+    #    image: ${DOCKER_IMAGE:-archivebox/archivebox:dev}
+    #    command: schedule --foreground
     #    environment:
-    #        - USE_COLOR=True
-    #        - SHOW_PROGRESS=False
+    #        - MEDIA_MAX_SIZE=750m               # increase this number to allow archiving larger audio/video files
+    #        # - TIMEOUT=60                      # increase if you see timeouts often during archiving / on slow networks
+    #        # - ONLY_NEW=True                   # set to False to retry previously failed URLs when re-adding instead of skipping them
+    #        # - CHECK_SSL_VALIDITY=True         # set to False to allow saving URLs w/ broken SSL certs
+    #        # - SAVE_ARCHIVE_DOT_ORG=True       # set to False to disable submitting URLs to Archive.org when archiving
+    #        # - PUID=502                        # set to your host user's UID & GID if you encounter permissions issues
+    #        # - PGID=20
     #    volumes:
     #        - ./data:/data
+    #        - ./etc/crontabs:/var/spool/cron/crontabs
+    #    # cpus: 2                               # uncomment / edit these values to limit container resource consumption
+    #    # mem_limit: 2048m
+    #    # shm_size: 1024m
+
+
+    ### Example: Put Nginx in front of the ArchiveBox server for SSL termination
 
-    # Example: Put Nginx in front of the ArchiveBox server for SSL termination
     # nginx:
     #     image: nginx:alpine
     #     ports:
     #         - 443:443
     #         - 80:80
     #     volumes:
-    #         - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf
+    #         - ./etc/nginx.conf:/etc/nginx/nginx.conf
     #         - ./data:/var/www
 
-    # Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel
+
+    ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel
+
     # wireguard:
-    #   image: linuxserver/wireguard
+    #   image: linuxserver/wireguard:latest
     #   network_mode: 'service:archivebox'
     #   cap_add:
     #     - NET_ADMIN
@@ -78,14 +134,26 @@ services:
     #     - /lib/modules:/lib/modules
     #     - ./wireguard.conf:/config/wg0.conf:ro
 
-    # Example: Run PYWB in parallel and auto-import WARCs from ArchiveBox
+
+    ### Example: Run PYWB in parallel and auto-import WARCs from ArchiveBox
+
     # pywb:
     #     image: webrecorder/pywb:latest
-    #     entrypoint: /bin/sh 'wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback --proxy;'
+    #     entrypoint: /bin/sh -c '(wb-manager init default || test $$? -eq 2) && wb-manager add default /archivebox/archive/*/warc/*.warc.gz; wayback;'
     #     environment:
     #         - INIT_COLLECTION=archivebox
     #     ports:
     #         - 8080:8080
     #     volumes:
-    #         ./data:/archivebox
-    #         ./data/wayback:/webarchive
+    #         - ./data:/archivebox
+    #         - ./data/wayback:/webarchive
+
+
+networks:
+
+    # network needed for pihole container to offer :53 dns resolving on fixed ip for archivebox container
+    dns:
+        ipam:
+            driver: default
+            config:
+                - subnet: 172.20.0.0/24

etc/ArchiveBox.conf.default

@@ -55,7 +55,7 @@
 # CURL_BINARY = curl
 # GIT_BINARY = git
 # WGET_BINARY = wget
-# YOUTUBEDL_BINARY = youtube-dl
+# YOUTUBEDL_BINARY = yt-dlp
 # CHROME_BINARY = chromium
 
 # CHROME_USER_DATA_DIR="~/.config/google-chrome/Default"

etc/README.md

@@ -2,7 +2,7 @@
 
 In this folder are some example config files you can use for setting up ArchiveBox on your machine.
 
-E.g. see `etc/nginx` for an example nginx config to serve your archive, or `etc/cron` for an example cron job that crawls a feed every 24 hours.
+E.g. see `nginx.conf` for an example nginx config to serve your archive with SSL, or `fly.toml` for an example deployment to the Fly.io hosting platform.
 
 Please contribute your etc files here! Example contributions
 

etc/archivebox.service

@@ -0,0 +1,29 @@
+# This is an example systemd service config definition for ArchiveBox.
+#
+# Link it into place on your system to use systemd to auto-start the ArchiveBox server on boot:
+#   https://unix.stackexchange.com/questions/224992/where-do-i-put-my-systemd-unit-file
+#
+# Review and change these lines as-needed for your specific environment and needs:
+#   WorkingDirectory, ExecStart, User, Group
+
+[Unit]
+Description=Open source self-hosted web archiving
+Documentation=https://github.com/ArchiveBox/ArchiveBox/wiki
+
+[Service]
+Type=simple
+WorkingDirectory=/home/archivebox/archivebox/
+ExecStart=/usr/local/bin/archivebox server 0.0.0.0:8000
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s QUIT $MAINPID
+Restart=always
+RestartSec=2
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=archivebox
+User=archivebox
+Group=archivebox
+
+
+[Install]
+WantedBy=multi-user.target

etc/cron.d/ArchiveBox

@@ -1 +0,0 @@
-0 24 * * * www-data /opt/ArchiveBox/bin/archive "https://getpocket.com/users/example/feed/all" >> /var/log/ArchiveBox.log

etc/crontabs/archivebox

@@ -0,0 +1,8 @@
+# DO NOT EDIT THIS FILE - edit the master and reinstall.
+# (/tmp/tmpe3dawo9u installed on Tue Jun 13 23:21:48 2023)
+# (Cron version -- $Id: crontab.c,v 2.13 1994/01/17 03:20:37 vixie Exp $)
+
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "https://example.com/3" >> /data/logs/schedule.log 2>&1 # archivebox_schedule
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "https://example.com/2" >> /data/logs/schedule.log 2>&1 # archivebox_schedule
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "https://example.com" >> /data/logs/schedule.log 2>&1 # archivebox_schedule
+@daily cd /data && /usr/local/bin/archivebox add --depth=0 "update" >> /data/logs/schedule.log 2>&1 # archivebox_schedule

etc/fly.toml

@@ -0,0 +1,40 @@
+# fly.toml file generated for archivebox on 2021-04-23T16:35:11-04:00
+
+app = "archivebox"
+
+kill_signal = "SIGINT"
+kill_timeout = 5
+
+[env]
+
+[mounts]
+source="archivebox_data"
+destination="/data"
+
+[experimental]
+  auto_rollback = true
+
+[[services]]
+  http_checks = []
+  internal_port = 8000
+  protocol = "tcp"
+  script_checks = []
+
+  [services.concurrency]
+    hard_limit = 25
+    soft_limit = 20
+    type = "connections"
+
+  [[services.ports]]
+    handlers = ["http"]
+    port = 80
+
+  [[services.ports]]
+    handlers = ["tls", "http"]
+    port = 443
+
+  [[services.tcp_checks]]
+    grace_period = "1s"
+    interval = "15s"
+    restart_limit = 6
+    timeout = "2s"

etc/nginx.conf

@@ -34,12 +34,14 @@ http {
     server {
         listen                      80 default_server;
         server_name                 _;
-
-        root                        /var/www;
+        
         index                       index.html;
         autoindex                   on;
-
         try_files                   $uri $uri/ $uri.html =404;
+
+        location /archive {
+            root                    /var/www/archive;
+        }
     }
 }