remove checks import and bump pydantic pkgr version

.dockerignore

@@ -9,6 +9,7 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
+.vscode/
 !.git/HEAD
 !.git/refs/heads/*
 
@@ -31,6 +32,8 @@ deb_dist/
 pip_dist/
 assets/
 docker/
+website/
+typings/
 
 data/
 data*/

.gitattributes

@@ -1,2 +0,0 @@
-**/*.lock
-**/*-lock.json

.github/workflows/gh-pages.yml

@@ -1,5 +1,5 @@
-# Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: Build GitHub Pages website
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
 
 on:
   # Runs on pushes targeting the default branch
@@ -19,40 +19,25 @@ permissions:
 # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
 concurrency:
   group: "pages"
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 jobs:
-  # Build job
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v3
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
-
-  # Deployment job
+  # Single deploy job since we're just deploying
   deploy:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
-    needs: build
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload entire repository
+          path: './website'
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4

.gitmodules

@@ -1,31 +1,9 @@
 [submodule "docs"]
     path = docs
     url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
-
-[submodule "deb_dist"]
-    path = deb_dist
-    url = https://github.com/ArchiveBox/debian-archivebox.git
-[submodule "brew_dist"]
-	path = brew_dist
-	url = https://github.com/ArchiveBox/homebrew-archivebox.git
-[submodule "pip_dist"]
-	path = pip_dist
-	url = https://github.com/ArchiveBox/pip-archivebox.git
-[submodule "docker"]
-	path = docker
-	url = https://github.com/ArchiveBox/docker-archivebox.git
-[submodule "archivebox/vendor/base32-crockford"]
-	path = archivebox/vendor/base32-crockford
-	url = https://github.com/jbittel/base32-crockford
 [submodule "archivebox/vendor/pocket"]
 	path = archivebox/vendor/pocket
 	url = https://github.com/tapanpandita/pocket
-[submodule "archivebox/vendor/django-taggit"]
-	path = archivebox/vendor/django-taggit
-	url = https://github.com/jazzband/django-taggit
-[submodule "archivebox/vendor/python-atomicwrites"]
-	path = archivebox/vendor/python-atomicwrites
-	url = https://github.com/untitaker/python-atomicwrites
 [submodule "archivebox/vendor/pydantic-pkgr"]
 	path = archivebox/vendor/pydantic-pkgr
 	url = https://github.com/ArchiveBox/pydantic-pkgr

.npmignore

@@ -1,19 +0,0 @@
-tests/
-archivebox/
-archivebox.egg-info/
-build/
-dist/
-docs/
-etc/
-.github
-.gitmodules
-.dockerignore
-.flake8
-CNAME
-_config.yml
-docker-compose.yaml
-docker-compose.yml
-Dockerfile
-MANIFEST.in
-Pipfile
-setup.py

CNAME

@@ -0,0 +1 @@
+archivebox.io

Dockerfile

@@ -18,6 +18,7 @@
 
 # Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
 FROM python:3.11-slim-bookworm
+# FROM debian:bookworm-backports
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -60,33 +61,29 @@ ENV TZ=UTC \
 
 # Version config
 ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=20
+    NODE_VERSION=22
 
 # User config
 ENV ARCHIVEBOX_USER="archivebox" \
     DEFAULT_PUID=911 \
-    DEFAULT_PGID=911
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
 
 # Global paths
 ENV CODE_DIR=/app \
     DATA_DIR=/data \
-    GLOBAL_VENV=/venv \
     PLAYWRIGHT_BROWSERS_PATH=/browsers
+    # GLOBAL_VENV=/venv \
     # TODO: add TMP_DIR and LIB_DIR?
 
-# Application-level paths
-ENV APP_VENV=/app/.venv \
-    NODE_MODULES=/app/node_modules
-
 # Build shell config
-ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
 
 ######### System Environment ####################################
 
-# Detect ArchiveBox version number by reading package.json
-COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
-RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
+# Detect ArchiveBox version number by reading pyproject.toml
+COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN grep '^version = ' "${CODE_DIR}/pyproject.toml" | awk -F'"' '{print $2}' > /VERSION.txt
 
 # Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
@@ -99,7 +96,6 @@ RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
     && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && echo \
-    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
     && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
     && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
     && echo \
@@ -150,10 +146,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
     # && apt-get update -qq \
     # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
     # && rm -rf /var/lib/apt/lists/* \
     # tell PDM to allow using global system python site packages
     # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
     # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
     # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
     # && python3 -m venv --system-site-packages $GLOBAL_VENV \
@@ -171,7 +168,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 
 # Install Node environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    echo "[+] Installing Node $NODE_VERSION environment..." \
     && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
     && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
     && apt-get update -qq \
@@ -189,6 +186,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     ) | tee -a /VERSION.txt
 
 
+
 ######### Extractor Dependencies ##################################
 
 # Install apt dependencies
@@ -196,7 +194,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     echo "[+] Installing APT extractor dependencies globally using apt..." \
     && apt-get update -qq \
     && apt-get install -qq -y -t bookworm-backports \
-        curl wget git yt-dlp ffmpeg ripgrep \
+        curl wget git ffmpeg ripgrep \
         # Packages we have also needed in the past:
         # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
     && rm -rf /var/lib/apt/lists/* \
@@ -204,7 +202,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && ( \
         which curl && curl --version | head -n1 \
         && which wget && wget --version 2>&1 | head -n1 \
-        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
         && which git && git --version 2>&1 | head -n1 \
         && which rg && rg --version 2>&1 | head -n1 \
         && echo -e '\n\n' \
@@ -220,7 +217,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
         libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
         libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
         # xfonts-scalable xfonts-utils xserver-common xvfb \
         # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
         # libxss1 dbus dbus-x11 upower \
@@ -233,7 +230,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && rm -rf /var/lib/apt/lists/* \
     && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
     && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
     && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
     && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
     # Save version info
@@ -243,14 +240,26 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     ) | tee -a /VERSION.txt
 
 # Install Node dependencies
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
-    && npm ci --prefer-offline --no-audit --cache /root/.npm \
-    && ( \
+ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
+USER $ARCHIVEBOX_USER
+WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
+RUN --mount=type=cache,target=/home/$ARCHIVEBOX_USER/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
+    echo "[+] Installing NPM extractor dependencies in /home/$ARCHIVEBOX_USER/.npm..." \
+    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
+    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
+        "@postlight/parser@^2.2.3" \
+        "readability-extractor@github:ArchiveBox/readability-extractor" \
+        "single-file-cli@^1.1.54" \
+        "puppeteer@^23.5.0" \
+        "@puppeteer/browsers@^2.4.0"
+USER root
+RUN ( \
         which node && node --version \
         && which npm && npm version \
+        && which postlight-parser \
+        && which readability-extractor && readability-extractor --version \
+        && which single-file && single-file --version \
+        && which puppeteer && puppeteer --version \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
@@ -282,30 +291,19 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
 RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    && pip install -e "${CODE_DIR}[sonic,ldap]" \
+    && pip install -e "${CODE_DIR}[all]" \
     && rm -rf /var/lib/apt/lists/*
 
 ####################################################
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True \
-    DISPLAY=novnc:0.0 \
-    CUSTOM_TEMPLATES_DIR=/data/user_templates \
-    GOOGLE_API_KEY=no \
+RUN openssl rand -hex 16 > /etc/machine-id \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/tmp"
+ENV GOOGLE_API_KEY=no \
     GOOGLE_DEFAULT_CLIENT_ID=no \
     GOOGLE_DEFAULT_CLIENT_SECRET=no \
     ALLOWED_HOSTS=*
-    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
-    # WGET_BINARY="wget" \
-    # YOUTUBEDL_BINARY="yt-dlp" \
-    # CHROME_BINARY="/usr/bin/chromium-browser" \
-    # USE_SINGLEFILE=True \
-    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
-    # USE_READABILITY=True \
-    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
-    # USE_MERCURY=True \
-    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
 
 # Print version for nice docker finish summary
 RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \

README.md

@@ -1319,8 +1319,8 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 ## Developers
 
 - [Developer Documentation](https://github.com/ArchiveBox/ArchiveBox#archivebox-development)
-- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
-- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)
+- [Python API](https://docs.archivebox.io/)
+- [REST API](https://demo.archivebox.io/api) (alpha)
 
 ## More Info
 
@@ -1365,18 +1365,12 @@ git pull --recurse-submodules
 
 ```bash
 # Install ArchiveBox + python dependencies
-python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]'
-# or: pipenv install --dev && pipenv shell
+pip install uv
+uv venv
+uv sync
 
-# Install node dependencies
-npm install
-# or
-archivebox setup
-
-# Check to see if anything is missing
-archivebox --version
-# install any missing dependencies manually, or use the helper script:
-./bin/setup.sh
+archivebox init
+archivebox install
 ```
 
 #### 2. Option B: Build the docker container and use that for development instead
@@ -1413,6 +1407,8 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 archivebox config --set DEBUG=True
 # or
 archivebox server --debug ...
+# faster dev version wo/ bg workers enabled:
+daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```
 
 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1447,12 +1443,12 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init --setup
+docker run -it -v $PWD:/data archivebox:dev init
 
 # or with pip:
 pip install 'git+https://github.com/pirate/ArchiveBox@dev'
 npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
-archivebox init --setup
+archivebox install
 ```
 
 </details>
@@ -1546,6 +1542,9 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 
 **The process to contribute a new extractor is like this:**
 
+> [!IMPORTANT]
+> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
+
 1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
 2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
    (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
@@ -1574,8 +1573,6 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/build_docs.sh
 ./bin/build_pip.sh
-./bin/build_deb.sh
-./bin/build_brew.sh
 ./bin/build_docker.sh
 ```
 
@@ -1592,8 +1589,6 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/release_docs.sh
 ./bin/release_pip.sh
-./bin/release_deb.sh
-./bin/release_brew.sh
 ./bin/release_docker.sh
 ```
 

archivebox/LICENSE

@@ -1 +0,0 @@
-../LICENSE

archivebox/__init__.py

@@ -1,12 +1,5 @@
 #!/usr/bin/env python3
-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
+
 # Welcome to the ArchiveBox source code! Thanks for checking it out!
 #
 # "We are swimming upstream against a great torrent of disorganization.
@@ -20,32 +13,48 @@ __package__ = 'archivebox'
 
 import os
 import sys
+
 from pathlib import Path
 
-PACKAGE_DIR = Path(__file__).resolve().parent           # archivebox source code dir
-DATA_DIR = Path(os.curdir).resolve()                    # archivebox user data dir
-ARCHIVE_DIR = DATA_DIR / 'archive'                      # archivebox snapshot data dir
+ASCII_LOGO = """
+ █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
+██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
+███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
+██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
+██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
+╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
+"""
 
 # make sure PACKAGE_DIR is in sys.path so we can import all subfolders
 # without necessarily waiting for django to load them thorugh INSTALLED_APPS
+PACKAGE_DIR = Path(__file__).resolve().parent
 if str(PACKAGE_DIR) not in sys.path:
     sys.path.append(str(PACKAGE_DIR))
-
-from .config.constants import CONSTANTS, VERSION  # noqa
-
-os.environ['ARCHIVEBOX_PACKAGE_DIR'] = str(PACKAGE_DIR)
-os.environ['ARCHIVEBOX_DATA_DIR'] = str(DATA_DIR)
 os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
 
+# detect ArchiveBox user's UID/GID based on data dir ownership
+from .config.permissions import drop_privileges                 # noqa
+drop_privileges()
+
+from .misc.checks import check_not_root, check_io_encoding      # noqa
+check_not_root()
+check_io_encoding()
+
 # print('INSTALLING MONKEY PATCHES')
 from .monkey_patches import *                    # noqa
 # print('DONE INSTALLING MONKEY PATCHES')
 
+
 # print('LOADING VENDORED LIBRARIES')
 from .vendor import load_vendored_libs           # noqa
 load_vendored_libs()
 # print('DONE LOADING VENDORED LIBRARIES')
 
+
+from .config.constants import CONSTANTS                         # noqa
+from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
+from .config.version import VERSION                             # noqa
+
 __version__ = VERSION
 __author__ = 'Nick Sweeting'
 __license__ = 'MIT'

archivebox/__main__.py

@@ -2,7 +2,7 @@
 """This is the main entry point for the ArchiveBox CLI."""
 __package__ = 'archivebox'
 
-import archivebox      # noqa  # import archivebox/__init__.py to apply monkey patches, load vendored libs, etc.
+import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys
 
 from .cli import main

archivebox/_ansible_unused/ansible_utils.py

@@ -1,59 +0,0 @@
-# import os
-
-# from pathlib import Path
-
-# from benedict import benedict
-# from rich.pretty import pprint
-
-# from ansible_runner import Runner, RunnerConfig
-
-# GLOBAL_CACHE = {}
-
-
-# def run_playbook(playbook_path, data_dir, quiet=False, **kwargs):
-#     ANSIBLE_TMP_DIR = str(Path(data_dir) / "tmp" / "ansible")
-#     os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False'
-#     os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False'
-#     os.environ["ANSIBLE_HOME"] = ANSIBLE_TMP_DIR
-#     # os.environ["ANSIBLE_COLLECTIONS_PATH"] = str(Path(data_dir).parent / 'archivebox')
-#     os.environ["ANSIBLE_ROLES_PATH"] = (
-#         './roles'
-#     )
-    
-#     rc = RunnerConfig(
-#         private_data_dir=ANSIBLE_TMP_DIR,
-#         playbook=str(playbook_path),
-#         rotate_artifacts=50000,
-#         host_pattern="localhost",
-#         extravars={
-#             "DATA_DIR": str(data_dir),
-#             **kwargs,
-#         },
-#         quiet=quiet,
-#     )
-#     rc.prepare()
-#     r = Runner(config=rc)
-#     r.set_fact_cache('localhost', GLOBAL_CACHE)
-#     r.run()
-#     last_run_facts = r.get_fact_cache('localhost')
-#     GLOBAL_CACHE.update(filtered_facts(last_run_facts))
-#     return benedict({
-#         key: val
-#         for key, val in last_run_facts.items()
-#         if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
-#     })
-
-# def filtered_facts(facts):
-#     return benedict({
-#         key: val
-#         for key, val in facts.items()
-#         if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
-#     })
-
-# def print_globals():
-#     pprint(filtered_facts(GLOBAL_CACHE), expand_all=True)
-
-
-
-# # YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'})
-# # pprint(YTDLP_OUTPUT)

archivebox/_ansible_unused/install_puppeteer.yml

@@ -1,125 +0,0 @@
-#!/usr/bin/env ansible-playbook
----
-- name: "Install puppeteer, puppeteer/browsers, and chrome"
-  hosts: localhost
-  gather_facts: true
-  vars:
-    DATA_DIR: '{{playbook_dir}}'
-    LIB_DIR: '{{DATA_DIR}}/lib'
-    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
-    LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers'
-    CHROME_RELEASE_CHANNEL: 'chrome@stable'
-    CHROME_VERSION_MIN: '128.0.6613.137'
-  tasks:
-    - include_role:
-        name: setup_lib_npm
-      vars:
-        TARGET_NODE_VERSION: '21'
-        MIN_NODE_VERSION: '20.0.0'
-        MIN_NPM_VERSION: '10.0.0'
-
-    - name: "Install npm packages: [puppeteer, @puppeteer/browsers]"
-      community.general.npm:
-        name: '{{item}}'
-        state: "present"
-        path: '{{BINPROVIDERS.npm.lib_dir_npm}}'
-      loop:
-        - 'puppeteer'
-        - '@puppeteer/browsers'
-      
-    - name: Make sure prerequisite folders exist
-      file:
-        path: '{{LIB_DIR_BROWSERS}}'
-        state: directory
-        recurse: true
-
-    - name: Load puppeteer binary from installed NPM package
-      include_role:
-        name: load_binary
-      vars:
-        name: puppeteer
-        PATH: '{{BINPROVIDERS.npm.PATH}}'
-
-    # - name: Find existing chrome binaries in environment PATH
-    #   include_role:
-    #     name: load_binary
-    #   vars:
-    #     name: 'chrome'
-    #     bin_name: '{{chrome_executable}}'
-    #     PATH: '{{ansible_env.PATH}}'
-    #   loop:
-    #     - chrome
-    #     - chrome-browser
-    #     - chromium
-    #     - chromium-browser
-    #     - google-chrome
-    #     - google-chrome-browser
-    #     - google-chrome-stable
-    #     - google-chrome-beta
-    #     - google-chrome-canary
-    #     - google-chrome-unstable
-    #     - google-chrome-dev
-    #   loop_control:
-    #     loop_var: chrome_executable
-    #     break_when: 
-    #       - BINARIES.chrome.version|default('')
-
-
-    ###################################################################################
-    - name: 'Install Chrome browser: npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}}'
-      command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}'
-      register: CHROME_VERSION_FULL
-      environment:
-        PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}"
-      changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout
-      when: not BINARIES.chrome.version|default('')
-      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
-
-    ###################################################################################
-    - name: Parse Chrome version and abspath from npx @puppeteer/browsers install output
-      set_fact:
-        CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}"
-        CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}"
-      when: not BINARIES.chrome.version|default('')
-
-    - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary
-      # normal symlink doesn't work for .app on macOS because it fails to load ../Framworks/..., so we create a tiny bash script to open it in its correct cwd instead
-      copy:
-        content: |
-                 #!/bin/bash
-                 exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@"
-        dest: "{{LIB_DIR_BIN}}/chrome"
-      changed_when: False
-    
-    - name: Ensure ./bin/chrome symlink is executable
-      file:
-        path: "{{LIB_DIR_BIN}}/chrome"
-        mode: u+rx,g-rx,o-rwx
-        state: 'file'
-      changed_when: False
-
-    ###################################################################################
-    - set_fact:
-        PUPPETEER_BINARIES:
-          chrome:
-            name: 'chrome'
-            bin_name: 'chrome'
-            abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath) or None}}"
-            version: "{{CHROME_VERSION|default(BINARIES.chrome.version) or None}}"
-            symlink: "{{LIB_DIR_BIN}}/chrome"
-            version_cmd: "chrome --version"
-            version_stdout: "{{CHROME_VERSION_FULL.stdout}}"
-            binprovider: 'puppeteer'
-            PATH: "{{LIB_DIR_BIN}}"
-
-    - name: Check that installed Chrome matches expected version
-      assert:
-        that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=')
-        quiet: true
-    
-    - set_fact:
-        BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}"
-        cacheable: true
-
-    - debug:
-        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/install_singlefile.yml

@@ -1,40 +0,0 @@
-#!/usr/bin/env ansible-playbook
----
-
-- import_playbook: ../puppeteer/install_puppeteer.yml
-  when: not BINARIES.chrome.version|default('')
-
-- name: "Install Singlefile"
-  hosts: localhost
-  gather_facts: no
-  vars:
-    SINGLEFILE_VERSION_EXACT: '1.1.54'
-  tasks:
-    - include_role:
-        name: setup_lib_npm
-      vars:
-        MIN_NODE_VERSION: '20.0.0'
-        MIN_NPM_VERSION: '10.0.0'
-
-    - name: "Install npm packages: [single-file-cli]"
-      community.general.npm:
-        name: 'single-file-cli@1.1.54'
-        state: "present"
-        path: '{{BINPROVIDERS.npm.lib_dir_npm}}'
-
-    - name: Load single-file binary from installed NPM package
-      include_role:
-        name: load_binary
-      vars:
-        name: singlefile
-        bin_name: 'single-file'
-        version_cmd: 'single-file --version'
-        PATH: '{{BINPROVIDERS.npm.PATH}}'
-
-    - name: Check that installed Singlefile version matches expected version
-      assert:
-        that: SINGLEFILE_VERSION_EXACT is version(BINARIES.singlefile.version, '==')
-        quiet: true
-
-    - debug:
-        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/install_ytdlp.yml

@@ -1,45 +0,0 @@
-#!/usr/bin/env ansible-playbook
----
-- name: "Install YT-DLP"
-  hosts: localhost
-  gather_facts: True
-  vars:
-    YTDLP_VERSION_MIN: '2024.8.6'
-  tasks:
-    - include_role:
-        name: setup_lib_pip
-      vars:
-        MIN_PYTHON_VERSION: '3.10.0'
-        MIN_PIP_VERSION: '22.0'
-    
-    - name: "Install pip packages: yt-dlp"
-      ansible.builtin.pip:
-        name: 'yt-dlp'
-        state: 'latest'
-        virtualenv: '{{BINPROVIDERS.pip.virtualenv}}'
-        virtualenv_python: "{{BINPROVIDERS.pip.virtualenv_python}}"
-        virtualenv_site_packages: no
-
-    - name: Load YTDLP binary
-      include_role:
-        name: load_binary
-      vars:
-        name: ytdlp
-        bin_name: yt-dlp
-        PATH: '{{BINPROVIDERS.pip.PATH}}'
-
-    - name: Load ffmpeg binary
-      include_role:
-        name: load_binary
-      vars:
-        name: ffmpeg
-        version_cmd: 'ffmpeg -version'
-        PATH: '{{BINPROVIDERS.pip.PATH}}:{{ansible_env.PATH}}'
-
-    - name: Check that installed YT-DLP matches expected version
-      assert:
-        that: BINARIES.ytdlp.version is version(YTDLP_VERSION_MIN, '>=')
-        quiet: true
-
-    - debug:
-        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/roles/load_binary/meta/argument_specs.yml

@@ -1,31 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Load a specified binary from the environment/PATH into BINARIES fact.
-
-    options:
-
-      name:
-        type: "str"
-        required: true
-        description: "A string key for the binary"
-
-      bin_name:
-        type: "str"
-        required: false
-        description: "The basename of the binary file (optional, defaults to name)"
-
-      abspath:
-        type: "str"
-        required: false
-        description: "An absolute path to the binary (overrides any auto-detected one)"
-
-      version_cmd:
-        type: "str"
-        required: false
-        description: "The command to run to get the binary's version (optional, defaults to $ <bin_name> --version)"
-
-      PATH:
-        type: "str"
-        required: false
-        description: "The PATH to search for the binary (optional, defaults to environment $PATH)"

archivebox/_ansible_unused/roles/load_binary/tasks/main.yml

@@ -1,75 +0,0 @@
-
----
-- name: Make sure ./data/lib/bin folder exists
-  file:
-    path: '{{LIB_DIR_BIN}}'
-    state: directory
-    recurse: true
-
-###################################################################################
-
-- set_fact:
-    PATH: "{{PATH or DEFAULT_PATH}}"
-    bin_name: "{{bin_name or name}}"
-    BINARY_ABSPATH: null
-    BINARY_VERSION: null
-
-- name: 'Get installed binary abspath: {{name}}'
-  command: 'env PATH="{{PATH}}:$PATH" which {{bin_name}}'
-  register: BINARY_ABSPATH
-  changed_when: False
-
-- set_fact:
-    bin_name: "{{BINARY_ABSPATH.stdout|basename or bin_name}}"
-- set_fact:
-    version_cmd: "{{version_cmd or (bin_name + ' --version')}}"
-
-- name: 'Get installed binary version: {{name}}'
-  command: 'env PATH="{{PATH}}:$PATH" {{version_cmd}}'
-  register: BINARY_VERSION
-  changed_when: False
-
-
-- name: 'Updating BINARIES with loaded abspaths & versions: {{name}}'
-  set_fact:
-    BINARIES: "{{
-      BINARIES
-      | default({})
-      | combine({
-          name: {
-            'name': name,
-            'bin_name': bin_name,
-            'version_cmd': version_cmd,
-            'symlink': LIB_DIR_BIN + '/' + name,
-            'abspath': BINARY_ABSPATH.stdout or abspath or None,
-            'version': BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1') or version or None,
-            'version_stdout': BINARY_VERSION.stdout or BINARY_VERSION.stderr,
-            'PATH': BINARY_ABSPATH.stdout|dirname or PATH,
-          },
-      })
-    }}"
-    cacheable: true
-  changed_when: True
-
-- name: 'Symlink installed binary into lib bin folder: {{name}}'
-  file:
-    src: "{{ BINARY_ABSPATH.stdout }}"
-    dest: "{{ LIB_DIR_BIN }}/{{ name }}"
-    state: link
-    force: true
-  when: BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1')|length
-
-- debug:
-    msg:
-      - '{{BINARIES}}'
-
-- name: Unset variables
-  set_fact:
-    name:
-    bin_name:
-    version_cmd:
-    abspath:
-    version:
-    PATH:
-    BINARY_ABSPATH:
-    BINARY_VERSION:

archivebox/_ansible_unused/roles/load_binary/vars/main.yml

@@ -1,12 +0,0 @@
-DATA_DIR: '{{playbook_dir}}'
-
-LIB_DIR: '{{DATA_DIR}}/lib'
-LIB_DIR_BIN: '{{LIB_DIR}}/bin'
-DEFAULT_PATH: /bin
-
-name:
-bin_name:
-version_cmd:
-PATH:
-abspath:
-version:

archivebox/_ansible_unused/roles/setup_lib_npm/meta/argument_specs.yml

@@ -1,31 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Main entry point for the npm role
-
-    options:
-    
-      state:
-        type: "str"
-        required: false
-        default: 'present'
-        description:
-          - "The desired state: present | latest"
-
-      npm_packages:
-        type: "list"
-        elements: "dict"
-        required: false
-        default: []
-        description: "A list of dicts with a defined structure and with default a value."
-        options:
-          key:
-            type: "str"
-            required: true
-            description: "A string name for the dependency"
-
-          packages:
-            type: "list"
-            elements: "str"
-            required: true
-            description: "What npm packages to install for the given dependency."

archivebox/_ansible_unused/roles/setup_lib_npm/tasks/main.yml

@@ -1,99 +0,0 @@
----
-- name: Make sure lib folders exist
-  file:
-    path: '{{item}}'
-    state: directory
-    recurse: true
-  loop:
-    - '{{LIB_DIR_NPM_BIN}}'
-    - '{{LIB_DIR_BIN}}'
-
-###################################################################################
-
-- name: Ensure dependencies are present.
-  when: ansible_facts['os_family']|lower == 'debian'
-  ansible.builtin.apt:
-    name:
-      - apt-transport-https
-      - python3-debian
-      - gnupg2
-    state: present
-
-- name: Download NodeSource's signing key.
-  # NodeSource's web server discriminates the User-Agent used by the deb822_repository module.
-  # https://github.com/nodesource/distributions/issues/1723
-  when: ansible_facts['os_family']|lower == 'debian'
-  ansible.builtin.get_url:
-    url: https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key
-    dest: /etc/apt/signing-key-nodesource-repo.asc
-    owner: root
-    group: root
-    mode: '0444'
-  register: node_signing_key
-
-- name: Add NodeSource repositories for Node.js.
-  when: ansible_facts['os_family']|lower == 'debian'
-  ansible.builtin.deb822_repository:
-    name: nodesource_{{ TARGET_NODE_VERSION }}
-    uris: "https://deb.nodesource.com/node_{{ TARGET_NODE_VERSION }}.x"
-    types: deb
-    suites: nodistro
-    components: main
-    signed_by: "{{ node_signing_key.dest }}"
-    state: present
-  register: node_repo
-
-- name: Update apt cache if repo was added.
-  ansible.builtin.apt: update_cache=yes
-  when: ansible_facts['os_family']|lower == 'debian' and node_repo is changed
-
-- name: Ensure Node.js and npm are installed.
-  when: ansible_facts['os_family']|lower == 'debian'
-  ansible.builtin.apt:
-    name: "nodejs={{ TARGET_NODE_VERSION | regex_replace('x', '') }}*"
-    state: present
-
-- name: Load NPM and Node binaries
-  include_role:
-    name: load_binary
-  vars:
-    name: '{{item}}'
-  loop:
-    - node
-    - npm
-
-- name: Check that installed Node version matches expected version
-  assert:
-    that:
-      - BINARIES.node.version is version(MIN_NODE_VERSION, '>=')
-      - BINARIES.npm.version is version(MIN_NPM_VERSION, '>=')
-    quiet: true
-
-###################################################################################
-
-
-# - name: "Install npm packages: {{install_npm}}"
-#   community.general.npm:
-#     name: '{{item}}'
-#     state: "{{state}}"
-#     path: '{{LIB_DIR_NPM}}'
-#   loop: "{{install_npm|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
-
-###################################################################################
-
-
-###################################################################################
-- set_fact:
-    NODE_BINPROVIDERS:
-      npm:
-        installer_abspath: "{{BINARIES.npm.abspath}}"
-        installer_version: "{{BINARIES.npm.version}}"
-        PATH: "{{LIB_DIR_NPM_BIN}}"
-        lib_dir_npm: "{{LIB_DIR_NPM}}"
-
-- set_fact:
-    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}"
-    cacheable: true
-
-- debug:
-    msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/roles/setup_lib_npm/vars/main.yml

@@ -1,10 +0,0 @@
-DATA_DIR: '{{playbook_dir}}'
-
-LIB_DIR: '{{DATA_DIR}}/lib'
-LIB_DIR_BIN: '{{LIB_DIR}}/bin'
-LIB_DIR_NPM: '{{LIB_DIR}}/npm'
-LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin'
-
-TARGET_NODE_VERSION: '21'
-MIN_NODE_VERSION: '20.0.0'
-MIN_NPM_VERSION: '10.0.0'

archivebox/_ansible_unused/roles/setup_lib_pip/meta/argument_specs.yml

@@ -1,31 +0,0 @@
----
-argument_specs:
-  main:
-    short_description: Main entry point for the npm role
-
-    options:
-    
-      state:
-        type: "str"
-        required: false
-        default: 'present'
-        description:
-          - "The desired state: present | latest"
-
-      npm_packages:
-        type: "list"
-        elements: "dict"
-        required: false
-        default: []
-        description: "A list of dicts with a defined structure and with default a value."
-        options:
-          key:
-            type: "str"
-            required: true
-            description: "A string name for the dependency"
-
-          packages:
-            type: "list"
-            elements: "str"
-            required: true
-            description: "What npm packages to install for the given dependency."

archivebox/_ansible_unused/roles/setup_lib_pip/tasks/main.yml

@@ -1,69 +0,0 @@
----
-- name: Make sure lib folders exist
-  file:
-    path: '{{item}}'
-    state: directory
-    recurse: true
-  loop:
-    - '{{LIB_DIR_PIP}}'
-    - '{{LIB_DIR_BIN}}'
-  when: BINPROVIDERS.pip is not defined
-
-- name: Load Python and Pip binaries
-  include_role:
-    name: load_binary
-  vars:
-    name: '{{item}}'
-  loop:
-    - python
-    - pip
-  when: BINARIES.python is not defined or BINARIES.pip is not defined
-
-- assert:
-    that:
-      - BINARIES.python.version is version(MIN_PYTHON_VERSION, '>=')
-      - BINARIES.pip.version is version(MIN_PIP_VERSION, '>=')
-    quiet: true
-  when: BINPROVIDERS.pip is not defined
-
-###################################################################################
-
-
-# - name: "Install pip packages: {{install_pip}}"
-#   ansible.builtin.pip:
-#     name: '{{item}}'
-#     state: "{{state}}"
-#     virtualenv: '{{LIB_DIR_PIP}}/venv'
-#     virtualenv_python: "{{BINARIES.python.abspath}}"
-#     virtualenv_site_packages: yes
-#   loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
-
-
-###################################################################################
-- set_fact:
-    PIP_BINPROVIDERS:
-      pip:
-        installer_abspath: "{{BINARIES.pip.abspath}}"
-        installer_version: "{{BINARIES.pip.version}}"
-        PATH: "{{LIB_DIR_PIP_BIN}}"
-        virtualenv: "{{LIB_DIR_PIP}}/venv"
-        virtualenv_python: "{{BINARIES.python.abspath}}"
-  when: BINPROVIDERS.pip is not defined
-
-- set_fact:
-    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}"
-    cacheable: true
-  changed_when: False
-
-- name: Load Python and Pip binaries from venv
-  include_role:
-    name: load_binary
-  vars:
-    name: '{{item}}'
-    PATH: '{{BINPROVIDERS.pip.PATH}}'
-  loop:
-    - python
-    - pip
-
-- debug:
-    msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/roles/setup_lib_pip/vars/main.yml

@@ -1,9 +0,0 @@
-DATA_DIR: '{{playbook_dir}}'
-
-LIB_DIR: '{{DATA_DIR}}/lib'
-LIB_DIR_BIN: '{{LIB_DIR}}/bin'
-LIB_DIR_PIP: '{{LIB_DIR}}/pip'
-LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin'
-
-MIN_PYTHON_VERSION: '3.10.0'
-MIN_PIP_VERSION: '22.0'

archivebox/abx/__init__.py

@@ -14,6 +14,15 @@ pm.add_hookspecs(base_spec)
 
 ###### PLUGIN DISCOVERY AND LOADING ########################################################
 
+def get_plugin_order(plugin_entrypoint: Path):
+    order = 999
+    try:
+        # if .plugin_order file exists, use it to set the load priority
+        order = int((plugin_entrypoint.parent / '.plugin_order').read_text())
+    except FileNotFoundError:
+        pass
+    return (order, plugin_entrypoint)
+
 def register_hookspecs(hookspecs):
     for hookspec_import_path in hookspecs:
         hookspec_module = importlib.import_module(hookspec_import_path)
@@ -23,7 +32,7 @@ def register_hookspecs(hookspecs):
 def find_plugins_in_dir(plugins_dir: Path, prefix: str) -> Dict[str, Path]:
     return {
         f"{prefix}.{plugin_entrypoint.parent.name}": plugin_entrypoint.parent
-        for plugin_entrypoint in sorted(plugins_dir.glob("*/apps.py"))  # key=get_plugin_order  # Someday enforcing plugin import order may be required, but right now it's not needed
+        for plugin_entrypoint in sorted(plugins_dir.glob("*/apps.py"), key=get_plugin_order)
     }   # "plugins_pkg.pip": "/app/archivebox/plugins_pkg/pip"
 
 

archivebox/abx/archivebox/__init__.py

@@ -1,5 +1,6 @@
 __package__ = 'abx.archivebox'
 
+import os
 import importlib
 
 from typing import Dict
@@ -21,7 +22,7 @@ def load_archivebox_plugins(pm, plugins_dict: Dict[str, Path]):
             archivebox_plugins_found.append(plugin_module_loaded.PLUGIN)
         
         # 2. then try to import plugin_module.apps as well
-        if (plugin_dir / 'apps.py').exists():
+        if os.access(plugin_dir / 'apps.py', os.R_OK):
             plugin_apps = importlib.import_module(plugin_module + '.apps')
             pm.register(plugin_apps)                                           # register the whole .apps  in case it contains loose hookimpls (not in a class)
             if hasattr(plugin_apps, 'PLUGIN'):

archivebox/abx/archivebox/base_binary.py

@@ -1,6 +1,7 @@
 __package__ = "abx.archivebox"
 
-from typing import Dict, List
+import os
+from typing import Dict, List, Optional
 from typing_extensions import Self
 
 from pydantic import Field, InstanceOf, validate_call
@@ -15,6 +16,7 @@ from pydantic_pkgr import (
 )
 
 from archivebox.config import CONSTANTS
+from archivebox.config.permissions import ARCHIVEBOX_USER
 
 import abx
 from .base_hook import BaseHook, HookType
@@ -57,7 +59,7 @@ class BaseBinary(BaseHook, Binary):
     def symlink_to_lib(binary, bin_dir=None) -> None:
         bin_dir = bin_dir or CONSTANTS.LIB_BIN_DIR
         
-        if not (binary.abspath and binary.abspath.exists()):
+        if not (binary.abspath and os.access(binary.abspath, os.R_OK)):
             return
         
         try:
@@ -78,9 +80,14 @@ class BaseBinary(BaseHook, Binary):
             self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
         else:
             # get cached binary from db
-            from machine.models import InstalledBinary
-            installed_binary = InstalledBinary.objects.get_from_db_or_cache(self)
-            binary = InstalledBinary.load_from_db(installed_binary)
+            try:
+                from machine.models import InstalledBinary
+                installed_binary = InstalledBinary.objects.get_from_db_or_cache(self)
+                binary = InstalledBinary.load_from_db(installed_binary)
+            except Exception:
+                # maybe we are not in a DATA dir so there is no db, fallback to reading from fs
+                # (e.g. when archivebox version is run outside of a DATA dir)
+                binary = super().load(**kwargs)
         return binary
     
     @validate_call
@@ -90,10 +97,15 @@ class BaseBinary(BaseHook, Binary):
         return binary
     
     @validate_call
-    def load_or_install(self, **kwargs) -> Self:
-        binary = super().load_or_install(**kwargs)
-        self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
-        return binary
+    def load_or_install(self, fresh=False, **kwargs) -> Self:
+        try:
+            binary = self.load(fresh=fresh)
+            if binary and binary.version:
+                self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
+                return binary
+        except Exception:
+            pass
+        return self.install(**kwargs)
     
     @property
     def admin_url(self) -> str:
@@ -113,6 +125,8 @@ class BrewBinProvider(BrewProvider, BaseBinProvider):
     
 class EnvBinProvider(EnvProvider, BaseBinProvider):
     name: BinProviderName = "env"
+    
+    euid: Optional[int] = ARCHIVEBOX_USER
 
 apt = AptBinProvider()
 brew = BrewBinProvider()

archivebox/abx/archivebox/base_check.py

@@ -1,44 +0,0 @@
-__package__ = "abx.archivebox"
-
-from typing import List
-
-from django.core.checks import Warning, Tags, register
-
-import abx
-
-from .base_hook import BaseHook, HookType
-
-
-class BaseCheck(BaseHook):
-    hook_type: HookType = "CHECK"
-    
-    tag: str = Tags.database
-    
-    @staticmethod
-    def check(settings, logger) -> List[Warning]:
-        """Override this method to implement your custom runtime check."""
-        errors = []
-        # if not hasattr(settings, 'SOME_KEY'):
-        #     errors.extend(Error(
-        #         'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
-        #         id='core.C001',
-        #         hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
-        #     ))
-        # logger.debug('[√] Loaded settings.PLUGINS succesfully.')
-        return errors
-
-    @abx.hookimpl
-    def get_CHECKS(self):
-        return [self]
-
-    @abx.hookimpl
-    def register_checks(self):
-        """Tell django that this check exists so it can be run automatically by django."""
-        def run_check(**kwargs):
-            from django.conf import settings
-            import logging
-            return self.check(settings, logging.getLogger("checks"))
-        
-        run_check.__name__ = self.id
-        run_check.tags = [self.tag]
-        register(self.tag)(run_check)

archivebox/abx/archivebox/base_configset.py

@@ -18,7 +18,7 @@ from . import toml_util
 
 
 PACKAGE_DIR = Path(__file__).resolve().parent.parent
-DATA_DIR = Path(os.curdir).resolve()
+DATA_DIR = Path(os.getcwd()).resolve()
 
 
 

archivebox/abx/archivebox/base_extractor.py

@@ -1,6 +1,7 @@
 __package__ = 'abx.archivebox'
 
 import json
+import os
 
 from typing import Optional, List, Literal, Annotated, Dict, Any, Tuple
 from typing_extensions import Self
@@ -83,7 +84,7 @@ class BaseExtractor(BaseHook):
         assert uplink.machine == installed_binary.machine  # it would be *very* weird if this wasn't true
         
         # output_dir = self.get_output_path(snapshot) or CONSTANTS.TMP_DIR
-        output_dir = CONSTANTS.TMP_DIR / 'test'
+        output_dir = CONSTANTS.DATA_DIR / '.tmp' / 'extractors' / self.name / str(snapshot.abid)
         output_dir.mkdir(parents=True, exist_ok=True)
 
         # execute the extractor binary with the given args
@@ -189,7 +190,7 @@ class BaseExtractor(BaseHook):
 
     # TODO: move this to a hookimpl
     def exec(self, args: CmdArgsList=(), cwd: Optional[Path]=None, installed_binary=None):
-        cwd = cwd or Path('.')
+        cwd = cwd or Path(os.getcwd())
         binary = self.load_binary(installed_binary=installed_binary)
         
         return binary.exec(cmd=args, cwd=cwd)

archivebox/abx/archivebox/hookspec.py

@@ -17,10 +17,6 @@ def get_EXTRACTORS():
 def get_REPLAYERS():
     return {}
 
-@hookspec
-def get_CHECKS():
-    return {}
-
 @hookspec
 def get_ADMINDATAVIEWS():
     return {}

archivebox/abx/archivebox/use.py

@@ -13,7 +13,6 @@ if TYPE_CHECKING:
     from .base_binary import BaseBinary, BaseBinProvider
     from .base_extractor import BaseExtractor
     from .base_replayer import BaseReplayer
-    from .base_check import BaseCheck
     from .base_queue import BaseQueue
     from .base_admindataview import BaseAdminDataView
     from .base_searchbackend import BaseSearchBackend
@@ -79,13 +78,6 @@ def get_REPLAYERS() -> Dict[str, 'BaseReplayer']:
             for replayer in plugin_replayers
     })
 
-def get_CHECKS() -> Dict[str, 'BaseCheck']:
-    return benedict({
-        check.id: check
-        for plugin_checks in pm.hook.get_CHECKS()
-            for check in plugin_checks
-    })
-
 def get_ADMINDATAVIEWS() -> Dict[str, 'BaseAdminDataView']:
     return benedict({
         admin_dataview.id: admin_dataview

archivebox/api/tests.py

@@ -1,30 +1,30 @@
 __package__ = 'archivebox.api'
 
-from django.test import TestCase
-from ninja.testing import TestClient
+# from django.test import TestCase
+# from ninja.testing import TestClient
 
-from .routes_cli import router
+# from .routes_cli import router
 
-class ArchiveBoxCLIAPITestCase(TestCase):
-    def setUp(self):
-        self.client = TestClient(router)
+# class ArchiveBoxCLIAPITestCase(TestCase):
+#     def setUp(self):
+#         self.client = TestClient(router)
 
-    def test_add_endpoint(self):
-        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_add_endpoint(self):
+#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_remove_endpoint(self):
-        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_remove_endpoint(self):
+#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_update_endpoint(self):
-        response = self.client.post("/update", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_update_endpoint(self):
+#         response = self.client.post("/update", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_list_all_endpoint(self):
-        response = self.client.post("/list_all", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_list_all_endpoint(self):
+#         response = self.client.post("/list_all", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])

archivebox/api/v1_api.py

@@ -12,12 +12,13 @@ from ninja import NinjaAPI, Swagger
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
-from archivebox.config import SHELL_CONFIG, VERSION
+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
 
 from api.auth import API_AUTH_METHODS
 
 
-COMMIT_HASH = SHELL_CONFIG.COMMIT_HASH or 'unknown'
+COMMIT_HASH = get_COMMIT_HASH() or 'unknown'
 
 html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>

archivebox/api/v1_cli.py

@@ -13,7 +13,7 @@ from ..main import (
     schedule,
 )
 from archivebox.misc.util import ansi_to_html
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG
 
 
 from .auth import API_AUTH_METHODS

archivebox/cli/__init__.py

@@ -1,6 +1,7 @@
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox'
 
+import os
 import sys
 import argparse
 import threading
@@ -8,13 +9,11 @@ import threading
 from time import sleep
 from collections.abc import Mapping
 
+from rich import print
+
 from typing import Optional, List, IO, Union, Iterable
 from pathlib import Path
 
-from archivebox.config import DATA_DIR
-from archivebox.misc.checks import check_migrations
-from archivebox.misc.logging import stderr
-
 from importlib import import_module
 
 BUILTIN_LIST = list
@@ -27,6 +26,10 @@ if len(sys.argv) > 1 and sys.argv[1] == 'setup':
     print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
     sys.argv[1] = 'install'
 
+if '--debug' in sys.argv:
+    os.environ['DEBUG'] = 'True'
+    sys.argv.remove('--debug')
+
 
 # def list_subcommands() -> Dict[str, str]:
 #     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
@@ -52,8 +55,8 @@ SUBCOMMAND_MODULES = {
     
     'init': 'archivebox_init',
     'install': 'archivebox_install',
+    ##############################################
     'config': 'archivebox_config',
-    
     'add': 'archivebox_add',
     'remove': 'archivebox_remove',
     'update': 'archivebox_update',
@@ -65,7 +68,7 @@ SUBCOMMAND_MODULES = {
     'shell': 'archivebox_shell',
     'manage': 'archivebox_manage',
 
-    'oneshot': 'archivebox_oneshot',
+    # 'oneshot': 'archivebox_oneshot',
 }
 
 # every imported command module must have these properties in order to be valid
@@ -104,11 +107,11 @@ CLI_SUBCOMMANDS = LazySubcommands()
 
 # these common commands will appear sorted before any others for ease-of-use
 meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-main_cmds = ('init', 'config', 'setup', 'install')            # dont require existing db present
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+setup_cmds = ('init', 'setup', 'install')                      # require valid data folder, but dont require DB present in it yet
+archive_cmds = ('add', 'remove', 'update', 'list', 'status', 'schedule', 'server', 'shell', 'manage')  # require valid data folder + existing db present
 fake_db = ("oneshot",)                                        # use fake in-memory db
 
-display_first = (*meta_cmds, *main_cmds, *archive_cmds)
+display_first = (*meta_cmds, *setup_cmds, *archive_cmds)
 
 
 IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
@@ -135,9 +138,10 @@ def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: It
         if blocking_threads:
             sleep(1)
             if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                stderr(
+                print(
                     f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
                     threads_summary,
+                    file=sys.stderr,
                 )
         else:
             return tries
@@ -154,14 +158,20 @@ def run_subcommand(subcommand: str,
 
     subcommand_args = subcommand_args or []
 
-    if subcommand not in meta_cmds:
-        from archivebox.config.legacy import setup_django
+    from archivebox.misc.checks import check_migrations
+    from archivebox.config.legacy import setup_django
+    
+    # print('DATA_DIR is', DATA_DIR)
+    # print('pwd is', os.getcwd())    
 
-        cmd_requires_db = subcommand in archive_cmds
-        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+    cmd_requires_db = subcommand in archive_cmds
+    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
 
-        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
+    check_db = cmd_requires_db and not init_pending
 
+    setup_django(in_memory_db=subcommand in fake_db, check_db=check_db)
+
+    if subcommand in archive_cmds:
         if cmd_requires_db:
             check_migrations()
 
@@ -237,12 +247,13 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
             subcommand=command.subcommand,
             subcommand_args=command.subcommand_args,
             stdin=stdin or None,
-            pwd=pwd or DATA_DIR,
         )
 
-    run_subcommand(
-        subcommand=command.subcommand,
-        subcommand_args=command.subcommand_args,
-        stdin=stdin or None,
-        pwd=pwd or DATA_DIR,
-    )
+    try:
+        run_subcommand(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin or None,
+        )
+    except KeyboardInterrupt:
+        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')

archivebox/cli/archivebox_add.py

@@ -9,7 +9,8 @@ import argparse
 from typing import List, Optional, IO
 
 from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR, ARCHIVING_CONFIG
+from archivebox.config import DATA_DIR
+from archivebox.config.common import ARCHIVING_CONFIG
 
 from ..main import add
 from ..parsers import PARSERS

archivebox/cli/archivebox_server.py

@@ -9,7 +9,8 @@ from pathlib import Path
 from typing import Optional, List, IO
 
 from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR, SERVER_CONFIG
+from archivebox.config import DATA_DIR
+from archivebox.config.common import SERVER_CONFIG
 from ..logging_util import SmartFormatter, reject_stdin
 from ..main import server
 

archivebox/config/__init__.py

@@ -1,27 +1,9 @@
 __package__ = 'archivebox.config'
 
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR, VERSION
-from .defaults import (
-    SHELL_CONFIG,
-    STORAGE_CONFIG,
-    GENERAL_CONFIG,
-    SERVER_CONFIG,
-    ARCHIVING_CONFIG,
-    SEARCH_BACKEND_CONFIG,
+from .paths import (
+    PACKAGE_DIR,                                    # noqa
+    DATA_DIR,                                       # noqa
+    ARCHIVE_DIR,                                    # noqa
 )
-
-
-__all__ = [
-    'CONSTANTS',
-    'PACKAGE_DIR',
-    'DATA_DIR',
-    'ARCHIVE_DIR',
-    'VERSION',
-    'SHELL_CONFIG',
-    'STORAGE_CONFIG',
-    'GENERAL_CONFIG',
-    'SERVER_CONFIG',
-    'ARCHIVING_CONFIG',
-    'SEARCH_BACKEND_CONFIG',
-    'CONSTANTS_CONFIG',
-]
+from .constants import CONSTANTS, CONSTANTS_CONFIG  # noqa
+from .version import VERSION                        # noqa

archivebox/config/apps.py

@@ -8,7 +8,7 @@ from abx.archivebox.base_hook import BaseHook
 
 
 from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR      # noqa
-from .defaults import (
+from .common import (
     ShellConfig,                    # noqa: F401
     StorageConfig,                  # noqa: F401
     GeneralConfig,                  # noqa: F401

archivebox/config/check_for_update.py

@@ -1,47 +0,0 @@
-# def get_versions_available_on_github(config):
-#     """
-#     returns a dictionary containing the ArchiveBox GitHub release info for
-#     the recommended upgrade version and the currently installed version
-#     """
-    
-#     # we only want to perform the (relatively expensive) check for new versions
-#     # when its most relevant, e.g. when the user runs a long-running command
-#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
-#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
-#     if subcommand_run_by_user not in long_running_commands:
-#         return None
-    
-#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
-#     response = requests.get(github_releases_api)
-#     if response.status_code != 200:
-#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
-#         return None
-#     all_releases = response.json()
-
-#     installed_version = parse_version_string(config['VERSION'])
-
-#     # find current version or nearest older version (to link to)
-#     current_version = None
-#     for idx, release in enumerate(all_releases):
-#         release_version = parse_version_string(release['tag_name'])
-#         if release_version <= installed_version:
-#             current_version = release
-#             break
-
-#     current_version = current_version or all_releases[-1]
-    
-#     # recommended version is whatever comes after current_version in the release list
-#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
-#     try:
-#         recommended_version = all_releases[idx+1]
-#     except IndexError:
-#         recommended_version = None
-
-#     return {'recommended_version': recommended_version, 'current_version': current_version}
-
-# def can_upgrade(config):
-#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
-#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
-#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
-#         return recommended_version > current_version
-#     return False

archivebox/config/common.py

@@ -1,21 +1,21 @@
 __package__ = 'archivebox.config'
 
-import os
 import sys
 import shutil
 
 from typing import Dict, Optional
-from datetime import datetime
 from pathlib import Path
 
 from rich import print
-from pydantic import Field, field_validator, model_validator, computed_field
+from pydantic import Field, field_validator, computed_field
 from django.utils.crypto import get_random_string
 
 from abx.archivebox.base_configset import BaseConfigSet
 
 
-from .constants import CONSTANTS, PACKAGE_DIR
+from .constants import CONSTANTS
+from .version import get_COMMIT_HASH, get_BUILD_TIME
+from .permissions import IN_DOCKER
 
 ###################### Config ##########################
 
@@ -27,14 +27,8 @@ class ShellConfig(BaseConfigSet):
     USE_COLOR: bool                     = Field(default=lambda c: c.IS_TTY)
     SHOW_PROGRESS: bool                 = Field(default=lambda c: c.IS_TTY)
     
-    IN_DOCKER: bool                     = Field(default=False)
+    IN_DOCKER: bool                     = Field(default=IN_DOCKER)
     IN_QEMU: bool                       = Field(default=False)
-    
-    USER: str                           = Field(default=Path('~').expanduser().resolve().name)
-    PUID: int                           = Field(default=os.getuid())
-    PGID: int                           = Field(default=os.getgid())
-    
-    PYTHON_ENCODING: str                = Field(default=(sys.__stdout__ or sys.stdout or sys.__stderr__ or sys.stderr).encoding.upper().replace('UTF8', 'UTF-8'))
 
     ANSI: Dict[str, str]                = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
 
@@ -52,63 +46,12 @@ class ShellConfig(BaseConfigSet):
     @computed_field
     @property
     def COMMIT_HASH(self) -> Optional[str]:
-        try:
-            git_dir = PACKAGE_DIR / '../.git'
-            ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
-            commit_hash = git_dir.joinpath(ref).read_text().strip()
-            return commit_hash
-        except Exception:
-            pass
-    
-        try:
-            return list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
-        except Exception:
-            pass
-        
-        return None
+        return get_COMMIT_HASH()
     
     @computed_field
     @property
     def BUILD_TIME(self) -> str:
-        if self.IN_DOCKER:
-            docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
-            return docker_build_end_time
-    
-        src_last_modified_unix_timestamp = (PACKAGE_DIR / 'package.json').stat().st_mtime
-        return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
-    
-
-    @model_validator(mode='after')
-    def validate_not_running_as_root(self):
-        attempted_command = ' '.join(sys.argv[:3])
-        if self.PUID == 0 and attempted_command not in ('setup', 'install'):
-            # stderr('[!] ArchiveBox should never be run as root!', color='red')
-            # stderr('    For more information, see the security overview documentation:')
-            # stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root')
-            print('[red][!] ArchiveBox should never be run as root![/red]', file=sys.stderr)
-            print('    For more information, see the security overview documentation:', file=sys.stderr)
-            print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root', file=sys.stderr)
-            
-            if self.IN_DOCKER:
-                print('[red][!] When using Docker, you must run commands with [green]docker run[/green] instead of [yellow3]docker exec[/yellow3], e.g.:', file=sys.stderr)
-                print('        docker compose run archivebox {attempted_command}', file=sys.stderr)
-                print(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}', file=sys.stderr)
-                print('        or:', file=sys.stderr)
-                print(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
-                print(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
-            raise SystemExit(2)
-        
-        # check python locale
-        if self.PYTHON_ENCODING != 'UTF-8':
-            print(f'[red][X] Your system is running python3 scripts with a bad locale setting: {self.PYTHON_ENCODING} (it should be UTF-8).[/red]', file=sys.stderr)
-            print('    To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)', file=sys.stderr)
-            print('    Or if you\'re using ubuntu/debian, run "dpkg-reconfigure locales"', file=sys.stderr)
-            print('')
-            print('    Confirm that it\'s fixed by opening a new shell and running:', file=sys.stderr)
-            print('        python3 -c "import sys; print(sys.stdout.encoding)"   # should output UTF-8', file=sys.stderr)
-            raise SystemExit(2)
-        
-        return self
+        return get_BUILD_TIME()
 
 SHELL_CONFIG = ShellConfig()
 
@@ -182,13 +125,14 @@ class ArchivingConfig(BaseConfigSet):
 
     @field_validator('TIMEOUT', mode='after')
     def validate_timeout(cls, v):
-        print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={v} seconds)[/red]', file=sys.stderr)
-        print('    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
-        print('    (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
-        print(file=sys.stderr)
-        print('    If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
-        print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
-        print(file=sys.stderr)
+        if int(v) < 5:
+            print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={v} seconds)[/red]', file=sys.stderr)
+            print('    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
+            print('    (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
+            print(file=sys.stderr)
+            print('    If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
+            print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
+            print(file=sys.stderr)
         return v
     
     @field_validator('CHECK_SSL_VALIDITY', mode='after')

archivebox/config/config_stubs.py

@@ -1,115 +0,0 @@
-from pathlib import Path
-from typing import Optional, Dict, Union, Tuple, Callable, Pattern, Type, Any, List
-from mypy_extensions import TypedDict
-
-from benedict import benedict
-
-SimpleConfigValue = Union[str, bool, int, None, Pattern, Dict[str, Any]]
-SimpleConfigValueDict = Dict[str, SimpleConfigValue]
-SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
-ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
-
-
-
-class BaseConfig(TypedDict):
-    pass
-
-class ConfigDict(BaseConfig, benedict, total=False):
-    """
-    # Regenerate by pasting this quine into `archivebox shell` 🥚
-    from archivebox.config import ConfigDict, CONFIG_DEFAULTS
-    print('class ConfigDict(BaseConfig, total=False):')
-    print('    ' + '"'*3 + ConfigDict.__doc__ + '"'*3)
-    for section, configs in CONFIG_DEFAULTS.items():
-        for key, attrs in configs.items():
-            Type, default = attrs['type'], attrs['default']
-            if default is None:
-                print(f'    {key}: Optional[{Type.__name__}]')
-            else:
-                print(f'    {key}: {Type.__name__}')
-        print()
-    """
-
-    IS_TTY: bool
-    USE_COLOR: bool
-    SHOW_PROGRESS: bool
-    IN_DOCKER: bool
-
-    PACKAGE_DIR: Path
-    CONFIG_FILE: Path
-    ONLY_NEW: bool
-    TIMEOUT: int
-    MEDIA_TIMEOUT: int
-    OUTPUT_PERMISSIONS: str
-    RESTRICT_FILE_NAMES: str
-    URL_DENYLIST: str
-
-    SECRET_KEY: Optional[str]
-    BIND_ADDR: str
-    ALLOWED_HOSTS: str
-    DEBUG: bool
-    PUBLIC_INDEX: bool
-    PUBLIC_SNAPSHOTS: bool
-    FOOTER_INFO: str
-
-    SAVE_TITLE: bool
-    SAVE_FAVICON: bool
-    SAVE_WGET: bool
-    SAVE_WGET_REQUISITES: bool
-    SAVE_SINGLEFILE: bool
-    SAVE_READABILITY: bool
-    SAVE_MERCURY: bool
-    SAVE_PDF: bool
-    SAVE_SCREENSHOT: bool
-    SAVE_DOM: bool
-    SAVE_WARC: bool
-    SAVE_GIT: bool
-    SAVE_MEDIA: bool
-    SAVE_ARCHIVE_DOT_ORG: bool
-
-    RESOLUTION: str
-    GIT_DOMAINS: str
-    CHECK_SSL_VALIDITY: bool
-    CURL_USER_AGENT: str
-    WGET_USER_AGENT: str
-    CHROME_USER_AGENT: str
-    COOKIES_FILE: Union[str, Path, None]
-    CHROME_USER_DATA_DIR: Union[str, Path, None]
-    CHROME_TIMEOUT: int
-    CHROME_HEADLESS: bool
-    CHROME_SANDBOX: bool
-
-    USE_CURL: bool
-    USE_WGET: bool
-    USE_SINGLEFILE: bool
-    USE_READABILITY: bool
-    USE_MERCURY: bool
-    USE_GIT: bool
-    USE_CHROME: bool
-    USE_YOUTUBEDL: bool
-    CURL_BINARY: str
-    GIT_BINARY: str
-    WGET_BINARY: str
-    SINGLEFILE_BINARY: str
-    READABILITY_BINARY: str
-    MERCURY_BINARY: str
-    YOUTUBEDL_BINARY: str
-    CHROME_BINARY: Optional[str]
-
-    YOUTUBEDL_ARGS: List[str]
-    WGET_ARGS: List[str]
-    CURL_ARGS: List[str]
-    GIT_ARGS: List[str]
-    TAG_SEPARATOR_PATTERN: str
-
-
-ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]
-ConfigDefaultValue = Union[ConfigValue, ConfigDefaultValueGetter]
-
-ConfigDefault = TypedDict('ConfigDefault', {
-    'default': ConfigDefaultValue,
-    'type': Optional[Type],
-    'aliases': Optional[Tuple[str, ...]],
-}, total=False)
-
-ConfigDefaultDict = Dict[str, ConfigDefault]

archivebox/config/constants.py

@@ -1,119 +1,117 @@
 __package__ = 'archivebox.config'
 
-
 import os
 import re
-import platform
+import sys
 
 from typing import Dict
 from pathlib import Path
-import importlib.metadata
 from collections.abc import Mapping
 
 from benedict import benedict
 
 from ..misc.logging import DEFAULT_CLI_COLORS
 
+from .paths import (
+    PACKAGE_DIR,
+    DATA_DIR,
+    ARCHIVE_DIR,
+    get_collection_id,
+    get_machine_id,
+    get_machine_type,
+)
+from .permissions import (
+    IS_ROOT,
+    IN_DOCKER,
+    RUNNING_AS_UID,
+    RUNNING_AS_GID,
+    DEFAULT_PUID,
+    DEFAULT_PGID,
+    ARCHIVEBOX_USER,
+    ARCHIVEBOX_GROUP,
+)
+from .version import detect_installed_version
+
 ###################### Config ##########################
 
-PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
-DATA_DIR: Path = Path(os.curdir).resolve()                    # archivebox user data dir
-ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
-
-def _detect_installed_version(PACKAGE_DIR: Path):
-    """Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file"""
-    try:
-        # if in production install, use pip-installed package metadata
-        return importlib.metadata.version(__package__ or 'archivebox')
-    except importlib.metadata.PackageNotFoundError:
-        pass
-
-    try:
-        # if in dev Git repo dir, use pyproject.toml file
-        pyproject_config = (PACKAGE_DIR.parent / 'pyproject.toml').read_text().split('\n')
-        for line in pyproject_config:
-            if line.startswith('version = '):
-                return line.split(' = ', 1)[-1].strip('"')
-    except FileNotFoundError:
-        # building docs, pyproject.toml is not available
-        pass
-    
-    try:
-        # if in dev but not in Git repo dir, fallback to using package.json file
-        package_json = (PACKAGE_DIR / 'package.json').read_text().split('\n')
-        for line in package_json:
-            if '"version": "' in line:
-                return line.replace('"', '').split(':')[-1].strip(',')
-    except FileNotFoundError:
-        pass
-
-    # raise Exception('Failed to detect installed archivebox version!')
-    return 'dev'
-
-VERSION: str = _detect_installed_version(PACKAGE_DIR)
-
 
 class ConstantsDict(Mapping):
-    IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'yes')
-    OS = platform.system().lower()      # darwin, linux, etc.
-    ARCH = platform.machine().lower()   # arm64, x86_64, etc.
-    LIB_DIR_SCOPE = f'{ARCH}-{OS}' + ('-docker' if IN_DOCKER else '')
-
-
-    PACKAGE_DIR: Path = PACKAGE_DIR     # archivebox source code dir
-    DATA_DIR: Path = DATA_DIR           # archivebox user data dir
-    ARCHIVE_DIR: Path = ARCHIVE_DIR     # archivebox snapshot data dir
-    VERSION: str = VERSION
+    PACKAGE_DIR: Path                   = PACKAGE_DIR
+    DATA_DIR: Path                      = DATA_DIR
+    ARCHIVE_DIR: Path                   = ARCHIVE_DIR
     
+    MACHINE_TYPE: str                   = get_machine_type()
+    MACHINE_ID: str                     = get_machine_id()
+    COLLECTION_ID: str                  = get_collection_id(DATA_DIR)
+    
+    # Host system
+    VERSION: str                        = detect_installed_version(PACKAGE_DIR)
+    IN_DOCKER: bool                     = IN_DOCKER
+    
+    # Permissions
+    IS_ROOT: bool                       = IS_ROOT
+    ARCHIVEBOX_USER: int                = ARCHIVEBOX_USER
+    ARCHIVEBOX_GROUP: int               = ARCHIVEBOX_GROUP
+    RUNNING_AS_UID: int                 = RUNNING_AS_UID
+    RUNNING_AS_GID: int                 = RUNNING_AS_GID
+    DEFAULT_PUID: int                   = DEFAULT_PUID
+    DEFAULT_PGID: int                   = DEFAULT_PGID
+    IS_INSIDE_VENV: bool                = sys.prefix != sys.base_prefix
+    
+    # Source code dirs
     PACKAGE_DIR_NAME: str               = PACKAGE_DIR.name
     TEMPLATES_DIR_NAME: str             = 'templates'
     TEMPLATES_DIR: Path                 = PACKAGE_DIR / TEMPLATES_DIR_NAME
-    STATIC_DIR: Path                    = TEMPLATES_DIR / 'static'
+    STATIC_DIR_NAME: str                = 'static'
+    STATIC_DIR: Path                    = TEMPLATES_DIR / STATIC_DIR_NAME
+
+    # Data dirs
+    ARCHIVE_DIR_NAME: str               = 'archive'
+    SOURCES_DIR_NAME: str               = 'sources'
+    PERSONAS_DIR_NAME: str              = 'personas'
+    CRONTABS_DIR_NAME: str              = 'crontabs'
+    CACHE_DIR_NAME: str                 = 'cache'
+    LOGS_DIR_NAME: str                  = 'logs'
     USER_PLUGINS_DIR_NAME: str          = 'user_plugins'
     CUSTOM_TEMPLATES_DIR_NAME: str      = 'user_templates'
-
-    ARCHIVE_DIR_NAME: str = 'archive'
-    SOURCES_DIR_NAME: str = 'sources'
-    PERSONAS_DIR_NAME: str = 'personas'
-    CRONTABS_DIR_NAME: str = 'crontabs'
-    CACHE_DIR_NAME: str = 'cache'
-    LOGS_DIR_NAME: str = 'logs'
-    LIB_DIR_NAME: str = 'lib'
-    TMP_DIR_NAME: str = 'tmp'
-
     ARCHIVE_DIR: Path                   = DATA_DIR / ARCHIVE_DIR_NAME
     SOURCES_DIR: Path                   = DATA_DIR / SOURCES_DIR_NAME
     PERSONAS_DIR: Path                  = DATA_DIR / PERSONAS_DIR_NAME
-    CACHE_DIR: Path                     = DATA_DIR / CACHE_DIR_NAME
     LOGS_DIR: Path                      = DATA_DIR / LOGS_DIR_NAME
-    LIB_DIR: Path                       = DATA_DIR / LIB_DIR_NAME / LIB_DIR_SCOPE   # e.g. data/lib/arm64-darwin-docker
-    TMP_DIR: Path                       = (Path('/tmp') if IN_DOCKER else DATA_DIR) / TMP_DIR_NAME
+    CACHE_DIR: Path                     = DATA_DIR / CACHE_DIR_NAME
     CUSTOM_TEMPLATES_DIR: Path          = DATA_DIR / CUSTOM_TEMPLATES_DIR_NAME
     USER_PLUGINS_DIR: Path              = DATA_DIR / USER_PLUGINS_DIR_NAME
 
+    # Data dir files
+    CONFIG_FILENAME: str                = 'ArchiveBox.conf'
+    SQL_INDEX_FILENAME: str             = 'index.sqlite3'
+    QUEUE_DATABASE_FILENAME: str        = 'queue.sqlite3'
+    CONFIG_FILE: Path                   = DATA_DIR / CONFIG_FILENAME
+    DATABASE_FILE: Path                 = DATA_DIR / SQL_INDEX_FILENAME
+    QUEUE_DATABASE_FILE: Path           = DATA_DIR / QUEUE_DATABASE_FILENAME
+    
+    JSON_INDEX_FILENAME: str            = 'index.json'
+    HTML_INDEX_FILENAME: str            = 'index.html'
+    ROBOTS_TXT_FILENAME: str            = 'robots.txt'
+    FAVICON_FILENAME: str               = 'favicon.ico'
+    
+    # Runtime dirs
+    TMP_DIR_NAME: str                   = 'tmp'
+    TMP_DIR: Path                       = DATA_DIR / TMP_DIR_NAME / MACHINE_ID
+    LIB_DIR_NAME: str                   = 'lib'
+    LIB_DIR: Path                       = DATA_DIR / LIB_DIR_NAME / MACHINE_TYPE
     LIB_PIP_DIR: Path                   = LIB_DIR / 'pip'
     LIB_NPM_DIR: Path                   = LIB_DIR / 'npm'
     LIB_BROWSERS_DIR: Path              = LIB_DIR / 'browsers'
     LIB_BIN_DIR: Path                   = LIB_DIR / 'bin'
     BIN_DIR: Path                       = LIB_BIN_DIR
 
-    CONFIG_FILENAME: str = 'ArchiveBox.conf'
-    SQL_INDEX_FILENAME: str = 'index.sqlite3'
+    # Config constants
+    TIMEZONE: str                       = 'UTC'
+    DEFAULT_CLI_COLORS: Dict[str, str]  = DEFAULT_CLI_COLORS
+    DISABLED_CLI_COLORS: Dict[str, str] = benedict({k: '' for k in DEFAULT_CLI_COLORS})
 
-    CONFIG_FILE: Path                   = DATA_DIR / CONFIG_FILENAME
-    DATABASE_FILE: Path                 = DATA_DIR / SQL_INDEX_FILENAME
-    QUEUE_DATABASE_FILE: Path           = DATA_DIR / SQL_INDEX_FILENAME.replace('index.', 'queue.')
-
-    JSON_INDEX_FILENAME: str = 'index.json'
-    HTML_INDEX_FILENAME: str = 'index.html'
-    ROBOTS_TXT_FILENAME: str = 'robots.txt'
-    FAVICON_FILENAME: str = 'favicon.ico'
-
-    TIMEZONE: str                             = 'UTC'
-    DEFAULT_CLI_COLORS: Dict[str, str]        = DEFAULT_CLI_COLORS
-    DISABLED_CLI_COLORS: Dict[str, str]       = benedict({k: '' for k in DEFAULT_CLI_COLORS})
-
-    ALLOWDENYLIST_REGEX_FLAGS: int = re.IGNORECASE | re.UNICODE | re.MULTILINE
+    ALLOWDENYLIST_REGEX_FLAGS: int      = re.IGNORECASE | re.UNICODE | re.MULTILINE
 
     STATICFILE_EXTENSIONS: frozenset[str] = frozenset((
         # 99.999% of the time, URLs ending in these extensions are static files
@@ -137,16 +135,6 @@ class ConstantsDict(Mapping):
         # html, htm, shtml, xhtml, xml, aspx, php, cgi
     ))
 
-    INGORED_PATHS: frozenset[str] = frozenset((
-        ".git",
-        ".svn",
-        ".DS_Store",
-        ".gitignore",
-        "lost+found",
-        ".DS_Store",
-        ".env",
-        "Dockerfile",
-    ))
     PIP_RELATED_NAMES: frozenset[str] = frozenset((
         ".venv",
         "venv",
@@ -160,25 +148,38 @@ class ConstantsDict(Mapping):
         "yarn.lock",
     ))
 
-    DATA_DIR_NAMES: frozenset[str] = frozenset((
+    # When initializing archivebox in a new directory, we check to make sure the dir is
+    # actually empty so that we dont clobber someone's home directory or desktop by accident.
+    # These files are exceptions to the is_empty check when we're trying to init a new dir,
+    # as they could be from a previous archivebox version, system artifacts, dependencies, etc.
+    ALLOWED_IN_DATA_DIR: frozenset[str] = frozenset((
+        *PIP_RELATED_NAMES,
+        *NPM_RELATED_NAMES,
+        
+        ### Dirs:
         ARCHIVE_DIR_NAME,
         SOURCES_DIR_NAME,
         LOGS_DIR_NAME,
         CACHE_DIR_NAME,
         LIB_DIR_NAME,
+        TMP_DIR_NAME,
         PERSONAS_DIR_NAME,
         CUSTOM_TEMPLATES_DIR_NAME,
         USER_PLUGINS_DIR_NAME,
-    ))
-    DATA_DIRS: frozenset[Path] = frozenset(DATA_DIR / dirname for dirname in DATA_DIR_NAMES)
-    DATA_FILE_NAMES: frozenset[str] = frozenset((
+        CRONTABS_DIR_NAME,
+        "static",                # created by old static exports <v0.6.0
+        "sonic",                 # created by docker bind mount / sonic FTS process
+        ".git",
+        ".svn",
+        
+        ### Files:
         CONFIG_FILENAME,
         SQL_INDEX_FILENAME,
         f"{SQL_INDEX_FILENAME}-wal",
         f"{SQL_INDEX_FILENAME}-shm",
-        "queue.sqlite3",
-        "queue.sqlite3-wal",
-        "queue.sqlite3-shm",
+        QUEUE_DATABASE_FILENAME,
+        f"{QUEUE_DATABASE_FILENAME}-wal",
+        f"{QUEUE_DATABASE_FILENAME}-shm",
         "search.sqlite3",
         JSON_INDEX_FILENAME,
         HTML_INDEX_FILENAME,
@@ -186,48 +187,43 @@ class ConstantsDict(Mapping):
         FAVICON_FILENAME,
         CONFIG_FILENAME,
         f"{CONFIG_FILENAME}.bak",
+        f".{CONFIG_FILENAME}.bak",
         "static_index.json",
-    ))
-
-    # When initializing archivebox in a new directory, we check to make sure the dir is
-    # actually empty so that we dont clobber someone's home directory or desktop by accident.
-    # These files are exceptions to the is_empty check when we're trying to init a new dir,
-    # as they could be from a previous archivebox version, system artifacts, dependencies, etc.
-    ALLOWED_IN_DATA_DIR: frozenset[str] = frozenset((
-        *INGORED_PATHS,
-        *PIP_RELATED_NAMES,
-        *NPM_RELATED_NAMES,
-        *DATA_DIR_NAMES,
-        *DATA_FILE_NAMES,
-        "static",                # created by old static exports <v0.6.0
-        "sonic",                 # created by docker bind mount
+        ".DS_Store",
+        ".gitignore",
+        "lost+found",
+        ".DS_Store",
+        ".env",
+        ".collection_id",
+        ".archivebox_id",
+        "Dockerfile",
     ))
 
     CODE_LOCATIONS = benedict({
         'PACKAGE_DIR': {
             'path': (PACKAGE_DIR).resolve(),
             'enabled': True,
-            'is_valid': (PACKAGE_DIR / '__main__.py').exists(),
+            'is_valid': os.access(PACKAGE_DIR / '__main__.py', os.X_OK),                                                                  # executable
         },
         'TEMPLATES_DIR': {
             'path': TEMPLATES_DIR.resolve(),
             'enabled': True,
-            'is_valid': STATIC_DIR.exists(),
+            'is_valid': os.access(STATIC_DIR, os.R_OK) and os.access(STATIC_DIR, os.X_OK),                                                # read + list
         },
         'CUSTOM_TEMPLATES_DIR': {
             'path': CUSTOM_TEMPLATES_DIR.resolve(),
-            'enabled': True,
-            'is_valid': CUSTOM_TEMPLATES_DIR.is_dir(),
+            'enabled': os.path.isdir(CUSTOM_TEMPLATES_DIR),
+            'is_valid': os.path.isdir(CUSTOM_TEMPLATES_DIR) and os.access(CUSTOM_TEMPLATES_DIR, os.R_OK),                                      # read
+        },
+        'USER_PLUGINS_DIR': {
+            'path': USER_PLUGINS_DIR.resolve(),
+            'enabled': os.path.isdir(USER_PLUGINS_DIR),
+            'is_valid': os.path.isdir(USER_PLUGINS_DIR) and os.access(USER_PLUGINS_DIR, os.R_OK),                                              # read
         },
         'LIB_DIR': {
             'path': LIB_DIR.resolve(),
             'enabled': True,
-            'is_valid': LIB_DIR.is_dir(),
-        },
-        'TMP_DIR': {
-            'path': TMP_DIR.resolve(),
-            'enabled': True,
-            'is_valid': TMP_DIR.is_dir(),
+            'is_valid': os.path.isdir(LIB_DIR) and os.access(LIB_DIR, os.R_OK) and os.access(LIB_DIR, os.W_OK),                      # read + write
         },
     })
         
@@ -235,52 +231,57 @@ class ConstantsDict(Mapping):
         "DATA_DIR": {
             "path": DATA_DIR.resolve(),
             "enabled": True,
-            "is_valid": DATABASE_FILE.exists(),
+            "is_valid": os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.R_OK) and os.access(DATA_DIR, os.W_OK),
             "is_mount": os.path.ismount(DATA_DIR.resolve()),
         },
         "CONFIG_FILE": {
             "path": CONFIG_FILE.resolve(),
             "enabled": True,
-            "is_valid": CONFIG_FILE.exists(),
+            "is_valid": os.path.isfile(CONFIG_FILE) and os.access(CONFIG_FILE, os.R_OK) and os.access(CONFIG_FILE, os.W_OK),
         },
         "SQL_INDEX": {
             "path": DATABASE_FILE.resolve(),
             "enabled": True,
-            "is_valid": DATABASE_FILE.exists(),
+            "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
             "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
         },
         "QUEUE_DATABASE": {
             "path": QUEUE_DATABASE_FILE.resolve(),
             "enabled": True,
-            "is_valid": QUEUE_DATABASE_FILE.exists(),
+            "is_valid": os.path.isfile(QUEUE_DATABASE_FILE) and os.access(QUEUE_DATABASE_FILE, os.R_OK) and os.access(QUEUE_DATABASE_FILE, os.W_OK),
             "is_mount": os.path.ismount(QUEUE_DATABASE_FILE.resolve()),
         },
         "ARCHIVE_DIR": {
             "path": ARCHIVE_DIR.resolve(),
             "enabled": True,
-            "is_valid": ARCHIVE_DIR.exists(),
+            "is_valid": os.path.isdir(ARCHIVE_DIR) and os.access(ARCHIVE_DIR, os.R_OK) and os.access(ARCHIVE_DIR, os.W_OK),
             "is_mount": os.path.ismount(ARCHIVE_DIR.resolve()),
         },
         "SOURCES_DIR": {
             "path": SOURCES_DIR.resolve(),
             "enabled": True,
-            "is_valid": SOURCES_DIR.exists(),
+            "is_valid": os.path.isdir(SOURCES_DIR) and os.access(SOURCES_DIR, os.R_OK) and os.access(SOURCES_DIR, os.W_OK),
         },
         "PERSONAS_DIR": {
             "path": PERSONAS_DIR.resolve(),
-            "enabled": PERSONAS_DIR.exists(),
-            "is_valid": PERSONAS_DIR.exists(),
+            "enabled": os.path.isdir(PERSONAS_DIR),
+            "is_valid": os.path.isdir(PERSONAS_DIR) and os.access(PERSONAS_DIR, os.R_OK) and os.access(PERSONAS_DIR, os.W_OK),                 # read + write
         },
         "LOGS_DIR": {
             "path": LOGS_DIR.resolve(),
             "enabled": True,
-            "is_valid": LOGS_DIR.is_dir(),
+            "is_valid": os.path.isdir(LOGS_DIR) and os.access(LOGS_DIR, os.R_OK) and os.access(LOGS_DIR, os.W_OK),                              # read + write
         },
-        "CACHE_DIR": {
-            "path": CACHE_DIR.resolve(),
-            "enabled": True,
-            "is_valid": CACHE_DIR.is_dir(),
+        'TMP_DIR': {
+            'path': TMP_DIR.resolve(),
+            'enabled': True,
+            'is_valid': os.path.isdir(TMP_DIR) and os.access(TMP_DIR, os.R_OK) and os.access(TMP_DIR, os.W_OK),                      # read + write
         },
+        # "CACHE_DIR": {
+        #     "path": CACHE_DIR.resolve(),
+        #     "enabled": True,
+        #     "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK),                        # read + write
+        # },
     })
 
     @classmethod

archivebox/config/legacy.py

@@ -22,41 +22,34 @@ Documentation:
 __package__ = 'archivebox.config'
 
 import os
-import io
 import re
 import sys
 import json
 import shutil
 
-from hashlib import md5
 from pathlib import Path
 from datetime import datetime, timezone
-from typing import Optional, Type, Tuple, Dict
-from subprocess import run, PIPE, DEVNULL, STDOUT, TimeoutExpired
+from typing import Optional, Type, Tuple, Dict, Any
+from subprocess import run, DEVNULL
 from configparser import ConfigParser
 
 from rich.progress import Progress
 from rich.console import Console
 from benedict import benedict
-from pydantic_pkgr import SemVer
 
 import django
 from django.db.backends.sqlite3.base import Database as sqlite3
 
 
-from .constants import CONSTANTS, TIMEZONE
+from .constants import CONSTANTS
 from .constants import *
-from .config_stubs import (
-    ConfigValue,
-    ConfigDefaultValue,
-    ConfigDefaultDict,
-)
+
 from ..misc.logging import (
     stderr,
     hint,      # noqa
 )
 
-from .defaults import SHELL_CONFIG, GENERAL_CONFIG, ARCHIVING_CONFIG, SERVER_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG
+from .common import SHELL_CONFIG, GENERAL_CONFIG, ARCHIVING_CONFIG, SERVER_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG
 from archivebox.plugins_auth.ldap.apps import LDAP_CONFIG
 from archivebox.plugins_extractor.favicon.apps import FAVICON_CONFIG
 from archivebox.plugins_extractor.wget.apps import WGET_CONFIG
@@ -67,7 +60,7 @@ LDAP = LDAP_CONFIG.LDAP_ENABLED
 
 ############################### Config Schema ##################################
 
-CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
+CONFIG_SCHEMA: Dict[str, Dict[str, Any]] = {
     'SHELL_CONFIG': SHELL_CONFIG.as_legacy_config_schema(),
 
     'SERVER_CONFIG': SERVER_CONFIG.as_legacy_config_schema(),
@@ -194,24 +187,10 @@ def get_real_name(key: str) -> str:
 
 # These are derived/computed values calculated *after* all user-provided config values are ingested
 # they appear in `archivebox config` output and are intended to be read-only for the user
-DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
-    'PACKAGE_DIR':              {'default': lambda c: CONSTANTS.PACKAGE_DIR.resolve()},
-    'TEMPLATES_DIR':            {'default': lambda c: c['PACKAGE_DIR'] / CONSTANTS.TEMPLATES_DIR_NAME},
-    # 'CUSTOM_TEMPLATES_DIR':     {'default': lambda c: c['CUSTOM_TEMPLATES_DIR'] and Path(c['CUSTOM_TEMPLATES_DIR'])},  # this is now a constant
-
-
+DYNAMIC_CONFIG_SCHEMA: Dict[str, Any] = {
     'URL_DENYLIST_PTN':         {'default': lambda c: c['URL_DENYLIST'] and re.compile(c['URL_DENYLIST'] or '', CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)},
     'URL_ALLOWLIST_PTN':        {'default': lambda c: c['URL_ALLOWLIST'] and re.compile(c['URL_ALLOWLIST'] or '', CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)},
 
-    # 'USE_GIT':                  {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']},
-    # 'GIT_VERSION':              {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None},
-    # 'SAVE_GIT':                 {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']},
-
-
-    # 'DEPENDENCIES':             {'default': lambda c: get_dependency_info(c)},
-    # 'CODE_LOCATIONS':           {'default': lambda c: get_code_locations(c)},
-    # 'DATA_LOCATIONS':           {'default': lambda c: get_data_locations(c)},
-
     'SAVE_ALLOWLIST_PTN':       {'default': lambda c: c['SAVE_ALLOWLIST'] and {re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_ALLOWLIST'].items()}},
     'SAVE_DENYLIST_PTN':        {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
 }
@@ -223,12 +202,12 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
 
 
 def load_config_val(key: str,
-                    default: ConfigDefaultValue=None,
+                    default: Any=None,
                     type: Optional[Type]=None,
                     aliases: Optional[Tuple[str, ...]]=None,
                     config: Optional[benedict]=None,
                     env_vars: Optional[os._Environ]=None,
-                    config_file_vars: Optional[Dict[str, str]]=None) -> ConfigValue:
+                    config_file_vars: Optional[Dict[str, str]]=None) -> Any:
     """parse bool, int, and str key=value pairs from env"""
 
     assert isinstance(config, dict)
@@ -291,7 +270,7 @@ def load_config_file(out_dir: str | None=CONSTANTS.DATA_DIR) -> Optional[benedic
     """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
 
     config_path = CONSTANTS.CONFIG_FILE
-    if config_path.exists():
+    if os.access(config_path, os.R_OK):
         config_file = ConfigParser()
         config_file.optionxform = str
         config_file.read(config_path)
@@ -328,7 +307,7 @@ def write_config_file(config: Dict[str, str], out_dir: str | None=CONSTANTS.DATA
 
     config_path = CONSTANTS.CONFIG_FILE
 
-    if not config_path.exists():
+    if not os.access(config_path, os.F_OK):
         atomic_write(config_path, CONFIG_HEADER)
 
     config_file = ConfigParser()
@@ -376,7 +355,7 @@ def write_config_file(config: Dict[str, str], out_dir: str | None=CONSTANTS.DATA
 
         raise
 
-    if Path(f'{config_path}.bak').exists():
+    if os.access(f'{config_path}.bak', os.F_OK):
         os.remove(f'{config_path}.bak')
 
     return benedict({
@@ -386,7 +365,7 @@ def write_config_file(config: Dict[str, str], out_dir: str | None=CONSTANTS.DATA
 
 
 
-def load_config(defaults: ConfigDefaultDict,
+def load_config(defaults: Dict[str, Any],
                 config: Optional[benedict]=None,
                 out_dir: Optional[str]=None,
                 env_vars: Optional[os._Environ]=None,
@@ -427,74 +406,6 @@ def load_config(defaults: ConfigDefaultDict,
 
 
 
-# Dependency Metadata Helpers
-def bin_version(binary: Optional[str], cmd: Optional[str]=None, timeout: int=3) -> Optional[str]:
-    """check the presence and return valid version line of a specified binary"""
-
-    abspath = bin_path(binary)
-    if not binary or not abspath:
-        return None
-    
-    return '999.999.999'
-
-    # Now handled by new BinProvider plugin system, no longer needed:
-
-    try:
-        bin_env = os.environ | {'LANG': 'C'}
-        is_cmd_str = cmd and isinstance(cmd, str)
-        version_str = (
-            run(cmd or [abspath, "--version"], timeout=timeout, shell=is_cmd_str, stdout=PIPE, stderr=STDOUT, env=bin_env)
-            .stdout.strip()
-            .decode()
-        )
-        if not version_str:
-            version_str = (
-                run(cmd or [abspath, "--version"], timeout=timeout, shell=is_cmd_str, stdout=PIPE, stderr=STDOUT)
-                .stdout.strip()
-                .decode()
-            )
-        
-        # take first 3 columns of first line of version info
-        semver = SemVer.parse(version_str)
-        if semver:
-            return str(semver)
-    except (OSError, TimeoutExpired):
-        pass
-        # stderr(f'[X] Unable to find working version of dependency: {binary}', color='red')
-        # stderr('    Make sure it\'s installed, then confirm it\'s working by running:')
-        # stderr(f'        {binary} --version')
-        # stderr()
-        # stderr('    If you don\'t want to install it, you can disable it via config. See here for more info:')
-        # stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Install')
-    return None
-
-def bin_path(binary: Optional[str]) -> Optional[str]:
-    if binary is None:
-        return None
-
-    node_modules_bin = Path('.') / 'node_modules' / '.bin' / binary
-    if node_modules_bin.exists():
-        return str(node_modules_bin.resolve())
-
-    return shutil.which(str(Path(binary).expanduser())) or shutil.which(str(binary)) or binary
-
-def bin_hash(binary: Optional[str]) -> Optional[str]:
-    return 'UNUSED'
-    # DEPRECATED: now handled by new BinProvider plugin system, no longer needed:
-
-    if binary is None:
-        return None
-    abs_path = bin_path(binary)
-    if abs_path is None or not Path(abs_path).exists():
-        return None
-
-    file_hash = md5()
-    with io.open(abs_path, mode='rb') as f:
-        for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b''):
-            file_hash.update(chunk)
-
-    return f'md5:{file_hash.hexdigest()}'
-
 def find_chrome_binary() -> Optional[str]:
     """find any installed chrome binaries in the default locations"""
     # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
@@ -551,7 +462,7 @@ def find_chrome_data_dir() -> Optional[str]:
     # )
     # for path in default_profile_paths:
     #     full_path = Path(path).resolve()
-    #     if full_path.exists():
+    #     if full_path.is_dir():
     #         return full_path
     return None
 
@@ -567,116 +478,6 @@ def wget_supports_compression(config):
         return False
 
 
-def get_dependency_info(config: benedict) -> ConfigValue:
-    return {
-        # 'PYTHON_BINARY': {
-        #     'path': bin_path(config['PYTHON_BINARY']),
-        #     'version': config['PYTHON_VERSION'],
-        #     'hash': bin_hash(config['PYTHON_BINARY']),
-        #     'enabled': True,
-        #     'is_valid': bool(config['PYTHON_VERSION']),
-        # },
-        # 'SQLITE_BINARY': {
-        #     'path': bin_path(config['SQLITE_BINARY']),
-        #     'version': config['SQLITE_VERSION'],
-        #     'hash': bin_hash(config['SQLITE_BINARY']),
-        #     'enabled': True,
-        #     'is_valid': bool(config['SQLITE_VERSION']),
-        # },
-        # 'DJANGO_BINARY': {
-        #     'path': bin_path(config['DJANGO_BINARY']),
-        #     'version': config['DJANGO_VERSION'],
-        #     'hash': bin_hash(config['DJANGO_BINARY']),
-        #     'enabled': True,
-        #     'is_valid': bool(config['DJANGO_VERSION']),
-        # },
-        # 'ARCHIVEBOX_BINARY': {
-        #     'path': bin_path(config['ARCHIVEBOX_BINARY']),
-        #     'version': config['VERSION'],
-        #     'hash': bin_hash(config['ARCHIVEBOX_BINARY']),
-        #     'enabled': True,
-        #     'is_valid': True,
-        # },
-        
-        # 'CURL_BINARY': {
-        #     'path': bin_path(config['CURL_BINARY']),
-        #     'version': config['CURL_VERSION'],
-        #     'hash': bin_hash(config['CURL_BINARY']),
-        #     'enabled': config['USE_CURL'],
-        #     'is_valid': bool(config['CURL_VERSION']),
-        # },
-        # 'WGET_BINARY': {
-        #     'path': bin_path(config['WGET_BINARY']),
-        #     'version': config['WGET_VERSION'],
-        #     'hash': bin_hash(config['WGET_BINARY']),
-        #     'enabled': config['USE_WGET'],
-        #     'is_valid': bool(config['WGET_VERSION']),
-        # },
-        # 'NODE_BINARY': {
-        #     'path': bin_path(config['NODE_BINARY']),
-        #     'version': config['NODE_VERSION'],
-        #     'hash': bin_hash(config['NODE_BINARY']),
-        #     'enabled': config['USE_NODE'],
-        #     'is_valid': bool(config['NODE_VERSION']),
-        # },
-        # 'MERCURY_BINARY': {
-        #     'path': bin_path(config['MERCURY_BINARY']),
-        #     'version': config['MERCURY_VERSION'],
-        #     'hash': bin_hash(config['MERCURY_BINARY']),
-        #     'enabled': config['USE_MERCURY'],
-        #     'is_valid': bool(config['MERCURY_VERSION']),
-        # },
-        # 'GIT_BINARY': {
-        #     'path': bin_path(config['GIT_BINARY']),
-        #     'version': config['GIT_VERSION'],
-        #     'hash': bin_hash(config['GIT_BINARY']),
-        #     'enabled': config['USE_GIT'],
-        #     'is_valid': bool(config['GIT_VERSION']),
-        # },
-        # 'SINGLEFILE_BINARY': {
-        #     'path': bin_path(config['SINGLEFILE_BINARY']),
-        #     'version': config['SINGLEFILE_VERSION'],
-        #     'hash': bin_hash(config['SINGLEFILE_BINARY']),
-        #     'enabled': config['USE_SINGLEFILE'],
-        #     'is_valid': bool(config['SINGLEFILE_VERSION']),
-        # },
-        # 'READABILITY_BINARY': {
-        #     'path': bin_path(config['READABILITY_BINARY']),
-        #     'version': config['READABILITY_VERSION'],
-        #     'hash': bin_hash(config['READABILITY_BINARY']),
-        #     'enabled': config['USE_READABILITY'],
-        #     'is_valid': bool(config['READABILITY_VERSION']),
-        # },
-        # 'YOUTUBEDL_BINARY': {
-        #     'path': bin_path(config['YOUTUBEDL_BINARY']),
-        #     'version': config['YOUTUBEDL_VERSION'],
-        #     'hash': bin_hash(config['YOUTUBEDL_BINARY']),
-        #     'enabled': config['USE_YOUTUBEDL'],
-        #     'is_valid': bool(config['YOUTUBEDL_VERSION']),
-        # },
-        # 'CHROME_BINARY': {
-        #     'path': bin_path(config['CHROME_BINARY']),
-        #     'version': config['CHROME_VERSION'],
-        #     'hash': bin_hash(config['CHROME_BINARY']),
-        #     'enabled': config['USE_CHROME'],
-        #     'is_valid': bool(config['CHROME_VERSION']),
-        # },
-        # 'RIPGREP_BINARY': {
-        #     'path': bin_path(config['RIPGREP_BINARY']),
-        #     'version': config['RIPGREP_VERSION'],
-        #     'hash': bin_hash(config['RIPGREP_BINARY']),
-        #     'enabled': config['USE_RIPGREP'],
-        #     'is_valid': bool(config['RIPGREP_VERSION']),
-        # },
-        # 'SONIC_BINARY': {
-        #     'path': bin_path(config['SONIC_BINARY']),
-        #     'version': config['SONIC_VERSION'],
-        #     'hash': bin_hash(config['SONIC_BINARY']),
-        #     'enabled': config['USE_SONIC'],
-        #     'is_valid': bool(config['SONIC_VERSION']),
-        # },
-    }
-
 # ******************************************************************************
 # ******************************************************************************
 # ******************************** Load Config *********************************
@@ -697,7 +498,7 @@ def load_all_config():
 # add all final config values in CONFIG to globals in this file
 CONFIG: benedict = load_all_config()
 globals().update(CONFIG)
-# this lets us do:  from .config import DEBUG, MEDIA_TIMEOUT, ...
+
 
 # print("FINISHED LOADING CONFIG USING SCHEMAS + FILE + ENV")
 
@@ -713,8 +514,8 @@ globals().update(CONFIG)
 
 
 # Set timezone to UTC and umask to OUTPUT_PERMISSIONS
-assert TIMEZONE == 'UTC', f'The server timezone should always be set to UTC (got {TIMEZONE})'  # noqa: F821
-os.environ["TZ"] = TIMEZONE                                                  # noqa: F821
+assert CONSTANTS.TIMEZONE == 'UTC', f'The server timezone should always be set to UTC (got {CONSTANTS.TIMEZONE})'  # noqa: F821
+os.environ["TZ"] = CONSTANTS.TIMEZONE                                                  # noqa: F821
 os.umask(0o777 - int(STORAGE_CONFIG.DIR_OUTPUT_PERMISSIONS, base=8))                        # noqa: F821
 
 ########################### Config Validity Checkers ###########################
@@ -725,7 +526,8 @@ if not SHELL_CONFIG.SHOW_PROGRESS:
     os.environ['TERM'] = 'dumb'
 
 # recreate rich console obj based on new config values
-CONSOLE = Console()
+STDOUT = CONSOLE = Console()
+STDERR = Console(stderr=True)
 from ..misc import logging
 logging.CONSOLE = CONSOLE
 
@@ -733,11 +535,11 @@ logging.CONSOLE = CONSOLE
 INITIAL_STARTUP_PROGRESS = None
 INITIAL_STARTUP_PROGRESS_TASK = 0
 
-def bump_startup_progress_bar():
+def bump_startup_progress_bar(advance=1):
     global INITIAL_STARTUP_PROGRESS
     global INITIAL_STARTUP_PROGRESS_TASK
     if INITIAL_STARTUP_PROGRESS:
-        INITIAL_STARTUP_PROGRESS.update(INITIAL_STARTUP_PROGRESS_TASK, advance=1)   # type: ignore
+        INITIAL_STARTUP_PROGRESS.update(INITIAL_STARTUP_PROGRESS_TASK, advance=advance)   # type: ignore
 
 
 def setup_django_minimal():
@@ -751,6 +553,8 @@ DJANGO_SET_UP = False
 
 
 def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CONFIG, in_memory_db=False) -> None:
+    from rich.panel import Panel
+    
     global INITIAL_STARTUP_PROGRESS
     global INITIAL_STARTUP_PROGRESS_TASK
     global DJANGO_SET_UP
@@ -760,12 +564,24 @@ def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CON
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
-    with Progress(transient=True, expand=True, console=CONSOLE) as INITIAL_STARTUP_PROGRESS:
+    with Progress(transient=True, expand=True, console=STDERR) as INITIAL_STARTUP_PROGRESS:
         INITIAL_STARTUP_PROGRESS_TASK = INITIAL_STARTUP_PROGRESS.add_task("[green]Loading modules...", total=25)
 
         output_dir = out_dir or CONSTANTS.DATA_DIR
 
         assert isinstance(output_dir, Path) and isinstance(CONSTANTS.PACKAGE_DIR, Path)
+        
+        from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
+        from archivebox.config.paths import _get_collection_id
+    
+        # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
+        if IS_ROOT:
+            with SudoPermission(uid=0):
+                os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}"')
+        _get_collection_id(DATA_DIR=CONSTANTS.DATA_DIR, force_create=True)
+        if IS_ROOT:
+            with SudoPermission(uid=0):
+                os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}"/* 2>/dev/null')
 
         bump_startup_progress_bar()
         try:
@@ -787,7 +603,25 @@ def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CON
             else:
                 # Otherwise use default sqlite3 file-based database and initialize django
                 # without running migrations automatically (user runs them manually by calling init)
-                django.setup()
+                try:
+                    django.setup()
+                except Exception as e:
+                    bump_startup_progress_bar(advance=1000)
+                    
+                    is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version', 'init'))
+                    if not is_using_meta_cmd:
+                        # show error message to user only if they're not running a meta command / just trying to get help
+                        STDERR.print()
+                        STDERR.print(Panel(
+                            f'\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n',
+                            title='\n\n[red][X] Error while trying to load database![/red]',
+                            subtitle='[grey53]NO WRITES CAN BE PERFORMED[/grey53]',
+                            expand=False,
+                            style='bold red',
+                        ))
+                        STDERR.print()
+                        STDERR.print_exception(show_locals=False)
+                    return
             
             bump_startup_progress_bar()
 
@@ -800,6 +634,17 @@ def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CON
                 f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n")
 
             if check_db:
+                # make sure the data dir is owned by a non-root user
+                if CONSTANTS.DATA_DIR.stat().st_uid == 0:
+                    STDERR.print('[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]')
+                    STDERR.print(f'    {CONSTANTS.DATA_DIR}')
+                    STDERR.print()
+                    STDERR.print('[violet]Hint:[/violet] Are you running archivebox in the right folder? (and as a non-root user?)')
+                    STDERR.print('    cd path/to/your/archive/data')
+                    STDERR.print('    archivebox [command]')
+                    STDERR.print()
+                    raise SystemExit(9)
+                
                 # Create cache table in DB if needed
                 try:
                     from django.core.cache import cache
@@ -817,7 +662,7 @@ def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CON
                     conn.close_if_unusable_or_obsolete()
 
                 sql_index_path = CONSTANTS.DATABASE_FILE
-                assert sql_index_path.exists(), (
+                assert os.access(sql_index_path, os.F_OK), (
                     f'No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)')
 
                 bump_startup_progress_bar()

archivebox/config/paths.py

@@ -0,0 +1,221 @@
+__package__ = 'archivebox.config'
+
+import os
+import hashlib
+import platform
+from pathlib import Path
+from functools import cache
+from datetime import datetime
+
+from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER
+
+#############################################################################################
+
+PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()                  # archivebox user data dir
+ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
+
+IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
+
+DATABASE_FILE = DATA_DIR / 'index.sqlite3'
+
+#############################################################################################
+
+def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
+    collection_id_file = DATA_DIR / '.archivebox_id'
+    
+    try:
+        return collection_id_file.read_text().strip()
+    except (OSError, FileNotFoundError, PermissionError):
+        pass
+    
+    # hash the machine_id + collection dir path + creation time to get a unique collection_id
+    machine_id = get_machine_id()
+    collection_path = DATA_DIR.resolve()
+    try:
+        creation_date = DATA_DIR.stat().st_ctime
+    except Exception:
+        creation_date = datetime.now().isoformat()
+    collection_id = hashlib.sha256(f'{machine_id}:{collection_path}@{creation_date}'.encode()).hexdigest()[:8]
+    
+    try:
+        # only persist collection_id file if we already have an index.sqlite3 file present
+        # otherwise we might be running in a directory that is not a collection, no point creating cruft files
+        if os.path.isfile(DATABASE_FILE) and os.access(DATA_DIR, os.W_OK) or force_create:
+            collection_id_file.write_text(collection_id)
+            
+            # if we're running as root right now, make sure the collection_id file is owned by the archivebox user
+            if IS_ROOT:
+                with SudoPermission(uid=0):
+                    if ARCHIVEBOX_USER == 0:
+                        os.system(f'chmod 777 "{collection_id_file}"')
+                    else:    
+                        os.system(f'chown {ARCHIVEBOX_USER} "{collection_id_file}"')
+    except (OSError, FileNotFoundError, PermissionError):
+        pass
+    return collection_id
+
+@cache
+def get_collection_id(DATA_DIR=DATA_DIR) -> str:
+    """Get a short, stable, unique ID for the current collection (e.g. abc45678)"""
+    return _get_collection_id(DATA_DIR=DATA_DIR)
+
+@cache
+def get_machine_id() -> str:
+    """Get a short, stable, unique ID for the current machine (e.g. abc45678)"""
+    
+    MACHINE_ID = 'unknown'
+    try:
+        import machineid
+        MACHINE_ID = machineid.hashed_id('archivebox')[:8]
+    except Exception:
+        try:
+            import uuid
+            import hashlib
+            MACHINE_ID = hashlib.sha256(str(uuid.getnode()).encode()).hexdigest()[:8]
+        except Exception:
+            pass
+    return MACHINE_ID
+
+@cache
+def get_machine_type() -> str:
+    """Get a short, stable, unique type identifier for the current machine (e.g. linux-x86_64-docker)"""
+    
+    OS: str                             = platform.system().lower()    # darwin, linux, etc.
+    ARCH: str                           = platform.machine().lower()   # arm64, x86_64, aarch64, etc.
+    LIB_DIR_SCOPE: str                  = f'{ARCH}-{OS}-docker' if IN_DOCKER else f'{ARCH}-{OS}'
+    return LIB_DIR_SCOPE
+
+
+def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = None, fallback=True) -> bool:
+    """Check if a given directory is writable by a specific user and group (fallback=try as current user is unable to check with provided uid)"""
+    current_uid, current_gid = os.geteuid(), os.getegid()
+    uid, gid = uid or current_uid, gid or current_gid
+
+    test_file = dir_path / '.permissions_test'
+    try:
+        with SudoPermission(uid=uid, fallback=fallback):
+            test_file.exists()
+            test_file.write_text(f'Checking if PUID={uid} PGID={gid} can write to dir')
+            test_file.unlink()
+            return True
+    except (IOError, OSError, PermissionError):
+        pass
+        
+    return False
+
+
+
+# @cache
+# def get_LIB_DIR():
+#     """
+#     - should be shared with other collections on the same host
+#     - must be scoped by CPU architecture, OS family, and archivebox version
+#     - should not be shared with other hosts/archivebox versions
+#     - must be writable by any archivebox user
+#     - should be persistent across reboots
+#     - can be on a docker bin mount but probably shouldnt be
+#     - ok to have a long path (doesnt contain SOCKETS)
+#     """
+#     from .version import detect_installed_version
+    
+#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
+    
+#     lib_dir = tempfile.gettempdir()
+#     try:
+#         if 'SYSTEM_LIB_DIR' in os.environ:
+#             lib_dir = Path(os.environ['SYSTEM_LIB_DIR'])
+#         else:
+#             with SudoPermission(uid=ARCHIVEBOX_USER, fallback=True):
+#                 lib_dir = HOST_DIRS.site_data_path
+        
+#         # Docker: /usr/local/share/archivebox/0.8.5
+#         # Ubuntu: /usr/local/share/archivebox/0.8.5
+#         # macOS: /Library/Application Support/archivebox
+#         try:
+#             with SudoPermission(uid=0, fallback=True):
+#                 lib_dir.mkdir(parents=True, exist_ok=True)
+#         except PermissionError:
+#             # our user cannot 
+#             lib_dir = HOST_DIRS.user_data_path
+#             lib_dir.mkdir(parents=True, exist_ok=True)
+        
+#         if IS_ROOT or not dir_is_writable(lib_dir, uid=ARCHIVEBOX_USER):
+#             if IS_ROOT:
+#                 # make sure lib dir is owned by the archivebox user, not root
+#                 with SudoPermission(uid=0):
+#                     if ARCHIVEBOX_USER == 0:
+#                         # print(f'[yellow]:warning:  Waring: Creating SYSTEM_LIB_DIR {lib_dir} with mode 777 so that non-root archivebox users can share it.[/yellow] (caches shared libs used by archivebox for performance)', file=sys.stderr)
+#                         os.system(f'chmod -R 777 "{lib_dir}"')
+#                     else:
+#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{lib_dir}"')
+#             else:
+#                 raise PermissionError()
+#     except (PermissionError, AssertionError):
+#         # raise PermissionError(f'SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
+#         print(f'[red]:cross_mark:  ERROR: SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
+        
+#     return lib_dir
+    
+# @cache
+# def get_TMP_DIR():
+#     """
+#     - must NOT be inside DATA_DIR / inside a docker volume bind mount
+#     - must NOT have a long PATH (UNIX socket path length restrictions)
+#     - must NOT be shared with other collections/hosts
+#     - must be writable by archivebox user & root
+#     - must be cleared on every boot / not persisted
+#     - must be cleared on every archivebox version upgrade
+#     """
+#     from .version import detect_installed_version
+    
+#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
+    
+#     # print('DATA_DIR OWNED BY:', ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
+#     # print('RUNNING AS:', self.PUID, self.PGID)
+#     run_dir = tempfile.gettempdir()
+#     try:
+#         if 'SYSTEM_TMP_DIR' in os.environ:
+#             run_dir = Path(os.environ['SYSTEM_TMP_DIR']).resolve() / get_collection_id(DATA_DIR=DATA_DIR)
+#             with SudoPermission(uid=0, fallback=True):
+#                 run_dir.mkdir(parents=True, exist_ok=True)
+#             if not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
+#                 if IS_ROOT:
+#                     with SudoPermission(uid=0, fallback=False):
+#                         if ARCHIVEBOX_USER == 0:
+#                             # print(f'[yellow]:warning:  Waring: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
+#                             os.system(f'chmod -R 777 "{run_dir}"')
+#                         else:
+#                             os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
+#                 else:
+#                     raise PermissionError()
+#             assert len(str(run_dir / 'supervisord.conf')) < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
+#             return run_dir
+        
+#         run_dir = (HOST_DIRS.site_runtime_path / get_collection_id(DATA_DIR=DATA_DIR)).resolve()
+#         try:
+#             assert len(str(run_dir)) + len('/supervisord.sock') < 95
+#         except AssertionError:
+#             run_dir = Path(tempfile.gettempdir()).resolve() / 'archivebox' / get_collection_id(DATA_DIR=DATA_DIR)
+#             assert len(str(run_dir)) + len('/supervisord.sock') < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
+        
+#         with SudoPermission(uid=0, fallback=True):
+#             run_dir.mkdir(parents=True, exist_ok=True)
+            
+#         if IS_ROOT or not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
+#             if IS_ROOT:
+#                 with SudoPermission(uid=0):
+#                     if ARCHIVEBOX_USER == 0:
+#                         # print(f'[yellow]:warning:  Waring: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
+#                         os.system(f'chmod -R 777 "{run_dir}"')
+#                     else:
+#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
+#             else:
+#                 raise PermissionError()
+            
+#     except (PermissionError, AssertionError):
+#         # raise PermissionError(f'SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
+#         print(f'[red]:cross_mark:  ERROR: SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
+        
+#     return run_dir
+

archivebox/config/permissions.py

@@ -0,0 +1,133 @@
+__package__ = 'archivebox.config'
+
+import os
+import pwd
+import sys
+
+from rich import print
+
+from pathlib import Path
+from contextlib import contextmanager
+
+#############################################################################################
+
+DATA_DIR = Path(os.getcwd())
+
+try:
+    DATA_DIR_STAT           = DATA_DIR.stat()
+    DATA_DIR_UID            = DATA_DIR_STAT.st_uid
+    DATA_DIR_GID            = DATA_DIR_STAT.st_gid
+except PermissionError:
+    DATA_DIR_UID            = 0
+    DATA_DIR_GID            = 0
+
+DEFAULT_PUID            = 911
+DEFAULT_PGID            = 911
+RUNNING_AS_UID          = os.getuid()
+RUNNING_AS_GID          = os.getgid()
+EUID                    = os.geteuid()
+EGID                    = os.getegid()
+SUDO_UID                = int(os.environ.get('SUDO_UID', 0))
+SUDO_GID                = int(os.environ.get('SUDO_GID', 0))
+USER: str               = Path('~').expanduser().resolve().name
+
+IS_ROOT = RUNNING_AS_UID == 0
+IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
+
+FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
+FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
+if RUNNING_AS_UID == 0:
+    try:
+        # if we are running as root it's really hard to figure out what the correct archivebox user should be
+        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
+        # check if 911:911 archivebox user exists on host system, and use it instead of 0
+        import pwd
+        if pwd.getpwuid(DEFAULT_PUID).pw_name == 'archivebox':
+            FALLBACK_UID = DEFAULT_PUID
+            FALLBACK_GID = DEFAULT_PGID
+    except Exception:
+        pass
+
+
+os.environ.setdefault('PUID', str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
+os.environ.setdefault('PGID', str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
+
+ARCHIVEBOX_USER = int(os.environ['PUID'])
+ARCHIVEBOX_GROUP = int(os.environ['PGID'])
+if not USER:
+    try:
+        # alternative method 1 to get username
+        USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name
+    except Exception:
+        pass
+        
+if not USER:
+    try:
+        # alternative method 2 to get username
+        import getpass
+        USER = getpass.getuser()
+    except Exception:
+        pass
+    
+if not USER:
+    try:
+        # alternative method 3 to get username
+        USER = os.getlogin() or 'archivebox'
+    except Exception:
+        USER = 'archivebox'
+        
+ARCHIVEBOX_USER_EXISTS = False
+try:
+    pwd.getpwuid(ARCHIVEBOX_USER)
+    ARCHIVEBOX_USER_EXISTS = True
+except Exception:
+    ARCHIVEBOX_USER_EXISTS = False
+    
+
+#############################################################################################
+
+def drop_privileges():
+    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    
+    # always run archivebox as the user that owns the data dir, never as root
+    if os.getuid() == 0:
+        # drop permissions to the user that owns the data dir / provided PUID
+        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
+            # drop our effective UID to the archivebox user's UID
+            os.seteuid(ARCHIVEBOX_USER)
+            
+            # update environment variables so that subprocesses dont try to write to /root
+            pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
+            os.environ['HOME']     = pw_record.pw_dir
+            os.environ['LOGNAME']  = pw_record.pw_name
+            os.environ['USER']     = pw_record.pw_name
+
+    if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS:
+        print('[yellow]:warning:  Running as [red]root[/red] is not recommended and may make your [blue]DATA_DIR[/blue] inaccessible to other users on your system.[/yellow] (use [blue]sudo[/blue] instead)', file=sys.stderr)
+
+
+@contextmanager
+def SudoPermission(uid=0, fallback=False):
+    """Attempt to run code with sudo permissions for a given user (or root)"""
+    
+    if os.geteuid() == uid:
+        # no need to change effective UID, we are already that user
+        yield
+        return
+
+    try:
+        # change our effective UID to the given UID
+        os.seteuid(uid)
+    except PermissionError as err:
+        if not fallback:
+            raise PermissionError(f'Not enough permissions to run code as uid={uid}, please retry with sudo') from err
+    try:
+        # yield back to the caller so they can run code inside context as root
+        yield
+    finally:
+        # then set effective UID back to DATA_DIR owner
+        try:
+            os.seteuid(ARCHIVEBOX_USER)
+        except PermissionError as err:
+            if not fallback:
+                raise PermissionError(f'Failed to revert uid={uid} back to {ARCHIVEBOX_USER} after running code with sudo') from err

archivebox/config/version.py

@@ -0,0 +1,121 @@
+__package__ = 'archivebox.config'
+
+import os
+import importlib.metadata
+
+from pathlib import Path
+from functools import cache
+from datetime import datetime
+from typing import Optional
+
+#############################################################################################
+
+IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
+
+PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()                  # archivebox user data dir
+ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
+
+#############################################################################################
+
+
+@cache
+def detect_installed_version(PACKAGE_DIR: Path=PACKAGE_DIR):
+    """Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file"""
+    try:
+        # if in production install, use pip-installed package metadata
+        return importlib.metadata.version('archivebox').strip()
+    except importlib.metadata.PackageNotFoundError:
+        pass
+
+    try:
+        # if in dev Git repo dir, use pyproject.toml file
+        pyproject_config = (PACKAGE_DIR.parent / 'pyproject.toml').read_text().split('\n')
+        for line in pyproject_config:
+            if line.startswith('version = '):
+                return line.split(' = ', 1)[-1].strip('"').strip()
+    except FileNotFoundError:
+        # building docs, pyproject.toml is not available
+        pass
+
+    # raise Exception('Failed to detect installed archivebox version!')
+    return 'dev'
+
+
+@cache
+def get_COMMIT_HASH() -> Optional[str]:
+    try:
+        git_dir = PACKAGE_DIR / '../.git'
+        ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
+        commit_hash = git_dir.joinpath(ref).read_text().strip()
+        return commit_hash
+    except Exception:
+        pass
+
+    try:
+        return list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
+    except Exception:
+        pass
+    
+    return None
+    
+@cache
+def get_BUILD_TIME() -> str:
+    if IN_DOCKER:
+        docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
+        return docker_build_end_time
+
+    src_last_modified_unix_timestamp = (PACKAGE_DIR / 'README.md').stat().st_mtime
+    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
+
+
+# def get_versions_available_on_github(config):
+#     """
+#     returns a dictionary containing the ArchiveBox GitHub release info for
+#     the recommended upgrade version and the currently installed version
+#     """
+    
+#     # we only want to perform the (relatively expensive) check for new versions
+#     # when its most relevant, e.g. when the user runs a long-running command
+#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
+#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
+#     if subcommand_run_by_user not in long_running_commands:
+#         return None
+    
+#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
+#     response = requests.get(github_releases_api)
+#     if response.status_code != 200:
+#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
+#         return None
+#     all_releases = response.json()
+
+#     installed_version = parse_version_string(config['VERSION'])
+
+#     # find current version or nearest older version (to link to)
+#     current_version = None
+#     for idx, release in enumerate(all_releases):
+#         release_version = parse_version_string(release['tag_name'])
+#         if release_version <= installed_version:
+#             current_version = release
+#             break
+
+#     current_version = current_version or all_releases[-1]
+    
+#     # recommended version is whatever comes after current_version in the release list
+#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
+#     try:
+#         recommended_version = all_releases[idx+1]
+#     except IndexError:
+#         recommended_version = None
+
+#     return {'recommended_version': recommended_version, 'current_version': current_version}
+
+# def can_upgrade(config):
+#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
+#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
+#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
+#         return recommended_version > current_version
+#     return False
+
+
+VERSION: str = detect_installed_version()

archivebox/config/views.py

@@ -319,7 +319,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert request.user.is_superuser, "Must be a superuser to view configuration settings."
 
     from queues.supervisor_util import get_existing_supervisord_process, get_worker
-    from queues.settings import CONFIG_FILE
+    from queues.settings import SUPERVISORD_CONFIG_FILE
 
     supervisor = get_existing_supervisord_process()
     if supervisor is None:
@@ -332,7 +332,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
 
     if key == 'supervisord':
-        relevant_config = CONFIG_FILE.read_text()
+        relevant_config = SUPERVISORD_CONFIG_FILE.read_text()
         relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
         start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
         uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]

archivebox/core/admin.py

@@ -475,7 +475,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         # ordering='archiveresult_count'
     )
     def size(self, obj):
-        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
+        archive_size = os.access(Path(obj.link_dir) / 'index.html', os.F_OK) and obj.archive_size
         if archive_size:
             size_txt = printable_filesize(archive_size)
             if archive_size > 52428800:
@@ -740,7 +740,7 @@ class ArchiveResultAdmin(ABIDModelAdmin):
         output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
         path_from_output_str = (snapshot_dir / result.output)
         output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
-        if path_from_output_str.exists():
+        if os.access(path_from_output_str, os.R_OK):
             root_dir = str(path_from_output_str)
         else:
             root_dir = str(snapshot_dir)

archivebox/core/middleware.py

@@ -5,7 +5,7 @@ from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.core.exceptions import ImproperlyConfigured
 
-from archivebox.config import SERVER_CONFIG
+from archivebox.config.common import SERVER_CONFIG
 
 
 def detect_timezone(request, activate: bool=True):

archivebox/core/migrations/0007_archiveresult.py

@@ -9,7 +9,7 @@ import django.db.models.deletion
 
 from index.json import to_json
 
-DATA_DIR = Path(os.curdir).resolve()                    # archivebox user data dir
+DATA_DIR = Path(os.getcwd()).resolve()                    # archivebox user data dir
 ARCHIVE_DIR = DATA_DIR / 'archive'                      # archivebox snapshot data dir
 
 

archivebox/core/models.py

@@ -4,6 +4,7 @@ __package__ = 'archivebox.core'
 from typing import Optional, Dict, Iterable
 from django_stubs_ext.db.models import TypedModelMeta
 
+import os
 import json
 
 from pathlib import Path
@@ -22,7 +23,7 @@ from archivebox.config import CONSTANTS
 
 from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
 from queues.tasks import bg_archive_snapshot
-from machine.models import Machine, NetworkInterface
+# from machine.models import Machine, NetworkInterface
 
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.util import parse_date, base_url
@@ -604,7 +605,7 @@ class ArchiveResult(ABIDModel):
         return link.canonical_outputs().get(f'{self.extractor}_path')
 
     def output_exists(self) -> bool:
-        return Path(self.output_path()).exists()
+        return os.access(self.output_path(), os.R_OK)
 
 
     # def get_storage_dir(self, create=True, symlink=True):

archivebox/core/serve_static.py

@@ -21,11 +21,11 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     assert document_root
     path = posixpath.normpath(path).lstrip("/")
     fullpath = Path(safe_join(document_root, path))
-    if fullpath.is_dir():
+    if os.access(fullpath, os.R_OK) and fullpath.is_dir():
         if show_indexes:
             return static.directory_index(path, fullpath)
         raise Http404(_("Directory indexes are not allowed here."))
-    if not fullpath.exists():
+    if not os.access(fullpath, os.R_OK):
         raise Http404(_("“%(path)s” does not exist") % {"path": fullpath})
     
     # Respect the If-Modified-Since header.

archivebox/core/settings.py

@@ -13,7 +13,8 @@ import abx.archivebox
 import abx.archivebox.use
 import abx.django.use
 
-from archivebox.config import VERSION, DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS, SHELL_CONFIG, SERVER_CONFIG      # noqa
+from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS
+from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG      # noqa
 
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
@@ -61,7 +62,6 @@ BINPROVIDERS = abx.archivebox.use.get_BINPROVIDERS()
 BINARIES = abx.archivebox.use.get_BINARIES()
 EXTRACTORS = abx.archivebox.use.get_EXTRACTORS()
 REPLAYERS = abx.archivebox.use.get_REPLAYERS()
-CHECKS = abx.archivebox.use.get_CHECKS()
 ADMINDATAVIEWS = abx.archivebox.use.get_ADMINDATAVIEWS()
 QUEUES = abx.archivebox.use.get_QUEUES()
 SEARCHBACKENDS = abx.archivebox.use.get_SEARCHBACKENDS()
@@ -168,8 +168,9 @@ AUTHENTICATION_BACKENDS = [
 
 STATIC_URL = '/static/'
 TEMPLATES_DIR_NAME = 'templates'
+CUSTOM_TEMPLATES_ENABLED = os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK) and CONSTANTS.CUSTOM_TEMPLATES_DIR.is_dir()
 STATICFILES_DIRS = [
-    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / 'static')] if CONSTANTS.CUSTOM_TEMPLATES_DIR.is_dir() else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'static')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -180,7 +181,7 @@ STATICFILES_DIRS = [
 ]
 
 TEMPLATE_DIRS = [
-    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CONSTANTS.CUSTOM_TEMPLATES_DIR.is_dir() else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'templates')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -217,11 +218,8 @@ TEMPLATES = [
 # CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME
 # CACHE_DB_TABLE = 'django_cache'
 
-DATABASE_FILE = DATA_DIR / CONSTANTS.SQL_INDEX_FILENAME
 DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
 
-QUEUE_DATABASE_NAME = DATABASE_NAME.replace('index.sqlite3', 'queue.sqlite3')
-
 SQLITE_CONNECTION_OPTIONS = {
     "ENGINE": "django.db.backends.sqlite3",
     "TIME_ZONE": CONSTANTS.TIMEZONE,
@@ -250,7 +248,7 @@ DATABASES = {
         **SQLITE_CONNECTION_OPTIONS,
     },
     "queue": {
-        "NAME": QUEUE_DATABASE_NAME,
+        "NAME": CONSTANTS.QUEUE_DATABASE_FILE,
         **SQLITE_CONNECTION_OPTIONS,
     },
     # 'cache': {
@@ -266,7 +264,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 HUEY = {
     "huey_class": "huey.SqliteHuey",
-    "filename": QUEUE_DATABASE_NAME,
+    "filename": CONSTANTS.QUEUE_DATABASE_FILENAME,
     "name": "system_tasks",
     "results": True,
     "store_none": True,
@@ -292,7 +290,7 @@ DJANGO_HUEY = {
     "queues": {
         HUEY["name"]: HUEY.copy(),
         # more registered here at plugin import-time by BaseQueue.register()
-        **abx.django.use.get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME=QUEUE_DATABASE_NAME),
+        **abx.django.use.get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME=CONSTANTS.QUEUE_DATABASE_FILENAME),
     },
 }
 
@@ -602,7 +600,7 @@ if DEBUG_REQUESTS_TRACKER:
 
 # # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
 # DEBUG_LOGFIRE = False
-# DEBUG_LOGFIRE = DEBUG_LOGFIRE and (DATA_DIR / '.logfire').is_dir()
+# DEBUG_LOGFIRE = DEBUG_LOGFIRE and os.access(DATA_DIR / '.logfire', os.W_OK) and (DATA_DIR / '.logfire').is_dir()
 
 
 # For usage with https://www.jetadmin.io/integrations/django

archivebox/core/settings_logging.py

@@ -1,5 +1,8 @@
 __package__ = 'archivebox.core'
+
 import re
+import os
+
 import shutil
 import tempfile
 import logging
@@ -54,7 +57,7 @@ ERROR_LOG = tempfile.NamedTemporaryFile().name
 
 LOGS_DIR = CONSTANTS.LOGS_DIR
 
-if LOGS_DIR.is_dir():
+if os.access(LOGS_DIR, os.W_OK) and LOGS_DIR.is_dir():
     ERROR_LOG = (LOGS_DIR / 'errors.log')
 else:
     # historically too many edge cases here around creating log dir w/ correct permissions early on

archivebox/core/views.py

@@ -1,5 +1,6 @@
 __package__ = 'archivebox.core'
 
+import os
 import inspect
 from typing import Callable, get_type_hints
 from pathlib import Path
@@ -27,7 +28,8 @@ from core.admin import result_url
 
 from queues.tasks import bg_add
 
-from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION, SHELL_CONFIG, SERVER_CONFIG
+from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION
+from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
 from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
 
 from .serve_static import serve_static_with_byterange_support
@@ -66,6 +68,7 @@ class SnapshotView(View):
             if (result.status == 'succeeded'
                 and (result.extractor not in HIDDEN_RESULTS)
                 and embed_path
+                and os.access(abs_path, os.R_OK)
                 and abs_path.exists()):
                 if abs_path.is_dir() and not any(abs_path.glob('*.*')):
                     continue
@@ -101,6 +104,8 @@ class SnapshotView(View):
 
         # iterate through all the files in the snapshot dir and add the biggest ones to1 the result list
         snap_dir = Path(snapshot.link_dir)
+        assert os.access(snap_dir, os.R_OK) and os.access(snap_dir, os.X_OK)
+        
         for result_file in (*snap_dir.glob('*'), *snap_dir.glob('*/*')):
             extension = result_file.suffix.lstrip('.').lower()
             if result_file.is_dir() or result_file.name.startswith('.') or extension not in allowed_extensions:

archivebox/core/wsgi.py

@@ -7,8 +7,8 @@ For more information on this file, see
 https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
 """
 
-
-from archivebox.config import setup_django
+import archivebox
+from archivebox.config.legacy import setup_django
 setup_django(in_memory_db=False, check_db=True)
 
 from django.core.wsgi import get_wsgi_application

archivebox/extractors/htmltotext.py

@@ -5,7 +5,8 @@ import io
 from pathlib import Path
 from typing import Optional
 
-from archivebox.config import VERSION, ARCHIVING_CONFIG
+from archivebox.config import VERSION
+from archivebox.config.common import ARCHIVING_CONFIG
 from archivebox.config.legacy import SAVE_HTMLTOTEXT
 from archivebox.misc.system import atomic_write
 from archivebox.misc.util import enforce_types, is_static_file

archivebox/extractors/wget.py

@@ -1,6 +1,7 @@
 __package__ = 'archivebox.extractors'
 
 import re
+import os
 from pathlib import Path
 
 from typing import Optional
@@ -147,23 +148,22 @@ def unsafe_wget_output_path(link: Link) -> Optional[str]:
     search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
     for _ in range(4):
         try:
-            if search_dir.exists():
-                if search_dir.is_dir():
-                    html_files = [
-                        f for f in search_dir.iterdir()
-                        if re.search(".+\\.[Ss]?[Hh][Tt][Mm][Ll]?$", str(f), re.I | re.M)
-                    ]
-                    if html_files:
-                        return str(html_files[0].relative_to(link.link_dir))
+            if os.access(search_dir, os.R_OK) and search_dir.is_dir():
+                html_files = [
+                    f for f in search_dir.iterdir()
+                    if re.search(".+\\.[Ss]?[Hh][Tt][Mm][Ll]?$", str(f), re.I | re.M)
+                ]
+                if html_files:
+                    return str(html_files[0].relative_to(link.link_dir))
 
-                    # sometimes wget'd URLs have no ext and return non-html
-                    # e.g. /some/example/rss/all -> some RSS XML content)
-                    #      /some/other/url.o4g   -> some binary unrecognized ext)
-                    # test this with archivebox add --depth=1 https://getpocket.com/users/nikisweeting/feed/all
-                    last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
-                    for file_present in search_dir.iterdir():
-                        if file_present == last_part_of_url:
-                            return str((search_dir / file_present).relative_to(link.link_dir))
+                # sometimes wget'd URLs have no ext and return non-html
+                # e.g. /some/example/rss/all -> some RSS XML content)
+                #      /some/other/url.o4g   -> some binary unrecognized ext)
+                # test this with archivebox add --depth=1 https://getpocket.com/users/nikisweeting/feed/all
+                last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
+                for file_present in search_dir.iterdir():
+                    if file_present == last_part_of_url:
+                        return str((search_dir / file_present).relative_to(link.link_dir))
         except OSError:
             # OSError 36 and others can happen here, caused by trying to check for impossible paths
             # (paths derived from URLs can often contain illegal unicode characters or be too long,
@@ -278,12 +278,12 @@ def wget_output_path(link: Link, nocache: bool=False) -> Optional[str]:
 
     # fallback to just the domain dir
     search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
-    if search_dir.is_dir():
+    if os.access(search_dir, os.R_OK) and search_dir.is_dir():
         return domain(link.url).replace(":", "+")
 
     # fallback to just the domain dir without port
     search_dir = Path(link.link_dir) / domain(link.url).split(":", 1)[0]
-    if search_dir.is_dir():
+    if os.access(search_dir, os.R_OK) and search_dir.is_dir():
         return domain(link.url).split(":", 1)[0]
 
     return None

archivebox/index/__init__.py

@@ -12,9 +12,11 @@ from urllib.parse import urlparse
 from django.db.models import QuerySet, Q
 
 
-from archivebox.config import DATA_DIR, CONSTANTS, ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
-from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
 from archivebox.misc.logging import stderr
+from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
+
+from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
 from archivebox.config.legacy import URL_DENYLIST_PTN, URL_ALLOWLIST_PTN
 
 from ..logging_util import (
@@ -247,7 +249,7 @@ def load_main_index(out_dir: Path | str=DATA_DIR, warn: bool=True) -> List[Link]
 @enforce_types
 def load_main_index_meta(out_dir: Path=DATA_DIR) -> Optional[dict]:
     index_path = out_dir / CONSTANTS.JSON_INDEX_FILENAME
-    if index_path.exists():
+    if os.access(index_path, os.F_OK):
         with open(index_path, 'r', encoding='utf-8') as f:
             meta_dict = pyjson.load(f)
             meta_dict.pop('links')

archivebox/index/html.py

@@ -16,7 +16,9 @@ from archivebox.misc.util import (
     htmlencode,
     urldecode,
 )
-from archivebox.config import CONSTANTS, DATA_DIR, VERSION, SHELL_CONFIG, SERVER_CONFIG
+from archivebox.config import CONSTANTS, DATA_DIR, VERSION
+from archivebox.config.common import SERVER_CONFIG
+from archivebox.config.version import get_COMMIT_HASH
 from archivebox.plugins_extractor.archivedotorg.apps import ARCHIVEDOTORG_CONFIG
 
 from .schema import Link
@@ -56,7 +58,7 @@ def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) ->
 
     return render_django_template(template, {
         'version': VERSION,
-        'git_sha': SHELL_CONFIG.COMMIT_HASH or VERSION,
+        'git_sha': get_COMMIT_HASH() or VERSION,
         'num_links': str(len(links)),
         'date_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d'),
         'time_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M'),

archivebox/index/json.py

@@ -8,7 +8,8 @@ from pathlib import Path
 from datetime import datetime, timezone
 from typing import List, Optional, Iterator, Any, Union
 
-from archivebox.config import VERSION, DATA_DIR, CONSTANTS, SERVER_CONFIG, SHELL_CONFIG
+from archivebox.config import VERSION, DATA_DIR, CONSTANTS
+from archivebox.config.common import SERVER_CONFIG, SHELL_CONFIG
 
 from .schema import Link
 from archivebox.misc.system import atomic_write
@@ -101,7 +102,7 @@ def parse_json_link_details(out_dir: Union[Path, str], guess: bool=False) -> Opt
     """load the json link index from a given directory"""
     
     existing_index = Path(out_dir) / CONSTANTS.JSON_INDEX_FILENAME
-    if existing_index.exists():
+    if os.access(existing_index, os.F_OK):
         with open(existing_index, 'r', encoding='utf-8') as f:
             try:
                 link_json = pyjson.load(f)
@@ -118,7 +119,7 @@ def parse_json_links_details(out_dir: Union[Path, str]) -> Iterator[Link]:
 
     for entry in os.scandir(CONSTANTS.ARCHIVE_DIR):
         if entry.is_dir(follow_symlinks=True):
-            if (Path(entry.path) / 'index.json').exists():
+            if os.access((Path(entry.path) / 'index.json'), os.F_OK):
                 try:
                     link = parse_json_link_details(entry.path)
                 except KeyError:

archivebox/index/sql.py

@@ -9,7 +9,8 @@ from django.db.models import QuerySet
 from django.db import transaction
 
 from archivebox.misc.util import enforce_types, parse_date
-from archivebox.config import DATA_DIR, GENERAL_CONFIG
+from archivebox.config import DATA_DIR
+from archivebox.config.common import GENERAL_CONFIG
 
 from .schema import Link
 

archivebox/logging_util.py

@@ -22,7 +22,8 @@ from rich.panel import Panel
 from rich_argparse import RichHelpFormatter
 from django.core.management.base import DjangoHelpFormatter
 
-from archivebox.config import CONSTANTS, DATA_DIR, VERSION, SHELL_CONFIG
+from archivebox.config import CONSTANTS, DATA_DIR, VERSION
+from archivebox.config.common import SHELL_CONFIG
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.util import enforce_types
 from archivebox.misc.logging import ANSI, stderr
@@ -227,7 +228,7 @@ def progress_bar(seconds: int, prefix: str='', ANSI: Dict[str, str]=ANSI) -> Non
         print()
 
 
-def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str | IO], pwd: str):
+def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str | IO], pwd: str='.'):
     args = ' '.join(subcommand_args)
     version_msg = '[dark_magenta]\\[{now}][/dark_magenta] [dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{VERSION}[/dark_goldenrod]: [green4]archivebox [green3]{subcommand}[green2] {args}[/green2]'.format(
         now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
@@ -477,7 +478,7 @@ def log_list_finished(links):
 def log_removal_started(links: List["Link"], yes: bool, delete: bool):
     print(f'[yellow3][i] Found {len(links)} matching URLs to remove.[/]')
     if delete:
-        file_counts = [link.num_outputs for link in links if Path(link.link_dir).exists()]
+        file_counts = [link.num_outputs for link in links if os.access(link.link_dir, os.R_OK)]
         print(
             f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
             f'    ({len(file_counts)} data folders with {sum(file_counts)} archived files will be deleted!)'
@@ -567,16 +568,19 @@ def printable_folder_status(name: str, folder: Dict) -> str:
         else:
             color, symbol, note, num_files = 'red', 'X', 'invalid', '?'
     else:
-        color, symbol, note, num_files = 'lightyellow', '-', 'disabled', '-'
+        color, symbol, note, num_files = 'grey53', '-', 'unused', '-'
 
 
     if folder['path']:
-        if Path(folder['path']).exists():
-            num_files = (
-                f'{len(os.listdir(folder["path"]))} files'
-                if Path(folder['path']).is_dir() else
-                printable_filesize(Path(folder['path']).stat().st_size)
-            )
+        if os.access(folder['path'], os.R_OK):
+            try:
+                num_files = (
+                    f'{len(os.listdir(folder["path"]))} files'
+                    if os.path.isdir(folder['path']) else
+                    printable_filesize(Path(folder['path']).stat().st_size)
+                )
+            except PermissionError:
+                num_files = 'error'
         else:
             num_files = 'missing'
         

archivebox/machine/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.machine'

archivebox/machine/apps.py

@@ -1,3 +1,5 @@
+__package__ = 'archivebox.machine'
+
 from django.apps import AppConfig
 
 

archivebox/machine/detect.py

@@ -15,7 +15,7 @@ import machineid             # https://github.com/keygen-sh/py-machineid
 from rich import print
 
 PACKAGE_DIR = Path(__file__).parent
-DATA_DIR = Path('.').resolve()
+DATA_DIR = Path(os.getcwd()).resolve()
 
 def get_vm_info():
     hw_in_docker = bool(os.getenv('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE'))

archivebox/main.py

@@ -14,13 +14,15 @@ from crontab import CronTab, CronSlices
 from django.db.models import QuerySet
 from django.utils import timezone
 
-from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR, SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
+from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
+from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
+from archivebox.config.permissions import SudoPermission, IN_DOCKER
 from .cli import (
     CLI_SUBCOMMANDS,
     run_subcommand,
     display_first,
     meta_cmds,
-    main_cmds,
+    setup_cmds,
     archive_cmds,
 )
 from .parsers import (
@@ -101,7 +103,7 @@ def help(out_dir: Path=DATA_DIR) -> None:
     ) + '\n\n    ' + '\n    '.join(
         f'[green]{cmd.ljust(20)}[/green] {func.__doc__}'
         for cmd, func in all_subcommands.items()
-        if cmd in main_cmds
+        if cmd in setup_cmds
     ) + '\n\n    ' + '\n    '.join(
         f'[green]{cmd.ljust(20)}[/green] {func.__doc__}'
         for cmd, func in all_subcommands.items()
@@ -119,10 +121,10 @@ def help(out_dir: Path=DATA_DIR) -> None:
 
     [grey53]# using Docker:[/grey53]
     [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-''' if SHELL_CONFIG.IN_DOCKER else ''
-    DOCKER_DOCS = '\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]' if SHELL_CONFIG.IN_DOCKER else ''
-    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if SHELL_CONFIG.IN_DOCKER else ''
-    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if SHELL_CONFIG.IN_DOCKER else ''
+''' if IN_DOCKER else ''
+    DOCKER_DOCS = '\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]' if IN_DOCKER else ''
+    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ''
+    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ''
 
     print(f'''{DOCKER_USAGE}
 [deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
@@ -138,7 +140,7 @@ def help(out_dir: Path=DATA_DIR) -> None:
 ''')
     
     
-    if CONSTANTS.DATABASE_FILE.exists():
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(out_dir).replace(str(Path('~').expanduser()), '~')
         EXAMPLE_USAGE = f'''
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
@@ -158,7 +160,7 @@ def help(out_dir: Path=DATA_DIR) -> None:
         print(Panel(EXAMPLE_USAGE, expand=False, border_style='grey53', title='[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]', subtitle='Commands run inside this dir will only apply to this collection.'))
     else:
         DATA_SETUP_HELP = '\n'
-        if SHELL_CONFIG.IN_DOCKER:
+        if IN_DOCKER:
             DATA_SETUP_HELP += '[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n'
             DATA_SETUP_HELP += '    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n'
         DATA_SETUP_HELP += 'To load an [dark_blue]existing[/dark_blue] collection:\n'
@@ -190,6 +192,10 @@ def version(quiet: bool=False,
     
     from plugins_auth.ldap.apps import LDAP_CONFIG
     from django.conf import settings
+    from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
+    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID
+
+    from abx.archivebox.base_binary import BaseBinary, apt, brew, env
 
     # 0.7.1
     # ArchiveBox v0.7.1+editable COMMIT_HASH=951bba5 BUILD_TIME=2023-12-17 16:46:05 1702860365
@@ -198,37 +204,42 @@ def version(quiet: bool=False,
     # DEBUG=False IS_TTY=True TZ=UTC SEARCH_BACKEND=ripgrep LDAP=False
     
     p = platform.uname()
+    COMMIT_HASH = get_COMMIT_HASH()
     prnt(
         '[dark_green]ArchiveBox[/dark_green] [dark_goldenrod]v{}[/dark_goldenrod]'.format(CONSTANTS.VERSION),
-        f'COMMIT_HASH={SHELL_CONFIG.COMMIT_HASH[:7] if SHELL_CONFIG.COMMIT_HASH else "unknown"}',
-        f'BUILD_TIME={SHELL_CONFIG.BUILD_TIME}',
+        f'COMMIT_HASH={COMMIT_HASH[:7] if COMMIT_HASH else "unknown"}',
+        f'BUILD_TIME={get_BUILD_TIME()}',
     )
     prnt(
-        f'IN_DOCKER={SHELL_CONFIG.IN_DOCKER}',
+        f'IN_DOCKER={IN_DOCKER}',
         f'IN_QEMU={SHELL_CONFIG.IN_QEMU}',
         f'ARCH={p.machine}',
         f'OS={p.system}',
         f'PLATFORM={platform.platform()}',
-        f'PYTHON={sys.implementation.name.title()}',
+        f'PYTHON={sys.implementation.name.title()}' + (' (venv)' if CONSTANTS.IS_INSIDE_VENV else ''),
     )
     OUTPUT_IS_REMOTE_FS = CONSTANTS.DATA_LOCATIONS.DATA_DIR.is_mount or CONSTANTS.DATA_LOCATIONS.ARCHIVE_DIR.is_mount
+    DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
     prnt(
+        f'EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}',
+        f'FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}',
+        f'FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}',
         f'FS_ATOMIC={STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES}',
         f'FS_REMOTE={OUTPUT_IS_REMOTE_FS}',
-        f'FS_USER={SHELL_CONFIG.PUID}:{SHELL_CONFIG.PGID}',
-        f'FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}',
     )
     prnt(
         f'DEBUG={SHELL_CONFIG.DEBUG}',
         f'IS_TTY={SHELL_CONFIG.IS_TTY}',
-        f'TZ={CONSTANTS.TIMEZONE}',
+        f'SUDO={CONSTANTS.IS_ROOT}',
+        f'ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}',
         f'SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}',
         f'LDAP={LDAP_CONFIG.LDAP_ENABLED}',
         #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
     )
     prnt()
 
-    prnt('[pale_green1][i] Dependency versions:[/pale_green1]')
+    prnt('[pale_green1][i] Binary Dependencies:[/pale_green1]')
+    failures = []
     for name, binary in reversed(list(settings.BINARIES.items())):
         if binary.name == 'archivebox':
             continue
@@ -239,7 +250,7 @@ def version(quiet: bool=False,
         except Exception as e:
             err = e
             loaded_bin = binary
-        provider_summary = f'[dark_sea_green3]{loaded_bin.binprovider.name.ljust(10)}[/dark_sea_green3]' if loaded_bin.binprovider else '[grey23]not found[/grey23]'
+        provider_summary = f'[dark_sea_green3]{loaded_bin.binprovider.name.ljust(10)}[/dark_sea_green3]' if loaded_bin.binprovider else '[grey23]not found[/grey23] '
         if loaded_bin.abspath:
             abspath = str(loaded_bin.abspath).replace(str(DATA_DIR), '[light_slate_blue].[/light_slate_blue]').replace(str(Path('~').expanduser()), '~')
             if ' ' in abspath:
@@ -247,6 +258,27 @@ def version(quiet: bool=False,
         else:
             abspath = f'[red]{err}[/red]'
         prnt('', '[green]√[/green]' if loaded_bin.is_valid else '[red]X[/red]', '', loaded_bin.name.ljust(21), str(loaded_bin.version).ljust(12), provider_summary, abspath, overflow='ignore', crop=False)
+        if not loaded_bin.is_valid:
+            failures.append(loaded_bin.name)
+            
+    prnt()
+    prnt('[gold3][i] Package Managers:[/gold3]')
+    for name, binprovider in reversed(list(settings.BINPROVIDERS.items())):
+        err = None
+        
+        # TODO: implement a BinProvider.BINARY() method that gets the loaded binary for a binprovider's INSTALLER_BIN
+        loaded_bin = binprovider.INSTALLER_BINARY or BaseBinary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
+        
+        abspath = None
+        if loaded_bin.abspath:
+            abspath = str(loaded_bin.abspath).replace(str(DATA_DIR), '.').replace(str(Path('~').expanduser()), '~')
+            if ' ' in abspath:
+                abspath = abspath.replace(' ', r'\ ')
+                
+        PATH = str(binprovider.PATH).replace(str(DATA_DIR), '[light_slate_blue].[/light_slate_blue]').replace(str(Path('~').expanduser()), '~')
+        ownership_summary = f'UID=[blue]{str(binprovider.EUID).ljust(4)}[/blue]'
+        provider_summary = f'[dark_sea_green3]{str(abspath).ljust(52)}[/dark_sea_green3]' if abspath else f'[grey23]{"not available".ljust(52)}[/grey23]'
+        prnt('', '[green]√[/green]' if binprovider.is_valid else '[red]X[/red]', '', binprovider.name.ljust(11), provider_summary, ownership_summary, f'PATH={PATH}', overflow='ellipsis', soft_wrap=True)
 
     prnt()
     prnt('[deep_sky_blue3][i] Source-code locations:[/deep_sky_blue3]')
@@ -254,16 +286,23 @@ def version(quiet: bool=False,
         prnt(printable_folder_status(name, path), overflow='ignore', crop=False)
 
     prnt()
-    if CONSTANTS.DATABASE_FILE.exists() or CONSTANTS.ARCHIVE_DIR.exists() or CONSTANTS.CONFIG_FILE.exists():
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
         prnt('[bright_yellow][i] Data locations:[/bright_yellow]')
         for name, path in CONSTANTS.DATA_LOCATIONS.items():
             prnt(printable_folder_status(name, path), overflow='ignore', crop=False)
+    
+        from archivebox.misc.checks import check_data_dir_permissions
+        
+        check_data_dir_permissions()
     else:
         prnt()
         prnt('[red][i] Data locations:[/red] (not in a data directory)')
-
+        
     prnt()
-
+    
+    if failures:
+        raise SystemExit(1)
+    raise SystemExit(0)
 
 @enforce_types
 def run(subcommand: str,
@@ -284,59 +323,57 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
     """Initialize a new ArchiveBox collection in the current directory"""
     
     from core.models import Snapshot
+    from rich import print
+    
+    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
+    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
+    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
 
-    out_dir.mkdir(exist_ok=True)
     is_empty = not len(set(os.listdir(out_dir)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-
-    if (out_dir / CONSTANTS.JSON_INDEX_FILENAME).exists():
-        stderr("[!] This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.", color="lightyellow")
-        stderr("    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.", color="lightyellow")
-
-    existing_index = CONSTANTS.DATABASE_FILE.exists()
-
+    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
-        print('{green}[+] Initializing a new ArchiveBox v{} collection...{reset}'.format(VERSION, **SHELL_CONFIG.ANSI))
-        print('{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI))
+        print(f'[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]')
+        print('[green]----------------------------------------------------------------------[/green]')
     elif existing_index:
         # TODO: properly detect and print the existing version in current index as well
-        print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **SHELL_CONFIG.ANSI))
-        print('{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI))
+        print(f'[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]')
+        print('[green]----------------------------------------------------------------------[/green]')
     else:
         if force:
-            stderr('[!] This folder appears to already have files in it, but no index.sqlite3 is present.', color='lightyellow')
-            stderr('    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).')
+            print('[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]')
+            print('[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]')
         else:
-            stderr(
-                ("{red}[X] This folder appears to already have files in it, but no index.sqlite3 present.{reset}\n\n"
+            print(
+                ("[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
                 "    You must run init in a completely empty directory, or an existing data folder.\n\n"
-                "    {lightred}Hint:{reset} To import an existing data folder make sure to cd into the folder first, \n"
+                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
                 "    then run and run 'archivebox init' to pick up where you left off.\n\n"
                 "    (Always make sure your data folder is backed up first before updating ArchiveBox)"
-                ).format(**SHELL_CONFIG.ANSI)
+                )
             )
             raise SystemExit(2)
 
     if existing_index:
-        print('\n{green}[*] Verifying archive folder structure...{reset}'.format(**SHELL_CONFIG.ANSI))
+        print('\n[green][*] Verifying archive folder structure...[/green]')
     else:
-        print('\n{green}[+] Building archive folder structure...{reset}'.format(**SHELL_CONFIG.ANSI))
+        print('\n[green][+] Building archive folder structure...[/green]')
     
     print(f'    + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...')
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
     Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
     print(f'    + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...')
-    write_config_file({}, out_dir=out_dir)
+    write_config_file({}, out_dir=str(out_dir))
 
-    if CONSTANTS.DATABASE_FILE.exists():
-        print('\n{green}[*] Verifying main SQL index and running any migrations needed...{reset}'.format(**SHELL_CONFIG.ANSI))
+    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
+        print('\n[green][*] Verifying main SQL index and running any migrations needed...[/green]')
     else:
-        print('\n{green}[+] Building main SQL index and running initial migrations...{reset}'.format(**SHELL_CONFIG.ANSI))
+        print('\n[green][+] Building main SQL index and running initial migrations...[/green]')
     
     for migration_line in apply_migrations(out_dir):
-        print(f'    {migration_line}')
+        sys.stdout.write(f'    {migration_line}\n')
 
-    assert CONSTANTS.DATABASE_FILE.exists()
+    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
     print(f'    √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}')
     
@@ -346,14 +383,14 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
     #     call_command("createsuperuser", interactive=True)
 
     print()
-    print('{green}[*] Checking links from indexes and archive folders (safe to Ctrl+C)...{reset}'.format(**SHELL_CONFIG.ANSI))
+    print('[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]')
 
     all_links = Snapshot.objects.none()
     pending_links: Dict[str, Link] = {}
 
     if existing_index:
         all_links = load_main_index(out_dir=out_dir, warn=False)
-        print('    √ Loaded {} links from existing main index.'.format(all_links.count()))
+        print(f'    √ Loaded {all_links.count()} links from existing main index.')
 
     if quick:
         print('    > Skipping full snapshot directory check (quick mode)')
@@ -362,9 +399,9 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
             # Links in data folders that dont match their timestamp
             fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir)
             if fixed:
-                print('    {lightyellow}√ Fixed {} data directory locations that didn\'t match their link timestamps.{reset}'.format(len(fixed), **SHELL_CONFIG.ANSI))
+                print(f'    [yellow]√ Fixed {len(fixed)} data directory locations that didn\'t match their link timestamps.[/yellow]')
             if cant_fix:
-                print('    {lightyellow}! Could not fix {} data directory locations due to conflicts with existing folders.{reset}'.format(len(cant_fix), **SHELL_CONFIG.ANSI))
+                print(f'    [red]! Could not fix {len(cant_fix)} data directory locations due to conflicts with existing folders.[/red]')
 
             # Links in JSON index but not in main index
             orphaned_json_links = {
@@ -374,7 +411,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
             }
             if orphaned_json_links:
                 pending_links.update(orphaned_json_links)
-                print('    {lightyellow}√ Added {} orphaned links from existing JSON index...{reset}'.format(len(orphaned_json_links), **SHELL_CONFIG.ANSI))
+                print(f'    [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]')
 
             # Links in data dir indexes but not in main index
             orphaned_data_dir_links = {
@@ -384,7 +421,7 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
             }
             if orphaned_data_dir_links:
                 pending_links.update(orphaned_data_dir_links)
-                print('    {lightyellow}√ Added {} orphaned links from existing archive directories.{reset}'.format(len(orphaned_data_dir_links), **SHELL_CONFIG.ANSI))
+                print(f'    [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]')
 
             # Links in invalid/duplicate data dirs
             invalid_folders = {
@@ -392,52 +429,56 @@ def init(force: bool=False, quick: bool=False, install: bool=False, out_dir: Pat
                 for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items()
             }
             if invalid_folders:
-                print('    {lightyellow}! Skipped adding {} invalid link data directories.{reset}'.format(len(invalid_folders), **SHELL_CONFIG.ANSI))
+                print(f'    [red]! Skipped adding {len(invalid_folders)} invalid link data directories.[/red]')
                 print('        X ' + '\n        X '.join(f'./{Path(folder).relative_to(DATA_DIR)} {link}' for folder, link in invalid_folders.items()))
                 print()
-                print('    {lightred}Hint:{reset} For more information about the link data directories that were skipped, run:'.format(**SHELL_CONFIG.ANSI))
+                print('    [violet]Hint:[/violet] For more information about the link data directories that were skipped, run:')
                 print('        archivebox status')
                 print('        archivebox list --status=invalid')
 
         except (KeyboardInterrupt, SystemExit):
-            stderr()
-            stderr('[x] Stopped checking archive directories due to Ctrl-C/SIGTERM', color='red')
-            stderr('    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.')
-            stderr()
-            stderr('    {lightred}Hint:{reset} In the future you can run a quick init without checking dirs like so:'.format(**SHELL_CONFIG.ANSI))
-            stderr('        archivebox init --quick')
+            print(file=sys.stderr)
+            print('[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]', file=sys.stderr)
+            print('    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.', file=sys.stderr)
+            print(file=sys.stderr)
+            print('    [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:', file=sys.stderr)
+            print('        archivebox init --quick', file=sys.stderr)
             raise SystemExit(1)
         
         write_main_index(list(pending_links.values()), out_dir=out_dir)
 
-    print('\n{green}----------------------------------------------------------------------{reset}'.format(**SHELL_CONFIG.ANSI))
+    print('\n[green]----------------------------------------------------------------------[/green]')
 
     from django.contrib.auth.models import User
 
     if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(username=SERVER_CONFIG.ADMIN_USERNAME).exists():
-        print('{green}[+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.{reset}'.format(**SHELL_CONFIG.ANSI))
+        print('[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]')
         User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
 
     if existing_index:
-        print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**SHELL_CONFIG.ANSI))
+        print('[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]')
     else:
-        print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links) + len(pending_links), **SHELL_CONFIG.ANSI))
+        print(f'[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]')
 
     json_index = out_dir / CONSTANTS.JSON_INDEX_FILENAME
     html_index = out_dir / CONSTANTS.HTML_INDEX_FILENAME
     index_name = f"{date.today()}_index_old"
-    if json_index.exists():
+    if os.access(json_index, os.F_OK):
         json_index.rename(f"{index_name}.json")
-    if html_index.exists():
+    if os.access(html_index, os.F_OK):
         html_index.rename(f"{index_name}.html")
+    
+    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.LIB_DIR.mkdir(parents=True, exist_ok=True)
 
     if install:
         run_subcommand('install', pwd=out_dir)
 
     if Snapshot.objects.count() < 25:     # hide the hints for experienced users
         print()
-        print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**SHELL_CONFIG.ANSI))
-        print(f'        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link]')
+        print('    [violet]Hint:[/violet] To view your archive index, run:')
+        print('        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]')
         print()
         print('    To add new links, you can run:')
         print("        archivebox add < ~/some/path/to/list_of_links.txt")
@@ -947,23 +988,68 @@ def list_folders(links: List[Link],
 @enforce_types
 def install(out_dir: Path=DATA_DIR) -> None:
     """Automatically install all ArchiveBox dependencies and extras"""
+    
+    # if running as root:
+    #    - run init to create index + lib dir
+    #    - chown -R 911 DATA_DIR
+    #    - install all binaries as root
+    #    - chown -R 911 LIB_DIR
+    # else:
+    #    - run init to create index + lib dir as current user
+    #    - install all binaries as current user
+    #    - recommend user re-run with sudo if any deps need to be installed as root
 
     from rich import print
     from django.conf import settings
+    
+    from archivebox import CONSTANTS
+    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
 
-    if not ARCHIVE_DIR.exists():
-        run_subcommand('init', stdin=None, pwd=out_dir)
-
-    stderr('\n[+] Installing ArchiveBox dependencies automatically...', color='green')
+    if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
+        run_subcommand('init', stdin=None, pwd=out_dir)  # must init full index because we need a db to store InstalledBinary entries in
 
+    print('\n[green][+] Installing ArchiveBox dependencies automatically...[/green]')
+    
+    # we never want the data dir to be owned by root, detect owner of existing owner of DATA_DIR to try and guess desired non-root UID
+    if IS_ROOT:
+        EUID = os.geteuid()
+        
+        # if we have sudo/root permissions, take advantage of them just while installing dependencies
+        print()
+        print(f'[yellow]:warning:  Running as UID=[blue]{EUID}[/blue] with [red]sudo[/red] only for dependencies that need it.[/yellow]')
+        print(f'    DATA_DIR, LIB_DIR, and TMP_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].')
+        print()
+    
+    
+    package_manager_names = ', '.join(f'[yellow]{binprovider.name}[/yellow]' for binprovider in reversed(list(settings.BINPROVIDERS.values())))
+    print(f'[+] Setting up package managers {package_manager_names}...')
+    for binprovider in reversed(list(settings.BINPROVIDERS.values())):
+        try:
+            binprovider.setup()
+        except Exception:
+            # it's ok, installing binaries below will automatically set up package managers as needed
+            # e.g. if user does not have npm available we cannot set it up here yet, but once npm Binary is installed
+            # the next package that depends on npm will automatically call binprovider.setup() during its own install
+            pass
+    
+    print()
+    
     for binary in reversed(list(settings.BINARIES.values())):
         providers = ' [grey53]or[/grey53] '.join(provider.name for provider in binary.binproviders_supported)
-        print(f'[+] Locating / Installing [yellow]{binary.name}[/yellow] using [red]{providers}[/red]...')
+        print(f'[+] Detecting / Installing [yellow]{binary.name.ljust(22)}[/yellow] using [red]{providers}[/red]...')
         try:
-            print(binary.load_or_install().model_dump(exclude={'binproviders_supported', 'loaded_binprovider', 'provider_overrides', 'loaded_abspaths', 'bin_dir', 'loaded_respath', 'hook_type'}))
+            with SudoPermission(uid=0, fallback=True):
+                # print(binary.load_or_install(fresh=True).model_dump(exclude={'provider_overrides', 'bin_dir', 'hook_type'}))
+                binary.load_or_install(fresh=True).model_dump(exclude={'provider_overrides', 'bin_dir', 'hook_type'})
+            if IS_ROOT:
+                with SudoPermission(uid=0):
+                    if ARCHIVEBOX_USER == 0:
+                        os.system(f'chmod -R 777 "{CONSTANTS.LIB_DIR.resolve()}"')
+                    else:    
+                        os.system(f'chown -R {ARCHIVEBOX_USER} "{CONSTANTS.LIB_DIR.resolve()}"')
         except Exception as e:
-            print(f'[X] Failed to install {binary.name}: {e}')
-
+            print(f'[red]:cross_mark: Failed to install {binary.name} as user {ARCHIVEBOX_USER}: {e}[/red]')
+                
 
     from django.contrib.auth import get_user_model
     User = get_user_model()
@@ -973,11 +1059,13 @@ def install(out_dir: Path=DATA_DIR) -> None:
         stderr('    archivebox manage createsuperuser')
         # run_subcommand('manage', subcommand_args=['createsuperuser'], pwd=out_dir)
     
-    stderr('\n[√] Set up ArchiveBox and its dependencies successfully.', color='green')
+    print('\n[green][√] Set up ArchiveBox and its dependencies successfully.[/green]\n', file=sys.stderr)
     
     from plugins_pkg.pip.apps import ARCHIVEBOX_BINARY
     
-    run_shell([ARCHIVEBOX_BINARY.load().abspath, 'version'], capture_output=False, cwd=out_dir)
+    proc = run_shell([ARCHIVEBOX_BINARY.load().abspath, 'version'], capture_output=False, cwd=out_dir)
+    raise SystemExit(proc.returncode)
+
 
 # backwards-compatibility:
 setup = install
@@ -1099,6 +1187,7 @@ def schedule(add: bool=False,
     
     check_data_folder()
     from archivebox.plugins_pkg.pip.apps import ARCHIVEBOX_BINARY
+    from archivebox.config.permissions import USER
 
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
 
@@ -1155,7 +1244,7 @@ def schedule(add: bool=False,
         existing_jobs = list(cron.find_comment(CRON_COMMENT))
 
         print()
-        print('{green}[√] Scheduled new ArchiveBox cron job for user: {} ({} jobs are active).{reset}'.format(SHELL_CONFIG.USER, len(existing_jobs), **SHELL_CONFIG.ANSI))
+        print('{green}[√] Scheduled new ArchiveBox cron job for user: {} ({} jobs are active).{reset}'.format(USER, len(existing_jobs), **SHELL_CONFIG.ANSI))
         print('\n'.join(f'  > {cmd}' if str(cmd) == str(new_job) else f'    {cmd}' for cmd in existing_jobs))
         if total_runs > 60 and not quiet:
             stderr()
@@ -1169,7 +1258,7 @@ def schedule(add: bool=False,
         if existing_jobs:
             print('\n'.join(str(cmd) for cmd in existing_jobs))
         else:
-            stderr('{red}[X] There are no ArchiveBox cron jobs scheduled for your user ({}).{reset}'.format(SHELL_CONFIG.USER, **SHELL_CONFIG.ANSI))
+            stderr('{red}[X] There are no ArchiveBox cron jobs scheduled for your user ({}).{reset}'.format(USER, **SHELL_CONFIG.ANSI))
             stderr('    To schedule a new job, run:')
             stderr('        archivebox schedule --every=[timeperiod] --depth=1 https://example.com/some/rss/feed.xml')
         raise SystemExit(0)
@@ -1275,7 +1364,7 @@ def server(runserver_args: Optional[List[str]]=None,
         except IndexError:
             pass
 
-        print(f'    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link]')
+        print(f'    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]')
 
         from queues.supervisor_util import start_server_workers
 
@@ -1293,7 +1382,7 @@ def manage(args: Optional[List[str]]=None, out_dir: Path=DATA_DIR) -> None:
     check_data_folder()
     from django.core.management import execute_from_command_line
 
-    if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY):
+    if (args and "createsuperuser" in args) and (IN_DOCKER and not SHELL_CONFIG.IS_TTY):
         stderr('[!] Warning: you need to pass -it to use interactive commands in docker', color='lightyellow')
         stderr('    docker run -it archivebox manage {}'.format(' '.join(args or ['...'])), color='lightyellow')
         stderr('')

archivebox/misc/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.misc'

archivebox/misc/checks.py

@@ -1,41 +1,129 @@
 __package__ = 'archivebox.misc'
 
-from archivebox.config import DATA_DIR, ARCHIVE_DIR, CONSTANTS, SHELL_CONFIG
+import os
+import sys
+from pathlib import Path
 
-from .logging import stderr
+from rich import print
+
+# DO NOT ADD ANY TOP-LEVEL IMPORTS HERE
+# this file is imported by archivebox/__init__.py
+# and any imports here will be imported by EVERYTHING else
+# so this file should only be used for pure python checks
+# that don't need to import other parts of ArchiveBox
 
 
 def check_data_folder() -> None:
-
-    archive_dir_exists = ARCHIVE_DIR.exists()
+    from archivebox import DATA_DIR, ARCHIVE_DIR
+    
+    archive_dir_exists = os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()
     if not archive_dir_exists:
-        stderr('[X] No archivebox index found in the current directory.', color='red')
-        stderr(f'    {DATA_DIR}', color='lightyellow')
-        stderr()
-        stderr('    {lightred}Hint{reset}: Are you running archivebox in the right folder?'.format(**SHELL_CONFIG.ANSI))
-        stderr('        cd path/to/your/archive/folder')
-        stderr('        archivebox [command]')
-        stderr()
-        stderr('    {lightred}Hint{reset}: To create a new archive collection or import existing data in this folder, run:'.format(**SHELL_CONFIG.ANSI))
-        stderr('        archivebox init')
+        print('[red][X] No archivebox index found in the current directory.[/red]', file=sys.stderr)
+        print(f'    {DATA_DIR}', file=sys.stderr)
+        print(file=sys.stderr)
+        print('    [violet]Hint[/violet]: Are you running archivebox in the right folder?', file=sys.stderr)
+        print('        cd path/to/your/archive/folder', file=sys.stderr)
+        print('        archivebox [command]', file=sys.stderr)
+        print(file=sys.stderr)
+        print('    [violet]Hint[/violet]: To create a new archive collection or import existing data in this folder, run:', file=sys.stderr)
+        print('        archivebox init', file=sys.stderr)
         raise SystemExit(2)
-
-
+    
+    
 def check_migrations():
+    from archivebox import DATA_DIR, CONSTANTS
     from ..index.sql import list_migrations
 
     pending_migrations = [name for status, name in list_migrations() if not status]
 
     if pending_migrations:
-        stderr('[X] This collection was created with an older version of ArchiveBox and must be upgraded first.', color='lightyellow')
-        stderr(f'    {DATA_DIR}')
-        stderr()
-        stderr(f'    To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:')
-        stderr('        archivebox init')
+        print('[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]')
+        print(f'    {DATA_DIR}', file=sys.stderr)
+        print(file=sys.stderr)
+        print(f'    [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:', file=sys.stderr)
+        print('        archivebox init', file=sys.stderr)
         raise SystemExit(3)
 
     CONSTANTS.SOURCES_DIR.mkdir(exist_ok=True)
     CONSTANTS.LOGS_DIR.mkdir(exist_ok=True)
-    CONSTANTS.CACHE_DIR.mkdir(exist_ok=True)
+    # CONSTANTS.CACHE_DIR.mkdir(exist_ok=True)
     (CONSTANTS.LIB_DIR / 'bin').mkdir(exist_ok=True, parents=True)
     (CONSTANTS.PERSONAS_DIR / 'Default').mkdir(exist_ok=True, parents=True)
+
+
+def check_io_encoding():
+    PYTHON_ENCODING = (sys.__stdout__ or sys.stdout or sys.__stderr__ or sys.stderr).encoding.upper().replace('UTF8', 'UTF-8')
+            
+    if PYTHON_ENCODING != 'UTF-8':
+        print(f'[red][X] Your system is running python3 scripts with a bad locale setting: {PYTHON_ENCODING} (it should be UTF-8).[/red]', file=sys.stderr)
+        print('    To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)', file=sys.stderr)
+        print('    Or if you\'re using ubuntu/debian, run "dpkg-reconfigure locales"', file=sys.stderr)
+        print('')
+        print('    Confirm that it\'s fixed by opening a new shell and running:', file=sys.stderr)
+        print('        python3 -c "import sys; print(sys.stdout.encoding)"   # should output UTF-8', file=sys.stderr)
+        raise SystemExit(2)
+    
+    # # hard errors: check python version
+    # if sys.version_info[:3] < (3, 10, 0):
+    #     print('[red][X] Python version is not new enough: {sys.version} (>3.10 is required)[/red]', file=sys.stderr)
+    #     print('    See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.', file=sys.stderr)
+    #     raise SystemExit(2)
+    
+    # # hard errors: check django version
+    # if int(django.VERSION[0]) < 5:
+    #     print('[red][X] Django version is not new enough: {django.VERSION[:3]} (>=5.0 is required)[/red]', file=sys.stderr)
+    #     print('    Upgrade django using pip or your system package manager: pip3 install --upgrade django', file=sys.stderr)
+    #     raise SystemExit(2)
+
+
+def check_not_root():
+    from archivebox.config.permissions import IS_ROOT, IN_DOCKER
+    
+    attempted_command = ' '.join(sys.argv[1:]) if len(sys.argv) > 1 else ''
+    is_getting_help = '-h' in sys.argv or '--help' in sys.argv or 'help' in sys.argv
+    is_getting_version = '--version' in sys.argv or 'version' in sys.argv
+    is_installing = 'setup' in sys.argv or 'install' in sys.argv
+    
+    if IS_ROOT and not (is_getting_help or is_getting_version or is_installing):
+        print('[red][!] ArchiveBox should never be run as root![/red]', file=sys.stderr)
+        print('    For more information, see the security overview documentation:', file=sys.stderr)
+        print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root', file=sys.stderr)
+        
+        if IN_DOCKER:
+            print('[red][!] When using Docker, you must run commands with [green]docker run[/green] instead of [yellow3]docker exec[/yellow3], e.g.:', file=sys.stderr)
+            print('        docker compose run archivebox {attempted_command}', file=sys.stderr)
+            print(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}', file=sys.stderr)
+            print('        or:', file=sys.stderr)
+            print(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
+            print(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
+        raise SystemExit(2)
+
+
+def check_data_dir_permissions():
+    from archivebox import DATA_DIR
+    from archivebox.misc.logging import STDERR
+    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER
+    
+    data_dir_stat = Path(DATA_DIR).stat()
+    data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid
+    data_owned_by_root = data_dir_uid == 0
+    
+    # data_owned_by_default_user = data_dir_uid == DEFAULT_PUID or data_dir_gid == DEFAULT_PGID
+    data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False
+    data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK))     #  and os.access(CONSTANTS.LIB_DIR, os.W_OK) and os.access(CONSTANTS.TMP_DIR, os.W_OK))
+    if data_owned_by_root:
+        STDERR.print('\n[yellow]:warning: Warning: ArchiveBox [blue]DATA_DIR[/blue] is currently owned by [red]root[/red], it must be changed before archiving can run![/yellow]')
+    elif data_owner_doesnt_match or data_not_writable:
+        STDERR.print(f'\n[yellow]:warning: Warning: ArchiveBox [blue]DATA_DIR[/blue] is currently owned by [red]{data_dir_uid}:{data_dir_gid}[/red], but ArchiveBox user is [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue] ({USER})! (ArchiveBox may not be able to write to the data dir)[/yellow]')
+        
+    if data_owned_by_root or data_owner_doesnt_match or data_not_writable:
+        STDERR.print(f'[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} (PUID:PGID) to a non-root user & group that will run ArchiveBox, e.g.:')
+        STDERR.print(f'    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {DATA_DIR.resolve()}')
+        # STDERR.print(f'    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {CONSTANTS.LIB_DIR.resolve()}')
+        # STDERR.print(f'    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {CONSTANTS.TMP_DIR.resolve()}')
+        STDERR.print()
+        STDERR.print('[blue]More info:[/blue]')
+        STDERR.print('    [link=https://github.com/ArchiveBox/ArchiveBox#storage-requirements]https://github.com/ArchiveBox/ArchiveBox#storage-requirements[/link]')
+        STDERR.print('    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]')
+        STDERR.print('    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid[/link]')
+        STDERR.print('    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]')

archivebox/misc/logging.py

@@ -2,6 +2,7 @@ __package__ = 'archivebox.misc'
 
 # TODO: merge/dedupe this file with archivebox/logging_util.py
 
+
 import sys
 from typing import Optional, Union, Tuple, List
 from collections import defaultdict
@@ -13,9 +14,9 @@ from rich.highlighter import Highlighter
 
 # SETUP RICH CONSOLE / TTY detection / COLOR / PROGRESS BARS
 CONSOLE = Console()
+STDERR = Console(stderr=True)
 IS_TTY = CONSOLE.is_interactive
 
-
 class RainbowHighlighter(Highlighter):
     def highlight(self, text):
         for index in range(len(text)):
@@ -51,7 +52,7 @@ COLOR_DICT = defaultdict(lambda: [(0, 0, 0), (0, 0, 0)], {
     '37': [(255, 255, 255), (255, 255, 255)],
 })
 
-# Logging Helpers
+# Logging Helpers (DEPRECATED, use rich.print instead going forward)
 def stdout(*args, color: Optional[str]=None, prefix: str='', config: Optional[benedict]=None) -> None:
     ansi = DEFAULT_CLI_COLORS if (config or {}).get('USE_COLOR') else ANSI
 

archivebox/misc/system.py

@@ -4,7 +4,6 @@ __package__ = 'archivebox.misc'
 import os
 import signal
 import shutil
-import getpass
 
 from json import dump
 from pathlib import Path
@@ -14,7 +13,7 @@ from subprocess import _mswindows, PIPE, Popen, CalledProcessError, CompletedPro
 from crontab import CronTab
 from atomicwrites import atomic_write as lib_atomic_write
 
-from archivebox.config import STORAGE_CONFIG
+from archivebox.config.common import STORAGE_CONFIG
 from archivebox.misc.util import enforce_types, ExtendedEncoder
 
 
@@ -111,11 +110,11 @@ def atomic_write(path: Union[Path, str], contents: Union[dict, str, bytes], over
     os.chmod(path, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8))
 
 @enforce_types
-def chmod_file(path: str, cwd: str='.') -> None:
+def chmod_file(path: str, cwd: str='') -> None:
     """chmod -R <permissions> <cwd>/<path>"""
 
-    root = Path(cwd) / path
-    if not root.exists():
+    root = Path(cwd or os.getcwd()) / path
+    if not os.access(root, os.R_OK):
         raise Exception('Failed to chmod: {} does not exist (did the previous step fail?)'.format(path))
 
     if not root.is_dir():
@@ -133,6 +132,9 @@ def chmod_file(path: str, cwd: str='.') -> None:
 @enforce_types
 def copy_and_overwrite(from_path: Union[str, Path], to_path: Union[str, Path]):
     """copy a given file or directory to a given path, overwriting the destination"""
+    
+    assert os.access(from_path, os.R_OK)
+    
     if Path(from_path).is_dir():
         shutil.rmtree(to_path, ignore_errors=True)
         shutil.copytree(from_path, to_path)

archivebox/misc/tests.py

@@ -1,332 +1,332 @@
 __package__ = 'abx.archivebox'
 
-from django.test import TestCase
+# from django.test import TestCase
 
-from .toml_util import convert, TOML_HEADER
+# from .toml_util import convert, TOML_HEADER
 
-TEST_INPUT = """
-[SERVER_CONFIG]
-IS_TTY=False
-USE_COLOR=False
-SHOW_PROGRESS=False
-IN_DOCKER=False
-IN_QEMU=False
-PUID=501
-PGID=20
-CONFIG_FILE=/opt/archivebox/data/ArchiveBox.conf
-ONLY_NEW=True
-TIMEOUT=60
-MEDIA_TIMEOUT=3600
-OUTPUT_PERMISSIONS=644
-RESTRICT_FILE_NAMES=windows
-URL_DENYLIST=\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$
-URL_ALLOWLIST=None
-ADMIN_USERNAME=None
-ADMIN_PASSWORD=None
-ENFORCE_ATOMIC_WRITES=True
-TAG_SEPARATOR_PATTERN=[,]
-SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-BIND_ADDR=127.0.0.1:8000
-ALLOWED_HOSTS=*
-DEBUG=False
-PUBLIC_INDEX=True
-PUBLIC_SNAPSHOTS=True
-PUBLIC_ADD_VIEW=False
-FOOTER_INFO=Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.
-SNAPSHOTS_PER_PAGE=40
-CUSTOM_TEMPLATES_DIR=None
-TIME_ZONE=UTC
-TIMEZONE=UTC
-REVERSE_PROXY_USER_HEADER=Remote-User
-REVERSE_PROXY_WHITELIST=
-LOGOUT_REDIRECT_URL=/
-PREVIEW_ORIGINALS=True
-LDAP=False
-LDAP_SERVER_URI=None
-LDAP_BIND_DN=None
-LDAP_BIND_PASSWORD=None
-LDAP_USER_BASE=None
-LDAP_USER_FILTER=None
-LDAP_USERNAME_ATTR=None
-LDAP_FIRSTNAME_ATTR=None
-LDAP_LASTNAME_ATTR=None
-LDAP_EMAIL_ATTR=None
-LDAP_CREATE_SUPERUSER=False
-SAVE_TITLE=True
-SAVE_FAVICON=True
-SAVE_WGET=True
-SAVE_WGET_REQUISITES=True
-SAVE_SINGLEFILE=True
-SAVE_READABILITY=True
-SAVE_MERCURY=True
-SAVE_HTMLTOTEXT=True
-SAVE_PDF=True
-SAVE_SCREENSHOT=True
-SAVE_DOM=True
-SAVE_HEADERS=True
-SAVE_WARC=True
-SAVE_GIT=True
-SAVE_MEDIA=True
-SAVE_ARCHIVE_DOT_ORG=True
-RESOLUTION=1440,2000
-GIT_DOMAINS=github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht
-CHECK_SSL_VALIDITY=True
-MEDIA_MAX_SIZE=750m
-USER_AGENT=None
-CURL_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0)
-WGET_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5
-CHROME_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/)
-COOKIES_FILE=None
-CHROME_USER_DATA_DIR=None
-CHROME_TIMEOUT=0
-CHROME_HEADLESS=True
-CHROME_SANDBOX=True
-CHROME_EXTRA_ARGS=[]
-YOUTUBEDL_ARGS=['--restrict-filenames', '--trim-filenames', '128', '--write-description', '--write-info-json', '--write-annotations', '--write-thumbnail', '--no-call-home', '--write-sub', '--write-auto-subs', '--convert-subs=srt', '--yes-playlist', '--continue', '--no-abort-on-error', '--ignore-errors', '--geo-bypass', '--add-metadata', '--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)']
-YOUTUBEDL_EXTRA_ARGS=[]
-WGET_ARGS=['--no-verbose', '--adjust-extension', '--convert-links', '--force-directories', '--backup-converted', '--span-hosts', '--no-parent', '-e', 'robots=off']
-WGET_EXTRA_ARGS=[]
-CURL_ARGS=['--silent', '--location', '--compressed']
-CURL_EXTRA_ARGS=[]
-GIT_ARGS=['--recursive']
-SINGLEFILE_ARGS=[]
-SINGLEFILE_EXTRA_ARGS=[]
-MERCURY_ARGS=['--format=text']
-MERCURY_EXTRA_ARGS=[]
-FAVICON_PROVIDER=https://www.google.com/s2/favicons?domain={}
-USE_INDEXING_BACKEND=True
-USE_SEARCHING_BACKEND=True
-SEARCH_BACKEND_ENGINE=ripgrep
-SEARCH_BACKEND_HOST_NAME=localhost
-SEARCH_BACKEND_PORT=1491
-SEARCH_BACKEND_PASSWORD=SecretPassword
-SEARCH_PROCESS_HTML=True
-SONIC_COLLECTION=archivebox
-SONIC_BUCKET=snapshots
-SEARCH_BACKEND_TIMEOUT=90
-FTS_SEPARATE_DATABASE=True
-FTS_TOKENIZERS=porter unicode61 remove_diacritics 2
-FTS_SQLITE_MAX_LENGTH=1000000000
-USE_CURL=True
-USE_WGET=True
-USE_SINGLEFILE=True
-USE_READABILITY=True
-USE_MERCURY=True
-USE_GIT=True
-USE_CHROME=True
-USE_NODE=True
-USE_YOUTUBEDL=True
-USE_RIPGREP=True
-CURL_BINARY=curl
-GIT_BINARY=git
-WGET_BINARY=wget
-SINGLEFILE_BINARY=single-file
-READABILITY_BINARY=readability-extractor
-MERCURY_BINARY=postlight-parser
-YOUTUBEDL_BINARY=yt-dlp
-NODE_BINARY=node
-RIPGREP_BINARY=rg
-CHROME_BINARY=chrome
-POCKET_CONSUMER_KEY=None
-USER=squash
-PACKAGE_DIR=/opt/archivebox/archivebox
-TEMPLATES_DIR=/opt/archivebox/archivebox/templates
-ARCHIVE_DIR=/opt/archivebox/data/archive
-SOURCES_DIR=/opt/archivebox/data/sources
-LOGS_DIR=/opt/archivebox/data/logs
-PERSONAS_DIR=/opt/archivebox/data/personas
-URL_DENYLIST_PTN=re.compile('\\.(css|js|otf|ttf|woff|woff2|gstatic\\.com|googleapis\\.com/css)(\\?.*)?$', re.IGNORECASE|re.MULTILINE)
-URL_ALLOWLIST_PTN=None
-DIR_OUTPUT_PERMISSIONS=755
-ARCHIVEBOX_BINARY=/opt/archivebox/.venv/bin/archivebox
-VERSION=0.8.0
-COMMIT_HASH=102e87578c6036bb0132dd1ebd17f8f05ffc880f
-BUILD_TIME=2024-05-15 03:28:05 1715768885
-VERSIONS_AVAILABLE=None
-CAN_UPGRADE=False
-PYTHON_BINARY=/opt/archivebox/.venv/bin/python3.10
-PYTHON_VERSION=3.10.14
-DJANGO_BINARY=/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py
-DJANGO_VERSION=5.0.6 final (0)
-SQLITE_BINARY=/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py
-SQLITE_VERSION=2.6.0
-CURL_VERSION=curl 8.4.0 (x86_64-apple-darwin23.0)
-WGET_VERSION=GNU Wget 1.24.5
-WGET_AUTO_COMPRESSION=True
-RIPGREP_VERSION=ripgrep 14.1.0
-SINGLEFILE_VERSION=None
-READABILITY_VERSION=None
-MERCURY_VERSION=None
-GIT_VERSION=git version 2.44.0
-YOUTUBEDL_VERSION=2024.04.09
-CHROME_VERSION=Google Chrome 124.0.6367.207
-NODE_VERSION=v21.7.3
-"""
+# TEST_INPUT = """
+# [SERVER_CONFIG]
+# IS_TTY=False
+# USE_COLOR=False
+# SHOW_PROGRESS=False
+# IN_DOCKER=False
+# IN_QEMU=False
+# PUID=501
+# PGID=20
+# CONFIG_FILE=/opt/archivebox/data/ArchiveBox.conf
+# ONLY_NEW=True
+# TIMEOUT=60
+# MEDIA_TIMEOUT=3600
+# OUTPUT_PERMISSIONS=644
+# RESTRICT_FILE_NAMES=windows
+# URL_DENYLIST=\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$
+# URL_ALLOWLIST=None
+# ADMIN_USERNAME=None
+# ADMIN_PASSWORD=None
+# ENFORCE_ATOMIC_WRITES=True
+# TAG_SEPARATOR_PATTERN=[,]
+# SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+# BIND_ADDR=127.0.0.1:8000
+# ALLOWED_HOSTS=*
+# DEBUG=False
+# PUBLIC_INDEX=True
+# PUBLIC_SNAPSHOTS=True
+# PUBLIC_ADD_VIEW=False
+# FOOTER_INFO=Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.
+# SNAPSHOTS_PER_PAGE=40
+# CUSTOM_TEMPLATES_DIR=None
+# TIME_ZONE=UTC
+# TIMEZONE=UTC
+# REVERSE_PROXY_USER_HEADER=Remote-User
+# REVERSE_PROXY_WHITELIST=
+# LOGOUT_REDIRECT_URL=/
+# PREVIEW_ORIGINALS=True
+# LDAP=False
+# LDAP_SERVER_URI=None
+# LDAP_BIND_DN=None
+# LDAP_BIND_PASSWORD=None
+# LDAP_USER_BASE=None
+# LDAP_USER_FILTER=None
+# LDAP_USERNAME_ATTR=None
+# LDAP_FIRSTNAME_ATTR=None
+# LDAP_LASTNAME_ATTR=None
+# LDAP_EMAIL_ATTR=None
+# LDAP_CREATE_SUPERUSER=False
+# SAVE_TITLE=True
+# SAVE_FAVICON=True
+# SAVE_WGET=True
+# SAVE_WGET_REQUISITES=True
+# SAVE_SINGLEFILE=True
+# SAVE_READABILITY=True
+# SAVE_MERCURY=True
+# SAVE_HTMLTOTEXT=True
+# SAVE_PDF=True
+# SAVE_SCREENSHOT=True
+# SAVE_DOM=True
+# SAVE_HEADERS=True
+# SAVE_WARC=True
+# SAVE_GIT=True
+# SAVE_MEDIA=True
+# SAVE_ARCHIVE_DOT_ORG=True
+# RESOLUTION=1440,2000
+# GIT_DOMAINS=github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht
+# CHECK_SSL_VALIDITY=True
+# MEDIA_MAX_SIZE=750m
+# USER_AGENT=None
+# CURL_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0)
+# WGET_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5
+# CHROME_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/)
+# COOKIES_FILE=None
+# CHROME_USER_DATA_DIR=None
+# CHROME_TIMEOUT=0
+# CHROME_HEADLESS=True
+# CHROME_SANDBOX=True
+# CHROME_EXTRA_ARGS=[]
+# YOUTUBEDL_ARGS=['--restrict-filenames', '--trim-filenames', '128', '--write-description', '--write-info-json', '--write-annotations', '--write-thumbnail', '--no-call-home', '--write-sub', '--write-auto-subs', '--convert-subs=srt', '--yes-playlist', '--continue', '--no-abort-on-error', '--ignore-errors', '--geo-bypass', '--add-metadata', '--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)']
+# YOUTUBEDL_EXTRA_ARGS=[]
+# WGET_ARGS=['--no-verbose', '--adjust-extension', '--convert-links', '--force-directories', '--backup-converted', '--span-hosts', '--no-parent', '-e', 'robots=off']
+# WGET_EXTRA_ARGS=[]
+# CURL_ARGS=['--silent', '--location', '--compressed']
+# CURL_EXTRA_ARGS=[]
+# GIT_ARGS=['--recursive']
+# SINGLEFILE_ARGS=[]
+# SINGLEFILE_EXTRA_ARGS=[]
+# MERCURY_ARGS=['--format=text']
+# MERCURY_EXTRA_ARGS=[]
+# FAVICON_PROVIDER=https://www.google.com/s2/favicons?domain={}
+# USE_INDEXING_BACKEND=True
+# USE_SEARCHING_BACKEND=True
+# SEARCH_BACKEND_ENGINE=ripgrep
+# SEARCH_BACKEND_HOST_NAME=localhost
+# SEARCH_BACKEND_PORT=1491
+# SEARCH_BACKEND_PASSWORD=SecretPassword
+# SEARCH_PROCESS_HTML=True
+# SONIC_COLLECTION=archivebox
+# SONIC_BUCKET=snapshots
+# SEARCH_BACKEND_TIMEOUT=90
+# FTS_SEPARATE_DATABASE=True
+# FTS_TOKENIZERS=porter unicode61 remove_diacritics 2
+# FTS_SQLITE_MAX_LENGTH=1000000000
+# USE_CURL=True
+# USE_WGET=True
+# USE_SINGLEFILE=True
+# USE_READABILITY=True
+# USE_MERCURY=True
+# USE_GIT=True
+# USE_CHROME=True
+# USE_NODE=True
+# USE_YOUTUBEDL=True
+# USE_RIPGREP=True
+# CURL_BINARY=curl
+# GIT_BINARY=git
+# WGET_BINARY=wget
+# SINGLEFILE_BINARY=single-file
+# READABILITY_BINARY=readability-extractor
+# MERCURY_BINARY=postlight-parser
+# YOUTUBEDL_BINARY=yt-dlp
+# NODE_BINARY=node
+# RIPGREP_BINARY=rg
+# CHROME_BINARY=chrome
+# POCKET_CONSUMER_KEY=None
+# USER=squash
+# PACKAGE_DIR=/opt/archivebox/archivebox
+# TEMPLATES_DIR=/opt/archivebox/archivebox/templates
+# ARCHIVE_DIR=/opt/archivebox/data/archive
+# SOURCES_DIR=/opt/archivebox/data/sources
+# LOGS_DIR=/opt/archivebox/data/logs
+# PERSONAS_DIR=/opt/archivebox/data/personas
+# URL_DENYLIST_PTN=re.compile('\\.(css|js|otf|ttf|woff|woff2|gstatic\\.com|googleapis\\.com/css)(\\?.*)?$', re.IGNORECASE|re.MULTILINE)
+# URL_ALLOWLIST_PTN=None
+# DIR_OUTPUT_PERMISSIONS=755
+# ARCHIVEBOX_BINARY=/opt/archivebox/.venv/bin/archivebox
+# VERSION=0.8.0
+# COMMIT_HASH=102e87578c6036bb0132dd1ebd17f8f05ffc880f
+# BUILD_TIME=2024-05-15 03:28:05 1715768885
+# VERSIONS_AVAILABLE=None
+# CAN_UPGRADE=False
+# PYTHON_BINARY=/opt/archivebox/.venv/bin/python3.10
+# PYTHON_VERSION=3.10.14
+# DJANGO_BINARY=/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py
+# DJANGO_VERSION=5.0.6 final (0)
+# SQLITE_BINARY=/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py
+# SQLITE_VERSION=2.6.0
+# CURL_VERSION=curl 8.4.0 (x86_64-apple-darwin23.0)
+# WGET_VERSION=GNU Wget 1.24.5
+# WGET_AUTO_COMPRESSION=True
+# RIPGREP_VERSION=ripgrep 14.1.0
+# SINGLEFILE_VERSION=None
+# READABILITY_VERSION=None
+# MERCURY_VERSION=None
+# GIT_VERSION=git version 2.44.0
+# YOUTUBEDL_VERSION=2024.04.09
+# CHROME_VERSION=Google Chrome 124.0.6367.207
+# NODE_VERSION=v21.7.3
+# """
 
 
-EXPECTED_OUTPUT = TOML_HEADER + '''[SERVER_CONFIG]
-IS_TTY = false
-USE_COLOR = false
-SHOW_PROGRESS = false
-IN_DOCKER = false
-IN_QEMU = false
-PUID = 501
-PGID = 20
-CONFIG_FILE = "/opt/archivebox/data/ArchiveBox.conf"
-ONLY_NEW = true
-TIMEOUT = 60
-MEDIA_TIMEOUT = 3600
-OUTPUT_PERMISSIONS = 644
-RESTRICT_FILE_NAMES = "windows"
-URL_DENYLIST = "\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$"
-URL_ALLOWLIST = null
-ADMIN_USERNAME = null
-ADMIN_PASSWORD = null
-ENFORCE_ATOMIC_WRITES = true
-TAG_SEPARATOR_PATTERN = "[,]"
-SECRET_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-BIND_ADDR = "127.0.0.1:8000"
-ALLOWED_HOSTS = "*"
-DEBUG = false
-PUBLIC_INDEX = true
-PUBLIC_SNAPSHOTS = true
-PUBLIC_ADD_VIEW = false
-FOOTER_INFO = "Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests."
-SNAPSHOTS_PER_PAGE = 40
-CUSTOM_TEMPLATES_DIR = null
-TIME_ZONE = "UTC"
-TIMEZONE = "UTC"
-REVERSE_PROXY_USER_HEADER = "Remote-User"
-REVERSE_PROXY_WHITELIST = ""
-LOGOUT_REDIRECT_URL = "/"
-PREVIEW_ORIGINALS = true
-LDAP = false
-LDAP_SERVER_URI = null
-LDAP_BIND_DN = null
-LDAP_BIND_PASSWORD = null
-LDAP_USER_BASE = null
-LDAP_USER_FILTER = null
-LDAP_USERNAME_ATTR = null
-LDAP_FIRSTNAME_ATTR = null
-LDAP_LASTNAME_ATTR = null
-LDAP_EMAIL_ATTR = null
-LDAP_CREATE_SUPERUSER = false
-SAVE_TITLE = true
-SAVE_FAVICON = true
-SAVE_WGET = true
-SAVE_WGET_REQUISITES = true
-SAVE_SINGLEFILE = true
-SAVE_READABILITY = true
-SAVE_MERCURY = true
-SAVE_HTMLTOTEXT = true
-SAVE_PDF = true
-SAVE_SCREENSHOT = true
-SAVE_DOM = true
-SAVE_HEADERS = true
-SAVE_WARC = true
-SAVE_GIT = true
-SAVE_MEDIA = true
-SAVE_ARCHIVE_DOT_ORG = true
-RESOLUTION = [1440, 2000]
-GIT_DOMAINS = "github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht"
-CHECK_SSL_VALIDITY = true
-MEDIA_MAX_SIZE = "750m"
-USER_AGENT = null
-CURL_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0)"
-WGET_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5"
-CHROME_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/)"
-COOKIES_FILE = null
-CHROME_USER_DATA_DIR = null
-CHROME_TIMEOUT = false
-CHROME_HEADLESS = true
-CHROME_SANDBOX = true
-CHROME_EXTRA_ARGS = []
-YOUTUBEDL_ARGS = ["--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)"]
-YOUTUBEDL_EXTRA_ARGS = []
-WGET_ARGS = ["--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off"]
-WGET_EXTRA_ARGS = []
-CURL_ARGS = ["--silent", "--location", "--compressed"]
-CURL_EXTRA_ARGS = []
-GIT_ARGS = ["--recursive"]
-SINGLEFILE_ARGS = []
-SINGLEFILE_EXTRA_ARGS = []
-MERCURY_ARGS = ["--format=text"]
-MERCURY_EXTRA_ARGS = []
-FAVICON_PROVIDER = "https://www.google.com/s2/favicons?domain={}"
-USE_INDEXING_BACKEND = true
-USE_SEARCHING_BACKEND = true
-SEARCH_BACKEND_ENGINE = "ripgrep"
-SEARCH_BACKEND_HOST_NAME = "localhost"
-SEARCH_BACKEND_PORT = 1491
-SEARCH_BACKEND_PASSWORD = "SecretPassword"
-SEARCH_PROCESS_HTML = true
-SONIC_COLLECTION = "archivebox"
-SONIC_BUCKET = "snapshots"
-SEARCH_BACKEND_TIMEOUT = 90
-FTS_SEPARATE_DATABASE = true
-FTS_TOKENIZERS = "porter unicode61 remove_diacritics 2"
-FTS_SQLITE_MAX_LENGTH = 1000000000
-USE_CURL = true
-USE_WGET = true
-USE_SINGLEFILE = true
-USE_READABILITY = true
-USE_MERCURY = true
-USE_GIT = true
-USE_CHROME = true
-USE_NODE = true
-USE_YOUTUBEDL = true
-USE_RIPGREP = true
-CURL_BINARY = "curl"
-GIT_BINARY = "git"
-WGET_BINARY = "wget"
-SINGLEFILE_BINARY = "single-file"
-READABILITY_BINARY = "readability-extractor"
-MERCURY_BINARY = "postlight-parser"
-YOUTUBEDL_BINARY = "yt-dlp"
-NODE_BINARY = "node"
-RIPGREP_BINARY = "rg"
-CHROME_BINARY = "chrome"
-POCKET_CONSUMER_KEY = null
-USER = "squash"
-PACKAGE_DIR = "/opt/archivebox/archivebox"
-TEMPLATES_DIR = "/opt/archivebox/archivebox/templates"
-ARCHIVE_DIR = "/opt/archivebox/data/archive"
-SOURCES_DIR = "/opt/archivebox/data/sources"
-LOGS_DIR = "/opt/archivebox/data/logs"
-PERSONAS_DIR = "/opt/archivebox/data/personas"
-URL_DENYLIST_PTN = "re.compile(\'\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$\', re.IGNORECASE|re.MULTILINE)"
-URL_ALLOWLIST_PTN = null
-DIR_OUTPUT_PERMISSIONS = 755
-ARCHIVEBOX_BINARY = "/opt/archivebox/.venv/bin/archivebox"
-VERSION = "0.8.0"
-COMMIT_HASH = "102e87578c6036bb0132dd1ebd17f8f05ffc880f"
-BUILD_TIME = "2024-05-15 03:28:05 1715768885"
-VERSIONS_AVAILABLE = null
-CAN_UPGRADE = false
-PYTHON_BINARY = "/opt/archivebox/.venv/bin/python3.10"
-PYTHON_VERSION = "3.10.14"
-DJANGO_BINARY = "/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py"
-DJANGO_VERSION = "5.0.6 final (0)"
-SQLITE_BINARY = "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py"
-SQLITE_VERSION = "2.6.0"
-CURL_VERSION = "curl 8.4.0 (x86_64-apple-darwin23.0)"
-WGET_VERSION = "GNU Wget 1.24.5"
-WGET_AUTO_COMPRESSION = true
-RIPGREP_VERSION = "ripgrep 14.1.0"
-SINGLEFILE_VERSION = null
-READABILITY_VERSION = null
-MERCURY_VERSION = null
-GIT_VERSION = "git version 2.44.0"
-YOUTUBEDL_VERSION = "2024.04.09"
-CHROME_VERSION = "Google Chrome 124.0.6367.207"
-NODE_VERSION = "v21.7.3"'''
+# EXPECTED_OUTPUT = TOML_HEADER + '''[SERVER_CONFIG]
+# IS_TTY = false
+# USE_COLOR = false
+# SHOW_PROGRESS = false
+# IN_DOCKER = false
+# IN_QEMU = false
+# PUID = 501
+# PGID = 20
+# CONFIG_FILE = "/opt/archivebox/data/ArchiveBox.conf"
+# ONLY_NEW = true
+# TIMEOUT = 60
+# MEDIA_TIMEOUT = 3600
+# OUTPUT_PERMISSIONS = 644
+# RESTRICT_FILE_NAMES = "windows"
+# URL_DENYLIST = "\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$"
+# URL_ALLOWLIST = null
+# ADMIN_USERNAME = null
+# ADMIN_PASSWORD = null
+# ENFORCE_ATOMIC_WRITES = true
+# TAG_SEPARATOR_PATTERN = "[,]"
+# SECRET_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+# BIND_ADDR = "127.0.0.1:8000"
+# ALLOWED_HOSTS = "*"
+# DEBUG = false
+# PUBLIC_INDEX = true
+# PUBLIC_SNAPSHOTS = true
+# PUBLIC_ADD_VIEW = false
+# FOOTER_INFO = "Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests."
+# SNAPSHOTS_PER_PAGE = 40
+# CUSTOM_TEMPLATES_DIR = null
+# TIME_ZONE = "UTC"
+# TIMEZONE = "UTC"
+# REVERSE_PROXY_USER_HEADER = "Remote-User"
+# REVERSE_PROXY_WHITELIST = ""
+# LOGOUT_REDIRECT_URL = "/"
+# PREVIEW_ORIGINALS = true
+# LDAP = false
+# LDAP_SERVER_URI = null
+# LDAP_BIND_DN = null
+# LDAP_BIND_PASSWORD = null
+# LDAP_USER_BASE = null
+# LDAP_USER_FILTER = null
+# LDAP_USERNAME_ATTR = null
+# LDAP_FIRSTNAME_ATTR = null
+# LDAP_LASTNAME_ATTR = null
+# LDAP_EMAIL_ATTR = null
+# LDAP_CREATE_SUPERUSER = false
+# SAVE_TITLE = true
+# SAVE_FAVICON = true
+# SAVE_WGET = true
+# SAVE_WGET_REQUISITES = true
+# SAVE_SINGLEFILE = true
+# SAVE_READABILITY = true
+# SAVE_MERCURY = true
+# SAVE_HTMLTOTEXT = true
+# SAVE_PDF = true
+# SAVE_SCREENSHOT = true
+# SAVE_DOM = true
+# SAVE_HEADERS = true
+# SAVE_WARC = true
+# SAVE_GIT = true
+# SAVE_MEDIA = true
+# SAVE_ARCHIVE_DOT_ORG = true
+# RESOLUTION = [1440, 2000]
+# GIT_DOMAINS = "github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht"
+# CHECK_SSL_VALIDITY = true
+# MEDIA_MAX_SIZE = "750m"
+# USER_AGENT = null
+# CURL_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0)"
+# WGET_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5"
+# CHROME_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/)"
+# COOKIES_FILE = null
+# CHROME_USER_DATA_DIR = null
+# CHROME_TIMEOUT = false
+# CHROME_HEADLESS = true
+# CHROME_SANDBOX = true
+# CHROME_EXTRA_ARGS = []
+# YOUTUBEDL_ARGS = ["--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)"]
+# YOUTUBEDL_EXTRA_ARGS = []
+# WGET_ARGS = ["--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off"]
+# WGET_EXTRA_ARGS = []
+# CURL_ARGS = ["--silent", "--location", "--compressed"]
+# CURL_EXTRA_ARGS = []
+# GIT_ARGS = ["--recursive"]
+# SINGLEFILE_ARGS = []
+# SINGLEFILE_EXTRA_ARGS = []
+# MERCURY_ARGS = ["--format=text"]
+# MERCURY_EXTRA_ARGS = []
+# FAVICON_PROVIDER = "https://www.google.com/s2/favicons?domain={}"
+# USE_INDEXING_BACKEND = true
+# USE_SEARCHING_BACKEND = true
+# SEARCH_BACKEND_ENGINE = "ripgrep"
+# SEARCH_BACKEND_HOST_NAME = "localhost"
+# SEARCH_BACKEND_PORT = 1491
+# SEARCH_BACKEND_PASSWORD = "SecretPassword"
+# SEARCH_PROCESS_HTML = true
+# SONIC_COLLECTION = "archivebox"
+# SONIC_BUCKET = "snapshots"
+# SEARCH_BACKEND_TIMEOUT = 90
+# FTS_SEPARATE_DATABASE = true
+# FTS_TOKENIZERS = "porter unicode61 remove_diacritics 2"
+# FTS_SQLITE_MAX_LENGTH = 1000000000
+# USE_CURL = true
+# USE_WGET = true
+# USE_SINGLEFILE = true
+# USE_READABILITY = true
+# USE_MERCURY = true
+# USE_GIT = true
+# USE_CHROME = true
+# USE_NODE = true
+# USE_YOUTUBEDL = true
+# USE_RIPGREP = true
+# CURL_BINARY = "curl"
+# GIT_BINARY = "git"
+# WGET_BINARY = "wget"
+# SINGLEFILE_BINARY = "single-file"
+# READABILITY_BINARY = "readability-extractor"
+# MERCURY_BINARY = "postlight-parser"
+# YOUTUBEDL_BINARY = "yt-dlp"
+# NODE_BINARY = "node"
+# RIPGREP_BINARY = "rg"
+# CHROME_BINARY = "chrome"
+# POCKET_CONSUMER_KEY = null
+# USER = "squash"
+# PACKAGE_DIR = "/opt/archivebox/archivebox"
+# TEMPLATES_DIR = "/opt/archivebox/archivebox/templates"
+# ARCHIVE_DIR = "/opt/archivebox/data/archive"
+# SOURCES_DIR = "/opt/archivebox/data/sources"
+# LOGS_DIR = "/opt/archivebox/data/logs"
+# PERSONAS_DIR = "/opt/archivebox/data/personas"
+# URL_DENYLIST_PTN = "re.compile(\'\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$\', re.IGNORECASE|re.MULTILINE)"
+# URL_ALLOWLIST_PTN = null
+# DIR_OUTPUT_PERMISSIONS = 755
+# ARCHIVEBOX_BINARY = "/opt/archivebox/.venv/bin/archivebox"
+# VERSION = "0.8.0"
+# COMMIT_HASH = "102e87578c6036bb0132dd1ebd17f8f05ffc880f"
+# BUILD_TIME = "2024-05-15 03:28:05 1715768885"
+# VERSIONS_AVAILABLE = null
+# CAN_UPGRADE = false
+# PYTHON_BINARY = "/opt/archivebox/.venv/bin/python3.10"
+# PYTHON_VERSION = "3.10.14"
+# DJANGO_BINARY = "/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py"
+# DJANGO_VERSION = "5.0.6 final (0)"
+# SQLITE_BINARY = "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py"
+# SQLITE_VERSION = "2.6.0"
+# CURL_VERSION = "curl 8.4.0 (x86_64-apple-darwin23.0)"
+# WGET_VERSION = "GNU Wget 1.24.5"
+# WGET_AUTO_COMPRESSION = true
+# RIPGREP_VERSION = "ripgrep 14.1.0"
+# SINGLEFILE_VERSION = null
+# READABILITY_VERSION = null
+# MERCURY_VERSION = null
+# GIT_VERSION = "git version 2.44.0"
+# YOUTUBEDL_VERSION = "2024.04.09"
+# CHROME_VERSION = "Google Chrome 124.0.6367.207"
+# NODE_VERSION = "v21.7.3"'''
 
 
-class IniToTomlTests(TestCase):
-    def test_convert(self):
-        first_output = convert(TEST_INPUT)      # make sure ini -> toml parses correctly
-        second_output = convert(first_output)   # make sure toml -> toml parses/dumps consistently
-        assert first_output == second_output == EXPECTED_OUTPUT  # make sure parsing is indempotent
+# class IniToTomlTests(TestCase):
+#     def test_convert(self):
+#         first_output = convert(TEST_INPUT)      # make sure ini -> toml parses correctly
+#         second_output = convert(first_output)   # make sure toml -> toml parses/dumps consistently
+#         assert first_output == second_output == EXPECTED_OUTPUT  # make sure parsing is indempotent
 
 # # DEBUGGING
 # import sys

archivebox/misc/util.py

@@ -1,4 +1,4 @@
-__package__ = 'archivebox'
+__package__ = 'archivebox.misc'
 
 import re
 import requests
@@ -25,10 +25,10 @@ except ImportError:
     detect_encoding = lambda rawdata: "utf-8"
 
 
-from archivebox.config.constants import STATICFILE_EXTENSIONS
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config import CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG
 
-from .misc.logging import COLOR_DICT
+from .logging import COLOR_DICT
 
 
 ### Parsing Helpers
@@ -120,7 +120,7 @@ def find_all_urls(urls_str: str):
 
 def is_static_file(url: str):
     # TODO: the proper way is with MIME type detection + ext, not only extension
-    return extension(url).lower() in STATICFILE_EXTENSIONS
+    return extension(url).lower() in CONSTANTS.STATICFILE_EXTENSIONS
 
 
 def enforce_types(func):

archivebox/parsers/__init__.py

@@ -13,7 +13,8 @@ from typing import IO, Tuple, List, Optional
 from datetime import datetime, timezone
 from pathlib import Path 
 
-from archivebox.config import DATA_DIR, CONSTANTS, SHELL_CONFIG, ARCHIVING_CONFIG
+from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config.common import SHELL_CONFIG, ARCHIVING_CONFIG
 from archivebox.misc.system import atomic_write
 from archivebox.misc.logging import stderr, hint
 from archivebox.misc.util import (
@@ -148,12 +149,13 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir:
 
     referenced_texts = ''
 
-    for entry in raw_text.split():
-        try:
-            if Path(entry).exists():
-                referenced_texts += Path(entry).read_text()
-        except Exception as err:
-            print(err)
+    # dont attempt to read local files from the text, security risk:
+    # for entry in raw_text.split():
+    #     try:
+    #         if Path(entry).exists():
+    #             referenced_texts += Path(entry).read_text()
+    #     except Exception as err:
+    #         print(err)
 
     atomic_write(source_path, raw_text + '\n' + referenced_texts)
     log_source_saved(source_file=source_path)

archivebox/parsers/generic_txt.py

@@ -3,7 +3,6 @@ __description__ = 'Plain Text'
 
 from typing import IO, Iterable
 from datetime import datetime, timezone
-from pathlib import Path
 
 from ..index.schema import Link
 from archivebox.misc.util import (
@@ -22,19 +21,20 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
         if not line.strip():
             continue
 
-        # if the line is a local file path that resolves, then we can archive it
-        try:
-            if Path(line).exists():
-                yield Link(
-                    url=line,
-                    timestamp=str(datetime.now(timezone.utc).timestamp()),
-                    title=None,
-                    tags=None,
-                    sources=[text_file.name],
-                )
-        except (OSError, PermissionError):
-            # nvm, not a valid path...
-            pass
+        # # if the line is a local file path that resolves, then we can archive it
+        # if line.startswith('file://'):    
+        #     try:
+        #         if Path(line).exists():
+        #             yield Link(
+        #                 url=line,
+        #                 timestamp=str(datetime.now(timezone.utc).timestamp()),
+        #                 title=None,
+        #                 tags=None,
+        #                 sources=[text_file.name],
+        #             )
+        #     except (OSError, PermissionError):
+        #         # nvm, not a valid path...
+        #         pass
 
         # otherwise look for anything that looks like a URL in the line
         for url in find_all_urls(line):

archivebox/plugins_auth/ldap/apps.py

@@ -1,4 +1,5 @@
-__package__ = 'plugins_auth.ldap'
+__package__ = 'archivebox.plugins_auth.ldap'
+
 
 import inspect
 
@@ -10,30 +11,58 @@ from pydantic_pkgr import BinProviderName, ProviderLookupDict, SemVer
 
 from abx.archivebox.base_plugin import BasePlugin
 from abx.archivebox.base_hook import BaseHook
-from abx.archivebox.base_binary import BaseBinary, BaseBinProvider
+from abx.archivebox.base_binary import BaseBinary, BaseBinProvider, apt
 
-from plugins_pkg.pip.apps import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER
+from plugins_pkg.pip.apps import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
 from .settings import LDAP_CONFIG, get_ldap_lib
 
 
 ###################### Config ##########################
 
-LDAP_LIB = lambda: get_ldap_lib()[0]   # lazy load to avoid slow ldap lib import on startup
+def get_LDAP_LIB_path(paths=()):
+    LDAP_LIB = get_ldap_lib()[0]
+    if not LDAP_LIB:
+        return None
+    
+    # check that LDAP_LIB path is in one of the specified site packages dirs
+    lib_path = Path(inspect.getfile(LDAP_LIB))
+    if not paths:
+        return lib_path
+    
+    for site_packges_dir in paths:
+        if str(lib_path.parent.parent.resolve()) == str(Path(site_packges_dir).resolve()):
+            return lib_path
+    return None
 
+def get_LDAP_LIB_version():
+    LDAP_LIB = get_ldap_lib()[0]
+    return LDAP_LIB and SemVer(LDAP_LIB.__version__)
 
 class LdapBinary(BaseBinary):
     name: str = 'ldap'
     description: str = 'LDAP Authentication'
-    binproviders_supported: List[InstanceOf[BaseBinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER]
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
 
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
+        LIB_PIP_BINPROVIDER.name: {
+            "abspath": lambda: get_LDAP_LIB_path(LIB_SITE_PACKAGES),
+            "version": lambda: get_LDAP_LIB_version(),
+            "packages": lambda: ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
+        },
         VENV_PIP_BINPROVIDER.name: {
-            "abspath": lambda: LDAP_LIB() and Path(inspect.getfile(LDAP_LIB())),         # type: ignore
-            "version": lambda: LDAP_LIB() and SemVer(LDAP_LIB().__version__),            # type: ignore
+            "abspath": lambda: get_LDAP_LIB_path(VENV_SITE_PACKAGES),
+            "version": lambda: get_LDAP_LIB_version(),
+            "packages": lambda: ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
         },
         SYS_PIP_BINPROVIDER.name: {
-            "abspath": lambda: LDAP_LIB() and Path(inspect.getfile(LDAP_LIB())),         # type: ignore
-            "version": lambda: LDAP_LIB() and SemVer(LDAP_LIB().__version__),            # type: ignore
+            "abspath": lambda: get_LDAP_LIB_path((*USER_SITE_PACKAGES, *SYS_SITE_PACKAGES)),
+            "version": lambda: get_LDAP_LIB_version(),
+            "packages": lambda: ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
+        },
+        apt.name: {
+            "abspath": lambda: get_LDAP_LIB_path(),
+            "version": lambda: get_LDAP_LIB_version(),
+            "packages": lambda: ['libssl-dev', 'libldap2-dev', 'libsasl2-dev', 'python3-ldap', 'python3-msgpack', 'python3-mutagen'],
         },
     }
 

archivebox/plugins_auth/ldap/settings.py

@@ -10,11 +10,15 @@ from abx.archivebox.base_configset import BaseConfigSet
 LDAP_LIB = None
 LDAP_SEARCH = None
 
-def get_ldap_lib():
+def get_ldap_lib(extra_paths=()):
     global LDAP_LIB, LDAP_SEARCH
     if LDAP_LIB and LDAP_SEARCH:
         return LDAP_LIB, LDAP_SEARCH
     try:
+        for path in extra_paths:
+            if path not in sys.path:
+                sys.path.append(path)
+            
         import ldap
         from django_auth_ldap.config import LDAPSearch
         LDAP_LIB, LDAP_SEARCH = ldap, LDAPSearch

archivebox/plugins_extractor/chrome/apps.py

@@ -1,5 +1,6 @@
 __package__ = 'archivebox.plugins_extractor.chrome'
 
+import os
 import sys
 import platform
 from pathlib import Path
@@ -25,7 +26,8 @@ from abx.archivebox.base_binary import BaseBinary, env
 from abx.archivebox.base_hook import BaseHook
 
 # Depends on Other Plugins:
-from archivebox.config import CONSTANTS, ARCHIVING_CONFIG, SHELL_CONFIG
+from archivebox.config import CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG
 from plugins_pkg.puppeteer.apps import PUPPETEER_BINPROVIDER
 from plugins_pkg.playwright.apps import PLAYWRIGHT_BINPROVIDER
 
@@ -129,9 +131,9 @@ class ChromeConfig(BaseConfigSet):
             print(file=sys.stderr)
             
         # if user has specified a user data dir, make sure its valid
-        if self.CHROME_USER_DATA_DIR and self.CHROME_USER_DATA_DIR.exists():
+        if self.CHROME_USER_DATA_DIR and os.access(self.CHROME_USER_DATA_DIR, os.R_OK):
             # check to make sure user_data_dir/<profile_name> exists
-            if not (self.CHROME_USER_DATA_DIR / self.CHROME_PROFILE_NAME).exists():
+            if not (self.CHROME_USER_DATA_DIR / self.CHROME_PROFILE_NAME).is_dir():
                 print(f'[red][X] Could not find profile "{self.CHROME_PROFILE_NAME}" in CHROME_USER_DATA_DIR.[/red]', file=sys.stderr)
                 print(f'    {self.CHROME_USER_DATA_DIR}', file=sys.stderr)
                 print('    Make sure you set it to a Chrome user data directory containing a Default profile folder.', file=sys.stderr)
@@ -185,12 +187,13 @@ class ChromeConfig(BaseConfigSet):
         if options.CHROME_USER_AGENT:
             cmd_args += ('--user-agent={}'.format(options.CHROME_USER_AGENT),)
     
-        if options.CHROME_TIMEOUT:
-           cmd_args += ('--timeout={}'.format(options.CHROME_TIMEOUT * 1000),)
+        # this no longer works on newer chrome version for some reason, just causes chrome to hang indefinitely:
+        # if options.CHROME_TIMEOUT:
+        #   cmd_args += ('--timeout={}'.format(options.CHROME_TIMEOUT * 1000),)
     
         if options.CHROME_USER_DATA_DIR:
             cmd_args.append('--user-data-dir={}'.format(options.CHROME_USER_DATA_DIR))
-            cmd_args.append('--profile-directory={}'.format(options.CHROME_PROFILE_NAME))
+            cmd_args.append('--profile-directory={}'.format(options.CHROME_PROFILE_NAME or 'Default'))
     
         return dedupe(cmd_args)
 
@@ -215,7 +218,7 @@ class ChromeBinary(BaseBinary):
 
     @staticmethod
     def symlink_to_lib(binary, bin_dir=CONSTANTS.LIB_BIN_DIR) -> None:
-        if not (binary.abspath and binary.abspath.exists()):
+        if not (binary.abspath and os.access(binary.abspath, os.F_OK)):
             return
         
         bin_dir.mkdir(parents=True, exist_ok=True)
@@ -240,10 +243,14 @@ class ChromeBinary(BaseBinary):
         Cleans up any state or runtime files that chrome leaves behind when killed by
         a timeout or other error
         """
-        lock_file = Path("~/.config/chromium/SingletonLock")
+        lock_file = Path("~/.config/chromium/SingletonLock").expanduser()
 
-        if SHELL_CONFIG.IN_DOCKER and lock_file.exists():
+        if SHELL_CONFIG.IN_DOCKER and os.access(lock_file, os.F_OK):
             lock_file.unlink()
+        
+        if CHROME_CONFIG.CHROME_USER_DATA_DIR:
+            if os.access(CHROME_CONFIG.CHROME_USER_DATA_DIR / 'SingletonLock', os.F_OK):
+                lock_file.unlink()
 
 
 

archivebox/plugins_extractor/curl/apps.py

@@ -11,7 +11,7 @@ from abx.archivebox.base_configset import BaseConfigSet
 from abx.archivebox.base_binary import BaseBinary, env, apt, brew
 # from abx.archivebox.base_extractor import BaseExtractor, ExtractorName
 
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG
 from archivebox.plugins_extractor.favicon.apps import FAVICON_CONFIG
 from archivebox.plugins_extractor.archivedotorg.apps import ARCHIVEDOTORG_CONFIG
 

archivebox/plugins_extractor/git/apps.py

@@ -11,7 +11,7 @@ from abx.archivebox.base_configset import BaseConfigSet
 from abx.archivebox.base_binary import BaseBinary, env, apt, brew
 from abx.archivebox.base_extractor import BaseExtractor, ExtractorName
 
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG
 
 
 class GitConfig(BaseConfigSet):

archivebox/plugins_extractor/mercury/apps.py

@@ -2,17 +2,16 @@ __package__ = 'plugins_extractor.mercury'
 
 from typing import List, Optional, Dict
 from pathlib import Path
-from subprocess import run
 
 from pydantic import InstanceOf, Field
-from pydantic_pkgr import BinProvider, BinName, bin_abspath
+from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict, bin_abspath
 
 from abx.archivebox.base_plugin import BasePlugin, BaseHook
 from abx.archivebox.base_configset import BaseConfigSet
-from abx.archivebox.base_binary import BaseBinary, BinProviderName,ProviderLookupDict, env
+from abx.archivebox.base_binary import BaseBinary, env
 from abx.archivebox.base_extractor import BaseExtractor, ExtractorName
 
-from archivebox.config import ARCHIVING_CONFIG, STORAGE_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG
 from archivebox.plugins_pkg.npm.apps import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
 
 class MercuryConfig(BaseConfigSet):
@@ -42,11 +41,10 @@ class MercuryBinary(BaseBinary):
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
         LIB_NPM_BINPROVIDER.name: {
             'packages': lambda: ['@postlight/parser@^2.2.3'],
-            'version': lambda: run([str(LIB_NPM_BINPROVIDER.INSTALLER_BIN_ABSPATH), f'--prefix={LIB_NPM_BINPROVIDER.npm_prefix}', 'info', '@postlight/parser', 'version'], text=True, capture_output=True).stdout.strip(),
         },
         SYS_NPM_BINPROVIDER.name: {
-            'packages': lambda: [],   # never try to install things globally
-            'version': lambda: run([str(SYS_NPM_BINPROVIDER.INSTALLER_BIN_ABSPATH), '-g', 'info', '@postlight/parser', 'version'], text=True, capture_output=True).stdout.strip(),
+            'packages': lambda: ['@postlight/parser@^2.2.3'],
+            'install': lambda: False,                          # never try to install things into global prefix
         },
         env.name: {
             'version': lambda: '999.999.999' if bin_abspath('postlight-parser', PATH=env.PATH) else None,

archivebox/plugins_extractor/readability/apps.py

@@ -1,7 +1,7 @@
 __package__ = 'archivebox.plugins_extractor.readability'
 
 from pathlib import Path
-from typing import List, Dict, Optional, ClassVar
+from typing import List, Dict, Optional
 # from typing_extensions import Self
 
 # Depends on other PyPI/vendor packages:
@@ -16,7 +16,7 @@ from abx.archivebox.base_extractor import BaseExtractor
 from abx.archivebox.base_hook import BaseHook
 
 # Depends on Other Plugins:
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG
 from plugins_pkg.npm.apps import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
 
 ###################### Config ##########################
@@ -45,17 +45,17 @@ class ReadabilityBinary(BaseBinary):
     }
     
     @validate_call
-    def install(self, binprovider_name: Optional[BinProviderName]=None) -> ShallowBinary:
+    def install(self, binprovider_name: Optional[BinProviderName]=None, **kwargs) -> ShallowBinary:
         # force install to only use lib/npm provider, we never want to modify global NPM packages
-        return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name)
+        return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
     
     @validate_call
-    def load_or_install(self, binprovider_name: Optional[BinProviderName] = None) -> ShallowBinary:
-        # force install to only use lib/npm provider, we never want to modify global NPM packages
+    def load_or_install(self, binprovider_name: Optional[BinProviderName] = None, fresh=False, **kwargs) -> ShallowBinary:
         try:
-            return self.load()
+            return self.load(fresh=fresh)
         except Exception:
-            return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name)
+            # force install to only use lib/npm provider, we never want to modify global NPM packages
+            return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
 
 
 

archivebox/plugins_extractor/singlefile/apps.py

@@ -1,11 +1,11 @@
 __package__ = 'archivebox.plugins_extractor.singlefile'
 
 from pathlib import Path
-from typing import List, Dict, Optional, ClassVar
+from typing import List, Dict, Optional
 # from typing_extensions import Self
 
 # Depends on other PyPI/vendor packages:
-from pydantic import InstanceOf, Field, validate_call
+from pydantic import InstanceOf, Field
 from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, bin_abspath, ShallowBinary
 
 # Depends on other Django apps:
@@ -17,7 +17,7 @@ from abx.archivebox.base_queue import BaseQueue
 from abx.archivebox.base_hook import BaseHook
 
 # Depends on Other Plugins:
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG
 from plugins_pkg.npm.apps import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
 
 ###################### Config ##########################
@@ -46,38 +46,40 @@ class SinglefileBinary(BaseBinary):
     binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
 
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        env.name: {
-            'abspath': lambda:
-                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=env.PATH)
-                or bin_abspath('single-file', PATH=env.PATH)
-                or bin_abspath('single-file-node.js', PATH=env.PATH),
-        },
         LIB_NPM_BINPROVIDER.name: {
             "abspath": lambda:
-                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=env.PATH)
+                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=LIB_NPM_BINPROVIDER.PATH)
                 or bin_abspath("single-file", PATH=LIB_NPM_BINPROVIDER.PATH)
                 or bin_abspath("single-file-node.js", PATH=LIB_NPM_BINPROVIDER.PATH),
             "packages": lambda:
                 [f"single-file-cli@>={SINGLEFILE_MIN_VERSION} <{SINGLEFILE_MAX_VERSION}"],
         },
         SYS_NPM_BINPROVIDER.name: {
+            "abspath": lambda:
+                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=SYS_NPM_BINPROVIDER.PATH)
+                or bin_abspath("single-file", PATH=SYS_NPM_BINPROVIDER.PATH)
+                or bin_abspath("single-file-node.js", PATH=SYS_NPM_BINPROVIDER.PATH),
             "packages": lambda:
                 [],    # prevent modifying system global npm packages
         },
+        env.name: {
+            'abspath': lambda:
+                bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=env.PATH)
+                or bin_abspath('single-file', PATH=env.PATH)
+                or bin_abspath('single-file-node.js', PATH=env.PATH),
+        },
     }
     
-    @validate_call
-    def install(self, binprovider_name: Optional[BinProviderName]=None) -> ShallowBinary:
+    def install(self, binprovider_name: Optional[BinProviderName]=None, **kwargs) -> ShallowBinary:
         # force install to only use lib/npm provider, we never want to modify global NPM packages
-        return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name)
+        return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
     
-    @validate_call
-    def load_or_install(self, binprovider_name: Optional[BinProviderName] = None) -> ShallowBinary:
-        # force install to only use lib/npm provider, we never want to modify global NPM packages
+    def load_or_install(self, binprovider_name: Optional[BinProviderName]=None, fresh=False, **kwargs) -> ShallowBinary:
         try:
-            return self.load()
+            return self.load(fresh=fresh)
         except Exception:
-            return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name)
+            # force install to only use lib/npm provider, we never want to modify global NPM packages
+            return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
 
 
 

archivebox/plugins_extractor/wget/apps.py

@@ -14,7 +14,7 @@ from abx.archivebox.base_configset import BaseConfigSet
 from abx.archivebox.base_binary import BaseBinary, env, apt, brew
 from abx.archivebox.base_extractor import BaseExtractor, ExtractorName
 
-from archivebox.config import ARCHIVING_CONFIG, STORAGE_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG
 from .wget_util import wget_output_path
 
 

archivebox/plugins_extractor/wget/wget_util.py

@@ -1,6 +1,7 @@
 __package__ = 'archivebox.extractors'
 
 import re
+import os
 from pathlib import Path
 
 from typing import Optional
@@ -157,12 +158,12 @@ def wget_output_path(link, nocache: bool=False) -> Optional[str]:
 
     # fallback to just the domain dir
     search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
-    if search_dir.is_dir():
+    if os.access(search_dir, os.R_OK) and search_dir.is_dir():
         return domain(link.url).replace(":", "+")
 
     # fallback to just the domain dir without port
     search_dir = Path(link.link_dir) / domain(link.url).split(":", 1)[0]
-    if search_dir.is_dir():
+    if os.access(search_dir, os.R_OK) and search_dir.is_dir():
         return domain(link.url).split(":", 1)[0]
 
     return None

archivebox/plugins_extractor/ytdlp/apps.py

@@ -11,7 +11,7 @@ from abx.archivebox.base_configset import BaseConfigSet
 from abx.archivebox.base_binary import BaseBinary, env, apt, brew
 from abx.archivebox.base_hook import BaseHook
 
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config.common import ARCHIVING_CONFIG
 from plugins_pkg.pip.apps import pip
 
 ###################### Config ##########################

archivebox/plugins_pkg/npm/apps.py

@@ -1,4 +1,4 @@
-__package__ = 'plugins_pkg.npm'
+__package__ = 'archivebox.plugins_pkg.npm'
 
 from pathlib import Path
 from typing import List, Optional
@@ -35,14 +35,14 @@ NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
 OLD_NODE_BIN_PATH = DATA_DIR / 'node_modules' / '.bin'
 NEW_NODE_BIN_PATH = CONSTANTS.LIB_NPM_DIR / 'node_modules' / '.bin'
 
-class SystemNpmProvider(NpmProvider, BaseBinProvider):
+class SystemNpmBinProvider(NpmProvider, BaseBinProvider):
     name: BinProviderName = "sys_npm"
     
     npm_prefix: Optional[Path] = None
 
-class LibNpmProvider(NpmProvider, BaseBinProvider):
+class LibNpmBinProvider(NpmProvider, BaseBinProvider):
     name: BinProviderName = "lib_npm"
-    PATH: PATHStr = str(OLD_NODE_BIN_PATH)
+    PATH: PATHStr = f'{NEW_NODE_BIN_PATH}:{OLD_NODE_BIN_PATH}'
     
     npm_prefix: Optional[Path] = CONSTANTS.LIB_NPM_DIR
     
@@ -52,17 +52,10 @@ class LibNpmProvider(NpmProvider, BaseBinProvider):
         return self
 
 
-SYS_NPM_BINPROVIDER = SystemNpmProvider()
-LIB_NPM_BINPROVIDER = LibNpmProvider()
+SYS_NPM_BINPROVIDER = SystemNpmBinProvider()
+LIB_NPM_BINPROVIDER = LibNpmBinProvider()
 npm = LIB_NPM_BINPROVIDER
 
-class NpmBinary(BaseBinary):
-    name: BinName = 'npm'
-    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
-
-
-NPM_BINARY = NpmBinary()
-
 class NodeBinary(BaseBinary):
     name: BinName = 'node'
     binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
@@ -71,6 +64,22 @@ class NodeBinary(BaseBinary):
 NODE_BINARY = NodeBinary()
 
 
+class NpmBinary(BaseBinary):
+    name: BinName = 'npm'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+NPM_BINARY = NpmBinary()
+
+
+class NpxBinary(BaseBinary):
+    name: BinName = 'npx'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+NPX_BINARY = NpxBinary()
+
+
+
+
 
 class NpmPlugin(BasePlugin):
     app_label: str = 'npm'
@@ -82,6 +91,7 @@ class NpmPlugin(BasePlugin):
         LIB_NPM_BINPROVIDER,
         NODE_BINARY,
         NPM_BINARY,
+        NPX_BINARY,
     ]
 
 

archivebox/plugins_pkg/pip/.plugin_order

@@ -0,0 +1 @@
+0

archivebox/plugins_pkg/pip/apps.py

@@ -2,23 +2,21 @@ __package__ = 'archivebox.plugins_pkg.pip'
 
 import os
 import sys
-import inspect
+import site
 from pathlib import Path
 from typing import List, Dict, Optional
-from pydantic import InstanceOf, Field, model_validator
+from pydantic import InstanceOf, Field, model_validator, validate_call
 
 
 import django
+import django.db.backends.sqlite3.base
 from django.db.backends.sqlite3.base import Database as django_sqlite3     # type: ignore[import-type]
-from django.core.checks import Error, Tags
 from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, ProviderLookupDict, SemVer
 
 from archivebox.config import CONSTANTS, VERSION
 
-import abx
 from abx.archivebox.base_plugin import BasePlugin
 from abx.archivebox.base_configset import BaseConfigSet
-from abx.archivebox.base_check import BaseCheck
 from abx.archivebox.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
 from abx.archivebox.base_hook import BaseHook
 
@@ -35,12 +33,9 @@ class PipDependencyConfigs(BaseConfigSet):
     PIP_EXTRA_ARGS: List[str] = []
     PIP_DEFAULT_ARGS: List[str] = []
     
+PIP_CONFIG = PipDependencyConfigs()
 
 
-DEFAULT_GLOBAL_CONFIG = {
-}
-PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
-
 class SystemPipBinProvider(PipProvider, BaseBinProvider):
     name: BinProviderName = "sys_pip"
     INSTALLER_BIN: BinName = "pip"
@@ -58,12 +53,18 @@ class SystemPipxBinProvider(PipProvider, BaseBinProvider):
     pip_venv: Optional[Path] = None        # global pipx scope
 
 
+IS_INSIDE_VENV = sys.prefix != sys.base_prefix
+
 class VenvPipBinProvider(PipProvider, BaseBinProvider):
     name: BinProviderName = "venv_pip"
     INSTALLER_BIN: BinName = "pip"
 
-    pip_venv: Optional[Path] = Path(os.environ.get("VIRTUAL_ENV", None) or '/tmp/NotInsideAVenv')
-
+    pip_venv: Optional[Path] = Path(sys.prefix if IS_INSIDE_VENV else os.environ.get("VIRTUAL_ENV", '/tmp/NotInsideAVenv/lib'))
+    
+    def setup(self):
+        """never attempt to create a venv here, this is just used to detect if we are inside an existing one"""
+        return None
+    
 
 class LibPipBinProvider(PipProvider, BaseBinProvider):
     name: BinProviderName = "lib_pip"
@@ -77,6 +78,28 @@ VENV_PIP_BINPROVIDER = VenvPipBinProvider()
 LIB_PIP_BINPROVIDER = LibPipBinProvider()
 pip = LIB_PIP_BINPROVIDER
 
+# ensure python libraries are importable from these locations (if archivebox wasnt executed from one of these then they wont already be in sys.path)
+assert VENV_PIP_BINPROVIDER.pip_venv is not None
+assert LIB_PIP_BINPROVIDER.pip_venv is not None
+
+major, minor, patch = sys.version_info[:3]
+site_packages_dir = f'lib/python{major}.{minor}/site-packages'
+
+LIB_SITE_PACKAGES = (LIB_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
+VENV_SITE_PACKAGES = (VENV_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
+USER_SITE_PACKAGES = site.getusersitepackages()
+SYS_SITE_PACKAGES = site.getsitepackages()
+
+ALL_SITE_PACKAGES = (
+    *LIB_SITE_PACKAGES,
+    *VENV_SITE_PACKAGES,
+    *USER_SITE_PACKAGES,
+    *SYS_SITE_PACKAGES,
+)
+for site_packages_dir in ALL_SITE_PACKAGES:
+    if site_packages_dir not in sys.path:
+        sys.path.append(str(site_packages_dir))
+
 
 class ArchiveboxBinary(BaseBinary):
     name: BinName = 'archivebox'
@@ -88,6 +111,14 @@ class ArchiveboxBinary(BaseBinary):
         apt.name:                   {'packages': lambda: [], 'version': lambda: VERSION},
         brew.name:                  {'packages': lambda: [], 'version': lambda: VERSION},
     }
+    
+    @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
 
 ARCHIVEBOX_BINARY = ArchiveboxBinary()
 
@@ -98,26 +129,37 @@ class PythonBinary(BaseBinary):
     binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
         SYS_PIP_BINPROVIDER.name: {
-            'abspath': lambda:
-                sys.executable,
-            'version': lambda: 
-                '{}.{}.{}'.format(*sys.version_info[:3]),
+            'abspath': lambda: sys.executable,
+            'version': lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
         },
     }
+    
+    @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
 
 PYTHON_BINARY = PythonBinary()
 
+
+LOADED_SQLITE_PATH = Path(django.db.backends.sqlite3.base.__file__)
+LOADED_SQLITE_VERSION = SemVer(django_sqlite3.version)
+LOADED_SQLITE_FROM_VENV = str(LOADED_SQLITE_PATH.absolute().resolve()).startswith(str(VENV_PIP_BINPROVIDER.pip_venv.absolute().resolve()))
+
 class SqliteBinary(BaseBinary):
     name: BinName = 'sqlite'
     binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
         VENV_PIP_BINPROVIDER.name: {
-            "abspath": lambda: Path(inspect.getfile(django_sqlite3)),
-            "version": lambda: SemVer(django_sqlite3.version),
+            "abspath": lambda: LOADED_SQLITE_PATH if LOADED_SQLITE_FROM_VENV else None,
+            "version": lambda: LOADED_SQLITE_VERSION if LOADED_SQLITE_FROM_VENV else None,
         },
         SYS_PIP_BINPROVIDER.name: {
-            "abspath": lambda: Path(inspect.getfile(django_sqlite3)),
-            "version": lambda: SemVer(django_sqlite3.version),
+            "abspath": lambda: LOADED_SQLITE_PATH if not LOADED_SQLITE_FROM_VENV else None,
+            "version": lambda: LOADED_SQLITE_VERSION if not LOADED_SQLITE_FROM_VENV else None,
         },
     }
     
@@ -134,24 +176,44 @@ class SqliteBinary(BaseBinary):
                 'https://code.djangoproject.com/wiki/JSON1Extension'
             ])
         return self
+    
+    @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
 
 SQLITE_BINARY = SqliteBinary()
 
 
+LOADED_DJANGO_PATH = Path(django.__file__)
+LOADED_DJANGO_VERSION = SemVer(django.VERSION[:3])
+LOADED_DJANGO_FROM_VENV = str(LOADED_DJANGO_PATH.absolute().resolve()).startswith(str(VENV_PIP_BINPROVIDER.pip_venv.absolute().resolve()))
+
 class DjangoBinary(BaseBinary):
     name: BinName = 'django'
 
     binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
         VENV_PIP_BINPROVIDER.name: {
-            "abspath": lambda: inspect.getfile(django),
-            "version": lambda: django.VERSION[:3],
+            "abspath": lambda: LOADED_DJANGO_PATH if LOADED_DJANGO_FROM_VENV else None,
+            "version": lambda: LOADED_DJANGO_VERSION if LOADED_DJANGO_FROM_VENV else None,
         },
         SYS_PIP_BINPROVIDER.name: {
-            "abspath": lambda: inspect.getfile(django),
-            "version": lambda: django.VERSION[:3],
+            "abspath": lambda: LOADED_DJANGO_PATH if not LOADED_DJANGO_FROM_VENV else None,
+            "version": lambda: LOADED_DJANGO_VERSION if not LOADED_DJANGO_FROM_VENV else None,
         },
     }
+    
+    @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
 
 DJANGO_BINARY = DjangoBinary()
 
@@ -159,65 +221,22 @@ class PipBinary(BaseBinary):
     name: BinName = "pip"
     binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
 
+    @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
 
 PIP_BINARY = PipBinary()
 
 
-class CheckUserIsNotRoot(BaseCheck):
-    label: str = 'CheckUserIsNotRoot'
-    tag: str = Tags.database
+class PipxBinary(BaseBinary):
+    name: BinName = "pipx"
+    binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
 
-    @staticmethod
-    def check(settings, logger) -> List[Warning]:
-        errors = []
-        if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
-            errors.append(
-                Error(
-                    "Cannot run as root!",
-                    id="core.S001",
-                    hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
-                )
-            )
-        # logger.debug('[√] UID is not root')
-        return errors
-
-    
-class CheckPipEnvironment(BaseCheck):
-    label: str = "CheckPipEnvironment"
-    tag: str = Tags.database
-
-    @staticmethod
-    def check(settings, logger) -> List[Warning]:
-        # hard errors: check python version
-        if sys.version_info[:3] < (3, 10, 0):
-            print('[red][X] Python version is not new enough: {sys.version} (>3.10 is required)[/red]', file=sys.stderr)
-            print('    See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.', file=sys.stderr)
-            raise SystemExit(2)
-        
-        # hard errors: check django version
-        if int(django.VERSION[0]) < 5:
-            print('[red][X] Django version is not new enough: {django.VERSION[:3]} (>=5.0 is required)[/red]', file=sys.stderr)
-            print('    Upgrade django using pip or your system package manager: pip3 install --upgrade django', file=sys.stderr)
-            raise SystemExit(2)
-        
-        # soft errors: check that lib/pip virtualenv is setup properly
-        errors = []
-        
-        LIB_PIP_BINPROVIDER.setup()
-        if not LIB_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH:
-            errors.append(
-                Error(
-                    "Failed to setup data/lib/pip virtualenv for runtime dependencies!",
-                    id="pip.P001",
-                    hint="Make sure the data dir is writable and make sure python3-pip and python3-venv are installed & available on the host.",
-                )
-            )
-        # logger.debug("[√] CheckPipEnvironment: data/lib/pip virtualenv is setup properly")
-        return errors
-
-
-USER_IS_NOT_ROOT_CHECK = CheckUserIsNotRoot()
-PIP_ENVIRONMENT_CHECK = CheckPipEnvironment()
+PIPX_BINARY = PipxBinary()
 
 
 class PipPlugin(BasePlugin):
@@ -231,20 +250,14 @@ class PipPlugin(BasePlugin):
         VENV_PIP_BINPROVIDER,
         LIB_PIP_BINPROVIDER,
         PIP_BINARY,
+        PIPX_BINARY,
         ARCHIVEBOX_BINARY,
         PYTHON_BINARY,
         SQLITE_BINARY,
         DJANGO_BINARY,
-        USER_IS_NOT_ROOT_CHECK,
-        PIP_ENVIRONMENT_CHECK,
     ]
 
+
 PLUGIN = PipPlugin()
 # PLUGIN.register(settings)
 DJANGO_APP = PLUGIN.AppConfig
-
-
-@abx.hookimpl
-def register_django_checks(settings):
-    USER_IS_NOT_ROOT_CHECK.register_with_django_check_system(settings)
-    PIP_ENVIRONMENT_CHECK.register_with_django_check_system(settings)

archivebox/plugins_pkg/playwright/apps.py

@@ -1,3 +1,6 @@
+__package__ = 'archivebox.plugins_pkg.playwright'
+
+import os
 import platform
 from pathlib import Path
 from typing import List, Optional, Dict, ClassVar
@@ -63,12 +66,12 @@ class PlaywrightBinProvider(BaseBinProvider):
 
     PATH: PATHStr = f"{CONSTANTS.LIB_BIN_DIR}:{DEFAULT_ENV_PATH}"
 
-    puppeteer_browsers_dir: Optional[Path] = (
+    playwright_browsers_dir: Optional[Path] = (
         Path("~/Library/Caches/ms-playwright").expanduser()      # macos playwright cache dir
         if OPERATING_SYSTEM == "darwin" else
         Path("~/.cache/ms-playwright").expanduser()              # linux playwright cache dir
     )
-    puppeteer_install_args: List[str] = ["install"]              # --with-deps
+    playwright_install_args: List[str] = ["install"]              # --with-deps
 
     packages_handler: ProviderLookupDict = Field(default={
         "chrome": lambda: ["chromium"],
@@ -84,8 +87,8 @@ class PlaywrightBinProvider(BaseBinProvider):
     def setup(self) -> None:
         assert SYS_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH, "Pip bin provider not initialized"
 
-        if self.puppeteer_browsers_dir:
-            self.puppeteer_browsers_dir.mkdir(parents=True, exist_ok=True)
+        if self.playwright_browsers_dir:
+            self.playwright_browsers_dir.mkdir(parents=True, exist_ok=True)
 
     def installed_browser_bins(self, browser_name: str = "*") -> List[Path]:
         if browser_name == 'chrome':
@@ -95,13 +98,13 @@ class PlaywrightBinProvider(BaseBinProvider):
         if platform.system().lower() == "darwin":
             # ~/Library/caches/ms-playwright/chromium-1097/chrome-mac/Chromium.app/Contents/MacOS/Chromium
             return sorted(
-                self.puppeteer_browsers_dir.glob(
+                self.playwright_browsers_dir.glob(
                     f"{browser_name}-*/*-mac*/*.app/Contents/MacOS/*"
                 )
             )
 
         # ~/Library/caches/ms-playwright/chromium-1097/chrome-linux/chromium
-        return sorted(self.puppeteer_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
+        return sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
 
     def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
         assert bin_name == "chrome", "Only chrome is supported using the @puppeteer/browsers install method currently."
@@ -110,7 +113,7 @@ class PlaywrightBinProvider(BaseBinProvider):
         if bin_name in self._browser_abspaths:
             return self._browser_abspaths[bin_name]
 
-        # first time loading, find browser in self.puppeteer_browsers_dir by searching filesystem for installed binaries
+        # first time loading, find browser in self.playwright_browsers_dir by searching filesystem for installed binaries
         matching_bins = [abspath for abspath in self.installed_browser_bins() if bin_name in str(abspath)]
         if matching_bins:
             newest_bin = matching_bins[-1]  # already sorted alphabetically, last should theoretically be highest version number
@@ -138,7 +141,7 @@ class PlaywrightBinProvider(BaseBinProvider):
 
         # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
 
-        install_args = [*self.puppeteer_install_args]
+        install_args = [*self.playwright_install_args]
 
         proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[*install_args, *packages])
 
@@ -148,12 +151,15 @@ class PlaywrightBinProvider(BaseBinProvider):
             raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}")
 
         # chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
-        output_info = proc.stdout.strip().split("\n")[-1]
-        browser_abspath = output_info.split(" ", 1)[-1]
-        # browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0]
-
-        self._browser_abspaths[bin_name] = Path(browser_abspath)
-
+        # playwright build v1010 downloaded to /home/squash/.cache/ms-playwright/ffmpeg-1010
+        output_lines = [line for line in proc.stdout.strip().split('\n') if '/chrome-' in line]
+        if output_lines:
+            relpath = output_lines[0].split(self.playwright_browsers_dir)[-1]
+            abspath = self.playwright_browsers_dir / relpath
+            if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
+                self._browser_abspaths[bin_name] = abspath
+                return abspath
+        
         return proc.stderr.strip() + "\n" + proc.stdout.strip()
 
 PLAYWRIGHT_BINPROVIDER = PlaywrightBinProvider()

archivebox/plugins_pkg/puppeteer/apps.py

@@ -1,3 +1,6 @@
+__package__ = 'archivebox.plugins_pkg.puppeteer'
+
+import os
 import platform
 from pathlib import Path
 from typing import List, Optional, Dict, ClassVar
@@ -15,6 +18,7 @@ from pydantic_pkgr import (
 )
 
 from archivebox.config import CONSTANTS
+from archivebox.config.permissions import ARCHIVEBOX_USER
 
 # Depends on other Django apps:
 from abx.archivebox.base_plugin import BasePlugin
@@ -58,6 +62,8 @@ class PuppeteerBinProvider(BaseBinProvider):
     INSTALLER_BIN: BinName = "npx"
 
     PATH: PATHStr = str(CONSTANTS.LIB_BIN_DIR)
+    
+    euid: Optional[int] = ARCHIVEBOX_USER
 
     puppeteer_browsers_dir: Optional[Path] = LIB_DIR_BROWSERS
     puppeteer_install_args: List[str] = ["@puppeteer/browsers", "install", "--path", str(LIB_DIR_BROWSERS)]
@@ -110,6 +116,7 @@ class PuppeteerBinProvider(BaseBinProvider):
                 f"{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)"
             )
         packages = packages or self.on_get_packages(bin_name)
+        assert packages, f"No packages specified for installation of {bin_name}"
 
         # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
 
@@ -122,12 +129,14 @@ class PuppeteerBinProvider(BaseBinProvider):
             print(proc.stderr.strip())
             raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}")
 
+        # to proceed? (y) chrome@129.0.6668.91 /tmp/test3/lib/x86_64-linux/browsers/chrome/linux-129.0.6668.91/chrome-linux64/chrome
         # chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
-        output_info = proc.stdout.strip().split('\n')[-1]
-        browser_abspath = output_info.split(' ', 1)[-1]
-        # browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0]
+        relpath = proc.stdout.strip().split(str(self.puppeteer_browsers_dir))[-1]
+        abspath = self.puppeteer_browsers_dir / relpath
         
-        self._browser_abspaths[bin_name] = Path(browser_abspath)
+        if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
+            self._browser_abspaths[bin_name] = abspath
+            return abspath
 
         return proc.stderr.strip() + "\n" + proc.stdout.strip()
 

archivebox/plugins_search/ripgrep/apps.py

@@ -18,7 +18,8 @@ from abx.archivebox.base_hook import BaseHook
 from abx.archivebox.base_searchbackend import BaseSearchBackend
 
 # Depends on Other Plugins:
-from archivebox.config import CONSTANTS, SEARCH_BACKEND_CONFIG
+from archivebox.config import CONSTANTS
+from archivebox.config.common import SEARCH_BACKEND_CONFIG
 
 ###################### Config ##########################
 

archivebox/plugins_search/sonic/apps.py

@@ -15,7 +15,7 @@ from abx.archivebox.base_hook import BaseHook
 from abx.archivebox.base_searchbackend import BaseSearchBackend
 
 # Depends on Other Plugins:
-from archivebox.config import SEARCH_BACKEND_CONFIG
+from archivebox.config.common import SEARCH_BACKEND_CONFIG
 
 SONIC_LIB = None
 try:

archivebox/plugins_search/sqlite/apps.py

@@ -17,7 +17,7 @@ from abx.archivebox.base_hook import BaseHook
 from abx.archivebox.base_searchbackend import BaseSearchBackend
 
 # Depends on Other Plugins:
-from archivebox.config import SEARCH_BACKEND_CONFIG
+from archivebox.config.common import SEARCH_BACKEND_CONFIG