Merge branch 'dev' into plugins-browsertrix

.dockerignore

@@ -9,7 +9,6 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
-.vscode/
 !.git/HEAD
 !.git/refs/heads/*
 
@@ -18,11 +17,6 @@ venv/
 .venv-old/
 .docker-venv/
 node_modules/
-chrome/
-chromeprofile/
-
-pdm.dev.lock
-pdm.lock
 
 docs/
 build/
@@ -32,11 +26,7 @@ deb_dist/
 pip_dist/
 assets/
 docker/
-website/
-typings/
 
 data/
 data*/
 output/
-index.sqlite3
-index.sqlite3-wal

.github/FUNDING.yml

@@ -1,2 +1,3 @@
-github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
+github: pirate
+patreon: theSquashSH
+custom: ["https://hcb.hackclub.com/donations/start/archivebox", "https://paypal.me/NicholasSweeting"]

.github/dependabot.yml

@@ -5,21 +5,8 @@
 
 version: 2
 updates:
-  - package-ecosystem: "pip"
+  - package-ecosystem: "pip" # See documentation for possible values
     directory: "/"
     target-branch: "dev"
     schedule:
-      interval: "monthly"
-    groups:
-      pip:
-        patterns:
-          - "*"
-  - package-ecosystem: "npm"
-    directory: "/"
-    target-branch: "dev"
-    schedule:
-      interval: "monthly"
-    groups:
-      npm:
-        patterns:
-          - "*"
+      interval: "weekly"

.github/workflows/docker.yml

@@ -81,7 +81,6 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
           platforms: linux/amd64,linux/arm64

.github/workflows/gh-pages.yml

@@ -1,64 +0,0 @@
-# Simple workflow for deploying static content to GitHub Pages
-name: Deploy static content to Pages
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  # Single deploy job since we're just deploying
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm -f ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v3
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
-
-      # - name: Checkout
-      #   uses: actions/checkout@v4
-      # - name: Setup Pages
-      #   uses: actions/configure-pages@v5
-      # - name: Upload artifact
-      #   uses: actions/upload-pages-artifact@v3
-      #   with:
-      #     # Upload entire repository
-      #     path: './website'
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4

.gitignore

@@ -11,11 +11,10 @@ venv/
 .venv/
 .docker-venv/
 node_modules/
-typings/
 
 # Ignore dev lockfiles (should always be built fresh)
-pdm.dev.lock
 requirements-dev.txt
+pdm.dev.lock
 
 # Packaging artifacts
 .pdm-python
@@ -26,17 +25,10 @@ build/
 dist/
 
 # Data folders
-lib/
-tmp/
 data/
 data*/
 output/
-index.sqlite3
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
 
 # vim
 *.sw?
-.vscode
+.vscode/

.gitmodules

@@ -1,9 +1,28 @@
 [submodule "docs"]
     path = docs
     url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
+
+[submodule "deb_dist"]
+    path = deb_dist
+    url = https://github.com/ArchiveBox/debian-archivebox.git
+[submodule "brew_dist"]
+	path = brew_dist
+	url = https://github.com/ArchiveBox/homebrew-archivebox.git
+[submodule "pip_dist"]
+	path = pip_dist
+	url = https://github.com/ArchiveBox/pip-archivebox.git
+[submodule "docker"]
+	path = docker
+	url = https://github.com/ArchiveBox/docker-archivebox.git
+[submodule "archivebox/vendor/base32-crockford"]
+	path = archivebox/vendor/base32-crockford
+	url = https://github.com/jbittel/base32-crockford
 [submodule "archivebox/vendor/pocket"]
 	path = archivebox/vendor/pocket
 	url = https://github.com/tapanpandita/pocket
-[submodule "archivebox/vendor/pydantic-pkgr"]
-	path = archivebox/vendor/pydantic-pkgr
-	url = https://github.com/ArchiveBox/pydantic-pkgr
+[submodule "archivebox/vendor/django-taggit"]
+	path = archivebox/vendor/django-taggit
+	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites

.npmignore

@@ -0,0 +1,19 @@
+tests/
+archivebox/
+archivebox.egg-info/
+build/
+dist/
+docs/
+etc/
+.github
+.gitmodules
+.dockerignore
+.flake8
+CNAME
+_config.yml
+docker-compose.yaml
+docker-compose.yml
+Dockerfile
+MANIFEST.in
+Pipfile
+setup.py

.readthedocs.yaml

@@ -1,5 +1,5 @@
-# Read the Docs config for https://docs.archivebox.io
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
 version: 2
 
@@ -7,20 +7,27 @@ submodules:
   include: all
   recursive: true
 
+# Set the OS, Python version and other tools you might need
 build:
   os: ubuntu-22.04
   tools:
     python: "3.12"
-    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
+    nodejs: "20"
 
+# Build documentation in the "docs/" directory with Sphinx
 sphinx:
   configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
 
+# Optionally build your docs in additional formats such as PDF and ePub
 formats:
   - pdf
   - epub
 
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
   install:
     - requirements: docs/requirements.txt

CNAME

@@ -1 +0,0 @@
-archivebox.io

Dockerfile

@@ -10,15 +10,13 @@
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:0.7.5 -t archivebox/archivebox:dev
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
 #
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
-#########################################################################################
 
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
 FROM python:3.11-slim-bookworm
-# FROM debian:bookworm-backports
+# Uses Debian 12 w/ faster-updating apt-lists added below: https://packages.debian.org/bookworm-backports/
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -39,7 +37,7 @@ LABEL name="archivebox" \
     com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
     com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
     com.docker.extension.categories='database,utility-tools'
-
+    
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
@@ -61,34 +59,35 @@ ENV TZ=UTC \
 
 # Version config
 ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=22
+    NODE_VERSION=20
 
 # User config
 ENV ARCHIVEBOX_USER="archivebox" \
     DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
+    DEFAULT_PGID=911
 
 # Global paths
 ENV CODE_DIR=/app \
     DATA_DIR=/data \
+    GLOBAL_VENV=/venv \
     PLAYWRIGHT_BROWSERS_PATH=/browsers
-    # GLOBAL_VENV=/venv \
-    # TODO: add TMP_DIR and LIB_DIR?
+
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
 
 # Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
 
 ######### System Environment ####################################
 
-# Detect ArchiveBox version number by reading pyproject.toml
-COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN grep '^version = ' "${CODE_DIR}/pyproject.toml" | awk -F'"' '{print $2}' > /VERSION.txt
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
 
 # Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
     && rm -f /etc/apt/apt.conf.d/docker-clean
 
 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
@@ -96,6 +95,7 @@ RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
     && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
     && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
     && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
     && echo \
@@ -120,37 +120,43 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
     && mkdir -p /etc/apt/keyrings \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
+        zlib1g-dev dumb-init gosu cron unzip grep ncat \
         # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
+        # nano iputils-ping dnsutils htop procps jq yq \
     && rm -rf /var/lib/apt/lists/*
 
-# Install sonic search backend
-COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
-
 ######### Language Environments ####################################
 
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        nodejs libatomic1 python3-minimal \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
 # Install Python environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
-    # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-    # && apt-get update -qq \
-    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
     # tell PDM to allow using global system python site packages
     # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
     # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
     # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
     # && python3 -m venv --system-site-packages $GLOBAL_VENV \
@@ -165,36 +171,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
-
-# Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment..." \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
-    && apt-get install -y -t bookworm-backports --no-upgrade \
-        nodejs \
-    && rm -rf /var/lib/apt/lists/* \
-    # Update NPM to latest version
-    && npm i -g npm --cache /root/.npm \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-
 ######### Extractor Dependencies ##################################
 
 # Install apt dependencies
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing APT extractor dependencies globally using apt..." \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        curl wget git ffmpeg ripgrep \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        curl wget git yt-dlp ffmpeg ripgrep \
         # Packages we have also needed in the past:
         # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
     && rm -rf /var/lib/apt/lists/* \
@@ -202,35 +186,39 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && ( \
         which curl && curl --version | head -n1 \
         && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
         && which git && git --version 2>&1 | head -n1 \
         && which rg && rg --version 2>&1 | head -n1 \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
-
 # Install chromium browser using playwright
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
         fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
-        # xfonts-scalable xfonts-utils xserver-common xvfb \
         # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
         # libxss1 dbus dbus-x11 upower \
     # && service dbus start \
-    # install Chromium using playwright
-    && pip install playwright \
-    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-    && playwright install chromium \
-    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
+    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
+        # install Chromium using playwright
+        pip install playwright \
+        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+        && playwright install --with-deps chromium \
+        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
+    else \
+        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.)
+        # apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        #     chromium \
+        # && export CHROME_BINARY="$(which chromium)"; \
+        echo 'armv7 no longer supported in versions after v0.7.3' \
+        exit 1; \
+    fi \
     && rm -rf /var/lib/apt/lists/* \
     && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
     && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
     && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
     && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
     # Save version info
@@ -240,26 +228,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     ) | tee -a /VERSION.txt
 
 # Install Node dependencies
-ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
-USER $ARCHIVEBOX_USER
-WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
-RUN --mount=type=cache,target=/home/$ARCHIVEBOX_USER/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
-    echo "[+] Installing NPM extractor dependencies in /home/$ARCHIVEBOX_USER/.npm..." \
-    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
-    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
-        "@postlight/parser@^2.2.3" \
-        "readability-extractor@github:ArchiveBox/readability-extractor" \
-        "single-file-cli@^1.1.54" \
-        "puppeteer@^23.5.0" \
-        "@puppeteer/browsers@^2.4.0"
-USER root
-RUN ( \
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
         which node && node --version \
         && which npm && npm version \
-        && which postlight-parser \
-        && which readability-extractor && readability-extractor --version \
-        && which single-file && single-file --version \
-        && which puppeteer && puppeteer --version \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
@@ -271,11 +247,10 @@ COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        build-essential gcc \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        build-essential \
         libssl-dev libldap2-dev libsasl2-dev \
         python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-        pipx \
     # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
     # && pdm use --venv in-project \
     # && pdm run python -m ensurepip \
@@ -284,33 +259,47 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     # && source $GLOBAL_VENV/bin/activate \
     && pip install -r requirements.txt \
     && apt-get purge -y \
-        build-essential gcc \
+        build-essential \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
 # Install ArchiveBox Python package from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    && pip install -e "${CODE_DIR}[all]" \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    #     build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
 ####################################################
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "/tmp/archivebox" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/tmp/archivebox" \
-    && mkdir -p "/usr/share/archivebox/lib" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/usr/share/archivebox/lib"
-
-ENV GOOGLE_API_KEY=no \
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chromium \
+    GOOGLE_API_KEY=no \
     GOOGLE_DEFAULT_CLIENT_ID=no \
     GOOGLE_DEFAULT_CLIENT_SECRET=no \
-    TMP_DIR=/tmp/archivebox \
-    LIB_DIR=/usr/share/archivebox/lib \
     ALLOWED_HOSTS=*
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
 
 # Print version for nice docker finish summary
 RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \

Dockerfile.simple

@@ -1,21 +0,0 @@
-# Example: Using ArchiveBox in your own project
-
-FROM python:3.12-slim
-
-WORKDIR /app
-
-RUN pip install archivebox==0.8.5rc44
-RUN archivebox install
-
-RUN useradd -ms /bin/bash archivebox && chown -R archivebox /app
-
-
-
-
-
-
-
-
-
-
-

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2020 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -22,7 +22,7 @@ curl -fsSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instruc
 
 **ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**
 
-Without active preservation effort, everything on the internet eventually disappears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
+Without active preservation effort, everything on the internet eventually dissapears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
 
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
@@ -93,7 +93,7 @@ docker run -it -v $PWD:/data archivebox/archivebox init --setup
 pip install archivebox
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init --setup
-# archivebox add 'https://example.com'
+# archviebox add 'https://example.com'
 # archivebox help
 # archivebox server 0.0.0.0:8000
 <br/>
@@ -125,12 +125,12 @@ curl -fsSL 'https://get.archivebox.io' | sh
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful command line interface**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -152,9 +152,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 - **Governments:**
   `snapshoting public service sites`, `recordkeeping compliance`
 
-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))  
-> We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
-> *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 
+> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
+> We offer: setup & support, hosting, custom features, security, hashing & audit logging/chain-of-custody, etc.  
+> *ArchiveBox has 🏛️ 501(c)(3) [nonprofit status](https://hackclub.com/hcb/) and all our work supports open-source development.* 
 
 <br/>
 
@@ -291,8 +291,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
-See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for instructions. ➡️
-<!--<ol>
+<ol>
 <li>Add the ArchiveBox repository to your sources.<br/>
 <pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
@@ -311,7 +310,7 @@ archivebox version                         # make sure all dependencies are inst
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init --setup
 </code></pre>
-<br/>
+<i>Note: If you encounter issues or want more granular instructions, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki.</i><br/><br/>
 </li>
 <li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
@@ -321,8 +320,9 @@ archivebox help
 </code></pre>
 </li>
 </ol>
+
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>-->
+<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
 <br/><br/>
 </details>
 
@@ -407,14 +407,11 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 > *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
 
 <ul>
-<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
+<li>TrueNAS: <a href="https://truecharts.org/charts/incubator/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
 <li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
-<li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
-<li><a href="https://portainer-templates.as93.net/archivebox">Portainer</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/pull/922/files#diff-00f0606e18b2618c3cc1667ca7c2b703b537af690ca71eba1330633587dcb1ee">AppImage</a></li>
-<li><a href="https://runtipi.io/docs/apps-available#:~:text=for%20AI%20Chats.-,ArchiveBox,Open%20source%20self%2Dhosted%20web%20archiving.,-Atuin%20Server">Runtipi</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/issues/986">Umbrel</a> (need contributors...)</li>
 
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -448,9 +445,6 @@ Other providers of paid ArchiveBox hosting (not officially endorsed):<br/>
 <li><a href="https://fly.io/">
  <img src="https://img.shields.io/badge/Unmanaged_App-Fly.io-%239a2de6.svg?style=flat" height="22px"/>
 </a> (USD $10-50+/mo, <a href="https://fly.io/docs/hands-on/start/">instructions</a>)</li>
-<li><a href="https://railway.app/template/2Vvhmy">
- <img src="https://img.shields.io/badge/Unmanaged_App-Railway-%23A11BE6.svg?style=flat" height="22px"/>
-</a> (USD $0-5+/mo)</li>
 <li><a href="https://aws.amazon.com/marketplace/pp/Linnovate-Open-Source-Innovation-Support-For-Archi/B08RVW6MJ2"><img src="https://img.shields.io/badge/Unmanaged_VPS-AWS-%23ee8135.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
 <li><a href="https://azuremarketplace.microsoft.com/en-us/marketplace/apps/meanio.archivebox?ocid=gtmrewards_whatsnewblog_archivebox_vol118"><img src="https://img.shields.io/badge/Unmanaged_VPS-Azure-%237cb300.svg?style=flat" height="22px"/></a> (USD $60-200+/mo)</li>
 <br/>
@@ -569,7 +563,7 @@ ls ./archive/*/index.html  # or inspect snapshot data directly on the filesystem
 <br/>
 
 <details>
-<summary><b>🖥&nbsp; Web UI & API Usage</b></summary>
+<summary><b>🖥&nbsp; Web UI Usage</b></summary>
 <pre lang="bash"><code style="white-space: pre-line">
 # Start the server on bare metal (pip/apt/brew/etc):
 archivebox manage createsuperuser              # create a new admin user via CLI
@@ -758,8 +752,8 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 # e.g. archivebox config --set TIMEOUT=120
 # or   docker compose run archivebox config --set TIMEOUT=120
 <br/>
-TIMEOUT=240                # default: 60    add more seconds on slower networks
-CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
+TIMEOUT=120                # default: 60    add more seconds on slower networks
+CHECK_SSL_VALIDITY=True    # default: False True = allow saving URLs w/ bad SSL
 SAVE_ARCHIVE_DOT_ORG=False # default: True  False = disable Archive.org saving
 MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
 <br/>
@@ -778,7 +772,7 @@ CURL_USER_AGENT="Mozilla/5.0 ..."
 
 To achieve high-fidelity archives in as many situations as possible, ArchiveBox depends on a variety of 3rd-party libraries and tools that specialize in extracting different types of content.
 
-> Under-the-hood, ArchiveBox uses [Django](https://www.djangoproject.com/start/overview/) to power its [Web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Django Ninja](https://django-ninja.dev/) for the REST API, and [SQlite](https://www.sqlite.org/locrsf.html) + the filesystem to provide [fast & durable metadata storage](https://www.sqlite.org/locrsf.html) w/ [deterministic upgrades](https://stackoverflow.com/a/39976321/2156113).
+> Under-the-hood, ArchiveBox uses [Django](https://www.djangoproject.com/start/overview/) to power its [Web UI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage) and [SQlite](https://www.sqlite.org/locrsf.html) + the filesystem to provide [fast & durable metadata storage](https://www.sqlite.org/locrsf.html) w/ [determinisitc upgrades](https://stackoverflow.com/a/39976321/2156113).
 
 ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install), [`wget`, `yt-dlp`, `readability`, etc.](#dependencies) internally, and its operation can be [tuned, secured, and extended](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration) as-needed for many different applications.
 
@@ -787,22 +781,10 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <summary><i>Expand to learn more about ArchiveBox's internals & dependencies...</i></summary><br/>
 
 <blockquote>
-<p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
+<p><em>TIP: For better security, easier updating, and to avoid polluting your host system with extra dependencies,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> with everything pre-installed for the best experience.</em></p>
 </blockquote>
 
-<ul>
-<li>Language: Python <code>&gt;=3.10</code></li>
-<li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
-<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
-<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
-<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
-<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
-<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
-</ul>
-
-
-These optional subdependencies used for archiving sites include:
+These optional dependencies used for archiving sites include:
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right" style="max-width: 100%;">
 
@@ -1026,7 +1008,7 @@ https://127.0.0.1:8000/archive/*
 
 ### Working Around Sites that Block Archiving
 
-For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) actively block archiving or bots in general. There are a number of approaches to work around this, and we also provide <a href="https://docs.monadical.com/s/archivebox-consulting-services">consulting services</a> to help here.
+For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) actively block archiving or bots in general. There are a number of approaches to work around this.
 
 <br/>
 <details>
@@ -1037,7 +1019,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 <ul>
 <li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
 <li>Set up a logged-in browser session for archiving using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile"><code>CHROME_USER_DATA_DIR</code> &amp; <code>COOKIES_FILE</code></a></li>
-<li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
+<li>Rewrite your URLs before archiving to swap in an alternative frontend thats more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
 
@@ -1193,7 +1175,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica
 - A. doesn't deprive the original creators of revenue and
 - B. is responsibly curated by an individual/institution.
 
-In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.
+In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's preservation work is covered under this exemption, as they are as a non-profit providing public service, and they respond to <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests.
 
 As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use sofware like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
 
@@ -1206,25 +1188,21 @@ As long as you A. don't try to profit off pirating copyrighted content and B. ha
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/4cac62a9-e8fb-425b-85a3-ca644aa6dd42" width="5%" align="right" alt="comparison" style="float: right"/> 
 
 
-> **Check out our [community wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) for a list of alternative web archiving tools and orgs.**
+> **Check out our [community wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) for a list of web archiving tools and orgs.**
 
-ArchiveBox gained momentum in the internet archiving industry because it uniquely combines 3 things:
-
-- **it's distributed:** users own their data instead of entrusting it to one big central provider
-- **it's future-proof:** saving in *multiple formats* and extracting out raw TXT, PNG, PDF, MP4, etc. files
-- **it's extensible:** with powerful APIs, flexible storage, and a big community adding new extractors regularly
+A variety of open and closed-source archiving projects exist, but few provide a nice UI and CLI to manage a large, high-fidelity collection over time.
 
 <br/>
 <details>
-<summary><i>Expand for a more direct comparison to Archive.org and specific open-source alternatives...</i></summary><br/>
+<summary><i>Click to read about how we differ from other centralized archiving services and open source tools...</i></summary><br/>
 
-ArchiveBox tries to be a robust, set-and-forget archiving solution suitable for archiving RSS feeds, bookmarks, or your entire browsing history (beware, it may be too big to store), including private/authenticated content that you wouldn't otherwise share with a centralized service like Archive.org.
+ArchiveBox tries to be a robust, set-and-forget archiving solution suitable for archiving RSS feeds, bookmarks, or your entire browsing history (beware, it may be too big to store), including private/authenticated content that you wouldn't otherwise share with a centralized service.
 
 <h3>Comparison With Centralized Public Archives</h3>
 
-Not all content is suitable to be archived on a centralized, publicly accessible platform. Archive.org doesn't offer the ability to save things behind login walls for good reason, as the content may not have been intended for a public audience. ArchiveBox exists to fill that gap by letting everyone save what they have access to on an individual basis, and to encourage decentralized archiving that's less succeptible to censorship or natural disasters.
+Not all content is suitable to be archived in a centralized collection, whether because it's private, copyrighted, too large, or too complex. ArchiveBox hopes to fill that gap.
 
-By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capcity handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.
+By having each user store their own content locally, we can save much larger portions of everyone's browsing history than a shared centralized service would be able to handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other.
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
@@ -1274,7 +1252,7 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
 
 **Need help building a custom archiving solution?**
 
-> ✨ **[Hire the team that built Archivebox](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102) to solve archiving for your org.** ([@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp))
+> ✨ **[Hire the team that built Archivebox](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102) to work on your project.** ([@ArchiveBoxApp](https://twitter.com/ArchiveBoxApp))
 
 <br/>
 
@@ -1287,11 +1265,9 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
 
 <img src="https://read-the-docs-guidelines.readthedocs-hosted.com/_images/logo-dark.png" width="13%" align="right" style="float: right"/>
 
-We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki) for documentation.
+We use the [GitHub wiki system](https://github.com/ArchiveBox/ArchiveBox/wiki) and [Read the Docs](https://archivebox.readthedocs.io/en/latest/) (WIP) for documentation.
 
-<sub>There is also a mirror available on <a href="https://archivebox.readthedocs.io/en/latest/">Read the Docs</a> (though it's sometimes outdated).</sub>
-
-> ✏️ You can submit docs changes & suggestions in our dedicated repo [`ArchiveBox/docs`](https://github.com/ArchiveBox/docs).
+You can also access the docs locally by looking in the [`ArchiveBox/docs/`](https://github.com/ArchiveBox/ArchiveBox/wiki/Home) folder.
 
 ## Getting Started
 
@@ -1302,25 +1278,22 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
 - [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
-- [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced
 
-- [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview)
-- [Cookies & Sessions Setup](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile) (archiving sites that require logins)
-- [Setting up the Search Backends](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Search) (choosing ripgrep, Sonic, or FTS5)
-- [Setting up Local/Remote Storages](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Storage) (S3/B2/Google Drive/SMB/NFS/etc.)
-- [Setting up Authentication & Permissions](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication) (SSO/LDAP/OAuth/API Keys/etc.)
-- [Publishing Your Archive](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive) (sharing your archive server with others)
-- [Chromium Install Options](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install) (installing and configuring ArchiveBox's Chrome)
-- [Upgrading or Merging Archives](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives)
 - [Troubleshooting](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
+- [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
+- [Publishing Your Archive](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive)
+- [Chromium Install](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install)
+- [Cookies & Sessions Setup](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile)
+- [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview)
+- [Upgrading or Merging Archives](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives)
 
 ## Developers
 
 - [Developer Documentation](https://github.com/ArchiveBox/ArchiveBox#archivebox-development)
-- [Python API](https://docs.archivebox.io/)
-- [REST API](https://demo.archivebox.io/api) (alpha)
+- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
+- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)
 
 ## More Info
 
@@ -1365,12 +1338,18 @@ git pull --recurse-submodules
 
 ```bash
 # Install ArchiveBox + python dependencies
-pip install uv
-uv venv
-uv sync
+python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]'
+# or: pipenv install --dev && pipenv shell
 
-archivebox init
+# Install node dependencies
+npm install
+# or
 archivebox setup
+
+# Check to see if anything is missing
+archivebox --version
+# install any missing dependencies manually, or use the helper script:
+./bin/setup.sh
 ```
 
 #### 2. Option B: Build the docker container and use that for development instead
@@ -1407,8 +1386,6 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 archivebox config --set DEBUG=True
 # or
 archivebox server --debug ...
-# faster dev version wo/ bg workers enabled:
-daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```
 
 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1443,12 +1420,12 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data archivebox:dev init --setup
 
 # or with pip:
 pip install 'git+https://github.com/pirate/ArchiveBox@dev'
 npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
-archivebox install
+archivebox init --setup
 ```
 
 </details>
@@ -1542,9 +1519,6 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 
 **The process to contribute a new extractor is like this:**
 
-> [!IMPORTANT]
-> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
-
 1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
 2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
    (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
@@ -1573,6 +1547,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/build_docs.sh
 ./bin/build_pip.sh
+./bin/build_deb.sh
+./bin/build_brew.sh
 ./bin/build_docker.sh
 ```
 
@@ -1589,6 +1565,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/release_docs.sh
 ./bin/release_pip.sh
+./bin/release_deb.sh
+./bin/release_brew.sh
 ./bin/release_docker.sh
 ```
 
@@ -1617,11 +1595,12 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a> &nbsp;
 <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
 <br/>
-<sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
+<sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
 <br/><br/>
 <a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>&nbsp;
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>&nbsp;
-<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/><br/>
+<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/>
+<sup>ArchiveBox was started by <a href="https://docs.sweeting.me/s/blog#About">Nick Sweeting</a> in 2017, and has <a href="https://star-history.com/#archivebox/archivebox&Date">grown steadily</a> with help from our <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">amazing contributors</a>.</sup>
 <hr/>
 <i>✨ Have spare CPU/disk/bandwidth after all your 网站存档爬 and want to help the world?<br/>Check out our <a href="https://github.com/ArchiveBox/good-karma-kit">Good Karma Kit</a>...</i>
 </div>

archivebox/Architecture.md

@@ -1,172 +0,0 @@
-# ArchiveBox UI
-
-## Page: Getting Started
-
-### What do you want to capture?
-
-- Save some URLs now -> [Add page]
-    - Paste some URLs to archive now
-    - Upload a file containing URLs (bookmarks.html export, RSS.xml feed, markdown file, word doc, PDF, etc.)
-    - Pull in URLs to archive from a remote location (e.g. RSS feed URL, remote TXT file, JSON file, etc.)
-
-- Import URLs from a browser -> [Import page]
-    - Desktop: Get the ArchiveBox Chrome/Firefox extension
-    - Mobile: Get the ArchiveBox iOS App / Android App
-    - Upload a bookmarks.html export file
-    - Upload a browser_history.sqlite3 export file
-
-- Import URLs from a 3rd party bookmarking service -> [Sync page]
-    - Pocket
-    - Pinboard
-    - Instapaper
-    - Wallabag
-    - Zapier, N8N, IFTTT, etc.
-    - Upload a bookmarks.html export, bookmarks.json, RSS, etc. file
-
-- Archive URLs on a schedule -> [Schedule page]
-
-- Archive an entire website -> [Crawl page]
-    - What starting URL/domain?
-    - How deep?
-    - Follow links to external domains?
-    - Follow links to parent URLs?
-    - Maximum number of pages to save?
-    - Maximum number of requests/minute?
-
-- Crawl for URLs with a search engine and save automatically
-    - 
-- Some URLs on a schedule
-- Save an entire website (e.g. `https://example.com`)
-- Save results matching a search query (e.g. "site:example.com")
-- Save a social media feed (e.g. `https://x.com/user/1234567890`)
-
---------------------------------------------------------------------------------
-
-### Crawls App
-
-- Archive an entire website -> [Crawl page]
-    - What are the seed URLs?
-    - How many hops to follow?
-    - Follow links to external domains?
-    - Follow links to parent URLs?
-    - Maximum number of pages to save?
-    - Maximum number of requests/minute?
-
-
---------------------------------------------------------------------------------
-
-### Scheduler App
-
-
-- Archive URLs on a schedule -> [Schedule page]
-    - What URL(s)?
-    - How often?
-    - Do you want to discard old snapshots after x amount of time?
-    - Any filter rules?
-    - Want to be notified when changes are detected -> redirect[Alerts app/create new alert(crawl=self)]
-
-
-* Choose Schedule check for new URLs: Schedule.objects.get(pk=xyz)
-    - 1 minute
-    - 5 minutes
-    - 1 hour
-    - 1 day
-
-    * Choose Destination Crawl to archive URLs using : Crawl.objects.get(pk=xyz)
-        - Tags
-        - Persona
-        - Created By ID
-        - Config
-        - Filters
-            - URL patterns to include
-            - URL patterns to exclude
-            - ONLY_NEW= Ignore URLs if already saved once / save URL each time it appears / only save is last save > x time ago
-
-
---------------------------------------------------------------------------------
-
-### Sources App (For managing sources that ArchiveBox pulls URLs in from)
-
-- Add a new source to pull URLs in from (WIZARD)
-    - Choose URI:
-        - [x] Web UI
-        - [x] CLI
-        - Local filesystem path (directory to monitor for new files containing URLs)
-        - Remote URL (RSS/JSON/XML feed)
-        - Chrome browser profile sync (login using gmail to pull bookmarks/history)
-        - Pocket, Pinboard, Instapaper, Wallabag, etc.
-        - Zapier, N8N, IFTTT, etc.
-        - Local server filesystem path (directory to monitor for new files containing URLs)
-        - Google drive (directory to monitor for new files containing URLs)
-        - Remote server FTP/SFTP/SCP path (directory to monitor for new files containing URLs)
-        - AWS/S3/B2/GCP bucket (directory to monitor for new files containing URLs)
-        - XBrowserSync (login to pull bookmarks)
-    - Choose extractor
-        - auto
-        - RSS
-        - Pocket
-        - etc.
-    - Specify extra Config, e.g.
-        - credentials
-        - extractor tuning options (e.g. verify_ssl, cookies, etc.)
-
-- Provide credentials for the source
-    - API Key
-    - Username / Password
-    - OAuth
-
---------------------------------------------------------------------------------
-
-### Alerts App
-
-- Create a new alert, choose condition
-    - Get notified when a site goes down (<x% success ratio for Snapshots)
-    - Get notified when a site changes visually more than x% (screenshot diff)
-    - Get notified when a site's text content changes more than x% (text diff)
-    - Get notified when a keyword appears
-    - Get notified when a keyword dissapears
-    - When an AI prompt returns some result
-
-- Choose alert threshold:
-    - any condition is met
-    - all conditions are met
-    - condition is met for x% of URLs
-    - condition is met for x% of time
-
-- Choose how to notify: (List[AlertDestination])
-    - maximum alert frequency
-    - destination type: email / Slack / Webhook / Google Sheet / logfile
-    - destination info:
-        - email address(es)
-        - Slack channel
-        - Webhook URL
-
-- Choose scope:
-    - Choose ArchiveResult scope (extractors): (a query that returns ArchiveResult.objects QuerySet)
-        - All extractors
-        - Only screenshots
-        - Only readability / mercury text
-        - Only video
-        - Only html
-        - Only headers
-
-    - Choose Snapshot scope (URL): (a query that returns Snapshot.objects QuerySet)
-        - All domains
-        - Specific domain
-        - All domains in a tag
-        - All domains in a tag category
-        - All URLs matching a certain regex pattern
-
-    - Choose crawl scope: (a query that returns Crawl.objects QuerySet)
-        - All crawls
-        - Specific crawls
-        - crawls by a certain user
-        - crawls using a certain persona
-
-
-class AlertDestination(models.Model):
-    destination_type: [email, slack, webhook, google_sheet, local logfile, b2/s3/gcp bucket, etc.]
-    maximum_frequency
-    filter_rules
-    credentials
-    alert_template: JINJA2 json/text template that gets populated with alert contents

archivebox/LICENSE

@@ -0,0 +1 @@
+../LICENSE

archivebox/__init__.py

@@ -1,106 +1 @@
-#!/usr/bin/env python3
-
-# Welcome to the ArchiveBox source code! Thanks for checking it out!
-#
-# "We are swimming upstream against a great torrent of disorganization.
-# In this, our main obligation is to establish arbitrary enclaves of order and system.
-# It is the greatest possible victory to be, to continue to be, and to have been.
-# No defeat can deprive us of the success of having existed for some moment of time
-# in a universe that seems indifferent to us."
-# --Norber Weiner
-
 __package__ = 'archivebox'
-
-import os
-import sys
-
-from pathlib import Path
-
-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
-
-# make sure PACKAGE_DIR is in sys.path so we can import all subfolders
-# without necessarily waiting for django to load them thorugh INSTALLED_APPS
-PACKAGE_DIR = Path(__file__).resolve().parent
-if str(PACKAGE_DIR) not in sys.path:
-    sys.path.append(str(PACKAGE_DIR))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
-os.environ['TZ'] = 'UTC'
-
-# detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges                 # noqa
-drop_privileges()
-
-from .misc.checks import check_not_root, check_io_encoding      # noqa
-check_not_root()
-check_io_encoding()
-
-# print('INSTALLING MONKEY PATCHES')
-from .monkey_patches import *                    # noqa
-# print('DONE INSTALLING MONKEY PATCHES')
-
-
-# print('LOADING VENDORED LIBRARIES')
-from .vendor import load_vendored_libs           # noqa
-load_vendored_libs()
-# print('DONE LOADING VENDORED LIBRARIES')
-
-
-from .config.constants import CONSTANTS                         # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
-from .config.version import VERSION                             # noqa
-
-__version__ = VERSION
-__author__ = 'ArchiveBox'
-__license__ = 'MIT'
-
-ASCII_ICON = """
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ████████████████████████████████████████████████████████████████████████████████          
-"""

archivebox/__main__.py

@@ -1,19 +1,11 @@
 #!/usr/bin/env python3
-"""This is the main entry point for the ArchiveBox CLI."""
+
 __package__ = 'archivebox'
 
-import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys
 
 from .cli import main
 
-ASCII_LOGO_MINI = r"""
-     _             _     _           ____            
-    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
-   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  < 
- /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
-"""
 
 if __name__ == '__main__':
     main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/abid_utils/__init__.py

@@ -1 +0,0 @@
-__package__ = 'abid_utils'

archivebox/abid_utils/abid.py

@@ -1,223 +0,0 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import NamedTuple, Any, Union, Dict
-
-import ulid
-import uuid6
-import hashlib
-from urllib.parse import urlparse
-
-from uuid import UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-
-from archivebox.misc.util import enforce_types
-
-
-ABID_PREFIX_LEN = 4
-ABID_SUFFIX_LEN = 26
-ABID_LEN = 30
-ABID_TS_LEN = 10
-ABID_URI_LEN = 8
-ABID_SUBTYPE_LEN = 2
-ABID_RAND_LEN = 6
-
-DEFAULT_ABID_PREFIX = 'obj_'
-
-# allows people to keep their uris secret on a per-instance basis by changing the salt.
-# the default means everyone can share the same namespace for URI hashes,
-# meaning anyone who has a URI and wants to check if you have it can guess the ABID
-DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
-
-
-class ABID(NamedTuple):
-    """
-    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
-    """
-    prefix: str            # e.g. obj_
-    ts: str                # e.g. 01HX9FPYTR
-    uri: str               # e.g. E4A5CCD9
-    subtype: str           # e.g. 01
-    rand: str              # e.g. ZYEBQE
-    
-    # salt: str = DEFAULT_ABID_URI_SALT
-
-    def __getattr__(self, attr: str) -> Any:
-        return getattr(self.ulid, attr)
-
-    def __eq__(self, other: Any) -> bool:
-        try:
-            return self.ulid == other.ulid
-        except AttributeError:
-            return NotImplemented
-
-    def __str__(self) -> str:
-        return self.prefix + self.suffix
-
-    def __len__(self) -> int:
-        return len(self.prefix + self.suffix)
-
-    @classmethod
-    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
-        assert buffer, f'Attempted to create ABID from null value {buffer}'
-
-        buffer = str(buffer)
-        if '_' in buffer:
-            prefix, suffix = buffer.split('_')
-        else:
-            prefix, suffix = prefix.strip('_'), buffer
-
-        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
-        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
-
-        return cls(
-            prefix=abid_part_from_prefix(prefix),
-            ts=suffix[0:10].upper(),
-            uri=suffix[10:18].upper(),
-            subtype=suffix[18:20].upper(),
-            rand=suffix[20:26].upper(),
-        )
-    
-    @property
-    def uri_salt(self) -> str:
-        return DEFAULT_ABID_URI_SALT
-
-    @property
-    def suffix(self):
-        return ''.join((self.ts, self.uri, self.subtype, self.rand))
-    
-    @property
-    def ulid(self) -> ulid.ULID:
-        return ulid.parse(self.suffix)
-
-    @property
-    def uuid(self) -> UUID:
-        return self.ulid.uuid
-
-    @property
-    def uuid6(self) -> uuid6.UUID:
-        return uuid6.UUID(hex=self.uuid.hex)
-
-    @property
-    def typeid(self) -> TypeID:
-        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
-
-    @property
-    def datetime(self) -> datetime:
-        return self.ulid.timestamp().datetime
-
-
-
-####################################################
-
-
-@enforce_types
-def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
-    """
-    if isinstance(uri, bytes):
-        uri_str: str = uri.decode()
-    else:
-        uri_str = str(uri)
-
-    # only hash the domain part of URLs
-    if '://' in uri_str:
-        try:
-            domain = urlparse(uri_str).netloc
-            if domain:
-                uri_str = domain
-        except AttributeError:
-            pass
-    
-    # the uri hash is the sha256 of the domain + salt
-    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
-
-    return hashlib.sha256(uri_bytes).hexdigest().upper()
-
-@enforce_types
-def abid_part_from_prefix(prefix: str) -> str:
-    """
-    'snp_'
-    """
-    # if prefix is None:
-    #     return 'obj_'
-
-    prefix = prefix.strip('_').lower()
-    assert len(prefix) == 3
-    return prefix + '_'
-
-@enforce_types
-def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9'     # takes first 8 characters of sha256(url)
-    """
-    uri = str(uri).strip()
-    assert uri not in ('None', '')
-    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
-
-@enforce_types
-def abid_part_from_ts(ts: datetime) -> str:
-    """
-    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
-    """
-    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
-
-@enforce_types
-def ts_from_abid(abid: str) -> datetime:
-    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
-
-@enforce_types
-def abid_part_from_subtype(subtype: str | int) -> str:
-    """
-    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
-    Also allows us to change the ulid spec later by putting special sigil values here.
-    """
-    subtype = str(subtype)
-    if len(subtype) == ABID_SUBTYPE_LEN:
-        return subtype
-
-    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
-
-@enforce_types
-def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
-    """
-    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
-    """
-    if rand is None:
-        # if it's None we generate a new random 6 character hex string
-        return str(ulid.new())[-ABID_RAND_LEN:]
-    elif isinstance(rand, UUID):
-        # if it's a uuid we take the last 6 characters of the ULID represation of it
-        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
-    elif isinstance(rand, int):
-        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
-        rand_str = str(rand)[-ABID_RAND_LEN:]
-        padding_needed = ABID_RAND_LEN - len(rand_str)
-        rand_str = ('0'*padding_needed) + rand_str
-        return rand_str
-
-    # otherwise treat it as a string, take the last 6 characters of it verbatim
-    return str(rand)[-ABID_RAND_LEN:].upper()
-
-
-@enforce_types
-def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
-    return {
-        'prefix': abid_part_from_prefix(prefix),
-        'ts': abid_part_from_ts(ts),
-        'uri': abid_part_from_uri(uri, salt=salt),
-        'subtype': abid_part_from_subtype(subtype),
-        'rand': abid_part_from_rand(rand),
-        # 'salt': don't add this, salt combined with uri above to form a single hash
-    }
-
-@enforce_types
-def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
-    """
-    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-    """
-
-    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
-    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
-    return abid

archivebox/abid_utils/admin.py

@@ -1,174 +0,0 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import Any
-
-from django.contrib import admin, messages
-from django.core.exceptions import ValidationError
-from django.utils.html import format_html
-from django.utils.safestring import mark_safe
-from django.shortcuts import redirect
-
-from django_object_actions import DjangoObjectActions, action
-
-from archivebox.misc.util import parse_date
-
-from .abid import ABID
-
-
-def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
-    """highlight each character in red that differs with the char at the same index in compare_val"""
-
-    display_val = str(display_val)
-    compare_val = str(compare_val)
-
-    if len(compare_val) < len(display_val):
-        compare_val += ' ' * (len(display_val) - len(compare_val))
-
-    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
-    if invert:
-        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
-
-    return mark_safe(''.join(
-        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
-        if display_val[i] != compare_val[i] else
-        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
-        for i in range(len(display_val))
-    ))
-
-def get_abid_info(self, obj, request=None):
-    from archivebox.api.auth import get_or_create_api_token
-    
-    try:
-        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
-
-        fresh_values = obj.ABID_FRESH_VALUES
-        fresh_hashes = obj.ABID_FRESH_HASHES
-        fresh_diffs = obj.ABID_FRESH_DIFFS
-        fresh_abid = ABID(**fresh_hashes)
-        
-        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
-        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
-
-        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
-
-        fresh_ts = parse_date(fresh_values['ts']) or None
-        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
-
-        derived_uri = fresh_hashes['uri']
-        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
-
-        derived_subtype = fresh_hashes['subtype']
-        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
-
-        derived_rand = fresh_hashes['rand']
-        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
-
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-            <br/><hr/>
-            <div style="opacity: 0.8">
-            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
-            <hr/>
-            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
-            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
-            <br/><hr/>
-            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
-            </div>
-            ''',
-            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
-            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
-            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
-            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
-            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
-            # str(fresh_abid), mark_safe(fresh_abid_diff),
-            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
-            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
-            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
-            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
-            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
-            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
-            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
-        )
-    except Exception as e:
-        # import ipdb; ipdb.set_trace()
-        return str(e)
-
-
-class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid')
-    sort_fields = ('created_at', 'created_by', 'abid')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    # fields = [*readonly_fields]
-    
-    change_actions = ("regenerate_abid",)
-    # changelist_actions = ("regenerate_abid",)
-
-    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
-        try:
-            object_pk = self.model.id_from_abid(object_id)
-            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
-        except (self.model.DoesNotExist, ValidationError):
-            pass
-        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
-    
-    def queryset(self, request):
-        self.request = request
-        return super().queryset(request)                                                # type: ignore
-    
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
-
-    def get_form(self, request, obj=None, **kwargs):
-        self.request = request
-        form = super().get_form(request, obj, **kwargs)
-        if 'created_by' in form.base_fields:
-            form.base_fields['created_by'].initial = request.user
-            
-        if obj:
-            if obj.ABID_FRESH_DIFFS:
-                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
-
-        return form
-
-    def get_formset(self, request, formset=None, obj=None, **kwargs):
-        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
-        formset.form.base_fields['created_at'].disabled = True
-        
-        return formset
-
-    def save_model(self, request, obj, form, change):
-        self.request = request
-
-        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
-
-        super().save_model(request, obj, form, change)
-        obj.refresh_from_db()
-
-        new_abid = obj.abid
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        # import ipdb; ipdb.set_trace()
-
-    @admin.display(description='API Identifiers')
-    def abid_info(self, obj):
-        return get_abid_info(self, obj, request=self.request)
-
-    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
-    def regenerate_abid(self, request, obj):
-        old_abid = str(obj.abid)
-        obj.abid = obj.issue_new_abid(overwrite=True)
-        obj.save()
-        obj.refresh_from_db()
-        new_abid = str(obj.abid)
-
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        else:
-            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")

archivebox/abid_utils/apps.py

@@ -1,7 +0,0 @@
-from django.apps import AppConfig
-
-
-class AbidUtilsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    
-    name = 'abid_utils'

archivebox/abid_utils/models.py

@@ -1,508 +0,0 @@
-"""
-This file provides the Django ABIDField and ABIDModel base model to inherit from.
-"""
-
-
-from typing import Any, Dict, Union, List, Set, cast
-
-from uuid import uuid4
-from functools import partial
-from charidfield import CharIDField  # type: ignore[import-untyped]
-
-from django.contrib import admin
-from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
-from django.db import models
-from django.utils import timezone
-from django.db.utils import OperationalError
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from .abid import (
-    ABID,
-    ABID_LEN,
-    ABID_RAND_LEN,
-    ABID_SUFFIX_LEN,
-    DEFAULT_ABID_PREFIX,
-    DEFAULT_ABID_URI_SALT,
-    abid_part_from_prefix,
-    abid_hashes_from_values,
-    ts_from_abid,
-    abid_part_from_ts,
-)
-
-####################################################
-
-
-# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
-ABIDField = partial(
-    CharIDField,
-    max_length=ABID_LEN,
-    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
-    default=None,
-    null=True,
-    blank=True,
-    db_index=True,
-    unique=True,
-)
-
-def get_or_create_system_user_pk(username='system'):
-    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
-
-    User = get_user_model()
-
-    # if only one user exists total, return that user
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
-
-    # otherwise, create a dedicated "system" user
-    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    # def pre_save(self, model_instance, add):
-    #     return timezone.now()
-    pass
-
-class ABIDError(Exception):
-    pass
-
-
-class ABIDModel(models.Model):
-    """
-    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
-    """
-    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
-    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
-    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
-    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
-    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
-    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
-    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
-
-    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    # abid = ABIDField(prefix=abid_prefix)
-
-    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    # modified_at = models.DateTimeField(auto_now=True)
-
-    _prefetched_objects_cache: Dict[str, Any]
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @admin.display(description='Summary')
-    def __str__(self) -> str:
-        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
-        super().__init__(*args, **kwargs)
-        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
-        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
-        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
-        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
-        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
-
-    def clean(self, abid_drift_allowed: bool | None=None) -> None:
-        if self._state.adding:
-            # only runs once when a new object is first saved to the DB
-            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
-            self._previous_abid = None
-            self.abid = str(self.issue_new_abid())
-
-        else:
-            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
-            abid_diffs = self.ABID_FRESH_DIFFS
-            if abid_diffs:
-                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
-
-                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
-                full_summary = (
-                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
-                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
-                )
-
-                change_error = ValidationError({
-                    **{
-                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
-                        diff['abid_src'].replace('self.', '')
-                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
-                            else NON_FIELD_ERRORS
-                        : ValidationError(
-                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
-                            code='ABIDConflict',
-                            params=diff,
-                        )
-                        for diff in abid_diffs.values()
-                    },
-                    NON_FIELD_ERRORS: ValidationError(full_summary),
-                })
-
-                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
-                if allowed_to_invalidate_abid:
-                    print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
-                    print(change_error)
-                    print('--------------------------------------------------------------------------------------------------')
-                else:
-                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
-                    print(change_error)
-                    print('--------------------------------------------------------------------------------------------------')
-                    raise change_error
-
-    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
-        """Overriden save method ensures new ABID is generated while a new object is first saving."""
-
-        self.clean(abid_drift_allowed=abid_drift_allowed)
-
-        return super().save(*args, **kwargs)
-    
-    @classmethod
-    def id_from_abid(cls, abid: str) -> str:
-        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
-
-    @property
-    def ABID_SOURCES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        assert self.abid_prefix
-        return {
-            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
-            'ts': self.abid_ts_src,
-            'uri': self.abid_uri_src,
-            'subtype': self.abid_subtype_src,
-            'rand': self.abid_rand_src,
-            'salt': 'self.abid_salt',               # defined as static class vars at build time
-        }
-
-    @property
-    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        abid_sources = self.ABID_SOURCES
-        assert all(src != 'None' for src in abid_sources.values())
-        return {
-            'prefix': eval(abid_sources['prefix']),
-            'ts': eval(abid_sources['ts']),
-            'uri': eval(abid_sources['uri']),
-            'subtype': eval(abid_sources['subtype']),
-            'rand': eval(abid_sources['rand']),
-            'salt': eval(abid_sources['salt']),
-        }
-    
-    @property
-    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
-        abid_values = self.ABID_FRESH_VALUES
-        assert all(val for val in abid_values.values())
-        return abid_hashes_from_values(
-            prefix=abid_values['prefix'],
-            ts=abid_values['ts'],
-            uri=abid_values['uri'],
-            subtype=abid_values['subtype'],
-            rand=abid_values['rand'],
-            salt=abid_values['salt'],
-        )
-    
-    @property
-    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
-        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
-        existing_abid = self.ABID
-        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
-        abid_sources = self.ABID_SOURCES
-        fresh_values = self.ABID_FRESH_VALUES
-        fresh_hashes = self.ABID_FRESH_HASHES
-        return {
-            key: {
-                'key': key,
-                'model': self.__class__.__name__,
-                'pk': self.pk,
-                'abid_src': abid_sources[key],
-                'old_val': existing_values.get(key, None),
-                'old_hash': getattr(existing_abid, key),
-                'new_val': fresh_values[key],
-                'new_hash': new_hash,
-                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
-            }
-            for key, new_hash in fresh_hashes.items()
-            if getattr(existing_abid, key) != new_hash
-        }
-
-    def issue_new_abid(self, overwrite=False) -> ABID:
-        """
-        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
-        """
-        if not overwrite:
-            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
-        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
-
-        # Setup Field defaults to be ready for ABID generation
-        self.abid = None
-        self.id = self.id or uuid4()
-        self.pk = self.id
-        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
-        self.modified_at = self.modified_at or self.created_at
-        self.created_by_id = (hasattr(self, 'created_by_id') and self.created_by_id) or get_or_create_system_user_pk()
-        
-        # Compute fresh ABID values & hashes based on object's live properties
-        abid_fresh_values = self.ABID_FRESH_VALUES
-        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
-        abid_fresh_hashes = self.ABID_FRESH_HASHES
-        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
-        
-        new_abid = ABID(**abid_fresh_hashes)
-        
-        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
-
-        return new_abid
-
-    @property
-    def ABID(self) -> ABID:
-        """
-        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
-        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
-        """
-
-        if self.abid:
-            return ABID.parse(cast(str, self.abid))
-        
-        return self.issue_new_abid()
-
-    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
-    # @property
-    # def UUID(self) -> UUID:
-    #     """
-    #     Get a uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return self.ABID.uuid
-    
-    # @property
-    # def uuid(self) -> str:
-    #     """
-    #     Get a str uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return str(self.ABID.uuid)
-    
-    # @property
-    # def ULID(self) -> ULID:
-    #     """
-    #     Get a ulid.ULID representation of the object's ABID.
-    #     """
-    #     return self.ABID.ulid
-
-    # @property
-    # def TypeID(self) -> TypeID:
-    #     """
-    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-    #     """
-    #     return self.ABID.typeid
-    
-    @property
-    def api_url(self) -> str:
-        """
-        Compute the REST API URL to access this object.
-        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
-        """
-        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
-
-    @property
-    def api_docs_url(self) -> str:
-        """
-        Compute the REST API Documentation URL to learn about accessing this object.
-        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
-        """
-        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    def get_absolute_url(self):
-        return self.api_docs_url
-
-
-
-class ModelWithHealthStats(models.Model):
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-    
-    class Meta:
-        abstract = True
-    
-    def record_health_failure(self) -> None:
-        self.num_uses_failed += 1
-        self.save()
-
-    def record_health_success(self) -> None:
-        self.num_uses_succeeded += 1
-        self.save()
-        
-    def reset_health(self) -> None:
-        # move all the failures to successes when resetting so we dont lose track of the total count
-        self.num_uses_succeeded = self.num_uses_failed + self.num_uses_succeeded
-        self.num_uses_failed = 0
-        self.save()
-        
-    @property
-    def health(self) -> int:
-        total_uses = max((self.num_uses_failed + self.num_uses_succeeded, 1))
-        success_pct = (self.num_uses_succeeded / total_uses) * 100
-        return round(success_pct)
-
-
-
-
-
-
-
-
-
-
-####################################################
-
-# Django helpers
-def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
-    """
-    Return the mapping of all ABID prefixes to their models.
-    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
-    """
-    import django.apps
-    prefix_map = {}
-
-    for model in django.apps.apps.get_models():
-        abid_prefix = getattr(model, 'abid_prefix', None)
-        if abid_prefix:
-            prefix_map[abid_prefix] = model
-    return prefix_map
-
-def find_prefix_for_abid(abid: ABID) -> str:
-    """
-    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
-    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
-    """
-    # if existing abid prefix is correct, lookup is easy
-    model = find_model_from_abid(abid)
-    if model:
-        assert issubclass(model, ABIDModel)
-        return model.abid_prefix
-
-    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
-    return find_obj_from_abid_rand(abid)[0].abid_prefix
-
-def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
-    """
-    Return the Django Model that corresponds to a given ABID prefix.
-    e.g. 'tag_' -> core.models.Tag
-    """
-    prefix = abid_part_from_prefix(prefix)
-
-    import django.apps
-
-    for model in django.apps.apps.get_models():
-        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
-        if not hasattr(model, 'objects'): continue      # skip abstract models
-
-        if (model.abid_prefix == prefix):
-            return model
-
-    return None
-
-def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
-    """
-    Shortcut for find_model_from_abid_prefix(abid.prefix)
-    """
-    return find_model_from_abid_prefix(abid.prefix)
-
-def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
-    """
-    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
-    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    Honestly should only be used for debugging, no reason to expose this ability to users.
-    """
-
-    # convert str to ABID if necessary
-    if isinstance(rand, ABID):
-        abid: ABID = rand
-    else:
-        rand = str(rand)
-        if len(rand) < ABID_SUFFIX_LEN:
-            padding_needed = ABID_SUFFIX_LEN - len(rand)
-            rand = ('0'*padding_needed) + rand
-        abid = ABID.parse(rand)
-
-    import django.apps
-
-    partial_matches: List[ABIDModel] = []
-
-    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
-        model,
-        find_model_from_abid(abid),
-        *django.apps.apps.get_models(),
-    ))))
-    # print(abid, abid.rand, abid.uuid, models_to_try)
-
-    for model in models_to_try:
-        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
-        if not hasattr(model, 'objects'): continue      # skip abstract Models
-        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
-
-        # continue on to try fuzzy searching by randomness portion derived from uuid field
-        try:
-            qs = []
-            if hasattr(model, 'abid'):
-                qs = model.objects.filter(abid__endswith=abid.rand)
-            elif hasattr(model, 'uuid'):
-                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-            elif hasattr(model, 'id'):
-                # NOTE: this only works on SQLite where every column is a string
-                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
-                
-                # try to search for uuid=...-2354352
-                # try to search for id=...2354352
-                # try to search for id=2354352
-                qs = model.objects.filter(
-                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-                    | models.Q(id__endswith=abid.rand)
-                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
-                )
-
-            for obj in qs:
-                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
-                    # found exact match, no need to keep iterating
-                    return [obj]
-                partial_matches.append(obj)
-        except OperationalError as err:
-            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
-
-    return partial_matches
-
-def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
-    """
-    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
-    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    model = model or find_model_from_abid(abid)
-    assert model, f'Could not find model that could match this ABID type: {abid}'
-
-    try:
-        if hasattr(model, 'abid'):
-            return model.objects.get(abid__endswith=abid.suffix)
-        if hasattr(model, 'uuid'):
-            return model.objects.get(uuid=abid.uuid)
-        return model.objects.get(id=abid.uuid)
-    except model.DoesNotExist:
-        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
-        if hasattr(model, 'abid') or (not fuzzy):
-            raise
-
-    # continue on to try fuzzy searching by randomness portion derived from uuid field
-    match_by_rand = find_obj_from_abid_rand(abid, model=model)
-    if match_by_rand:
-        if match_by_rand[0].abid_prefix != abid.prefix:
-            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
-        return match_by_rand
-
-    raise model.DoesNotExist
-

archivebox/abid_utils/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/abx/__init__.py

@@ -1,131 +0,0 @@
-__package__ = 'abx'
-
-import importlib
-from pathlib import Path
-from typing import Dict, Callable, List
-
-from . import hookspec as base_spec
-from abx.hookspec import hookimpl, hookspec           # noqa
-from abx.manager import pm, PluginManager             # noqa
-
-
-pm.add_hookspecs(base_spec)
-
-
-###### PLUGIN DISCOVERY AND LOADING ########################################################
-
-def get_plugin_order(plugin_entrypoint: Path):
-    order = 999
-    try:
-        # if .plugin_order file exists, use it to set the load priority
-        order = int((plugin_entrypoint.parent / '.plugin_order').read_text())
-    except FileNotFoundError:
-        pass
-    return (order, plugin_entrypoint)
-
-def register_hookspecs(hookspecs: List[str]):
-    """
-    Register all the hookspecs from a list of module names.
-    """
-    for hookspec_import_path in hookspecs:
-        hookspec_module = importlib.import_module(hookspec_import_path)
-        pm.add_hookspecs(hookspec_module)
-
-
-def find_plugins_in_dir(plugins_dir: Path, prefix: str) -> Dict[str, Path]:
-    """
-    Find all the plugins in a given directory. Just looks for an __init__.py file.
-    """
-    return {
-        f"{prefix}.{plugin_entrypoint.parent.name}": plugin_entrypoint.parent
-        for plugin_entrypoint in sorted(plugins_dir.glob("*/__init__.py"), key=get_plugin_order)
-        if plugin_entrypoint.parent.name != 'abx'
-    }   # "plugins_pkg.pip": "/app/archivebox/plugins_pkg/pip"
-
-
-def get_pip_installed_plugins(group='abx'):
-    """replaces pm.load_setuptools_entrypoints("abx"), finds plugins that registered entrypoints via pip"""
-    import importlib.metadata
-
-    DETECTED_PLUGINS = {}   # module_name: module_dir_path
-    for dist in list(importlib.metadata.distributions()):
-        for entrypoint in dist.entry_points:
-            if entrypoint.group != group or pm.is_blocked(entrypoint.name):
-                continue
-            DETECTED_PLUGINS[entrypoint.name] = Path(entrypoint.load().__file__).parent
-            # pm.register(plugin, name=ep.name)
-            # pm._plugin_distinfo.append((plugin, DistFacade(dist)))
-    return DETECTED_PLUGINS
-
-
-def get_plugins_in_dirs(plugin_dirs: Dict[str, Path]):
-    """
-    Get the mapping of dir_name: {plugin_id: plugin_dir} for all plugins in the given directories.
-    """
-    DETECTED_PLUGINS = {}
-    for plugin_prefix, plugin_dir in plugin_dirs.items():
-        DETECTED_PLUGINS.update(find_plugins_in_dir(plugin_dir, prefix=plugin_prefix))
-    return DETECTED_PLUGINS
-
-
-# Load all plugins from pip packages, archivebox built-ins, and user plugins
-
-def load_plugins(plugins_dict: Dict[str, Path]):
-    """
-    Load all the plugins from a dictionary of module names and directory paths.
-    """
-    LOADED_PLUGINS = {}
-    for plugin_module, plugin_dir in plugins_dict.items():
-        # print(f'Loading plugin: {plugin_module} from {plugin_dir}')
-        plugin_module_loaded = importlib.import_module(plugin_module)
-        pm.register(plugin_module_loaded)
-        LOADED_PLUGINS[plugin_module] = plugin_module_loaded.PLUGIN
-        # print(f'    √ Loaded plugin: {plugin_module}')
-    return LOADED_PLUGINS
-
-def get_registered_plugins():
-    """
-    Get all the plugins registered with Pluggy.
-    """
-    plugins = {}
-    plugin_to_distinfo = dict(pm.list_plugin_distinfo())
-    for plugin in pm.get_plugins():
-        plugin_info = {
-            "name": plugin.__name__,
-            "hooks": [h.name for h in pm.get_hookcallers(plugin) or ()],
-        }
-        distinfo = plugin_to_distinfo.get(plugin)
-        if distinfo:
-            plugin_info["version"] = distinfo.version
-            plugin_info["name"] = (
-                getattr(distinfo, "name", None) or distinfo.project_name
-            )
-        plugins[plugin_info["name"]] = plugin_info
-    return plugins
-
-
-
-
-def get_plugin_hooks(plugin_pkg: str | None) -> Dict[str, Callable]:
-    """
-    Get all the functions marked with @hookimpl on a module.
-    """
-    if not plugin_pkg:
-        return {}
-    
-    hooks = {}
-    
-    plugin_module = importlib.import_module(plugin_pkg)
-    for attr_name in dir(plugin_module):
-        if attr_name.startswith('_'):
-            continue
-        try:
-            attr = getattr(plugin_module, attr_name)
-            if isinstance(attr, Callable):
-                hooks[attr_name] = None
-                pm.parse_hookimpl_opts(plugin_module, attr_name)
-                hooks[attr_name] = attr
-        except Exception as e:
-            print(f'Error getting hookimpls for {plugin_pkg}: {e}')
-
-    return hooks

archivebox/abx/archivebox/__init__.py

@@ -1,30 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import os
-import importlib
-
-from typing import Dict
-from pathlib import Path
-
-
-def load_archivebox_plugins(pm, plugins_dict: Dict[str, Path]):
-    """Load archivebox plugins, very similar to abx.load_plugins but it looks for a pydantic PLUGIN model + hooks in apps.py"""
-    LOADED_PLUGINS = {}
-    for plugin_module, plugin_dir in reversed(plugins_dict.items()):
-        # print(f'Loading plugin: {plugin_module} from {plugin_dir}')
-        
-        # 1. register the plugin module directly in case it contains any look hookimpls (e.g. in __init__.py)
-        try:
-            plugin_module_loaded = importlib.import_module(plugin_module)
-            pm.register(plugin_module_loaded)
-        except Exception as e:
-            print(f'Error registering plugin: {plugin_module} - {e}')
-            
-        
-        # 2. then try to import plugin_module.apps as well
-        if os.access(plugin_dir / 'apps.py', os.R_OK):
-            plugin_apps = importlib.import_module(plugin_module + '.apps')
-            pm.register(plugin_apps)                                           # register the whole .apps  in case it contains loose hookimpls (not in a class)
-            
-        # print(f'    √ Loaded plugin: {plugin_module} {len(archivebox_plugins_found) * "🧩"}')
-    return LOADED_PLUGINS

archivebox/abx/archivebox/base_binary.py

@@ -1,117 +0,0 @@
-__package__ = "abx.archivebox"
-
-import os
-from typing import Optional, cast
-from typing_extensions import Self
-
-from pydantic import validate_call
-from pydantic_pkgr import (
-    Binary,
-    BinProvider,
-    BinProviderName,
-    AptProvider,
-    BrewProvider,
-    EnvProvider,
-)
-
-from archivebox.config import CONSTANTS
-from archivebox.config.permissions import ARCHIVEBOX_USER
-
-import abx
-
-
-class BaseBinProvider(BinProvider):
-    
-    # TODO: add install/load/load_or_install methods as abx.hookimpl methods
-    
-    @property
-    def admin_url(self) -> str:
-        # e.g. /admin/environment/binproviders/NpmBinProvider/   TODO
-        return "/admin/environment/binaries/"
-
-    @abx.hookimpl
-    def get_BINPROVIDERS(self):
-        return [self]
-
-class BaseBinary(Binary):
-
-    @staticmethod
-    def symlink_to_lib(binary, bin_dir=None) -> None:
-        from archivebox.config.common import STORAGE_CONFIG
-        bin_dir = bin_dir or STORAGE_CONFIG.LIB_DIR / 'bin'
-        
-        if not (binary.abspath and os.access(binary.abspath, os.R_OK)):
-            return
-        
-        try:
-            bin_dir.mkdir(parents=True, exist_ok=True)
-            symlink = bin_dir / binary.name
-            symlink.unlink(missing_ok=True)
-            symlink.symlink_to(binary.abspath)
-            symlink.chmod(0o777)   # make sure its executable by everyone
-        except Exception as err:
-            # print(f'[red]:warning: Failed to symlink {symlink} -> {binary.abspath}[/red] {err}')
-            # not actually needed, we can just run without it
-            pass
-        
-    @validate_call
-    def load(self, fresh=False, **kwargs) -> Self:
-        from archivebox.config.common import STORAGE_CONFIG
-        if fresh:
-            binary = super().load(**kwargs)
-            self.symlink_to_lib(binary=binary, bin_dir=STORAGE_CONFIG.LIB_DIR / 'bin')
-        else:
-            # get cached binary from db
-            try:
-                from machine.models import InstalledBinary
-                installed_binary = InstalledBinary.objects.get_from_db_or_cache(self)    # type: ignore
-                binary = InstalledBinary.load_from_db(installed_binary)
-            except Exception:
-                # maybe we are not in a DATA dir so there is no db, fallback to reading from fs
-                # (e.g. when archivebox version is run outside of a DATA dir)
-                binary = super().load(**kwargs)
-        return cast(Self, binary)
-    
-    @validate_call
-    def install(self, **kwargs) -> Self:
-        from archivebox.config.common import STORAGE_CONFIG
-        binary = super().install(**kwargs)
-        self.symlink_to_lib(binary=binary, bin_dir=STORAGE_CONFIG.LIB_DIR / 'bin')
-        return binary
-    
-    @validate_call
-    def load_or_install(self, fresh=False, **kwargs) -> Self:
-        from archivebox.config.common import STORAGE_CONFIG
-        try:
-            binary = self.load(fresh=fresh)
-            if binary and binary.version:
-                self.symlink_to_lib(binary=binary, bin_dir=STORAGE_CONFIG.LIB_DIR / 'bin')
-                return binary
-        except Exception:
-            pass
-        return self.install(**kwargs)
-    
-    @property
-    def admin_url(self) -> str:
-        # e.g. /admin/environment/config/LdapConfig/
-        return f"/admin/environment/binaries/{self.name}/"
-
-    @abx.hookimpl
-    def get_BINARIES(self):
-        return [self]
-
-
-class AptBinProvider(AptProvider, BaseBinProvider):
-    name: BinProviderName = "apt"
-    
-class BrewBinProvider(BrewProvider, BaseBinProvider):
-    name: BinProviderName = "brew"
-    
-class EnvBinProvider(EnvProvider, BaseBinProvider):
-    name: BinProviderName = "env"
-    
-    euid: Optional[int] = ARCHIVEBOX_USER
-
-apt = AptBinProvider()
-brew = BrewBinProvider()
-env = EnvBinProvider()

archivebox/abx/archivebox/base_configset.py

@@ -1,332 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import os
-import sys
-import re
-from pathlib import Path
-from typing import Type, Tuple, Callable, ClassVar, Dict, Any
-
-import toml
-from rich import print
-
-from benedict import benedict
-from pydantic import model_validator, TypeAdapter, AliasChoices
-from pydantic_settings import BaseSettings, SettingsConfigDict, PydanticBaseSettingsSource
-from pydantic_settings.sources import TomlConfigSettingsSource
-
-from pydantic_pkgr import func_takes_args_or_kwargs
-
-
-from . import toml_util
-
-
-PACKAGE_DIR = Path(__file__).resolve().parent.parent
-DATA_DIR = Path(os.getcwd()).resolve()
-
-ARCHIVEBOX_CONFIG_FILE = DATA_DIR / "ArchiveBox.conf"
-ARCHIVEBOX_CONFIG_FILE_BAK = ARCHIVEBOX_CONFIG_FILE.parent / ".ArchiveBox.conf.bak"
-
-AUTOFIXES_HEADER = "[AUTOFIXES]"
-AUTOFIXES_SUBHEADER = "# The following config was added automatically to fix problems detected at startup:"
-
-_ALREADY_WARNED_ABOUT_UPDATED_CONFIG = set()
-
-
-class FlatTomlConfigSettingsSource(TomlConfigSettingsSource):
-    """
-    A source class that loads variables from a TOML file
-    """
-
-    def __init__(
-        self,
-        settings_cls: type[BaseSettings],
-        toml_file: Path | None=None,
-    ):
-        self.toml_file_path = toml_file or settings_cls.model_config.get("toml_file")
-        
-        self.nested_toml_data = self._read_files(self.toml_file_path)
-        self.toml_data = {}
-        for top_level_key, top_level_value in self.nested_toml_data.items():
-            if isinstance(top_level_value, dict):
-                # value is nested, flatten it
-                for key, value in top_level_value.items():
-                    self.toml_data[key] = value
-            else:
-                # value is already flat, just set it as-is
-                self.toml_data[top_level_key] = top_level_value
-                
-        # filter toml_data to only include keys that are defined on this settings_cls
-        self.toml_data = {
-            key: value
-            for key, value in self.toml_data.items()
-            if key in settings_cls.model_fields
-        }
-            
-        super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data)
-
-
-class BaseConfigSet(BaseSettings):
-    """
-    This is the base class for an ArchiveBox ConfigSet.
-    It handles loading values from schema defaults, ArchiveBox.conf TOML config, and environment variables.
-
-    class WgetConfig(ArchiveBoxBaseConfig):
-        WGET_BINARY: str = Field(default='wget', alias='WGET_BINARY_PATH')
-
-    c = WgetConfig()
-    print(c.WGET_BINARY)                    # outputs: wget
-
-    # you can mutate process environment variable and reload config using .__init__()
-    os.environ['WGET_BINARY_PATH'] = 'wget2'
-    c.__init__()
-
-    print(c.WGET_BINARY)                    # outputs: wget2
-
-    """
-    
-    # these pydantic config options are all VERY carefully chosen, make sure to test thoroughly before changing!!!
-    model_config = SettingsConfigDict(
-        validate_default=False,
-        case_sensitive=True,
-        extra="ignore",
-        arbitrary_types_allowed=False,
-        populate_by_name=True,
-        from_attributes=True,
-        loc_by_alias=False,
-        validate_assignment=True,
-        validate_return=True,
-        revalidate_instances="subclass-instances",
-    )
-    
-    load_from_defaults: ClassVar[bool] = True
-    load_from_configfile: ClassVar[bool] = True
-    load_from_environment: ClassVar[bool] = True
-
-    @classmethod
-    def settings_customise_sources(
-        cls,
-        settings_cls: Type[BaseSettings],
-        init_settings: PydanticBaseSettingsSource,
-        env_settings: PydanticBaseSettingsSource,
-        dotenv_settings: PydanticBaseSettingsSource,
-        file_secret_settings: PydanticBaseSettingsSource,
-    ) -> Tuple[PydanticBaseSettingsSource, ...]:
-        """Defines the config precedence order: Schema defaults -> ArchiveBox.conf (TOML) -> Environment variables"""
-        
-        # import ipdb; ipdb.set_trace()
-        
-        precedence_order = {}
-        
-        # if ArchiveBox.conf does not exist yet, return defaults -> env order
-        if not ARCHIVEBOX_CONFIG_FILE.is_file():
-            precedence_order = {
-                'defaults': init_settings,
-                'environment': env_settings,
-            }
-        
-        # if ArchiveBox.conf exists and is in TOML format, return default -> TOML -> env order
-        try:
-            precedence_order = precedence_order or {
-                'defaults': init_settings,
-                'configfile': FlatTomlConfigSettingsSource(settings_cls, toml_file=ARCHIVEBOX_CONFIG_FILE),
-                'environment': env_settings,
-            }
-        except Exception as err:
-            if err.__class__.__name__ != "TOMLDecodeError":
-                raise
-            # if ArchiveBox.conf exists and is in INI format, convert it then return default -> TOML -> env order
-
-            # Convert ArchiveBox.conf in INI format to TOML and save original to .ArchiveBox.bak
-            original_ini = ARCHIVEBOX_CONFIG_FILE.read_text()
-            ARCHIVEBOX_CONFIG_FILE_BAK.write_text(original_ini)
-            new_toml = toml_util.convert(original_ini)
-            ARCHIVEBOX_CONFIG_FILE.write_text(new_toml)
-
-            precedence_order = {
-                'defaults': init_settings,
-                'configfile': FlatTomlConfigSettingsSource(settings_cls, toml_file=ARCHIVEBOX_CONFIG_FILE),
-                'environment': env_settings,
-            }
-            
-        if not cls.load_from_environment:
-            precedence_order.pop('environment')
-        if not cls.load_from_configfile:
-            precedence_order.pop('configfile')
-        if not cls.load_from_defaults:
-            precedence_order.pop('defaults')
-
-        return tuple(precedence_order.values())
-
-    @model_validator(mode="after")
-    def fill_defaults(self):
-        """Populate any unset values using function provided as their default"""
-
-        for key in self.model_fields.keys():
-            if isinstance(getattr(self, key), Callable):
-                if self.load_from_defaults:
-                    computed_default = self.get_default_value(key)
-                    # set generated default value as final validated value
-                    setattr(self, key, computed_default)
-        return self
-    
-    def validate(self):
-        """Manual validation method, to be called from plugin/__init__.py:get_CONFIG()"""
-        pass
-    
-    def get_default_value(self, key):
-        """Get the default value for a given config key"""
-        field = self.model_fields[key]
-        value = getattr(self, key)
-    
-        if isinstance(value, Callable):
-            # if value is a function, execute it to get the actual value, passing existing config as a dict arg if expected
-            if func_takes_args_or_kwargs(value):
-                # assemble dict of existing field values to pass to default factory functions
-                config_so_far = benedict(self.model_dump(include=set(self.model_fields.keys()), warnings=False))
-                computed_default = field.default(config_so_far)
-            else:
-                # otherwise it's a pure function with no args, just call it
-                computed_default = field.default()
-
-            # coerce/check to make sure default factory return value matches type annotation
-            TypeAdapter(field.annotation).validate_python(computed_default)
-
-            return computed_default
-        return value
-    
-    def update_in_place(self, warn=True, persist=False, hint='', **kwargs):
-        """
-        Update the config with new values. Use this sparingly! We should almost never be updating config at runtime.
-        Sets them in the environment so they propagate to spawned subprocesses / across future re-__init__()s and reload from environment
-
-        Example acceptable use case: user config says SEARCH_BACKEND_ENGINE=sonic but sonic_client pip library is not installed so we cannot use it.
-        SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep') can be used to reset it back to ripgrep so we can continue.
-        """
-        from archivebox.misc.toml_util import CustomTOMLEncoder
-        
-        # silence warnings if they've already been shown once
-        if all(key in _ALREADY_WARNED_ABOUT_UPDATED_CONFIG for key in kwargs.keys()):
-            warn = False
-        
-        if warn:
-            fix_scope = 'in ArchiveBox.conf' if persist else 'just for current run'
-            print(f'\n[yellow]:warning:  WARNING: Some config cannot be used as-is, fixing automatically {fix_scope}:[/yellow] {hint}', file=sys.stderr)
-        
-        # set the new values in the environment
-        for key, value in kwargs.items():
-            os.environ[key] = str(value)
-            original_value = getattr(self, key)
-            if warn:
-                print(f'    {key}={original_value} -> {value}')
-                _ALREADY_WARNED_ABOUT_UPDATED_CONFIG.add(key)
-        
-        # if persist=True, write config changes to data/ArchiveBox.conf [AUTOFIXES] section
-        try:
-            if persist and ARCHIVEBOX_CONFIG_FILE.is_file():
-                autofixes_to_add = benedict(kwargs).to_toml(encoder=CustomTOMLEncoder())
-                
-                existing_config = ARCHIVEBOX_CONFIG_FILE.read_text().split(AUTOFIXES_HEADER, 1)[0].strip()
-                if AUTOFIXES_HEADER in existing_config:
-                    existing_autofixes = existing_config.split(AUTOFIXES_HEADER, 1)[-1].strip().replace(AUTOFIXES_SUBHEADER, '').replace(AUTOFIXES_HEADER, '').strip()
-                else:
-                    existing_autofixes = ''
-                
-                new_config = '\n'.join(line for line in [
-                    existing_config,
-                    '\n' + AUTOFIXES_HEADER,
-                    AUTOFIXES_SUBHEADER,
-                    existing_autofixes,
-                    autofixes_to_add,
-                ] if line.strip()).strip() + '\n'
-                ARCHIVEBOX_CONFIG_FILE.write_text(new_config)
-        except Exception:
-            pass
-        self.__init__()
-        if warn:
-            print(file=sys.stderr)
-            
-        return self
-    
-    @property
-    def aliases(self) -> Dict[str, str]:
-        alias_map = {}
-        for key, field in self.model_fields.items():
-            alias_map[key] = key
-            
-            if field.validation_alias is None:
-                continue
-
-            if isinstance(field.validation_alias, AliasChoices):
-                for alias in field.validation_alias.choices:
-                    alias_map[alias] = key
-            elif isinstance(field.alias, str):
-                alias_map[field.alias] = key
-            else:
-                raise ValueError(f'Unknown alias type for field {key}: {field.alias}')
-        
-        return benedict(alias_map)
-    
-    
-    @property
-    def toml_section_header(self):
-        """Convert the class name to a TOML section header e.g. ShellConfig -> SHELL_CONFIG"""
-        class_name = self.__class__.__name__
-        return re.sub('([A-Z]+)', r'_\1', class_name).upper().strip('_')
-    
-    
-    def from_defaults(self) -> Dict[str, Any]:
-        """Get the dictionary of {key: value} config loaded from the default values"""
-        class OnlyDefaultsConfig(self.__class__):
-            load_from_defaults = True
-            load_from_configfile = False
-            load_from_environment = False
-        return benedict(OnlyDefaultsConfig().model_dump(exclude_unset=False, exclude_defaults=False, exclude=set(self.model_computed_fields.keys())))
-    
-    def from_configfile(self) -> Dict[str, Any]:
-        """Get the dictionary of {key: value} config loaded from the configfile ArchiveBox.conf"""
-        class OnlyConfigFileConfig(self.__class__):
-            load_from_defaults = False
-            load_from_configfile = True
-            load_from_environment = False
-        return benedict(OnlyConfigFileConfig().model_dump(exclude_unset=True, exclude_defaults=False, exclude=set(self.model_computed_fields.keys())))
-    
-    def from_environment(self) -> Dict[str, Any]:
-        """Get the dictionary of {key: value} config loaded from the environment variables"""
-        class OnlyEnvironmentConfig(self.__class__):
-            load_from_defaults = False
-            load_from_configfile = False
-            load_from_environment = True
-        return benedict(OnlyEnvironmentConfig().model_dump(exclude_unset=True, exclude_defaults=False, exclude=set(self.model_computed_fields.keys())))
-    
-    def from_computed(self) -> Dict[str, Any]:
-        """Get the dictionary of {key: value} config loaded from the computed fields"""
-        return benedict(self.model_dump(include=set(self.model_computed_fields.keys())))
-    
-
-    def to_toml_dict(self, defaults=False) -> Dict[str, Any]:
-        """Get the current config as a TOML-ready dict"""
-        config_dict = {}
-        for key, value in benedict(self).items():
-            if defaults or value != self.get_default_value(key):
-                config_dict[key] = value
-        
-        return benedict({self.toml_section_header: config_dict})
-    
-    def to_toml_str(self, defaults=False) -> str:
-        """Get the current config as a TOML string"""
-        from archivebox.misc.toml_util import CustomTOMLEncoder
-        
-        toml_dict = self.to_toml_dict(defaults=defaults)
-        if not toml_dict[self.toml_section_header]:
-            # if the section is empty, don't write it
-            toml_dict.pop(self.toml_section_header)
-        
-        return toml.dumps(toml_dict, encoder=CustomTOMLEncoder())
-    
-    def as_legacy_config_schema(self) -> Dict[str, Any]:
-        # shim for backwards compatibility with old config schema style
-        model_values = self.model_dump()
-        return benedict({
-            key: {'type': field.annotation, 'default': model_values[key]}
-            for key, field in self.model_fields.items()
-        })

archivebox/abx/archivebox/base_extractor.py

@@ -1,219 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import json
-import os
-
-from typing import Optional, List, Literal, Annotated, Dict, Any, Tuple
-from typing_extensions import Self
-from pathlib import Path
-
-from pydantic import model_validator, AfterValidator
-from pydantic_pkgr import BinName
-from django.utils.functional import cached_property
-from django.utils import timezone
-
-import abx
-
-from .base_binary import BaseBinary
-
-
-def no_empty_args(args: List[str]) -> List[str]:
-    assert all(len(arg) for arg in args)
-    return args
-
-ExtractorName = Literal['wget', 'warc', 'media', 'singlefile'] | str
-
-HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
-CmdArgsList = Annotated[List[str] | Tuple[str, ...], AfterValidator(no_empty_args)]
-
-
-class BaseExtractor:
-    
-    name: ExtractorName
-    binary: BinName
-
-    output_path_func: HandlerFuncStr = 'self.get_output_path'
-    should_extract_func: HandlerFuncStr = 'self.should_extract'
-    extract_func: HandlerFuncStr = 'self.extract'
-    exec_func: HandlerFuncStr = 'self.exec'
-
-    default_args: CmdArgsList = []
-    extra_args: CmdArgsList = []
-    args: Optional[CmdArgsList] = None
-
-    @model_validator(mode='after')
-    def validate_model(self) -> Self:
-        if self.args is None:
-            self.args = [*self.default_args, *self.extra_args]
-        return self
-
-
-    def get_output_path(self, snapshot) -> Path:
-        return Path(self.__class__.__name__.lower())
-
-    def should_extract(self, uri: str, config: dict | None=None) -> bool:
-        try:
-            assert self.detect_installed_binary().version
-        except Exception:
-            raise
-            # could not load binary
-            return False
-        
-        # output_dir = self.get_output_path(snapshot)
-        # if output_dir.glob('*.*'):
-        #     return False
-        return True
-
-    @abx.hookimpl
-    def extract(self, snapshot_id: str) -> Dict[str, Any]:
-        from core.models import Snapshot
-        from archivebox import CONSTANTS
-        
-        snapshot = Snapshot.objects.get(id=snapshot_id)
-        
-        if not self.should_extract(snapshot):
-            return {}
-        
-        status = 'failed'
-        start_ts = timezone.now()
-        uplink = self.detect_network_interface()
-        installed_binary = self.detect_installed_binary()
-        machine = installed_binary.machine
-        assert uplink.machine == installed_binary.machine  # it would be *very* weird if this wasn't true
-        
-        output_dir = CONSTANTS.DATA_DIR / '.tmp' / 'extractors' / self.name / str(snapshot.abid)
-        output_dir.mkdir(parents=True, exist_ok=True)
-
-        # execute the extractor binary with the given args
-        args = [snapshot.url, *self.args] if self.args is not None else [snapshot.url, *self.default_args, *self.extra_args]
-        cmd = [str(installed_binary.abspath), *args]
-        proc = self.exec(installed_binary=installed_binary, args=args, cwd=output_dir)
-
-        # collect the output
-        end_ts = timezone.now()
-        output_files = list(str(path.relative_to(output_dir)) for path in output_dir.glob('**/*.*'))
-        stdout = proc.stdout.strip()
-        stderr = proc.stderr.strip()
-        output_json = None
-        output_text = stdout
-        try:
-            output_json = json.loads(stdout.strip())
-            output_text = None
-        except json.JSONDecodeError:
-            pass
-        
-        errors = []
-        if proc.returncode == 0:
-            status = 'success'
-        else:
-            errors.append(f'{installed_binary.name} returned non-zero exit code: {proc.returncode}')   
-
-        # increment health stats counters
-        if status == 'success':
-            machine.record_health_success()
-            uplink.record_health_success()
-            installed_binary.record_health_success()
-        else:
-            machine.record_health_failure()
-            uplink.record_health_failure()
-            installed_binary.record_health_failure()
-
-        return {
-            'extractor': self.name,
-            
-            'snapshot': {
-                'id': snapshot.id,
-                'abid': snapshot.abid,
-                'url': snapshot.url,
-                'created_by_id': snapshot.created_by_id,
-            },
-            
-            'machine': {
-                'id': machine.id,
-                'abid': machine.abid,
-                'guid': machine.guid,
-                'hostname': machine.hostname,
-                'hw_in_docker': machine.hw_in_docker,
-                'hw_in_vm': machine.hw_in_vm,
-                'hw_manufacturer': machine.hw_manufacturer,
-                'hw_product': machine.hw_product,
-                'hw_uuid': machine.hw_uuid,
-                'os_arch': machine.os_arch,
-                'os_family': machine.os_family,
-                'os_platform': machine.os_platform,
-                'os_release': machine.os_release,
-                'os_kernel': machine.os_kernel,
-            },
-            
-            'uplink': { 
-                'id': uplink.id,
-                'abid': uplink.abid,
-                'mac_address': uplink.mac_address,
-                'ip_public': uplink.ip_public,
-                'ip_local': uplink.ip_local,
-                'dns_server': uplink.dns_server,
-                'hostname': uplink.hostname,
-                'iface': uplink.iface,
-                'isp': uplink.isp,
-                'city': uplink.city,
-                'region': uplink.region,
-                'country': uplink.country,
-            },
-            
-            'binary': {
-                'id': installed_binary.id,
-                'abid': installed_binary.abid,
-                'name': installed_binary.name,
-                'binprovider': installed_binary.binprovider,
-                'abspath': installed_binary.abspath,
-                'version': installed_binary.version,
-                'sha256': installed_binary.sha256,
-            },
-
-            'cmd': cmd,
-            'stdout': stdout,
-            'stderr': stderr,
-            'returncode': proc.returncode,
-            'start_ts': start_ts,
-            'end_ts': end_ts,
-            
-            'status': status,
-            'errors': errors,
-            'output_dir': str(output_dir.relative_to(CONSTANTS.DATA_DIR)),
-            'output_files': output_files,
-            'output_json': output_json or {},
-            'output_text': output_text or '',
-        }
-
-    # TODO: move this to a hookimpl
-    def exec(self, args: CmdArgsList=(), cwd: Optional[Path]=None, installed_binary=None):
-        cwd = cwd or Path(os.getcwd())
-        binary = self.load_binary(installed_binary=installed_binary)
-        
-        return binary.exec(cmd=args, cwd=cwd)
-    
-    @cached_property
-    def BINARY(self) -> BaseBinary:
-        import abx.archivebox.reads
-        for binary in abx.archivebox.reads.get_BINARIES().values():
-            if binary.name == self.binary:
-                return binary
-        raise ValueError(f'Binary {self.binary} not found')
-    
-    def detect_installed_binary(self):
-        from machine.models import InstalledBinary
-        # hydrates binary from DB/cache if record of installed version is recent enough
-        # otherwise it finds it from scratch by detecting installed version/abspath/sha256 on host
-        return InstalledBinary.objects.get_from_db_or_cache(self.BINARY)
-
-    def load_binary(self, installed_binary=None) -> BaseBinary:
-        installed_binary = installed_binary or self.detect_installed_binary()
-        return installed_binary.load_from_db()
-    
-    def detect_network_interface(self):
-        from machine.models import NetworkInterface
-        return NetworkInterface.objects.current()
-
-    @abx.hookimpl
-    def get_EXTRACTORS(self):
-        return [self]

archivebox/abx/archivebox/base_replayer.py

@@ -1,25 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import abx
-
-
-class BaseReplayer:
-    """Describes how to render an ArchiveResult in several contexts"""
-    
-    url_pattern: str = '*'
-
-    row_template: str = 'plugins/generic_replayer/templates/row.html'
-    embed_template: str = 'plugins/generic_replayer/templates/embed.html'
-    fullpage_template: str = 'plugins/generic_replayer/templates/fullpage.html'
-
-    # row_view: LazyImportStr = 'plugins.generic_replayer.views.row_view'
-    # embed_view: LazyImportStr = 'plugins.generic_replayer.views.embed_view'
-    # fullpage_view: LazyImportStr = 'plugins.generic_replayer.views.fullpage_view'
-    # icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
-    # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
-
-    @abx.hookimpl
-    def get_REPLAYERS(self):
-        return [self]
-
-    # TODO: add hookimpl methods for get_row_template, get_embed_template, get_fullpage_template, etc...

archivebox/abx/archivebox/base_searchbackend.py

@@ -1,25 +0,0 @@
-__package__ = 'abx.archivebox'
-
-from typing import Iterable, List
-import abc
-
-
-
-class BaseSearchBackend(abc.ABC):
-    name: str
-
-    @staticmethod
-    @abc.abstractmethod
-    def index(snapshot_id: str, texts: List[str]):
-        return
-
-    @staticmethod
-    @abc.abstractmethod
-    def flush(snapshot_ids: Iterable[str]):
-        return
-
-    @staticmethod
-    @abc.abstractmethod
-    def search(text: str) -> List[str]:
-        raise NotImplementedError("search method must be implemented by subclass")
-

archivebox/abx/archivebox/effects.py

@@ -1,20 +0,0 @@
-"""
-Hookspec for side-effects that ArchiveBox plugins can trigger.
-
-(e.g. network requests, binary execution, remote API calls, external library calls, etc.)
-"""
-
-__package__ = 'abx.archivebox'
-
-import abx
-
-
-@abx.hookspec
-def check_remote_seed_connection(urls, extractor, credentials, created_by):
-    pass
-
-
-@abx.hookspec
-def exec_extractor(url, extractor, credentials, config):
-    pass
-

archivebox/abx/archivebox/events.py

@@ -1,45 +0,0 @@
-"""
-Hookspec for ArchiveBox system events that plugins can hook into.
-
-Loosely modeled after Django's signals architecture.
-https://docs.djangoproject.com/en/5.1/ref/signals/
-"""
-
-__package__ = 'abx.archivebox'
-
-import abx
-
-
-
-@abx.hookspec
-def on_crawl_schedule_tick(crawl_schedule):
-    pass
-
-
-
-
-@abx.hookspec
-def on_seed_post_save(seed, created=False):
-    ...
-
-@abx.hookspec
-def on_crawl_post_save(crawl, created=False):
-    ...
-
-
-@abx.hookspec
-def on_snapshot_post_save(snapshot, created=False):
-    ...
-    
-# @abx.hookspec
-# def on_snapshot_post_delete(snapshot):
-#     ...
-
-
-@abx.hookspec
-def on_archiveresult_post_save(archiveresult, created=False):
-    ...
-
-# @abx.hookspec
-# def on_archiveresult_post_delete(archiveresult):
-#     ...

archivebox/abx/archivebox/hookspec.py

@@ -1,52 +0,0 @@
-__package__ = 'abx.archivebox'
-
-from typing import Dict, Any
-
-from .. import hookspec
-
-from .base_binary import BaseBinary, BaseBinProvider
-from .base_configset import BaseConfigSet
-from .base_extractor import BaseExtractor
-from .base_searchbackend import BaseSearchBackend
-
-
-@hookspec
-def get_PLUGIN() -> Dict[str, Dict[str, Any]]:
-    return {}
-
-@hookspec
-def get_CONFIG() -> Dict[str, BaseConfigSet]:
-    return {}
-
-
-
-@hookspec
-def get_EXTRACTORS() -> Dict[str, BaseExtractor]:
-    return {}
-
-@hookspec
-def get_SEARCHBACKENDS() -> Dict[str, BaseSearchBackend]:
-    return {}
-
-# @hookspec
-# def get_REPLAYERS() -> Dict[str, BaseReplayer]:
-#     return {}
-
-# @hookspec
-# def get_ADMINDATAVIEWS():
-#     return {}
-
-# @hookspec
-# def get_QUEUES():
-#     return {}
-
-
-##############################################################
-# provided by abx.pydantic_pkgr.hookspec:
-# @hookspec
-# def get_BINARIES() -> Dict[str, BaseBinary]:
-#     return {}
-
-# @hookspec
-# def get_BINPROVIDERS() -> Dict[str, BaseBinProvider]:
-#     return {}

archivebox/abx/archivebox/reads.py

@@ -1,160 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import importlib
-from typing import Dict, Set, Any, TYPE_CHECKING
-
-from benedict import benedict
-
-import abx
-from .. import pm
-
-if TYPE_CHECKING:
-    from .base_configset import BaseConfigSet
-    from .base_binary import BaseBinary, BaseBinProvider
-    from .base_extractor import BaseExtractor
-    from .base_searchbackend import BaseSearchBackend
-    # from .base_replayer import BaseReplayer
-    # from .base_queue import BaseQueue
-    # from .base_admindataview import BaseAdminDataView
-
-# API exposed to ArchiveBox code
-
-def get_PLUGINS() -> Dict[str, Dict[str, Any]]:
-    return benedict({
-        plugin_id: plugin
-        for plugin_dict in pm.hook.get_PLUGIN()
-            for plugin_id, plugin in plugin_dict.items()
-    })
-
-def get_PLUGIN(plugin_id: str) -> Dict[str, Any]:
-    plugin_info = get_PLUGINS().get(plugin_id, {})
-    package = plugin_info.get('package', plugin_info.get('PACKAGE', None))
-    if not package:
-        return {'id': plugin_id, 'hooks': {}}
-    module = importlib.import_module(package)
-    hooks = abx.get_plugin_hooks(module.__package__)
-    assert plugin_info and (plugin_info.get('id') or plugin_info.get('ID') or hooks)
-    
-    return benedict({
-        'id': plugin_id,
-        'label': getattr(module, '__label__', plugin_id),
-        'module': module,
-        'package': module.__package__,
-        'hooks': hooks,
-        'version': getattr(module, '__version__', '999.999.999'),
-        'author': getattr(module, '__author__', 'Unknown'),
-        'homepage': getattr(module, '__homepage__', 'https://github.com/ArchiveBox/ArchiveBox'),
-        'dependencies': getattr(module, '__dependencies__', []),
-        'source_code': module.__file__,
-        **plugin_info,
-    })
-    
-
-def get_HOOKS() -> Set[str]:
-    return {
-        hook_name
-        for plugin_id in get_PLUGINS().keys()
-            for hook_name in get_PLUGIN(plugin_id).hooks
-    }
-
-def get_CONFIGS() -> Dict[str, 'BaseConfigSet']:
-    return benedict({
-        config_id: configset
-        for plugin_configs in pm.hook.get_CONFIG()
-            for config_id, configset in plugin_configs.items()
-    })
-
-
-def get_FLAT_CONFIG() -> Dict[str, Any]:
-    return benedict({
-        key: value
-        for configset in get_CONFIGS().values()
-            for key, value in configset.model_dump().items()
-    })
-
-def get_BINPROVIDERS() -> Dict[str, 'BaseBinProvider']:
-    # TODO: move these to plugins
-    from abx.archivebox.base_binary import apt, brew, env
-    builtin_binproviders = {
-        'env': env,
-        'apt': apt,
-        'brew': brew,
-    }
-    
-    return benedict({
-        binprovider_id: binprovider
-        for plugin_binproviders in [builtin_binproviders, *pm.hook.get_BINPROVIDERS()]
-            for binprovider_id, binprovider in plugin_binproviders.items()
-    })
-
-def get_BINARIES() -> Dict[str, 'BaseBinary']:
-    return benedict({
-        binary_id: binary
-        for plugin_binaries in pm.hook.get_BINARIES()
-            for binary_id, binary in plugin_binaries.items()
-    })
-
-def get_EXTRACTORS() -> Dict[str, 'BaseExtractor']:
-    return benedict({
-        extractor_id: extractor
-        for plugin_extractors in pm.hook.get_EXTRACTORS()
-            for extractor_id, extractor in plugin_extractors.items()
-    })
-
-# def get_REPLAYERS() -> Dict[str, 'BaseReplayer']:
-#     return benedict({
-#         replayer.id: replayer
-#         for plugin_replayers in pm.hook.get_REPLAYERS()
-#             for replayer in plugin_replayers
-#     })
-
-# def get_ADMINDATAVIEWS() -> Dict[str, 'BaseAdminDataView']:
-#     return benedict({
-#         admin_dataview.id: admin_dataview
-#         for plugin_admin_dataviews in pm.hook.get_ADMINDATAVIEWS()
-#             for admin_dataview in plugin_admin_dataviews
-#     })
-
-# def get_QUEUES() -> Dict[str, 'BaseQueue']:
-#     return benedict({
-#         queue.id: queue
-#         for plugin_queues in pm.hook.get_QUEUES()
-#             for queue in plugin_queues
-#     })
-
-def get_SEARCHBACKENDS() -> Dict[str, 'BaseSearchBackend']:
-    return benedict({
-        searchbackend_id: searchbackend
-        for plugin_searchbackends in pm.hook.get_SEARCHBACKENDS()
-            for searchbackend_id,searchbackend in plugin_searchbackends.items()
-    })
-
-
-
-def get_scope_config(defaults: benedict | None = None, persona=None, seed=None, crawl=None, snapshot=None, archiveresult=None, extra_config=None):
-    """Get all the relevant config for the given scope, in correct precedence order"""
-    
-    from django.conf import settings
-    default_config: benedict = defaults or settings.CONFIG
-    
-    snapshot = snapshot or (archiveresult and archiveresult.snapshot)
-    crawl = crawl or (snapshot and snapshot.crawl)
-    seed = seed or (crawl and crawl.seed)
-    persona = persona or (crawl and crawl.persona)
-    
-    persona_config = persona.config if persona else {}
-    seed_config = seed.config if seed else {}
-    crawl_config = crawl.config if crawl else {}
-    snapshot_config = snapshot.config if snapshot else {}
-    archiveresult_config = archiveresult.config if archiveresult else {}
-    extra_config = extra_config or {}
-    
-    return {
-        **default_config,               # defaults / config file / environment variables
-        **persona_config,               # lowest precedence
-        **seed_config,
-        **crawl_config,
-        **snapshot_config,
-        **archiveresult_config,
-        **extra_config,                 # highest precedence
-    }

archivebox/abx/archivebox/toml_util.py

@@ -1,114 +0,0 @@
-from typing import Any, List, Callable
-
-import json
-import ast
-import inspect
-import toml
-import re
-import configparser
-
-from pathlib import Path, PosixPath
-
-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | List['JSONValue']
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ('true', 'yes', '1'):
-        return True
-    if val.lower() in ('false', 'no', '0'):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-    
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    config.optionxform = str  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-    def encode_default(self, default: Any) -> Any:
-        config = self._config
-        if isinstance(default, Callable):
-            return '{{lambda ' + inspect.getsource(default).split('=lambda ')[-1].strip()[:-1] + '}}'
-        return to_jsonable_python(
-           default,
-           timedelta_mode=config.ser_json_timedelta,
-           bytes_mode=config.ser_json_bytes,
-           serialize_unknown=True
-        )
-
-    # for computed_field properties render them like this instead:
-    # inspect.getsource(field.wrapped_property.fget).split('def ', 1)[-1].split('\n', 1)[-1].strip().strip('return '),
-
-
-def better_toml_dump_str(val: Any) -> str:
-    try:
-        return toml.encoder._dump_str(val)     # type: ignore
-    except Exception:
-        # if we hit any of toml's numerous encoding bugs,
-        # fall back to using json representation of string
-        return json.dumps(str(val))
-
-class CustomTOMLEncoder(toml.encoder.TomlEncoder):
-    """
-    Custom TomlEncoder to work around https://github.com/uiri/toml's many encoding bugs.
-    More info: https://github.com/fabiocaccamo/python-benedict/issues/439
-    >>> toml.dumps(value, encoder=CustomTOMLEncoder())
-    """
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.dump_funcs[Path] = lambda x: json.dumps(str(x))
-        self.dump_funcs[PosixPath] = lambda x: json.dumps(str(x))
-        self.dump_funcs[str] = better_toml_dump_str
-        self.dump_funcs[re.RegexFlag] = better_toml_dump_str

archivebox/abx/archivebox/writes.py

@@ -1,133 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import importlib
-from typing import Dict, Set, Any, TYPE_CHECKING
-
-from benedict import benedict
-
-from django.conf import settings
-
-import abx
-from .. import pm
-
-
-@abx.hookimpl
-def get_or_create_snapshot(crawl, url, config):
-    pass
-
-@abx.hookimpl
-def update_crawl_schedule_next_run_at(crawl_schedule, next_run_at):
-    pass
-
-@abx.hookimpl
-def create_crawl_copy(crawl_to_copy, schedule):
-    pass
-
-@abx.hookimpl
-def create_crawl(seed, depth, tags_str, persona, created_by, config, schedule):
-    pass
-
-
-
-
-def create_crawl_from_ui_action(urls, extractor, credentials, depth, tags_str, persona, created_by, crawl_config):
-    if seed_is_remote(urls, extractor, credentials):
-        # user's seed is a remote source that will provide the urls (e.g. RSS feed URL, Pocket API, etc.)
-        uri, extractor, credentials = abx.archivebox.effects.check_remote_seed_connection(urls, extractor, credentials, created_by)
-    else:
-        # user's seed is some raw text they provided to parse for urls, save it to a file then load the file as a Seed
-        uri = abx.archivebox.writes.write_raw_urls_to_local_file(urls, extractor, tags_str, created_by)  # file:///data/sources/some_import.txt
-    
-    seed = abx.archivebox.writes.get_or_create_seed(uri=remote_uri, extractor, credentials, created_by)
-    # abx.archivebox.events.on_seed_created(seed)
-        
-    crawl = abx.archivebox.writes.create_crawl(seed=seed, depth=depth, tags_str=tags_str, persona=persona, created_by=created_by, config=crawl_config, schedule=None)
-    abx.archivebox.events.on_crawl_created(crawl)
-
-
-@abx.hookimpl(specname='on_crawl_schedule_tick')
-def create_crawl_from_crawlschedule_if_due(crawl_schedule):
-    # make sure it's not too early to run this scheduled import (makes this function indepmpotent / safe to call multiple times / every second)
-    if timezone.now() < crawl_schedule.next_run_at:
-        # it's not time to run it yet, wait for the next tick
-        return
-    else:
-        # we're going to run it now, bump the next run time so that no one else runs it at the same time as us
-        abx.archivebox.writes.update_crawl_schedule_next_run_at(crawl_schedule, next_run_at=crawl_schedule.next_run_at + crawl_schedule.interval)
-    
-    crawl_to_copy = None
-    try:
-        crawl_to_copy = crawl_schedule.crawl_set.first()  # alternatively use .last() to copy most recent crawl instead of very first crawl
-    except Crawl.DoesNotExist:
-        # there is no template crawl to base the next one off of
-        # user must add at least one crawl to a schedule that serves as the template for all future repeated crawls
-        return
-    
-    new_crawl = abx.archivebox.writes.create_crawl_copy(crawl_to_copy=crawl_to_copy, schedule=crawl_schedule)
-    abx.archivebox.events.on_crawl_created(new_crawl)
-
-
-@abx.hookimpl(specname='on_crawl_post_save')
-def create_root_snapshot_from_seed(crawl):
-    # create a snapshot for the seed URI which kicks off the crawl
-    # only a single extractor will run on it, which will produce outlinks which get added back to the crawl
-    root_snapshot, created = abx.archivebox.writes.get_or_create_snapshot(crawl=crawl, url=crawl.seed.uri, config={
-        'extractors': (
-            abx.archivebox.reads.get_extractors_that_produce_outlinks()
-            if crawl.seed.extractor == 'auto' else
-            [crawl.seed.extractor]
-        ),
-        **crawl.seed.config,
-    })
-    if created:
-        abx.archivebox.events.on_snapshot_created(root_snapshot)
-        abx.archivebox.writes.update_crawl_stats(started_at=timezone.now())
-
-
-@abx.hookimpl(specname='on_snapshot_created')
-def create_archiveresults_pending_from_snapshot(snapshot, config):
-    config = get_scope_config(
-        # defaults=settings.CONFIG_FROM_DEFAULTS,
-        # configfile=settings.CONFIG_FROM_FILE,
-        # environment=settings.CONFIG_FROM_ENVIRONMENT,
-        persona=archiveresult.snapshot.crawl.persona,
-        seed=archiveresult.snapshot.crawl.seed,
-        crawl=archiveresult.snapshot.crawl,
-        snapshot=archiveresult.snapshot,
-        archiveresult=archiveresult,
-        # extra_config=extra_config,
-    )
-    
-    extractors = abx.archivebox.reads.get_extractors_for_snapshot(snapshot, config)
-    for extractor in extractors:
-        archiveresult, created = abx.archivebox.writes.get_or_create_archiveresult_pending(
-            snapshot=snapshot,
-            extractor=extractor,
-            status='pending'
-        )
-        if created:
-            abx.archivebox.events.on_archiveresult_created(archiveresult)
-
-
-
-@abx.hookimpl(specname='on_archiveresult_updated')
-def create_snapshots_pending_from_archiveresult_outlinks(archiveresult):
-    config = get_scope_config(...)
-    
-    # check if extractor has finished succesfully, if not, dont bother checking for outlinks
-    if not archiveresult.status == 'succeeded':
-        return
-    
-    # check if we have already reached the maximum recursion depth
-    hops_to_here = abx.archivebox.reads.get_outlink_parents(crawl_pk=archiveresult.snapshot.crawl_id, url=archiveresult.url, config=config)
-    if len(hops_to_here) >= archiveresult.crawl.max_depth +1:
-        return
-    
-    # parse the output to get outlink url_entries
-    discovered_urls = abx.archivebox.reads.get_archiveresult_discovered_url_entries(archiveresult, config=config)
-    
-    for url_entry in discovered_urls:
-        abx.archivebox.writes.create_outlink_record(src=archiveresult.snapshot.url, dst=url_entry.url, via=archiveresult)
-        abx.archivebox.writes.create_snapshot(crawl=archiveresult.snapshot.crawl, url_entry=url_entry)
-        
-    # abx.archivebox.events.on_crawl_updated(archiveresult.snapshot.crawl)

archivebox/abx/django/__init__.py

@@ -1 +0,0 @@
-__package__ = 'abx.django'

archivebox/abx/django/apps.py

@@ -1,13 +0,0 @@
-__package__ = 'abx.django'
-
-from django.apps import AppConfig
-
-
-class ABXConfig(AppConfig):
-    name = 'abx'
-
-    def ready(self):
-        import abx
-        from django.conf import settings
-        
-        abx.pm.hook.ready(settings=settings)

archivebox/abx/django/hookspec.py

@@ -1,125 +0,0 @@
-__package__ = 'abx.django'
-
-from ..hookspec import hookspec
-
-
-###########################################################################################
-
-@hookspec
-def get_INSTALLED_APPS():
-    """Return a list of apps to add to INSTALLED_APPS"""
-    # e.g. ['your_plugin_type.plugin_name']
-    return []
-
-# @hookspec
-# def register_INSTALLED_APPS(INSTALLED_APPS):
-#     """Mutate INSTALLED_APPS in place to add your app in a specific position"""
-#     # idx_of_contrib = INSTALLED_APPS.index('django.contrib.auth')
-#     # INSTALLED_APPS.insert(idx_of_contrib + 1, 'your_plugin_type.plugin_name')
-#     pass
-
-
-@hookspec
-def get_TEMPLATE_DIRS():
-    return []     # e.g. ['your_plugin_type/plugin_name/templates']
-
-# @hookspec
-# def register_TEMPLATE_DIRS(TEMPLATE_DIRS):
-#     """Install django settings"""
-#     # e.g. TEMPLATE_DIRS.insert(0, 'your_plugin_type/plugin_name/templates')
-#     pass
-
-
-@hookspec
-def get_STATICFILES_DIRS():
-    return []     # e.g. ['your_plugin_type/plugin_name/static']
-
-# @hookspec
-# def register_STATICFILES_DIRS(STATICFILES_DIRS):
-#     """Mutate STATICFILES_DIRS in place to add your static dirs in a specific position"""
-#     # e.g. STATICFILES_DIRS.insert(0, 'your_plugin_type/plugin_name/static')
-#     pass
-
-
-@hookspec
-def get_MIDDLEWARE():
-    return []     # e.g. ['your_plugin_type.plugin_name.middleware.YourMiddleware']
-
-# @hookspec
-# def register_MIDDLEWARE(MIDDLEWARE):
-#     """Mutate MIDDLEWARE in place to add your middleware in a specific position"""
-#     # e.g. MIDDLEWARE.insert(0, 'your_plugin_type.plugin_name.middleware.YourMiddleware')
-#     pass
-
-
-@hookspec
-def get_AUTHENTICATION_BACKENDS():
-    return []     # e.g. ['django_auth_ldap.backend.LDAPBackend']
-
-# @hookspec
-# def register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS):
-#     """Mutate AUTHENTICATION_BACKENDS in place to add your auth backends in a specific position"""
-#     # e.g. AUTHENTICATION_BACKENDS.insert(0, 'your_plugin_type.plugin_name.backend.YourBackend')
-#     pass
-
-@hookspec
-def get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME):
-    return []     # e.g. [{'name': 'your_plugin_type.plugin_name', 'HUEY': {...}}]
-
-# @hookspec
-# def register_DJANGO_HUEY(DJANGO_HUEY):
-#     """Mutate DJANGO_HUEY in place to add your huey queues in a specific position"""
-#     # e.g. DJANGO_HUEY['queues']['some_queue_name']['some_setting'] = 'some_value'
-#     pass
-
-
-@hookspec
-def get_ADMIN_DATA_VIEWS_URLS():
-    return []
-
-# @hookspec
-# def register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS):
-#     """Mutate ADMIN_DATA_VIEWS in place to add your admin data views in a specific position"""
-#     # e.g. ADMIN_DATA_VIEWS['URLS'].insert(0, 'your_plugin_type/plugin_name/admin_data_views.py')
-#     pass
-
-
-# @hookspec
-# def register_settings(settings):
-#     """Mutate settings in place to add your settings / modify existing settings"""
-#     # settings.SOME_KEY = 'some_value'
-#     pass
-
-
-###########################################################################################
-
-@hookspec
-def get_urlpatterns():
-    return []     # e.g. [path('your_plugin_type/plugin_name/url.py', your_view)]
-
-# @hookspec
-# def register_urlpatterns(urlpatterns):
-#     """Mutate urlpatterns in place to add your urlpatterns in a specific position"""
-#     # e.g. urlpatterns.insert(0, path('your_plugin_type/plugin_name/url.py', your_view))
-#     pass
-
-###########################################################################################
-
-@hookspec
-def register_checks():
-    """Register django checks with django system checks system"""
-    pass
-
-@hookspec
-def register_admin(admin_site):
-    """Register django admin views/models with the main django admin site instance"""
-    pass
-
-
-###########################################################################################
-
-
-@hookspec
-def ready():
-    """Called when Django apps app.ready() are triggered"""
-    pass

archivebox/abx/django/use.py

@@ -1,101 +0,0 @@
-__package__ = 'abx.django'
-
-import itertools
-# from benedict import benedict
-
-from .. import pm
-
-
-def get_INSTALLED_APPS():
-    return itertools.chain(*reversed(pm.hook.get_INSTALLED_APPS()))
-
-# def register_INSTALLLED_APPS(INSTALLED_APPS):
-#     pm.hook.register_INSTALLED_APPS(INSTALLED_APPS=INSTALLED_APPS)
-
-
-def get_MIDDLEWARES():
-    return itertools.chain(*reversed(pm.hook.get_MIDDLEWARE()))
-
-# def register_MIDDLEWARES(MIDDLEWARE):
-#     pm.hook.register_MIDDLEWARE(MIDDLEWARE=MIDDLEWARE)
-
-
-def get_AUTHENTICATION_BACKENDS():
-    return itertools.chain(*reversed(pm.hook.get_AUTHENTICATION_BACKENDS()))
-
-# def register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS):
-#     pm.hook.register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS=AUTHENTICATION_BACKENDS)
-
-
-def get_STATICFILES_DIRS():
-    return itertools.chain(*reversed(pm.hook.get_STATICFILES_DIRS()))
-
-# def register_STATICFILES_DIRS(STATICFILES_DIRS):
-#     pm.hook.register_STATICFILES_DIRS(STATICFILES_DIRS=STATICFILES_DIRS)
-
-
-def get_TEMPLATE_DIRS():
-    return itertools.chain(*reversed(pm.hook.get_TEMPLATE_DIRS()))
-
-# def register_TEMPLATE_DIRS(TEMPLATE_DIRS):
-#     pm.hook.register_TEMPLATE_DIRS(TEMPLATE_DIRS=TEMPLATE_DIRS)
-
-def get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME='queue.sqlite3'):
-    HUEY_QUEUES = {}
-    for plugin_result in pm.hook.get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME=QUEUE_DATABASE_NAME):
-        HUEY_QUEUES.update(plugin_result)
-    return HUEY_QUEUES
-
-# def register_DJANGO_HUEY(DJANGO_HUEY):
-#     pm.hook.register_DJANGO_HUEY(DJANGO_HUEY=DJANGO_HUEY)
-
-def get_ADMIN_DATA_VIEWS_URLS():
-    return itertools.chain(*reversed(pm.hook.get_ADMIN_DATA_VIEWS_URLS()))
-
-# def register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS):
-#     pm.hook.register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS=ADMIN_DATA_VIEWS)
-
-
-# def register_settings(settings):
-#     # convert settings dict to an benedict so we can set values using settings.attr = xyz notation
-#     settings_as_obj = benedict(settings, keypath_separator=None)
-    
-#     # set default values for settings that are used by plugins
-#     # settings_as_obj.INSTALLED_APPS = settings_as_obj.get('INSTALLED_APPS', [])
-#     # settings_as_obj.MIDDLEWARE = settings_as_obj.get('MIDDLEWARE', [])
-#     # settings_as_obj.AUTHENTICATION_BACKENDS = settings_as_obj.get('AUTHENTICATION_BACKENDS', [])
-#     # settings_as_obj.STATICFILES_DIRS = settings_as_obj.get('STATICFILES_DIRS', [])
-#     # settings_as_obj.TEMPLATE_DIRS = settings_as_obj.get('TEMPLATE_DIRS', [])
-#     # settings_as_obj.DJANGO_HUEY = settings_as_obj.get('DJANGO_HUEY', {'queues': {}})
-#     # settings_as_obj.ADMIN_DATA_VIEWS = settings_as_obj.get('ADMIN_DATA_VIEWS', {'URLS': []})
-    
-#     # # call all the hook functions to mutate the settings values in-place
-#     # register_INSTALLLED_APPS(settings_as_obj.INSTALLED_APPS)
-#     # register_MIDDLEWARES(settings_as_obj.MIDDLEWARE)
-#     # register_AUTHENTICATION_BACKENDS(settings_as_obj.AUTHENTICATION_BACKENDS)
-#     # register_STATICFILES_DIRS(settings_as_obj.STATICFILES_DIRS)
-#     # register_TEMPLATE_DIRS(settings_as_obj.TEMPLATE_DIRS)
-#     # register_DJANGO_HUEY(settings_as_obj.DJANGO_HUEY)
-#     # register_ADMIN_DATA_VIEWS(settings_as_obj.ADMIN_DATA_VIEWS)
-    
-#     # calls Plugin.settings(settings) on each registered plugin
-#     pm.hook.register_settings(settings=settings_as_obj)
-    
-#     # then finally update the settings globals() object will all the new settings
-#     # settings.update(settings_as_obj)
-
-
-def get_urlpatterns():
-    return list(itertools.chain(*pm.hook.urlpatterns()))
-
-def register_urlpatterns(urlpatterns):
-    pm.hook.register_urlpatterns(urlpatterns=urlpatterns)
-
-
-def register_checks():
-    """register any django system checks"""
-    pm.hook.register_checks()
-
-def register_admin(admin_site):
-    """register any django admin models/views with the main django admin site instance"""
-    pm.hook.register_admin(admin_site=admin_site)

archivebox/abx/hookspec.py

@@ -1,14 +0,0 @@
-from pathlib import Path
-
-from pluggy import HookimplMarker
-from pluggy import HookspecMarker
-
-spec = hookspec = HookspecMarker("abx")
-impl = hookimpl = HookimplMarker("abx")
-
-
-@hookspec
-@hookimpl
-def get_system_user() -> str:
-    return Path('~').expanduser().name
-

archivebox/abx/manager.py

@@ -1,30 +0,0 @@
-import inspect
-
-import pluggy
-
-
-class PluginManager(pluggy.PluginManager):
-    """
-    Patch to fix pluggy's PluginManager to work with pydantic models.
-    See: https://github.com/pytest-dev/pluggy/pull/536
-    """
-    def parse_hookimpl_opts(self, plugin, name: str) -> pluggy.HookimplOpts | None:
-        # IMPORTANT: @property methods can have side effects, and are never hookimpl
-        # if attr is a property, skip it in advance
-        plugin_class = plugin if inspect.isclass(plugin) else type(plugin)
-        if isinstance(getattr(plugin_class, name, None), property):
-            return None
-
-        # pydantic model fields are like attrs and also can never be hookimpls
-        plugin_is_pydantic_obj = hasattr(plugin, "__pydantic_core_schema__")
-        if plugin_is_pydantic_obj and name in getattr(plugin, "model_fields", {}):
-            # pydantic models mess with the class and attr __signature__
-            # so inspect.isroutine(...) throws exceptions and cant be used
-            return None
-        
-        try:
-            return super().parse_hookimpl_opts(plugin, name)
-        except AttributeError:
-            return super().parse_hookimpl_opts(type(plugin), name)
-
-pm = PluginManager("abx")

archivebox/abx/pydantic_pkgr/__init__.py

@@ -1 +0,0 @@
-__package__ = 'abx.pydantic_pkgr'

archivebox/abx/pydantic_pkgr/hookspec.py

@@ -1,13 +0,0 @@
-
-from ..hookspec import hookspec
-
-###########################################################################################
-
-@hookspec
-def get_BINPROVIDERS():
-    return {}
-
-@hookspec
-def get_BINARIES():
-    return {}
-

archivebox/api/__init__.py

@@ -1 +0,0 @@
-__package__ = 'archivebox.api'

archivebox/api/admin.py

@@ -1,31 +0,0 @@
-__package__ = 'archivebox.api'
-
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
-
-from abid_utils.admin import ABIDModelAdmin
-
-from api.models import APIToken
-
-
-class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    search_fields = ('id', 'abid', 'created_by__username', 'token')
-    fields = ('created_by', 'token', 'expires', *readonly_fields)
-
-    list_filter = ('created_by',)
-    ordering = ['-created_at']
-    list_per_page = 100
-
-
-class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
-
-
-def register_admin(admin_site):
-    admin_site.register(APIToken, APITokenAdmin)
-    admin_site.register(get_webhook_model(), CustomWebhookAdmin)

archivebox/api/apps.py

@@ -1,15 +1,5 @@
-__package__ = 'archivebox.api'
-
 from django.apps import AppConfig
 
-import abx
-
 
 class APIConfig(AppConfig):
     name = 'api'
-
-
-@abx.hookimpl
-def register_admin(admin_site):
-    from api.admin import register_admin
-    register_admin(admin_site)

archivebox/api/archive.py

@@ -0,0 +1,184 @@
+# archivebox_api.py
+from typing import List, Optional
+from enum import Enum
+from pydantic import BaseModel
+from ninja import Router
+from main import (
+    add,
+    remove,
+    update,
+    list_all,
+    ONLY_NEW,
+)  # Assuming these functions are defined in main.py
+
+
+# Schemas
+
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'
+
+
+class AddURLSchema(BaseModel):
+    urls: List[str]
+    tag: str = ""
+    depth: int = 0
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
+    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+
+class RemoveURLSchema(BaseModel):
+    yes: bool = False
+    delete: bool = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    filter_type: str = "exact"
+    filter_patterns: Optional[List[str]] = None
+
+
+class UpdateSchema(BaseModel):
+    resume: Optional[float] = None
+    only_new: Optional[bool] = None
+    index_only: Optional[bool] = False
+    overwrite: Optional[bool] = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    status: Optional[StatusChoices] = None
+    filter_type: Optional[str] = 'exact'
+    filter_patterns: Optional[List[str]] = None
+    extractors: Optional[str] = ""
+
+
+class ListAllSchema(BaseModel):
+    filter_patterns: Optional[List[str]] = None
+    filter_type: str = 'exact'
+    status: Optional[StatusChoices] = None
+    after: Optional[float] = None
+    before: Optional[float] = None
+    sort: Optional[str] = None
+    csv: Optional[str] = None
+    json: bool = False
+    html: bool = False
+    with_headers: bool = False
+
+
+# API Router
+router = Router()
+
+
+@router.post("/add", response={200: dict})
+def api_add(request, payload: AddURLSchema):
+    try:
+        result = add(
+            urls=payload.urls,
+            tag=payload.tag,
+            depth=payload.depth,
+            update=payload.update,
+            update_all=payload.update_all,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            init=payload.init,
+            extractors=payload.extractors,
+            parser=payload.parser,
+        )
+        # Currently the add function returns a list of ALL items in the DB, ideally only return new items
+        return {
+            "status": "success",
+            "message": "URLs added successfully.",
+            "result": str(result),
+        }
+    except Exception as e:
+        # Handle exceptions raised by the add function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/remove", response={200: dict})
+def api_remove(request, payload: RemoveURLSchema):
+    try:
+        result = remove(
+            yes=payload.yes,
+            delete=payload.delete,
+            before=payload.before,
+            after=payload.after,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+        )
+        return {
+            "status": "success",
+            "message": "URLs removed successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the remove function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/update", response={200: dict})
+def api_update(request, payload: UpdateSchema):
+    try:
+        result = update(
+            resume=payload.resume,
+            only_new=payload.only_new,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            before=payload.before,
+            after=payload.after,
+            status=payload.status,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+            extractors=payload.extractors,
+        )
+        return {
+            "status": "success",
+            "message": "Archive updated successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the update function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/list_all", response={200: dict})
+def api_list_all(request, payload: ListAllSchema):
+    try:
+        result = list_all(
+            filter_patterns=payload.filter_patterns,
+            filter_type=payload.filter_type,
+            status=payload.status,
+            after=payload.after,
+            before=payload.before,
+            sort=payload.sort,
+            csv=payload.csv,
+            json=payload.json,
+            html=payload.html,
+            with_headers=payload.with_headers,
+        )
+        # TODO: This is kind of bad, make the format a choice field
+        if payload.json:
+            return {"status": "success", "format": "json", "data": result}
+        elif payload.html:
+            return {"status": "success", "format": "html", "data": result}
+        elif payload.csv:
+            return {"status": "success", "format": "csv", "data": result}
+        else:
+            return {
+                "status": "success",
+                "message": "List generated successfully.",
+                "data": result,
+            }
+    except Exception as e:
+        # Handle exceptions raised by the list_all function or during processing
+        return {"status": "error", "message": str(e)}

archivebox/api/auth.py

@@ -1,139 +1,48 @@
-__package__ = 'archivebox.api'
-
-from typing import Any, Optional, cast
-from datetime import timedelta
-
-from django.http import HttpRequest
-from django.utils import timezone
-from django.contrib.auth import login
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import AbstractBaseUser
+from ninja import Form, Router, Schema
+from ninja.security import HttpBearer
 
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
-from ninja.errors import HttpError
+from api.models import Token
+
+router = Router()
 
 
-def get_or_create_api_token(user):
-    from api.models import APIToken
-    
-    if user and user.is_superuser:
-        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
-        if api_tokens.exists():
-            # unexpired token exists, use it
-            api_token = api_tokens.last()
-        else:
-            # does not exist, create a new one
-            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-        
-        assert api_token.is_valid(), f"API token is not valid {api_token}"
-
-        return api_token
-    return None
-
-
-def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
-    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
-    
-    user = None
-
-    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
-    if not submitted_empty_form:
+class GlobalAuth(HttpBearer):
+    def authenticate(self, request, token):
         try:
-            token = APIToken.objects.get(token=token)
-            if token.is_valid():
-                user = token.created_by
-                request._api_token = token
-        except APIToken.DoesNotExist:
+            return Token.objects.get(token=token).user
+        except Token.DoesNotExist:
             pass
 
-    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        return None
-    
-    return cast(AbstractBaseUser, user)
 
-def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
-    """Given a username and password, check if they are valid and return the corresponding user"""
-    user = None
-    
-    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
-    if not submitted_empty_form:
-        user = authenticate(
-            username=username,
-            password=password,
-        )
-
-    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        user = None
-
-    return cast(AbstractBaseUser | None, user)
+class AuthSchema(Schema):
+    email: str
+    password: str
 
 
-### Base Auth Types
+@router.post("/authenticate", auth=None)  # overriding global auth
+def get_token(request, auth_data: AuthSchema):
+    user = authenticate(username=auth_data.email, password=auth_data.password)
+    if user:
+        # Assuming a user can have multiple tokens and you want to create a new one every time
+        new_token = Token.objects.create(user=user)
+        return {"token": new_token.token, "expires": new_token.expiry_as_iso8601}
+    else:
+        return {"error": "Invalid credentials"}
 
 
-class APITokenAuthCheck:
-    """The base class for authentication methods that use an api.models.APIToken"""
-    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_token(
-            token=key,
-            request=request,
-        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-        return request.user
+class TokenValidationSchema(Schema):
+    token: str
 
 
-class UserPassAuthCheck:
-    """The base class for authentication methods that use a username & password"""
-    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_password(
-            username=username,
-            password=password,
-            request=request,
-        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-
-        return request.user
-
-
-### Django-Ninja-Provided Auth Methods
-
-class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
-    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-    param_name = "X-ArchiveBox-API-Key"
-
-class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
-    """Allow authenticating by passing Bearer=xyz as a request header"""
-    pass
-
-class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
-    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-    param_name = "api_key"
-
-class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-    pass
-
-### Enabled Auth Methods
-
-API_AUTH_METHODS = [
-    HeaderTokenAuth(),
-    BearerTokenAuth(),
-    QueryParamTokenAuth(), 
-    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
-    UsernameAndPasswordAuth(),
-]
+@router.post("/validate_token", auth=None) # No authentication required for this endpoint
+def validate_token(request, token_data: TokenValidationSchema):
+    try:
+        # Attempt to authenticate using the provided token
+        user = GlobalAuth().authenticate(request, token_data.token)
+        if user:
+            return {"status": "valid"}
+        else:
+            return {"status": "invalid"}
+    except Token.DoesNotExist:
+        return {"status": "invalid"}

archivebox/api/migrations/0001_initial.py

@@ -1,10 +1,9 @@
-# Generated by Django 4.2.11 on 2024-04-25 04:19
+# Generated by Django 3.1.14 on 2024-04-09 18:52
 
 import api.models
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import uuid
 
 
 class Migration(migrations.Migration):
@@ -17,13 +16,13 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.CreateModel(
-            name='APIToken',
+            name='Token',
             fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('token', models.CharField(default=auth.models.hex_uuid, max_length=32, unique=True)),
                 ('created', models.DateTimeField(auto_now_add=True)),
-                ('expires', models.DateTimeField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('expiry', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tokens', to=settings.AUTH_USER_MODEL)),
             ],
         ),
     ]

archivebox/api/migrations/0002_alter_apitoken_options.py

@@ -1,17 +0,0 @@
-# Generated by Django 5.0.4 on 2024-04-26 05:28
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='apitoken',
-            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
-        ),
-    ]

archivebox/api/migrations/0003_rename_user_apitoken_created_by_apitoken_abid_and_more.py

@@ -1,77 +0,0 @@
-# Generated by Django 5.0.6 on 2024-06-03 01:52
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import signal_webhooks.fields
-import signal_webhooks.utils
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_alter_apitoken_options'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='user',
-            new_name='created_by',
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='abid',
-            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='modified',
-            field=models.DateTimeField(auto_now=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, null=True, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
-        ),
-        migrations.CreateModel(
-            name='OutboundWebhook',
-            fields=[
-                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
-                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
-                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
-                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
-                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
-                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
-                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
-                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
-                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
-                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
-                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
-                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(blank=True, null=True, unique=True)),
-                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'verbose_name': 'API Outbound Webhook',
-                'abstract': False,
-            },
-        ),
-        migrations.AddConstraint(
-            model_name='outboundwebhook',
-            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
-        ),
-    ]

archivebox/api/migrations/0004_alter_apitoken_id_alter_apitoken_uuid.py

@@ -1,24 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 10:44
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
-        ),
-    ]

archivebox/api/migrations/0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:40
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='apitoken',
-            name='uuid',
-        ),
-        migrations.RemoveField(
-            model_name='outboundwebhook',
-            name='id',
-        ),
-    ]

archivebox/api/migrations/0006_remove_outboundwebhook_uuid_apitoken_id_and_more.py

@@ -1,29 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:43
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='uuid',
-            new_name='id'
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-    ]

archivebox/api/migrations/0007_alter_apitoken_created_by.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:52
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]

archivebox/api/migrations/0008_alter_apitoken_created_alter_apitoken_created_by_and_more.py

@@ -1,47 +0,0 @@
-# Generated by Django 5.1 on 2024-09-04 23:32
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0007_alter_apitoken_created_by'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-    ]

archivebox/api/migrations/0009_rename_created_apitoken_created_at_and_more.py

@@ -1,39 +0,0 @@
-# Generated by Django 5.1 on 2024-09-05 00:26
-
-import abid_utils.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='created',
-            new_name='created_at',
-        ),
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.AddField(
-            model_name='outboundwebhook',
-            name='created_at',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
-        ),
-    ]

archivebox/api/models.py

@@ -1,127 +1,30 @@
-__package__ = 'archivebox.api'
-
-import secrets
+import uuid
 from datetime import timedelta
 
 from django.conf import settings
 from django.db import models
 from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
 
-from signal_webhooks.models import WebhookBase
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
+def hex_uuid():
+    return uuid.uuid4().hex
 
 
-
-def generate_secret_token() -> str:
-    # returns cryptographically secure string with len() == 32
-    return secrets.token_hex(16)
-
-
-class APIToken(ABIDModel):
-    """
-    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
-    """
-    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
-    abid_prefix = 'apt_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.created_by_id'
-    abid_subtype_src = '"01"'
-    abid_rand_src = 'self.id'
-    abid_drift_allowed = True
-
-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-
-    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
-    expires = models.DateTimeField(null=True, blank=True)
-
-    class Meta(TypedModelMeta):
-        verbose_name = "API Key"
-        verbose_name_plural = "API Keys"
-
-    def __str__(self) -> str:
-        return self.token
-
-    def __repr__(self) -> str:
-        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
-
-    def __json__(self) -> dict:
-        return {
-            "TYPE":             "APIToken",    
-            "id":               str(self.pk),
-            "abid":             str(self.ABID),
-            "created_by_id":    str(self.created_by_id),
-            "token":            self.token,
-            "created_at":       self.created_at.isoformat(),
-            "expires":          self.expires_as_iso8601,
-        }
+class Token(models.Model):
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tokens"
+    )
+    token = models.CharField(max_length=32, default=hex_uuid, unique=True)
+    created = models.DateTimeField(auto_now_add=True)
+    expiry = models.DateTimeField(null=True, blank=True)
 
     @property
-    def expires_as_iso8601(self):
+    def expiry_as_iso8601(self):
         """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
-        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
-
+        expiry_date = (
+            self.expiry if self.expiry else timezone.now() + timedelta(days=365 * 100)
+        )
         return expiry_date.isoformat()
-    
-    @property
-    def token_redacted(self):
-        return f'************{self.token[-4:]}'
 
-    def is_valid(self, for_date=None):
-        for_date = for_date or timezone.now()
-
-        if self.expires and self.expires < for_date:
-            return False
-
-        return True
-
-
-
-
-
-
-# monkey patch django-signals-webhooks to change how it shows up in Admin UI
-
-class OutboundWebhook(ABIDModel, WebhookBase):
-    """
-    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
-        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
-    """
-    abid_prefix = 'whk_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.endpoint'
-    abid_subtype_src = 'self.ref'
-    abid_rand_src = 'self.id'
-    abid_drift_allowed = True
-
-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-
-    # More fields here: WebhookBase...
-
-    WebhookBase._meta.get_field('name').help_text = (
-        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
-    WebhookBase._meta.get_field('signal').help_text = (
-        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
-    WebhookBase._meta.get_field('ref').help_text = (
-        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
-    WebhookBase._meta.get_field('endpoint').help_text = (
-        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
-
-    class Meta(WebhookBase.Meta):
-        verbose_name = 'API Outbound Webhook'
-
-
-    def __str__(self) -> str:
-        return f'[{self.abid}] {self.ref} -> {self.endpoint}'
+    def __str__(self):
+        return self.token

archivebox/api/tests.py

@@ -1,30 +1,27 @@
-__package__ = 'archivebox.api'
+from django.test import TestCase
+from ninja.testing import TestClient
+from archivebox.api.archive import router as archive_router
 
-# from django.test import TestCase
-# from ninja.testing import TestClient
+class ArchiveBoxAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(archive_router)
 
-# from .routes_cli import router
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "test"})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
 
-# class ArchiveBoxCLIAPITestCase(TestCase):
-#     def setUp(self):
-#         self.client = TestClient(router)
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
 
-#     def test_add_endpoint(self):
-#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
 
-#     def test_remove_endpoint(self):
-#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_update_endpoint(self):
-#         response = self.client.post("/update", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_list_all_endpoint(self):
-#         response = self.client.post("/list_all", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue("success" in response.json()["status"])

archivebox/api/urls.py

@@ -1,17 +0,0 @@
-__package__ = 'archivebox.api'
-
-from django.urls import path
-from django.views.generic.base import RedirectView
-
-from .v1_api import urls as v1_api_urls
-
-urlpatterns = [
-    path("",                 RedirectView.as_view(url='/api/v1')),
-
-    path("v1/",              v1_api_urls),
-    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
-
-    # ... v2 can be added here ...
-    # path("v2/",              v2_api_urls),
-    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
-]

archivebox/api/v1_api.py

@@ -1,132 +0,0 @@
-__package__ = 'archivebox.api'
-
-
-from io import StringIO
-from traceback import format_exception
-from contextlib import redirect_stdout, redirect_stderr
-
-from django.http import HttpRequest, HttpResponse
-from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-
-from ninja import NinjaAPI, Swagger
-
-# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
-
-from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
-
-from api.auth import API_AUTH_METHODS
-
-
-COMMIT_HASH = get_COMMIT_HASH() or 'unknown'
-
-html_description=f'''
-<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
-<br/>
-<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
-<br/>
-<ul>
-<li>⬅️ Manage your server: <a href="/admin/api/"><b>Setup API Keys</b></a>, <a href="/admin/">Go to your Server Admin UI</a>, <a href="/">Go to your Snapshots list</a> 
-<li>💬 Ask questions and get help here: <a href="https://zulip.archivebox.io">ArchiveBox Chat Forum</a></li>
-<li>🐞 Report API bugs here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues">Github Issues</a></li>
-<li>📚 ArchiveBox Documentation: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Github Wiki</a></li>
-<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
-</ul>
-<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-'''
-
-
-def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router('/auth/',     'api.v1_auth.router')
-    api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/cli/',      'api.v1_cli.router')
-    return api
-
-
-class NinjaAPIWithIOCapture(NinjaAPI):    
-    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
-        stdout, stderr = StringIO(), StringIO()
-
-        with redirect_stderr(stderr):
-            with redirect_stdout(stdout):
-                request.stdout = stdout
-                request.stderr = stderr
-
-                response = super().create_temporal_response(request)
-
-        # Diable caching of API responses entirely
-        response['Cache-Control'] = 'no-store'
-
-        # Add debug stdout and stderr headers to response
-        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
-        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
-        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
-
-        # Add Auth Headers to response
-        api_token = getattr(request, '_api_token', None)
-        token_expiry = api_token.expires.isoformat() if api_token else 'Never'
-
-        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
-        response['X-ArchiveBox-Auth-Expires'] = token_expiry
-        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
-        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
-        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
-
-        # import ipdb; ipdb.set_trace()
-        # print('RESPONDING NOW', response)
-
-        return response
-
-
-api = NinjaAPIWithIOCapture(
-    title='ArchiveBox API',
-    description=html_description,
-    version='1.0.0',
-    csrf=False,
-    auth=API_AUTH_METHODS,
-    urls_namespace="api-1",
-    docs=Swagger(settings={"persistAuthorization": True}),
-    # docs_decorator=login_required,
-    # renderer=ORJSONRenderer(),
-)
-api = register_urls(api)
-urls = api.urls
-
-
-@api.exception_handler(Exception)
-def generic_exception_handler(request, err):
-    status = 503
-    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
-        status = 404
-
-    print(''.join(format_exception(err)))
-
-    return api.create_response(
-        request,
-        {
-            "succeeded": False,
-            "message": f'{err.__class__.__name__}: {err}',
-            "errors": [
-                ''.join(format_exception(err)),
-                # or send simpler parent-only traceback:
-                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
-            ],
-        },
-        status=status,
-    )
-
-
-
-# import orjson
-# from ninja.renderers import BaseRenderer
-# class ORJSONRenderer(BaseRenderer):
-#     media_type = "application/json"
-#     def render(self, request, data, *, response_status):
-#         return {
-#             "success": True,
-#             "errors": [],
-#             "result": data,
-#             "stdout": ansi_to_html(stdout.getvalue().strip()),
-#             "stderr": ansi_to_html(stderr.getvalue().strip()),
-#         }
-#         return orjson.dumps(data)

archivebox/api/v1_auth.py

@@ -1,53 +0,0 @@
-__package__ = 'archivebox.api'
-
-from typing import Optional
-
-from ninja import Router, Schema
-from django.utils import timezone
-from datetime import timedelta
-
-from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password, get_or_create_api_token
-
-
-router = Router(tags=['Authentication'], auth=None)
-
-
-class PasswordAuthSchema(Schema):
-    """Schema for a /get_api_token request"""
-    username: Optional[str] = None
-    password: Optional[str] = None
-
-
-@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
-def get_api_token(request, auth_data: PasswordAuthSchema):
-    user = auth_using_password(
-        username=auth_data.username,
-        password=auth_data.password,
-        request=request,
-    )
-
-    if user and user.is_superuser:
-        api_token = get_or_create_api_token(user)
-        assert api_token is not None, "Failed to create API token"
-        return api_token.__json__()
-    
-    return {"success": False, "errors": ["Invalid credentials"]}
-
-
-
-class TokenAuthSchema(Schema):
-    """Schema for a /check_api_token request"""
-    token: str
-
-
-@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
-def check_api_token(request, token_data: TokenAuthSchema):
-    user = auth_using_token(
-        token=token_data.token,
-        request=request,
-    )
-    if user:
-        return {"success": True, "user_id": str(user.pk)}
-    
-    return {"success": False, "user_id": None}

archivebox/api/v1_cli.py

@@ -1,236 +0,0 @@
-__package__ = 'archivebox.api'
-
-from typing import List, Dict, Any, Optional
-from enum import Enum
-
-from ninja import Router, Schema
-
-from ..main import (
-    add,
-    remove,
-    update,
-    list_all,
-    schedule,
-)
-from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import ARCHIVING_CONFIG
-
-
-from .auth import API_AUTH_METHODS
-
-# router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
-
-
-# Schemas
-
-JSONType = List[Any] | Dict[str, Any] | bool | int | str | None
-
-class CLICommandResponseSchema(Schema):
-    success: bool
-    errors: List[str]
-    result: JSONType
-    stdout: str
-    stderr: str
-
-class FilterTypeChoices(str, Enum):
-    exact = 'exact'
-    substring = 'substring'
-    regex = 'regex'
-    domain = 'domain'
-    tag = 'tag'
-    timestamp = 'timestamp'
-
-class StatusChoices(str, Enum):
-    indexed = 'indexed'
-    archived = 'archived'
-    unarchived = 'unarchived'
-    present = 'present'
-    valid = 'valid'
-    invalid = 'invalid'
-    duplicate = 'duplicate'
-    orphaned = 'orphaned'
-    corrupted = 'corrupted'
-    unrecognized = 'unrecognized'
-
-
-class AddCommandSchema(Schema):
-    urls: List[str]
-    tag: str = ""
-    depth: int = 0
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
-    update_all: bool = False
-    index_only: bool = False
-    overwrite: bool = False
-    init: bool = False
-    extractors: str = ""
-    parser: str = "auto"
-
-class UpdateCommandSchema(Schema):
-    resume: Optional[float] = 0
-    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
-    index_only: bool = False
-    overwrite: bool = False
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    status: Optional[StatusChoices] = StatusChoices.unarchived
-    filter_type: Optional[str] = FilterTypeChoices.substring
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-    extractors: Optional[str] = ""
-
-class ScheduleCommandSchema(Schema):
-    import_path: Optional[str] = None
-    add: bool = False
-    every: Optional[str] = None
-    tag: str = ''
-    depth: int = 0
-    overwrite: bool = False
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
-    clear: bool = False
-
-class ListCommandSchema(Schema):
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-    filter_type: str = FilterTypeChoices.substring
-    status: Optional[StatusChoices] = StatusChoices.indexed
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    sort: str = 'bookmarked_at'
-    as_json: bool = True
-    as_html: bool = False
-    as_csv: str | bool = 'timestamp,url'
-    with_headers: bool = False
-
-class RemoveCommandSchema(Schema):
-    delete: bool = True
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-
-
-
-
-
-@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
-def cli_add(request, args: AddCommandSchema):
-    result = add(
-        urls=args.urls,
-        tag=args.tag,
-        depth=args.depth,
-        update=args.update,
-        update_all=args.update_all,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        init=args.init,
-        extractors=args.extractors,
-        parser=args.parser,
-    )
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
-def cli_update(request, args: UpdateCommandSchema):
-    result = update(
-        resume=args.resume,
-        only_new=args.only_new,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        before=args.before,
-        after=args.after,
-        status=args.status,
-        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
-        extractors=args.extractors,
-    )
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-
-
-@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
-def cli_schedule(request, args: ScheduleCommandSchema):
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
-    )
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-
-
-
-@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
-def cli_list(request, args: ListCommandSchema):
-    result = list_all(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
-
-    result_format = 'txt'
-    if args.as_json:
-        result_format = "json"
-    elif args.as_html:
-        result_format = "html"
-    elif args.as_csv:
-        result_format = "csv"
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": result_format,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-    
-
-
-@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
-def cli_remove(request, args: RemoveCommandSchema):
-    result = remove(
-        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
-    )
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-    

archivebox/api/v1_core.py

@@ -1,486 +0,0 @@
-__package__ = 'archivebox.api'
-
-import math
-from uuid import UUID
-from typing import List, Optional, Union, Any
-from datetime import datetime
-
-from django.db.models import Q
-from django.core.exceptions import ValidationError
-from django.contrib.auth import get_user_model
-
-from ninja import Router, Schema, FilterSchema, Field, Query
-from ninja.pagination import paginate, PaginationBase
-from ninja.errors import HttpError
-
-from core.models import Snapshot, ArchiveResult, Tag
-from api.models import APIToken, OutboundWebhook
-
-from .auth import API_AUTH_METHODS
-
-router = Router(tags=['Core Models'], auth=API_AUTH_METHODS)
-
-
-
-class CustomPagination(PaginationBase):
-    class Input(Schema):
-        limit: int = 200
-        offset: int = 0
-        page: int = 0
-
-
-    class Output(Schema):
-        total_items: int
-        total_pages: int
-        page: int
-        limit: int
-        offset: int
-        num_items: int
-        items: List[Any]
-
-    def paginate_queryset(self, queryset, pagination: Input, **params):
-        limit = min(pagination.limit, 500)
-        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
-        total_pages = math.ceil(total / limit)
-        current_page = math.ceil(offset / (limit + 1))
-        items = queryset[offset : offset + limit]
-        return {
-            'total_items': total,
-            'total_pages': total_pages,
-            'page': current_page,
-            'limit': limit,
-            'offset': offset,
-            'num_items': len(items),
-            'items': items,
-        }
-
-
-### ArchiveResult #########################################################################
-
-class MinimalArchiveResultSchema(Schema):
-    TYPE: str = 'core.models.ArchiveResult'
-
-    id: UUID
-    abid: str
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-
-    extractor: str
-    cmd_version: Optional[str]
-    cmd: List[str]
-    pwd: str
-    status: str
-    output: str
-
-    start_ts: Optional[datetime]
-    end_ts: Optional[datetime]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-
-    @staticmethod
-    def resolve_abid(obj):
-        return str(obj.ABID)
-
-    @staticmethod
-    def resolve_created_at(obj):
-        return obj.start_ts
-
-    @staticmethod
-    def resolve_snapshot_timestamp(obj):
-        return obj.snapshot.timestamp
-    
-    @staticmethod
-    def resolve_snapshot_url(obj):
-        return obj.snapshot.url
-
-    @staticmethod
-    def resolve_snapshot_id(obj):
-        return str(obj.snapshot_id)
-    
-    @staticmethod
-    def resolve_snapshot_abid(obj):
-        return str(obj.snapshot.ABID)
-
-    @staticmethod
-    def resolve_snapshot_tags(obj):
-        return sorted(tag.name for tag in obj.snapshot.tags.all())
-
-class ArchiveResultSchema(MinimalArchiveResultSchema):
-    TYPE: str = 'core.models.ArchiveResult'
-
-    # ... Extends MinimalArchiveResultSchema fields ...
-
-    snapshot_id: UUID
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: List[str]
-
-
-class ArchiveResultFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-
-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-    snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
-    snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
-    
-    status: Optional[str] = Field(None, q='status')
-    output: Optional[str] = Field(None, q='output__icontains')
-    extractor: Optional[str] = Field(None, q='extractor__icontains')
-    cmd: Optional[str] = Field(None, q='cmd__0__icontains')
-    pwd: Optional[str] = Field(None, q='pwd__icontains')
-    cmd_version: Optional[str] = Field(None, q='cmd_version')
-
-    created_at: Optional[datetime] = Field(None, q='created_at')
-    created_at__gte: Optional[datetime] = Field(None, q='created_at__gte')
-    created_at__lt: Optional[datetime] = Field(None, q='created_at__lt')
-
-
-@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
-@paginate(CustomPagination)
-def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
-    """List all ArchiveResult entries matching these filters."""
-    qs = ArchiveResult.objects.all()
-    results = filters.filter(qs).distinct()
-    return results
-
-
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
-def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by id or abid."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id))
-
-
-# @router.post("/archiveresult", response=ArchiveResultSchema)
-# def create_archiveresult(request, payload: ArchiveResultSchema):
-#     archiveresult = ArchiveResult.objects.create(**payload.dict())
-#     return archiveresult
-#
-# @router.put("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
-# def update_archiveresult(request, archiveresult_id: str, payload: ArchiveResultSchema):
-#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
-#   
-#     for attr, value in payload.dict().items():
-#         setattr(archiveresult, attr, value)
-#     archiveresult.save()
-#
-#     return archiveresult
-#
-# @router.delete("/archiveresult/{archiveresult_id}")
-# def delete_archiveresult(request, archiveresult_id: str):
-#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
-#     archiveresult.delete()
-#     return {"success": True}
-
-
-
-
-
-### Snapshot #########################################################################
-
-
-class SnapshotSchema(Schema):
-    TYPE: str = 'core.models.Snapshot'
-
-    id: UUID
-    abid: str
-
-    created_by_id: str
-    created_by_username: str
-    created_at: datetime
-    modified_at: datetime
-
-    bookmarked_at: datetime
-    downloaded_at: Optional[datetime]
-
-    url: str
-    tags: List[str]
-    title: Optional[str]
-    timestamp: str
-    archive_path: str
-
-    # url_for_admin: str
-    # url_for_view: str
-
-    num_archiveresults: int
-    archiveresults: List[MinimalArchiveResultSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-
-    @staticmethod
-    def resolve_abid(obj):
-        return str(obj.ABID)
-
-    @staticmethod
-    def resolve_tags(obj):
-        return sorted(tag.name for tag in obj.tags.all())
-
-    # @staticmethod
-    # def resolve_url_for_admin(obj):
-    #     return f"/admin/core/snapshot/{obj.id}/change/"
-    
-    # @staticmethod
-    # def resolve_url_for_view(obj):
-    #     return f"/{obj.archive_path}"
-
-    @staticmethod
-    def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_archiveresults(obj, context):
-        if context['request'].with_archiveresults:
-            return obj.archiveresult_set.all().distinct()
-        return ArchiveResult.objects.none()
-
-
-class SnapshotFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'timestamp__startswith'])
-    abid: Optional[str] = Field(None, q='abid__icontains')
-
-    created_by_id: str = Field(None, q='created_by_id')
-    created_by_username: str = Field(None, q='created_by__username__icontains')
-
-    created_at__gte: datetime = Field(None, q='created_at__gte')
-    created_at__lt: datetime = Field(None, q='created_at__lt')
-    created_at: datetime = Field(None, q='created_at')
-    modified_at: datetime = Field(None, q='modified_at')
-    modified_at__gte: datetime = Field(None, q='modified_at__gte')
-    modified_at__lt: datetime = Field(None, q='modified_at__lt')
-
-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'timestamp__startswith'])
-    url: Optional[str] = Field(None, q='url')
-    tag: Optional[str] = Field(None, q='tags__name')
-    title: Optional[str] = Field(None, q='title__icontains')
-    timestamp: Optional[str] = Field(None, q='timestamp__startswith')
-    
-    bookmarked_at__gte: Optional[datetime] = Field(None, q='bookmarked_at__gte')
-    bookmarked_at__lt: Optional[datetime] = Field(None, q='bookmarked_at__lt')
-
-
-
-@router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
-@paginate(CustomPagination)
-def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
-    """List all Snapshot entries matching these filters."""
-    request.with_archiveresults = with_archiveresults
-
-    qs = Snapshot.objects.all()
-    results = filters.filter(qs).distinct()
-    return results
-
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid or id."""
-    request.with_archiveresults = with_archiveresults
-    snapshot = None
-    try:
-        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        pass
-
-    try:
-        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
-    except Snapshot.DoesNotExist:
-        pass
-
-    if not snapshot:
-        raise Snapshot.DoesNotExist
-
-    return snapshot
-
-
-# @router.post("/snapshot", response=SnapshotSchema)
-# def create_snapshot(request, payload: SnapshotSchema):
-#     snapshot = Snapshot.objects.create(**payload.dict())
-#     return snapshot
-#
-# @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
-# def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
-#
-#     for attr, value in payload.dict().items():
-#         setattr(snapshot, attr, value)
-#     snapshot.save()
-#
-#     return snapshot
-#
-# @router.delete("/snapshot/{snapshot_id}")
-# def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
-#     snapshot.delete()
-#     return {"success": True}
-
-
-
-### Tag #########################################################################
-
-
-class TagSchema(Schema):
-    TYPE: str = 'core.models.Tag'
-
-    id: UUID
-    abid: str
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-
-    name: str
-    slug: str
-    num_snapshots: int
-    snapshots: List[SnapshotSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-    
-    @staticmethod
-    def resolve_num_snapshots(obj, context):
-        return obj.snapshot_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if context['request'].with_snapshots:
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-@router.get("/tags", response=List[TagSchema], url_name="get_tags")
-@paginate(CustomPagination)
-def get_tags(request):
-    request.with_snapshots = False
-    request.with_archiveresults = False
-    return Tag.objects.all().distinct()
-
-@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request, tag_id: str, with_snapshots: bool=True):
-    request.with_snapshots = with_snapshots
-    request.with_archiveresults = False
-    tag = None
-    try:
-        tag = Tag.objects.get(abid__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        pass
-
-    try:
-        tag = tag or Tag.objects.get(id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        pass
-    return tag
-
-
-
-# class CrawlSchema(Schema):
-#     TYPE: str = 'core.models.Crawl'
-
-#     id: UUID
-#     abid: str
-
-#     modified_at: datetime
-#     created_at: datetime
-#     created_by_id: str
-#     created_by_username: str
-
-#     urls: str
-#     depth: int
-#     parser: str
-    
-#     # snapshots: List[SnapshotSchema]
-
-#     @staticmethod
-#     def resolve_created_by_id(obj):
-#         return str(obj.created_by_id)
-    
-#     @staticmethod
-#     def resolve_created_by_username(obj):
-#         User = get_user_model()
-#         return User.objects.get(id=obj.created_by_id).username
-    
-#     @staticmethod
-#     def resolve_snapshots(obj, context):
-#         if context['request'].with_snapshots:
-#             return obj.snapshot_set.all().distinct()
-#         return Snapshot.objects.none()
-
-
-# @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-# def get_crawl(request, crawl_id: str, with_snapshots: bool=False, with_archiveresults: bool=False):
-#     """Get a specific Crawl by id or abid."""
-#     crawl = None
-#     request.with_snapshots = with_snapshots
-#     request.with_archiveresults = with_archiveresults
-    
-#     try:
-#         crawl = Crawl.objects.get(abid__icontains=crawl_id)
-#     except Exception:
-#         pass
-
-#     try:
-#         crawl = crawl or Crawl.objects.get(id__icontains=crawl_id)
-#     except Exception:
-#         pass
-#     return crawl
-
-
-# [..., CrawlSchema]
-@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
-def get_any(request, abid: str):
-    request.with_snapshots = False
-    request.with_archiveresults = False
-
-    if abid.startswith(APIToken.abid_prefix):
-        raise HttpError(403, 'APIToken objects are not accessible via REST API')
-    
-    if abid.startswith(OutboundWebhook.abid_prefix):
-        raise HttpError(403, 'OutboundWebhook objects are not accessible via REST API')
-    
-    response = None
-    try:
-        response = response or get_snapshot(request, abid)
-    except Exception:
-        pass
-
-    try:
-        response = response or get_archiveresult(request, abid)
-    except Exception:
-        pass
-
-    try:
-        response = response or get_tag(request, abid)
-    except Exception:
-        pass
-    
-    # try:
-    #     response = response or get_crawl(request, abid)
-    # except Exception:
-    #     pass
-
-    if not response:
-        raise HttpError(404, 'Object with given ABID not found')
-
-    return response

archivebox/cli/__init__.py

@@ -4,72 +4,23 @@ __command__ = 'archivebox'
 import os
 import sys
 import argparse
-import threading
 
-from time import sleep
-from collections.abc import Mapping
-
-from rich import print
-
-from typing import Optional, List, IO, Union, Iterable
+from typing import Optional, Dict, List, IO, Union
 from pathlib import Path
 
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations
+
 from importlib import import_module
 
-BUILTIN_LIST = list
-
 CLI_DIR = Path(__file__).resolve().parent
 
-# rewrite setup -> install for backwards compatibility
-if len(sys.argv) > 1 and sys.argv[1] == 'setup':
-    from rich import print
-    print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
-    sys.argv[1] = 'install'
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
 
-if '--debug' in sys.argv:
-    os.environ['DEBUG'] = 'True'
-    sys.argv.remove('--debug')
-
-
-# def list_subcommands() -> Dict[str, str]:
-#     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
-#     COMMANDS = []
-#     for filename in os.listdir(CLI_DIR):
-#         if is_cli_module(filename):
-#             subcommand = filename.replace('archivebox_', '').replace('.py', '')
-#             module = import_module('.archivebox_{}'.format(subcommand), __package__)
-#             assert is_valid_cli_module(module, subcommand)
-#             COMMANDS.append((subcommand, module.main.__doc__))
-#             globals()[subcommand] = module.main
-#     display_order = lambda cmd: (
-#         display_first.index(cmd[0])
-#         if cmd[0] in display_first else
-#         100 + len(cmd[0])
-#     )
-#     return dict(sorted(COMMANDS, key=display_order))
-
-# just define it statically, it's much faster:
-SUBCOMMAND_MODULES = {
-    'help': 'archivebox_help',
-    'version': 'archivebox_version' ,
-    
-    'init': 'archivebox_init',
-    'install': 'archivebox_install',
-    ##############################################
-    'config': 'archivebox_config',
-    'add': 'archivebox_add',
-    'remove': 'archivebox_remove',
-    'update': 'archivebox_update',
-    'list': 'archivebox_list',
-    'status': 'archivebox_status',
-    
-    'schedule': 'archivebox_schedule',
-    'server': 'archivebox_server',
-    'shell': 'archivebox_shell',
-    'manage': 'archivebox_manage',
-
-    # 'oneshot': 'archivebox_oneshot',
-}
+display_first = (*meta_cmds, *main_cmds, *archive_cmds)
 
 # every imported command module must have these properties in order to be valid
 required_attrs = ('__package__', '__command__', 'main')
@@ -81,134 +32,65 @@ is_valid_cli_module = lambda module, subcommand: (
     and module.__command__.split(' ')[-1] == subcommand
 )
 
-class LazySubcommands(Mapping):
-    def keys(self):
-        return SUBCOMMAND_MODULES.keys()
-    
-    def values(self):
-        return [self[key] for key in self.keys()]
-    
-    def items(self):
-        return [(key, self[key]) for key in self.keys()]
-    
-    def __getitem__(self, key):
-        module = import_module(f'.{SUBCOMMAND_MODULES[key]}', __package__)
-        assert is_valid_cli_module(module, key)
-        return module.main
-    
-    def __iter__(self):
-        return iter(SUBCOMMAND_MODULES.keys())
-    
-    def __len__(self):
-        return len(SUBCOMMAND_MODULES)
 
-CLI_SUBCOMMANDS = LazySubcommands()
+def list_subcommands() -> Dict[str, str]:
+    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
 
+    COMMANDS = []
+    for filename in os.listdir(CLI_DIR):
+        if is_cli_module(filename):
+            subcommand = filename.replace('archivebox_', '').replace('.py', '')
+            module = import_module('.archivebox_{}'.format(subcommand), __package__)
+            assert is_valid_cli_module(module, subcommand)
+            COMMANDS.append((subcommand, module.main.__doc__))
+            globals()[subcommand] = module.main
 
-# these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-setup_cmds = ('init', 'setup', 'install')                      # require valid data folder, but dont require DB present in it yet
-archive_cmds = ('add', 'remove', 'update', 'list', 'status', 'schedule', 'server', 'shell', 'manage')  # require valid data folder + existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
-
-display_first = (*meta_cmds, *setup_cmds, *archive_cmds)
-
-
-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
-
-
-def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
-    """
-    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
-    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
-    """
-
-    wait_for_all: bool = thread_names == ()
-
-    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
-
-    should_wait = lambda thread: (
-        not thread_matches(thread, ignore_names)
-        and (wait_for_all or thread_matches(thread, thread_names)))
-
-    for tries in range(timeout):
-        all_threads = [*threading.enumerate()]
-        blocking_threads = [*filter(should_wait, all_threads)]
-        threads_summary = ', '.join(repr(t) for t in blocking_threads)
-        if blocking_threads:
-            sleep(1)
-            if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                print(
-                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
-                    threads_summary,
-                    file=sys.stderr,
-                )
-        else:
-            return tries
-
-    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
+    display_order = lambda cmd: (
+        display_first.index(cmd[0])
+        if cmd[0] in display_first else
+        100 + len(cmd[0])
+    )
 
+    return dict(sorted(COMMANDS, key=display_order))
 
 
 def run_subcommand(subcommand: str,
-                   subcommand_args: List[str] | None = None,
+                   subcommand_args: List[str]=None,
                    stdin: Optional[IO]=None,
                    pwd: Union[Path, str, None]=None) -> None:
     """Run a given ArchiveBox subcommand with the given list of args"""
 
     subcommand_args = subcommand_args or []
 
-    from archivebox.misc.checks import check_migrations
-    from archivebox.config.django import setup_django
-    
-    # print('DATA_DIR is', DATA_DIR)
-    # print('pwd is', os.getcwd())    
+    if subcommand not in meta_cmds:
+        from ..config import setup_django
 
-    cmd_requires_db = (subcommand in archive_cmds)
-    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
 
-    check_db = cmd_requires_db and not init_pending
+        if cmd_requires_db:
+            check_data_folder(pwd)
 
-    setup_django(in_memory_db=subcommand in fake_db, check_db=check_db)
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
 
-    for ignore_pattern in ('help', '-h', '--help', 'version', '--version'):
-        if ignore_pattern in sys.argv[:4]:
-            cmd_requires_db = False
-            break
-    
-    if subcommand in archive_cmds:
         if cmd_requires_db:
             check_migrations()
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
 
-    # wait for webhooks, signals, and other background jobs to finish before exit
-    wait_for_bg_threads_to_exit(timeout=60)
-
-
-
 
+SUBCOMMANDS = list_subcommands()
 
 class NotProvided:
-    def __len__(self):
-        return 0
-    def __bool__(self):
-        return False
-    def __repr__(self):
-        return '<not provided>'
-
-Omitted = Union[None, NotProvided]
-
-OMITTED = NotProvided()
+    pass
 
 
-def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: str | None=None) -> None:
-    # print('STARTING CLI MAIN ENTRYPOINT')
-    
-    args = sys.argv[1:] if args is OMITTED else args
-    stdin = sys.stdin if stdin is OMITTED else stdin
+def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
+    args = sys.argv[1:] if args is NotProvided else args
+    stdin = sys.stdin if stdin is NotProvided else stdin
 
+    subcommands = list_subcommands()
     parser = argparse.ArgumentParser(
         prog=__command__,
         description='ArchiveBox: The self-hosted internet archive',
@@ -218,19 +100,19 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
     group.add_argument(
         '--help', '-h',
         action='store_true',
-        help=CLI_SUBCOMMANDS['help'].__doc__,
+        help=subcommands['help'],
     )
     group.add_argument(
         '--version',
         action='store_true',
-        help=CLI_SUBCOMMANDS['version'].__doc__,
+        help=subcommands['version'],
     )
     group.add_argument(
         "subcommand",
         type=str,
         help= "The name of the subcommand to run",
         nargs='?',
-        choices=CLI_SUBCOMMANDS.keys(),
+        choices=subcommands.keys(),
         default=None,
     )
     parser.add_argument(
@@ -245,20 +127,29 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
     elif command.help or command.subcommand is None:
         command.subcommand = 'help'
 
-    if command.subcommand not in ('version',):
+    if command.subcommand not in ('help', 'version', 'status'):
         from ..logging_util import log_cli_command
 
         log_cli_command(
             subcommand=command.subcommand,
             subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
+            stdin=stdin,
+            pwd=pwd or OUTPUT_DIR
         )
 
-    try:
-        run_subcommand(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
-        )
-    except KeyboardInterrupt:
-        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')
+    run_subcommand(
+        subcommand=command.subcommand,
+        subcommand_args=command.subcommand_args,
+        stdin=stdin,
+        pwd=pwd or OUTPUT_DIR,
+    )
+
+
+__all__ = (
+    'SUBCOMMANDS',
+    'list_subcommands',
+    'run_subcommand',
+    *SUBCOMMANDS.keys(),
+)
+
+

archivebox/cli/archivebox_add.py

@@ -8,12 +8,10 @@ import argparse
 
 from typing import List, Optional, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from archivebox.config.common import ARCHIVING_CONFIG
-
 from ..main import add
+from ..util import docstring
 from ..parsers import PARSERS
+from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr
 
 
@@ -34,7 +32,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--update', #'-u',
         action='store_true',
-        default=not ARCHIVING_CONFIG.ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
     parser.add_argument(
@@ -119,7 +117,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         init=command.init,
         extractors=command.extract,
         parser=command.parser,
-        out_dir=pwd or DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
 
 

archivebox/cli/archivebox_config.py

@@ -5,13 +5,12 @@ __command__ = 'archivebox config'
 
 import sys
 import argparse
-from pathlib import Path
 
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import config
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, accept_stdin
 
 
@@ -24,11 +23,6 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         formatter_class=SmartFormatter,
     )
     group = parser.add_mutually_exclusive_group()
-    parser.add_argument(
-        '--search',
-        action='store_true',
-        help="Search config KEYs, VALUEs, and ALIASES for the given term",
-    )
     group.add_argument(
         '--get', #'-g',
         action='store_true',
@@ -59,11 +53,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     config(
         config_options_str=config_options_str,
         config_options=command.config_options,
-        search=command.search,
         get=command.get,
         set=command.set,
         reset=command.reset,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
 
 

archivebox/cli/archivebox_help.py

@@ -5,12 +5,12 @@ __command__ = 'archivebox help'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import help
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, reject_stdin
 
 
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
-    help(out_dir=Path(pwd) if pwd else DATA_DIR)
+    help(out_dir=pwd or OUTPUT_DIR)
 
 
 if __name__ == '__main__':

archivebox/cli/archivebox_init.py

@@ -8,10 +8,10 @@ import argparse
 
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import init
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
 
 
 @docstring(init.__doc__)
@@ -32,15 +32,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Run any updates or migrations without rechecking all snapshot dirs',
     )
-    parser.add_argument(
-        '--install', #'-s',
-        action='store_true',
-        help='Automatically install dependencies and extras used for archiving',
-    )
     parser.add_argument(
         '--setup', #'-s',
         action='store_true',
-        help='DEPRECATED: equivalent to --install',
+        help='Automatically install dependencies and extras used for archiving',
     )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
@@ -48,8 +43,8 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     init(
         force=command.force,
         quick=command.quick,
-        install=command.install or command.setup,
-        out_dir=pwd or DATA_DIR,
+        setup=command.setup,
+        out_dir=pwd or OUTPUT_DIR,
     )
     
 

archivebox/cli/archivebox_install.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox install'
-
-import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO
-
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import install
-
-
-@docstring(install.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=install.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--binproviders', '-p',
-        type=str,
-        help='Select binproviders to use DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--binaries', '-b',
-        type=str,
-        help='Select binaries to install DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--dry-run', '-d',
-        action='store_true',
-        help='Show what would be installed without actually installing anything',
-        default=False,
-    )
-    command = parser.parse_args(args or ())   # noqa
-    reject_stdin(__command__, stdin)
-
-    install(
-        # force=command.force,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        binaries=command.binaries.split(',') if command.binaries else None,
-        binproviders=command.binproviders.split(',') if command.binproviders else None,
-        dry_run=command.dry_run,
-    )
-    
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_list.py

@@ -5,12 +5,12 @@ __command__ = 'archivebox list'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import list_all
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..index import (
     LINK_FILTERS,
     get_indexed_folders,
@@ -131,7 +131,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         json=command.json,
         html=command.html,
         with_headers=command.with_headers,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
     raise SystemExit(not matching_folders)
 

archivebox/cli/archivebox_manage.py

@@ -4,19 +4,19 @@ __package__ = 'archivebox.cli'
 __command__ = 'archivebox manage'
 
 import sys
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import manage
+from ..util import docstring
+from ..config import OUTPUT_DIR
 
 
 @docstring(manage.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     manage(
         args=args,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
 
 

archivebox/cli/archivebox_oneshot.py

@@ -9,10 +9,10 @@ import argparse
 from pathlib import Path
 from typing import List, Optional, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, accept_stdin, stderr
 from ..main import oneshot
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin, stderr
 
 
 @docstring(oneshot.__doc__)
@@ -46,7 +46,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--out-dir',
         type=str,
-        default=DATA_DIR,
+        default=OUTPUT_DIR,
         help= "Path to save the single archive folder to, e.g. ./example.com_archive"
     )
     command = parser.parse_args(args or ())

archivebox/cli/archivebox_remove.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox remove'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, accept_stdin
 from ..main import remove
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin
 
 
 @docstring(remove.__doc__)
@@ -74,7 +74,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         after=command.after,
         yes=command.yes,
         delete=command.delete,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
     
 

archivebox/cli/archivebox_schedule.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox schedule'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import schedule
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
 
 
 @docstring(schedule.__doc__)
@@ -108,7 +108,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         overwrite=command.overwrite,
         update=command.update,
         import_path=command.import_path,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
 
 

archivebox/cli/archivebox_server.py

@@ -5,14 +5,13 @@ __command__ = 'archivebox server'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from archivebox.config.common import SERVER_CONFIG
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import server
+from ..util import docstring
+from ..config import OUTPUT_DIR, BIND_ADDR
+from ..logging_util import SmartFormatter, reject_stdin
 
 @docstring(server.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
@@ -26,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         'runserver_args',
         nargs='*',
         type=str,
-        default=[SERVER_CONFIG.BIND_ADDR],
+        default=[BIND_ADDR],
         help='Arguments to pass to Django runserver'
     )
     parser.add_argument(
@@ -59,11 +58,6 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Run archivebox manage createsuperuser before starting the server',
     )
-    parser.add_argument(
-        '--daemonize',
-        action='store_true',
-        help='Run the server in the background as a daemon',
-    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
@@ -74,8 +68,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         init=command.init,
         quick_init=command.quick_init,
         createsuperuser=command.createsuperuser,
-        daemonize=command.daemonize,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
 
 

archivebox/cli/archivebox_setup.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox setup'
+
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import setup
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+
+
+@docstring(setup.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=setup.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    # parser.add_argument(
+    #     '--force', # '-f',
+    #     action='store_true',
+    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
+    # )
+    command = parser.parse_args(args or ())   # noqa
+    reject_stdin(__command__, stdin)
+
+    setup(
+        # force=command.force,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_shell.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox shell'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import shell
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
 
 
 @docstring(shell.__doc__)
@@ -26,7 +26,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     reject_stdin(__command__, stdin)
     
     shell(
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
     )
     
 

archivebox/cli/archivebox_status.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox status'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import status
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
 
 
 @docstring(status.__doc__)
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
-    status(out_dir=Path(pwd) if pwd else DATA_DIR)
+    status(out_dir=pwd or OUTPUT_DIR)
 
 
 if __name__ == '__main__':

archivebox/cli/archivebox_update.py

@@ -5,11 +5,12 @@ __command__ = 'archivebox update'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import List, Optional, IO
 
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
+from ..main import update
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..index import (
     LINK_FILTERS,
     get_indexed_folders,
@@ -24,7 +25,6 @@ from ..index import (
     get_unrecognized_folders,
 )
 from ..logging_util import SmartFormatter, accept_stdin
-from ..main import update
 
 
 @docstring(update.__doc__)
@@ -127,7 +127,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         status=command.status,
         after=command.after,
         before=command.before,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
         extractors=command.extract,
     )
     

archivebox/cli/archivebox_version.py

@@ -5,20 +5,20 @@ __command__ = 'archivebox version'
 
 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO
 
-# from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR, VERSION
+from ..main import version
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, reject_stdin
 
 
-# @docstring(version.__doc__)
+@docstring(version.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    """Print the ArchiveBox version and dependency information"""
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description="Print the ArchiveBox version and dependency information",   # version.__doc__,
+        description=version.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
@@ -27,33 +27,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Only print ArchiveBox version number and nothing else.',
     )
-    parser.add_argument(
-        '--binproviders', '-p',
-        type=str,
-        help='Select binproviders to detect DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--binaries', '-b',
-        type=str,
-        help='Select binaries to detect DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
-        default=None,
-    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
-    # for speed reasons, check if quiet flag was set and just return simple version immediately if so
-    if command.quiet:
-        print(VERSION)
-        return
-    
-    # otherwise do big expensive import to get the full version
-    from ..main import version
     version(
         quiet=command.quiet,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        binproviders=command.binproviders.split(',') if command.binproviders else None,
-        binaries=command.binaries.split(',') if command.binaries else None,
+        out_dir=pwd or OUTPUT_DIR,
     )
 
 

archivebox/cli/tests.py

@@ -15,7 +15,7 @@ TEST_CONFIG = {
     'USE_COLOR': 'False',
     'SHOW_PROGRESS': 'False',
 
-    'DATA_DIR': 'data.tests',
+    'OUTPUT_DIR': 'data.tests',
     
     'SAVE_ARCHIVE_DOT_ORG': 'False',
     'SAVE_TITLE': 'False',
@@ -27,12 +27,12 @@ TEST_CONFIG = {
     'USE_YOUTUBEDL': 'False',
 }
 
-DATA_DIR = 'data.tests'
+OUTPUT_DIR = 'data.tests'
 os.environ.update(TEST_CONFIG)
 
 from ..main import init
 from ..index import load_main_index
-from archivebox.config.constants import (
+from ..config import (
     SQL_INDEX_FILENAME,
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,
@@ -101,22 +101,22 @@ def output_hidden(show_failing=True):
 
 class TestInit(unittest.TestCase):
     def setUp(self):
-        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
 
     def tearDown(self):
-        shutil.rmtree(DATA_DIR, ignore_errors=True)
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
 
     def test_basic_init(self):
         with output_hidden():
             archivebox_init.main([])
 
-        assert (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
-        assert (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
-        assert (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
-        assert len(load_main_index(out_dir=DATA_DIR)) == 0
+        assert (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
+        assert (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
+        assert (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
+        assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
 
     def test_conflicting_init(self):
-        with open(Path(DATA_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
             f.write('test')
 
         try:
@@ -126,11 +126,11 @@ class TestInit(unittest.TestCase):
         except SystemExit:
             pass
 
-        assert not (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
-        assert not (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
-        assert not (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
         try:
-            load_main_index(out_dir=DATA_DIR)
+            load_main_index(out_dir=OUTPUT_DIR)
             assert False, 'load_main_index should raise an exception when no index is present'
         except Exception:
             pass
@@ -138,36 +138,36 @@ class TestInit(unittest.TestCase):
     def test_no_dirty_state(self):
         with output_hidden():
             init()
-        shutil.rmtree(DATA_DIR, ignore_errors=True)
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
         with output_hidden():
             init()
 
 
 class TestAdd(unittest.TestCase):
     def setUp(self):
-        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
         with output_hidden():
             init()
 
     def tearDown(self):
-        shutil.rmtree(DATA_DIR, ignore_errors=True)
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
 
     def test_add_arg_url(self):
         with output_hidden():
             archivebox_add.main(['https://getpocket.com/users/nikisweeting/feed/all'])
 
-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
         assert len(all_links) == 30
 
     def test_add_arg_file(self):
-        test_file = Path(DATA_DIR) / 'test.txt'
+        test_file = Path(OUTPUT_DIR) / 'test.txt'
         with open(test_file, 'w+', encoding='utf') as f:
             f.write(test_urls)
 
         with output_hidden():
             archivebox_add.main([test_file])
 
-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
         assert len(all_links) == 12
         os.remove(test_file)
 
@@ -175,40 +175,40 @@ class TestAdd(unittest.TestCase):
         with output_hidden():
             archivebox_add.main([], stdin=test_urls)
 
-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
         assert len(all_links) == 12
 
 
 class TestRemove(unittest.TestCase):
     def setUp(self):
-        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
         with output_hidden():
             init()
             archivebox_add.main([], stdin=test_urls)
 
     # def tearDown(self):
-        # shutil.rmtree(DATA_DIR, ignore_errors=True)
+        # shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
 
 
     def test_remove_exact(self):
         with output_hidden():
             archivebox_remove.main(['--yes', '--delete', 'https://example5.com/'])
 
-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
         assert len(all_links) == 11
 
     def test_remove_regex(self):
         with output_hidden():
             archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)'])
 
-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
         assert len(all_links) == 4
 
     def test_remove_domain(self):
         with output_hidden():
             archivebox_remove.main(['--yes', '--delete', '--filter-type=domain', 'example5.com', 'example6.com'])
 
-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
         assert len(all_links) == 10
 
     def test_remove_none(self):

archivebox/config/__init__.py

@@ -1,38 +0,0 @@
-__package__ = 'archivebox.config'
-
-from .paths import (
-    PACKAGE_DIR,                                    # noqa
-    DATA_DIR,                                       # noqa
-    ARCHIVE_DIR,                                    # noqa
-)
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR      # noqa
-from .version import VERSION                        # noqa
-
-
-import abx
-
-
-# @abx.hookimpl
-# def get_INSTALLED_APPS():
-#     return ['config']
-
-
-@abx.hookimpl
-def get_CONFIG():
-    from .common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-    return {
-        'SHELL_CONFIG': SHELL_CONFIG,
-        'STORAGE_CONFIG': STORAGE_CONFIG,
-        'GENERAL_CONFIG': GENERAL_CONFIG,
-        'SERVER_CONFIG': SERVER_CONFIG,
-        'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
-        'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
-    }
-

archivebox/config/common.py

@@ -1,193 +0,0 @@
-__package__ = 'archivebox.config'
-
-import re
-import sys
-import shutil
-from typing import Dict, Optional, List
-from pathlib import Path
-
-from rich import print
-from pydantic import Field, field_validator, computed_field
-from django.utils.crypto import get_random_string
-
-from abx.archivebox.base_configset import BaseConfigSet
-
-from .constants import CONSTANTS
-from .version import get_COMMIT_HASH, get_BUILD_TIME
-from .permissions import IN_DOCKER
-
-###################### Config ##########################
-
-
-class ShellConfig(BaseConfigSet):
-    DEBUG: bool                         = Field(default=lambda: '--debug' in sys.argv)
-    
-    IS_TTY: bool                        = Field(default=sys.stdout.isatty())
-    USE_COLOR: bool                     = Field(default=lambda c: c.IS_TTY)
-    SHOW_PROGRESS: bool                 = Field(default=lambda c: c.IS_TTY)
-    
-    IN_DOCKER: bool                     = Field(default=IN_DOCKER)
-    IN_QEMU: bool                       = Field(default=False)
-
-    ANSI: Dict[str, str]                = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
-
-    VERSIONS_AVAILABLE: bool = False             # .check_for_update.get_versions_available_on_github(c)},
-    CAN_UPGRADE: bool = False                    # .check_for_update.can_upgrade(c)},
-
-    @computed_field
-    @property
-    def TERM_WIDTH(self) -> int:
-        if not self.IS_TTY:
-            return 200
-        return shutil.get_terminal_size((140, 10)).columns
-    
-    @computed_field
-    @property
-    def COMMIT_HASH(self) -> Optional[str]:
-        return get_COMMIT_HASH()
-    
-    @computed_field
-    @property
-    def BUILD_TIME(self) -> str:
-        return get_BUILD_TIME()
-
-SHELL_CONFIG = ShellConfig()
-
-
-class StorageConfig(BaseConfigSet):
-    # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
-    # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
-    TMP_DIR: Path                       = Field(default=CONSTANTS.DEFAULT_TMP_DIR)
-    
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be able to contain executable binaries (up to 5GB size)
-    # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path                       = Field(default=CONSTANTS.DEFAULT_LIB_DIR)
-    
-    OUTPUT_PERMISSIONS: str             = Field(default='644')
-    RESTRICT_FILE_NAMES: str            = Field(default='windows')
-    ENFORCE_ATOMIC_WRITES: bool         = Field(default=True)
-    
-    # not supposed to be user settable:
-    DIR_OUTPUT_PERMISSIONS: str         = Field(default=lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5'))
-
-
-STORAGE_CONFIG = StorageConfig()
-
-
-class GeneralConfig(BaseConfigSet):
-    TAG_SEPARATOR_PATTERN: str          = Field(default=r'[,]')
-
-
-GENERAL_CONFIG = GeneralConfig()
-
-
-class ServerConfig(BaseConfigSet):
-    SECRET_KEY: str                     = Field(default=lambda: get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_'))
-    BIND_ADDR: str                      = Field(default=lambda: ['127.0.0.1:8000', '0.0.0.0:8000'][SHELL_CONFIG.IN_DOCKER])
-    ALLOWED_HOSTS: str                  = Field(default='*')
-    CSRF_TRUSTED_ORIGINS: str           = Field(default=lambda c: 'http://localhost:8000,http://127.0.0.1:8000,http://0.0.0.0:8000,http://{}'.format(c.BIND_ADDR))
-    
-    SNAPSHOTS_PER_PAGE: int             = Field(default=40)
-    FOOTER_INFO: str                    = Field(default='Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.')
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
-    PUBLIC_INDEX: bool                  = Field(default=True)
-    PUBLIC_SNAPSHOTS: bool              = Field(default=True)
-    PUBLIC_ADD_VIEW: bool               = Field(default=False)
-    
-    ADMIN_USERNAME: str                 = Field(default=None)
-    ADMIN_PASSWORD: str                 = Field(default=None)
-    REVERSE_PROXY_USER_HEADER: str      = Field(default='Remote-User')
-    REVERSE_PROXY_WHITELIST: str        = Field(default='')
-    LOGOUT_REDIRECT_URL: str            = Field(default='/')
-    PREVIEW_ORIGINALS: bool             = Field(default=True)
-    
-SERVER_CONFIG = ServerConfig()
-
-
-class ArchivingConfig(BaseConfigSet):
-    ONLY_NEW: bool                        = Field(default=True)
-    
-    TIMEOUT: int                          = Field(default=60)
-    MEDIA_TIMEOUT: int                    = Field(default=3600)
-
-    MEDIA_MAX_SIZE: str                   = Field(default='750m')
-    RESOLUTION: str                       = Field(default='1440,2000')
-    CHECK_SSL_VALIDITY: bool              = Field(default=True)
-    USER_AGENT: str                       = Field(default='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
-    COOKIES_FILE: Path | None             = Field(default=None)
-    
-    URL_DENYLIST: str                     = Field(default=r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', alias='URL_BLACKLIST')
-    URL_ALLOWLIST: str | None             = Field(default=None, alias='URL_WHITELIST')
-    
-    SAVE_ALLOWLIST: Dict[str, List[str]]  = Field(default={})  # mapping of regex patterns to list of archive methods
-    SAVE_DENYLIST: Dict[str, List[str]]   = Field(default={})
-    
-    # GIT_DOMAINS: str                    = Field(default='github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht')
-    # WGET_USER_AGENT: str                = Field(default=lambda c: c['USER_AGENT'] + ' wget/{WGET_VERSION}')
-    # CURL_USER_AGENT: str                = Field(default=lambda c: c['USER_AGENT'] + ' curl/{CURL_VERSION}')
-    # CHROME_USER_AGENT: str              = Field(default=lambda c: c['USER_AGENT'])
-    # CHROME_USER_DATA_DIR: str | None    = Field(default=None)
-    # CHROME_TIMEOUT: int                 = Field(default=0)
-    # CHROME_HEADLESS: bool               = Field(default=True)
-    # CHROME_SANDBOX: bool                = Field(default=lambda: not SHELL_CONFIG.IN_DOCKER)
-
-    @field_validator('TIMEOUT', mode='after')
-    def validate_timeout(cls, v):
-        if int(v) < 5:
-            print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={v} seconds)[/red]', file=sys.stderr)
-            print('    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
-            print('    (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
-            print(file=sys.stderr)
-            print('    If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
-            print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
-            print(file=sys.stderr)
-        return v
-    
-    @field_validator('CHECK_SSL_VALIDITY', mode='after')
-    def validate_check_ssl_validity(cls, v):
-        """SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests"""
-        if not v:
-            import requests
-            import urllib3
-            requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
-            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-        return v
-    
-    @property
-    def URL_ALLOWLIST_PTN(self) -> re.Pattern | None:
-        return re.compile(self.URL_ALLOWLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) if self.URL_ALLOWLIST else None
-    
-    @property
-    def URL_DENYLIST_PTN(self) -> re.Pattern:
-        return re.compile(self.URL_DENYLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)
-    
-    @property
-    def SAVE_ALLOWLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
-        return {
-            re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v
-            for k, v in self.SAVE_ALLOWLIST.items()
-        } if self.SAVE_ALLOWLIST else {}
-    
-    @property
-    def SAVE_DENYLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
-        return {
-            re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v
-            for k, v in self.SAVE_DENYLIST.items()
-        } if self.SAVE_DENYLIST else {}
-
-ARCHIVING_CONFIG = ArchivingConfig()
-
-
-class SearchBackendConfig(BaseConfigSet):
-    USE_INDEXING_BACKEND: bool          = Field(default=True)
-    USE_SEARCHING_BACKEND: bool         = Field(default=True)
-    
-    SEARCH_BACKEND_ENGINE: str          = Field(default='ripgrep')
-    SEARCH_PROCESS_HTML: bool           = Field(default=True)
-    SEARCH_BACKEND_TIMEOUT: int         = Field(default=10)
-
-SEARCH_BACKEND_CONFIG = SearchBackendConfig()
-

archivebox/config/configfile.py

@@ -1,245 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import json
-from typing import Any, Optional, Type, Tuple, Dict
-
-from pathlib import Path
-from configparser import ConfigParser
-
-from benedict import benedict
-
-from archivebox.config.constants import CONSTANTS
-
-from archivebox.misc.logging import stderr
-
-
-def get_real_name(key: str) -> str:
-    """get the current canonical name for a given deprecated config key"""
-    from django.conf import settings
-    
-    for section in settings.CONFIGS.values():
-        try:
-            return section.aliases[key]
-        except KeyError:
-            pass
-    return key
-
-
-def load_config_val(key: str,
-                    default: Any=None,
-                    type: Optional[Type]=None,
-                    aliases: Optional[Tuple[str, ...]]=None,
-                    config: Optional[benedict]=None,
-                    env_vars: Optional[os._Environ]=None,
-                    config_file_vars: Optional[Dict[str, str]]=None) -> Any:
-    """parse bool, int, and str key=value pairs from env"""
-
-    assert isinstance(config, dict)
-
-    is_read_only = type is None
-    if is_read_only:
-        if callable(default):
-            return default(config)
-        return default
-
-    # get value from environment variables or config files
-    config_keys_to_check = (key, *(aliases or ()))
-    val = None
-    for key in config_keys_to_check:
-        if env_vars:
-            val = env_vars.get(key)
-            if val:
-                break
-
-        if config_file_vars:
-            val = config_file_vars.get(key)
-            if val:
-                break
-
-    is_unset = val is None
-    if is_unset:
-        if callable(default):
-            return default(config)
-        return default
-
-    # calculate value based on expected type
-    BOOL_TRUEIES = ('true', 'yes', '1')
-    BOOL_FALSEIES = ('false', 'no', '0')
-
-    if type is bool:
-        if val.lower() in BOOL_TRUEIES:
-            return True
-        elif val.lower() in BOOL_FALSEIES:
-            return False
-        else:
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)')
-
-    elif type is str:
-        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)')
-        return val.strip()
-
-    elif type is int:
-        if not val.strip().isdigit():
-            raise ValueError(f'Invalid configuration option {key}={val} (expected an integer)')
-        return int(val.strip())
-
-    elif type is list or type is dict:
-        return json.loads(val)
-    
-    elif type is Path:
-        return Path(val)
-
-    raise Exception('Config values can only be str, bool, int, or json')
-
-
-def load_config_file() -> Optional[benedict]:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
-    config_path = CONSTANTS.CONFIG_FILE
-    if os.access(config_path, os.R_OK):
-        config_file = ConfigParser()
-        config_file.optionxform = str
-        config_file.read(config_path)
-        # flatten into one namespace
-        config_file_vars = benedict({
-            key.upper(): val
-            for section, options in config_file.items()
-                for key, val in options.items()
-        })
-        # print('[i] Loaded config file', os.path.abspath(config_path))
-        # print(config_file_vars)
-        return config_file_vars
-    return None
-
-
-def section_for_key(key: str) -> Any:
-    from django.conf import settings
-    for config_section in settings.CONFIGS.values():
-        if hasattr(config_section, key):
-            return config_section
-    return None
-
-
-def write_config_file(config: Dict[str, str]) -> benedict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
-    import abx.archivebox.reads
-    from archivebox.misc.system import atomic_write
-
-    CONFIG_HEADER = (
-    """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
-
-    """)
-
-    config_path = CONSTANTS.CONFIG_FILE
-
-    if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
-
-    config_file = ConfigParser()
-    config_file.optionxform = str
-    config_file.read(config_path)
-
-    with open(config_path, 'r', encoding='utf-8') as old:
-        atomic_write(f'{config_path}.bak', old.read())
-
-    # Set up sections in empty config file
-    for key, val in config.items():
-        section = section_for_key(key)
-        assert section is not None
-        
-        section_name = section.toml_section_header
-        
-        if section_name in config_file:
-            existing_config = dict(config_file[section_name])
-        else:
-            existing_config = {}
-        
-        config_file[section_name] = benedict({**existing_config, key: val})
-        section.update_in_place(warn=False, persist=False, **{key: val})
-
-    with open(config_path, 'w+', encoding='utf-8') as new:
-        config_file.write(new)
-
-    updated_config = {}
-    try:
-        # validate the updated_config by attempting to re-parse it
-        updated_config = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
-    except BaseException:                                                       # lgtm [py/catch-base-exception]
-        # something went horribly wrong, revert to the previous version
-        with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
-            atomic_write(config_path, old.read())
-
-        raise
-
-    if os.access(f'{config_path}.bak', os.F_OK):
-        os.remove(f'{config_path}.bak')
-
-    return benedict({
-        key.upper(): updated_config.get(key.upper())
-        for key in config.keys()
-    })
-
-
-
-def load_config(defaults: Dict[str, Any],
-                config: Optional[benedict]=None,
-                out_dir: Optional[str]=None,
-                env_vars: Optional[os._Environ]=None,
-                config_file_vars: Optional[Dict[str, str]]=None) -> benedict:
-
-    env_vars = env_vars or os.environ
-    config_file_vars = config_file_vars or load_config_file()
-
-    extended_config = benedict(config.copy() if config else {})
-    for key, default in defaults.items():
-        try:
-            # print('LOADING CONFIG KEY:', key, 'DEFAULT=', default)
-            extended_config[key] = load_config_val(
-                key,
-                default=default['default'],
-                type=default.get('type'),
-                aliases=default.get('aliases'),
-                config=extended_config,
-                env_vars=env_vars,
-                config_file_vars=config_file_vars,
-            )
-        except KeyboardInterrupt:
-            raise SystemExit(0)
-        except Exception as e:
-            stderr()
-            stderr(f'[X] Error while loading configuration value: {key}', color='red', config=extended_config)
-            stderr('    {}: {}'.format(e.__class__.__name__, e))
-            stderr()
-            stderr('    Check your config for mistakes and try again (your archive data is unaffected).')
-            stderr()
-            stderr('    For config documentation and examples see:')
-            stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration')
-            stderr()
-            # raise
-            # raise SystemExit(2)
-
-    return benedict(extended_config)
-
-def load_all_config():
-    import abx.archivebox.reads
-    
-    flat_config = benedict()
-    
-    for config_section in abx.archivebox.reads.get_CONFIGS().values():
-        config_section.__init__()
-        flat_config.update(config_section.model_dump())
-        
-    return flat_config
-

archivebox/config/constants.py

@@ -1,218 +0,0 @@
-__package__ = 'archivebox.config'
-
-import re
-import sys
-
-from typing import Dict
-from pathlib import Path
-from collections.abc import Mapping
-
-from benedict import benedict
-
-from ..misc.logging import DEFAULT_CLI_COLORS
-
-from .paths import (
-    PACKAGE_DIR,
-    DATA_DIR,
-    ARCHIVE_DIR,
-    get_collection_id,
-    get_machine_id,
-    get_machine_type,
-)
-from .permissions import (
-    IS_ROOT,
-    IN_DOCKER,
-    RUNNING_AS_UID,
-    RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
-    ARCHIVEBOX_USER,
-    ARCHIVEBOX_GROUP,
-)
-from .version import detect_installed_version
-
-###################### Config ##########################
-
-
-class ConstantsDict(Mapping):
-    PACKAGE_DIR: Path                   = PACKAGE_DIR
-    DATA_DIR: Path                      = DATA_DIR
-    ARCHIVE_DIR: Path                   = ARCHIVE_DIR
-    
-    MACHINE_TYPE: str                   = get_machine_type()
-    MACHINE_ID: str                     = get_machine_id()
-    COLLECTION_ID: str                  = get_collection_id(DATA_DIR)
-    
-    # Host system
-    VERSION: str                        = detect_installed_version(PACKAGE_DIR)
-    IN_DOCKER: bool                     = IN_DOCKER
-    
-    # Permissions
-    IS_ROOT: bool                       = IS_ROOT
-    ARCHIVEBOX_USER: int                = ARCHIVEBOX_USER
-    ARCHIVEBOX_GROUP: int               = ARCHIVEBOX_GROUP
-    RUNNING_AS_UID: int                 = RUNNING_AS_UID
-    RUNNING_AS_GID: int                 = RUNNING_AS_GID
-    DEFAULT_PUID: int                   = DEFAULT_PUID
-    DEFAULT_PGID: int                   = DEFAULT_PGID
-    IS_INSIDE_VENV: bool                = sys.prefix != sys.base_prefix
-    
-    # Source code dirs
-    PACKAGE_DIR_NAME: str               = PACKAGE_DIR.name
-    TEMPLATES_DIR_NAME: str             = 'templates'
-    TEMPLATES_DIR: Path                 = PACKAGE_DIR / TEMPLATES_DIR_NAME
-    STATIC_DIR_NAME: str                = 'static'
-    STATIC_DIR: Path                    = TEMPLATES_DIR / STATIC_DIR_NAME
-
-    # Data dirs
-    ARCHIVE_DIR_NAME: str               = 'archive'
-    SOURCES_DIR_NAME: str               = 'sources'
-    PERSONAS_DIR_NAME: str              = 'personas'
-    CRONTABS_DIR_NAME: str              = 'crontabs'
-    CACHE_DIR_NAME: str                 = 'cache'
-    LOGS_DIR_NAME: str                  = 'logs'
-    USER_PLUGINS_DIR_NAME: str          = 'user_plugins'
-    CUSTOM_TEMPLATES_DIR_NAME: str      = 'user_templates'
-    ARCHIVE_DIR: Path                   = DATA_DIR / ARCHIVE_DIR_NAME
-    SOURCES_DIR: Path                   = DATA_DIR / SOURCES_DIR_NAME
-    PERSONAS_DIR: Path                  = DATA_DIR / PERSONAS_DIR_NAME
-    LOGS_DIR: Path                      = DATA_DIR / LOGS_DIR_NAME
-    CACHE_DIR: Path                     = DATA_DIR / CACHE_DIR_NAME
-    CUSTOM_TEMPLATES_DIR: Path          = DATA_DIR / CUSTOM_TEMPLATES_DIR_NAME
-    USER_PLUGINS_DIR: Path              = DATA_DIR / USER_PLUGINS_DIR_NAME
-
-    # Data dir files
-    CONFIG_FILENAME: str                = 'ArchiveBox.conf'
-    SQL_INDEX_FILENAME: str             = 'index.sqlite3'
-    QUEUE_DATABASE_FILENAME: str        = 'queue.sqlite3'
-    CONFIG_FILE: Path                   = DATA_DIR / CONFIG_FILENAME
-    DATABASE_FILE: Path                 = DATA_DIR / SQL_INDEX_FILENAME
-    QUEUE_DATABASE_FILE: Path           = DATA_DIR / QUEUE_DATABASE_FILENAME
-    
-    JSON_INDEX_FILENAME: str            = 'index.json'
-    HTML_INDEX_FILENAME: str            = 'index.html'
-    ROBOTS_TXT_FILENAME: str            = 'robots.txt'
-    FAVICON_FILENAME: str               = 'favicon.ico'
-    
-    # Runtime dirs
-    TMP_DIR_NAME: str                   = 'tmp'
-    DEFAULT_TMP_DIR: Path               = DATA_DIR / TMP_DIR_NAME / MACHINE_ID    # ./data/tmp/abc3244323
-    
-    LIB_DIR_NAME: str                   = 'lib'
-    DEFAULT_LIB_DIR: Path               = DATA_DIR / LIB_DIR_NAME / MACHINE_TYPE  # ./data/lib/arm64-linux-docker
-
-    # Config constants
-    TIMEZONE: str                       = 'UTC'
-    DEFAULT_CLI_COLORS: Dict[str, str]  = DEFAULT_CLI_COLORS
-    DISABLED_CLI_COLORS: Dict[str, str] = benedict({k: '' for k in DEFAULT_CLI_COLORS})
-
-    ALLOWDENYLIST_REGEX_FLAGS: int      = re.IGNORECASE | re.UNICODE | re.MULTILINE
-
-    STATICFILE_EXTENSIONS: frozenset[str] = frozenset((
-        # 99.999% of the time, URLs ending in these extensions are static files
-        # that can be downloaded as-is, not html pages that need to be rendered
-        'gif', 'jpeg', 'jpg', 'png', 'tif', 'tiff', 'wbmp', 'ico', 'jng', 'bmp',
-        'svg', 'svgz', 'webp', 'ps', 'eps', 'ai',
-        'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v',
-        'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8',
-        'pdf', 'txt', 'rtf', 'rtfd', 'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx',
-        'atom', 'rss', 'css', 'js', 'json',
-        'dmg', 'iso', 'img',
-        'rar', 'war', 'hqx', 'zip', 'gz', 'bz2', '7z',
-
-        # Less common extensions to consider adding later
-        # jar, swf, bin, com, exe, dll, deb
-        # ear, hqx, eot, wmlc, kml, kmz, cco, jardiff, jnlp, run, msi, msp, msm,
-        # pl pm, prc pdb, rar, rpm, sea, sit, tcl tk, der, pem, crt, xpi, xspf,
-        # ra, mng, asx, asf, 3gpp, 3gp, mid, midi, kar, jad, wml, htc, mml
-
-        # These are always treated as pages, not as static files, never add them:
-        # html, htm, shtml, xhtml, xml, aspx, php, cgi
-    ))
-
-    PIP_RELATED_NAMES: frozenset[str] = frozenset((
-        ".venv",
-        "venv",
-        "virtualenv",
-        ".virtualenv",
-    ))
-    NPM_RELATED_NAMES: frozenset[str] = frozenset((
-        "node_modules",
-        "package.json",
-        "package-lock.json",
-        "yarn.lock",
-    ))
-
-    # When initializing archivebox in a new directory, we check to make sure the dir is
-    # actually empty so that we dont clobber someone's home directory or desktop by accident.
-    # These files are exceptions to the is_empty check when we're trying to init a new dir,
-    # as they could be from a previous archivebox version, system artifacts, dependencies, etc.
-    ALLOWED_IN_DATA_DIR: frozenset[str] = frozenset((
-        *PIP_RELATED_NAMES,
-        *NPM_RELATED_NAMES,
-        
-        ### Dirs:
-        ARCHIVE_DIR_NAME,
-        SOURCES_DIR_NAME,
-        LOGS_DIR_NAME,
-        CACHE_DIR_NAME,
-        LIB_DIR_NAME,
-        TMP_DIR_NAME,
-        PERSONAS_DIR_NAME,
-        CUSTOM_TEMPLATES_DIR_NAME,
-        USER_PLUGINS_DIR_NAME,
-        CRONTABS_DIR_NAME,
-        "static",                # created by old static exports <v0.6.0
-        "sonic",                 # created by docker bind mount / sonic FTS process
-        ".git",
-        ".svn",
-        
-        ### Files:
-        CONFIG_FILENAME,
-        SQL_INDEX_FILENAME,
-        f"{SQL_INDEX_FILENAME}-wal",
-        f"{SQL_INDEX_FILENAME}-shm",
-        QUEUE_DATABASE_FILENAME,
-        f"{QUEUE_DATABASE_FILENAME}-wal",
-        f"{QUEUE_DATABASE_FILENAME}-shm",
-        "search.sqlite3",
-        JSON_INDEX_FILENAME,
-        HTML_INDEX_FILENAME,
-        ROBOTS_TXT_FILENAME,
-        FAVICON_FILENAME,
-        CONFIG_FILENAME,
-        f"{CONFIG_FILENAME}.bak",
-        f".{CONFIG_FILENAME}.bak",
-        "static_index.json",
-        ".DS_Store",
-        ".gitignore",
-        "lost+found",
-        ".DS_Store",
-        ".env",
-        ".collection_id",
-        ".archivebox_id",
-        "Dockerfile",
-    ))
-        
-
-    @classmethod
-    def __getitem__(cls, key: str):
-        return getattr(cls, key)
-    
-    @classmethod
-    def __benedict__(cls):
-        return benedict({key: value for key, value in cls.__dict__.items() if key.isupper() and not key.startswith('_')})
-    
-    @classmethod
-    def __len__(cls):
-        return len(cls.__benedict__())
-
-    @classmethod
-    def __iter__(cls):
-        return iter(cls.__benedict__())
-
-CONSTANTS = ConstantsDict()
-CONSTANTS_CONFIG = CONSTANTS.__benedict__()
-
-# add all key: values to globals() for easier importing
-globals().update(CONSTANTS)

archivebox/config/django.py

@@ -1,176 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import sys
-
-from datetime import datetime, timezone
-
-from rich.progress import Progress
-from rich.console import Console
-
-import django
-
-from . import CONSTANTS
-
-from .common import SHELL_CONFIG
-from ..misc import logging
-
-
-if not SHELL_CONFIG.USE_COLOR:
-    os.environ['NO_COLOR'] = '1'
-if not SHELL_CONFIG.SHOW_PROGRESS:
-    os.environ['TERM'] = 'dumb'
-
-# recreate rich console obj based on new config values
-STDOUT = CONSOLE = Console()
-STDERR = Console(stderr=True)
-logging.CONSOLE = CONSOLE
-
-
-INITIAL_STARTUP_PROGRESS = None
-INITIAL_STARTUP_PROGRESS_TASK = 0
-
-def bump_startup_progress_bar(advance=1):
-    global INITIAL_STARTUP_PROGRESS
-    global INITIAL_STARTUP_PROGRESS_TASK
-    if INITIAL_STARTUP_PROGRESS:
-        INITIAL_STARTUP_PROGRESS.update(INITIAL_STARTUP_PROGRESS_TASK, advance=advance)   # type: ignore
-
-
-def setup_django_minimal():
-    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
-    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
-    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-    # django.setup()
-    raise Exception('dont use this anymore')
-
-DJANGO_SET_UP = False
-
-
-def setup_django(check_db=False, in_memory_db=False) -> None:
-    from rich.panel import Panel
-    
-    global INITIAL_STARTUP_PROGRESS
-    global INITIAL_STARTUP_PROGRESS_TASK
-    global DJANGO_SET_UP
-
-    if DJANGO_SET_UP:
-        # raise Exception('django is already set up!')
-        # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
-        return
-
-    with Progress(transient=True, expand=True, console=STDERR) as INITIAL_STARTUP_PROGRESS:
-        INITIAL_STARTUP_PROGRESS_TASK = INITIAL_STARTUP_PROGRESS.add_task("[green]Loading modules...", total=25)
-        
-        from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
-    
-        # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
-        if IS_ROOT and ARCHIVEBOX_USER != 0:
-            with SudoPermission(uid=0):
-                # running as root is a special case where it's ok to be a bit slower
-                # make sure data dir is always owned by the correct user
-                os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}" 2>/dev/null')
-                os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}"/* 2>/dev/null')
-
-        bump_startup_progress_bar()
-        try:
-            from django.core.management import call_command
-                
-            bump_startup_progress_bar()
-
-            if in_memory_db:
-                raise Exception('dont use this anymore')
-            
-                # some commands (e.g. oneshot) dont store a long-lived sqlite3 db file on disk.
-                # in those cases we create a temporary in-memory db and run the migrations
-                # immediately to get a usable in-memory-database at startup
-                os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
-                django.setup()
-                
-                bump_startup_progress_bar()
-                call_command("migrate", interactive=False, verbosity=0)
-            else:
-                # Otherwise use default sqlite3 file-based database and initialize django
-                # without running migrations automatically (user runs them manually by calling init)
-                try:
-                    django.setup()
-                except Exception as e:
-                    bump_startup_progress_bar(advance=1000)
-                    
-                    is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version', 'init'))
-                    if not is_using_meta_cmd:
-                        # show error message to user only if they're not running a meta command / just trying to get help
-                        STDERR.print()
-                        STDERR.print(Panel(
-                            f'\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n',
-                            title='\n\n[red][X] Error while trying to load database![/red]',
-                            subtitle='[grey53]NO WRITES CAN BE PERFORMED[/grey53]',
-                            expand=False,
-                            style='bold red',
-                        ))
-                        STDERR.print()
-                        STDERR.print_exception(show_locals=False)
-                    return
-            
-            bump_startup_progress_bar()
-
-            from django.conf import settings
-            
-            # log startup message to the error log
-            with open(settings.ERROR_LOG, "a", encoding='utf-8') as f:
-                command = ' '.join(sys.argv)
-                ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
-                f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n")
-
-            if check_db:
-                # make sure the data dir is owned by a non-root user
-                if CONSTANTS.DATA_DIR.stat().st_uid == 0:
-                    STDERR.print('[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]')
-                    STDERR.print(f'    {CONSTANTS.DATA_DIR}')
-                    STDERR.print()
-                    STDERR.print('[violet]Hint:[/violet] Are you running archivebox in the right folder? (and as a non-root user?)')
-                    STDERR.print('    cd path/to/your/archive/data')
-                    STDERR.print('    archivebox [command]')
-                    STDERR.print()
-                    raise SystemExit(9)
-                
-                # Create cache table in DB if needed
-                try:
-                    from django.core.cache import cache
-                    cache.get('test', None)
-                except django.db.utils.OperationalError:
-                    call_command("createcachetable", verbosity=0)
-
-                bump_startup_progress_bar()
-
-                # if archivebox gets imported multiple times, we have to close
-                # the sqlite3 whenever we init from scratch to avoid multiple threads
-                # sharing the same connection by accident
-                from django.db import connections
-                for conn in connections.all():
-                    conn.close_if_unusable_or_obsolete()
-
-                sql_index_path = CONSTANTS.DATABASE_FILE
-                assert os.access(sql_index_path, os.F_OK), (
-                    f'No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)')
-
-                bump_startup_progress_bar()
-
-                # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
-                # if settings.DEBUG_LOGFIRE:
-                #     from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
-                #     SQLite3Instrumentor().instrument()
-
-                #     import logfire
-
-                #     logfire.configure()
-                #     logfire.instrument_django(is_sql_commentor_enabled=True)
-                #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
-
-        except KeyboardInterrupt:
-            raise SystemExit(2)
-        
-    DJANGO_SET_UP = True
-
-    INITIAL_STARTUP_PROGRESS = None
-    INITIAL_STARTUP_PROGRESS_TASK = None

archivebox/config/paths.py

@@ -1,413 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import socket
-import hashlib
-import tempfile
-import platform
-from pathlib import Path
-from functools import cache
-from datetime import datetime
-
-from benedict import benedict
-
-from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER
-
-#############################################################################################
-
-PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
-DATA_DIR: Path = Path(os.getcwd()).resolve()                  # archivebox user data dir
-ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
-
-IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
-
-DATABASE_FILE = DATA_DIR / 'index.sqlite3'
-
-#############################################################################################
-
-def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
-    collection_id_file = DATA_DIR / '.archivebox_id'
-    
-    try:
-        return collection_id_file.read_text().strip()
-    except (OSError, FileNotFoundError, PermissionError):
-        pass
-    
-    # hash the machine_id + collection dir path + creation time to get a unique collection_id
-    machine_id = get_machine_id()
-    collection_path = DATA_DIR.resolve()
-    try:
-        creation_date = DATA_DIR.stat().st_ctime
-    except Exception:
-        creation_date = datetime.now().isoformat()
-    collection_id = hashlib.sha256(f'{machine_id}:{collection_path}@{creation_date}'.encode()).hexdigest()[:8]
-    
-    try:
-        # only persist collection_id file if we already have an index.sqlite3 file present
-        # otherwise we might be running in a directory that is not a collection, no point creating cruft files
-        collection_is_active = os.path.isfile(DATABASE_FILE) and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
-        if collection_is_active or force_create:
-            collection_id_file.write_text(collection_id)
-            
-            # if we're running as root right now, make sure the collection_id file is owned by the archivebox user
-            if IS_ROOT:
-                with SudoPermission(uid=0):
-                    if ARCHIVEBOX_USER == 0:
-                        os.system(f'chmod 777 "{collection_id_file}"')
-                    else:    
-                        os.system(f'chown {ARCHIVEBOX_USER} "{collection_id_file}"')
-    except (OSError, FileNotFoundError, PermissionError):
-        pass
-    return collection_id
-
-@cache
-def get_collection_id(DATA_DIR=DATA_DIR) -> str:
-    """Get a short, stable, unique ID for the current collection (e.g. abc45678)"""
-    return _get_collection_id(DATA_DIR=DATA_DIR)
-
-@cache
-def get_machine_id() -> str:
-    """Get a short, stable, unique ID for the current machine (e.g. abc45678)"""
-    
-    MACHINE_ID = 'unknown'
-    try:
-        import machineid
-        MACHINE_ID = machineid.hashed_id('archivebox')[:8]
-    except Exception:
-        try:
-            import uuid
-            import hashlib
-            MACHINE_ID = hashlib.sha256(str(uuid.getnode()).encode()).hexdigest()[:8]
-        except Exception:
-            pass
-    return MACHINE_ID
-
-@cache
-def get_machine_type() -> str:
-    """Get a short, stable, unique type identifier for the current machine (e.g. linux-x86_64-docker)"""
-    
-    OS: str                             = platform.system().lower()    # darwin, linux, etc.
-    ARCH: str                           = platform.machine().lower()   # arm64, x86_64, aarch64, etc.
-    LIB_DIR_SCOPE: str                  = f'{ARCH}-{OS}-docker' if IN_DOCKER else f'{ARCH}-{OS}'
-    return LIB_DIR_SCOPE
-
-
-def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = None, fallback=True, chown=True) -> bool:
-    """Check if a given directory is writable by a specific user and group (fallback=try as current user is unable to check with provided uid)"""
-    current_uid, current_gid = os.geteuid(), os.getegid()
-    uid, gid = uid or current_uid, gid or current_gid
-
-    test_file = dir_path / '.permissions_test'
-    try:
-        with SudoPermission(uid=uid, fallback=fallback):
-            test_file.exists()
-            test_file.write_text(f'Checking if PUID={uid} PGID={gid} can write to dir')
-            test_file.unlink()
-            return True
-    except (IOError, OSError, PermissionError):
-        if chown:    
-            # try fixing it using sudo permissions
-            with SudoPermission(uid=uid, fallback=fallback):
-                os.system(f'chown {uid}:{gid} "{dir_path}" 2>/dev/null')
-            return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
-    return False
-
-def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:
-    """Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)"""
-    from archivebox.logging_util import pretty_path
-    
-    try:
-        socket_path = str(dir_path / '.test_socket.sock')
-        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        try:
-            os.remove(socket_path)
-        except OSError:
-            pass
-        s.bind(socket_path)
-        s.close()
-        try:
-            os.remove(socket_path)
-        except OSError:
-            pass
-    except Exception as e:
-        raise Exception(f'ArchiveBox failed to create a test UNIX socket file in {pretty_path(dir_path, color=False)}') from e
-    
-    return True
-
-
-def create_and_chown_dir(dir_path: Path) -> None:
-    with SudoPermission(uid=0, fallback=True):
-        dir_path.mkdir(parents=True, exist_ok=True)
-        os.system(f'chown {ARCHIVEBOX_USER} "{dir_path}" 2>/dev/null')
-        os.system(f'chown {ARCHIVEBOX_USER} "{dir_path}"/* 2>/dev/null &')
-
-@cache
-def get_or_create_working_tmp_dir(autofix=True, quiet=False):
-    from archivebox import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    from archivebox.misc.checks import check_tmp_dir
-
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        STORAGE_CONFIG.TMP_DIR,                                                # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,                                             # ./data/tmp/<machine_id>
-        Path('/var/run/archivebox') / get_collection_id(),                     # /var/run/archivebox/abc5d8512
-        Path('/tmp') / 'archivebox' / get_collection_id(),                     # /tmp/archivebox/abc5d8512
-        Path('~/.tmp/archivebox').expanduser() / get_collection_id(),          # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir()) / 'archivebox' / get_collection_id(),      # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir()) / 'archivebox' / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / 'abx' / get_collection_id()[:4],         # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
-    for candidate in CANDIDATES:
-        try:
-            create_and_chown_dir(candidate)
-        except Exception:
-            pass
-        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and STORAGE_CONFIG.TMP_DIR != candidate:
-                STORAGE_CONFIG.update_in_place(TMP_DIR=candidate, warn=not quiet)
-            return candidate
-    
-    if not quiet:
-        raise OSError(f'ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!')
-
-@cache
-def get_or_create_working_lib_dir(autofix=True, quiet=False):
-    from archivebox import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    from archivebox.misc.checks import check_lib_dir
-    
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        STORAGE_CONFIG.LIB_DIR,                                                   # <user-specified>
-        CONSTANTS.DEFAULT_LIB_DIR,                                                # ./data/lib/arm64-linux-docker
-        Path('/usr/local/share/archivebox') / get_collection_id(),                # /usr/local/share/archivebox/abc5
-        *([Path('/opt/homebrew/share/archivebox') / get_collection_id()] if os.path.isfile('/opt/homebrew/bin/archivebox') else []),  # /opt/homebrew/share/archivebox/abc5
-        Path('~/.local/share/archivebox').expanduser() / get_collection_id(),     # ~/.local/share/archivebox/abc5
-    ]
-    
-    for candidate in CANDIDATES:
-        try:
-            create_and_chown_dir(candidate)
-        except Exception:
-            pass
-        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and STORAGE_CONFIG.LIB_DIR != candidate:
-                STORAGE_CONFIG.update_in_place(LIB_DIR=candidate, warn=not quiet)
-            return candidate
-    
-    if not quiet:
-        raise OSError(f'ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!')
-
-
-
-@cache
-def get_data_locations():
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    
-    return benedict({
-        "DATA_DIR": {
-            "path": DATA_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.R_OK) and os.access(DATA_DIR, os.W_OK),
-            "is_mount": os.path.ismount(DATA_DIR.resolve()),
-        },
-        "CONFIG_FILE": {
-            "path": CONSTANTS.CONFIG_FILE.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isfile(CONSTANTS.CONFIG_FILE) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.W_OK),
-        },
-        "SQL_INDEX": {
-            "path": DATABASE_FILE.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
-            "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
-        },
-        "QUEUE_DATABASE": {
-            "path": CONSTANTS.QUEUE_DATABASE_FILE,
-            "enabled": True,
-            "is_valid": os.path.isfile(CONSTANTS.QUEUE_DATABASE_FILE) and os.access(CONSTANTS.QUEUE_DATABASE_FILE, os.R_OK) and os.access(CONSTANTS.QUEUE_DATABASE_FILE, os.W_OK),
-            "is_mount": os.path.ismount(CONSTANTS.QUEUE_DATABASE_FILE),
-        },
-        "ARCHIVE_DIR": {
-            "path": ARCHIVE_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(ARCHIVE_DIR) and os.access(ARCHIVE_DIR, os.R_OK) and os.access(ARCHIVE_DIR, os.W_OK),
-            "is_mount": os.path.ismount(ARCHIVE_DIR.resolve()),
-        },
-        "SOURCES_DIR": {
-            "path": CONSTANTS.SOURCES_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(CONSTANTS.SOURCES_DIR) and os.access(CONSTANTS.SOURCES_DIR, os.R_OK) and os.access(CONSTANTS.SOURCES_DIR, os.W_OK),
-        },
-        "PERSONAS_DIR": {
-            "path": CONSTANTS.PERSONAS_DIR.resolve(),
-            "enabled": os.path.isdir(CONSTANTS.PERSONAS_DIR),
-            "is_valid": os.path.isdir(CONSTANTS.PERSONAS_DIR) and os.access(CONSTANTS.PERSONAS_DIR, os.R_OK) and os.access(CONSTANTS.PERSONAS_DIR, os.W_OK),                 # read + write
-        },
-        "LOGS_DIR": {
-            "path": CONSTANTS.LOGS_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(CONSTANTS.LOGS_DIR) and os.access(CONSTANTS.LOGS_DIR, os.R_OK) and os.access(CONSTANTS.LOGS_DIR, os.W_OK),                             # read + write
-        },
-        'TMP_DIR': {
-            'path': STORAGE_CONFIG.TMP_DIR.resolve(),
-            'enabled': True,
-            'is_valid': os.path.isdir(STORAGE_CONFIG.TMP_DIR) and os.access(STORAGE_CONFIG.TMP_DIR, os.R_OK) and os.access(STORAGE_CONFIG.TMP_DIR, os.W_OK),        # read + write
-        },
-        # "CACHE_DIR": {
-        #     "path": CACHE_DIR.resolve(),
-        #     "enabled": True,
-        #     "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK),                        # read + write
-        # },
-    })
-
-@cache
-def get_code_locations():
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    
-    return benedict({
-        'PACKAGE_DIR': {
-            'path': (PACKAGE_DIR).resolve(),
-            'enabled': True,
-            'is_valid': os.access(PACKAGE_DIR / '__main__.py', os.X_OK),                                                                  # executable
-        },
-        'TEMPLATES_DIR': {
-            'path': CONSTANTS.TEMPLATES_DIR.resolve(),
-            'enabled': True,
-            'is_valid': os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),                                                # read + list
-        },
-        'CUSTOM_TEMPLATES_DIR': {
-            'path': CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
-            'enabled': os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
-            'is_valid': os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),                                      # read
-        },
-        'USER_PLUGINS_DIR': {
-            'path': CONSTANTS.USER_PLUGINS_DIR.resolve(),
-            'enabled': os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
-            'is_valid': os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),                                              # read
-        },
-        'LIB_DIR': {
-            'path': STORAGE_CONFIG.LIB_DIR.resolve(),
-            'enabled': True,
-            'is_valid': os.path.isdir(STORAGE_CONFIG.LIB_DIR) and os.access(STORAGE_CONFIG.LIB_DIR, os.R_OK) and os.access(STORAGE_CONFIG.LIB_DIR, os.W_OK),                      # read + write
-        },
-    })
-
-
-
-# @cache
-# def get_LIB_DIR():
-#     """
-#     - should be shared with other collections on the same host
-#     - must be scoped by CPU architecture, OS family, and archivebox version
-#     - should not be shared with other hosts/archivebox versions
-#     - must be writable by any archivebox user
-#     - should be persistent across reboots
-#     - can be on a docker bin mount but probably shouldnt be
-#     - ok to have a long path (doesnt contain SOCKETS)
-#     """
-#     from .version import detect_installed_version
-    
-#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
-    
-#     lib_dir = tempfile.gettempdir()
-#     try:
-#         if 'SYSTEM_LIB_DIR' in os.environ:
-#             lib_dir = Path(os.environ['SYSTEM_LIB_DIR'])
-#         else:
-#             with SudoPermission(uid=ARCHIVEBOX_USER, fallback=True):
-#                 lib_dir = HOST_DIRS.site_data_path
-        
-#         # Docker: /usr/local/share/archivebox/0.8.5
-#         # Ubuntu: /usr/local/share/archivebox/0.8.5
-#         # macOS: /Library/Application Support/archivebox
-#         try:
-#             with SudoPermission(uid=0, fallback=True):
-#                 lib_dir.mkdir(parents=True, exist_ok=True)
-#         except PermissionError:
-#             # our user cannot 
-#             lib_dir = HOST_DIRS.user_data_path
-#             lib_dir.mkdir(parents=True, exist_ok=True)
-        
-#         if IS_ROOT or not dir_is_writable(lib_dir, uid=ARCHIVEBOX_USER):
-#             if IS_ROOT:
-#                 # make sure lib dir is owned by the archivebox user, not root
-#                 with SudoPermission(uid=0):
-#                     if ARCHIVEBOX_USER == 0:
-#                         # print(f'[yellow]:warning:  Waring: Creating SYSTEM_LIB_DIR {lib_dir} with mode 777 so that non-root archivebox users can share it.[/yellow] (caches shared libs used by archivebox for performance)', file=sys.stderr)
-#                         os.system(f'chmod -R 777 "{lib_dir}"')
-#                     else:
-#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{lib_dir}"')
-#             else:
-#                 raise PermissionError()
-#     except (PermissionError, AssertionError):
-#         # raise PermissionError(f'SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
-#         print(f'[red]:cross_mark:  ERROR: SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
-        
-#     return lib_dir
-    
-# @cache
-# def get_TMP_DIR():
-#     """
-#     - must NOT be inside DATA_DIR / inside a docker volume bind mount
-#     - must NOT have a long PATH (UNIX socket path length restrictions)
-#     - must NOT be shared with other collections/hosts
-#     - must be writable by archivebox user & root
-#     - must be cleared on every boot / not persisted
-#     - must be cleared on every archivebox version upgrade
-#     """
-#     from .version import detect_installed_version
-    
-#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
-    
-#     # print('DATA_DIR OWNED BY:', ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
-#     # print('RUNNING AS:', self.PUID, self.PGID)
-#     run_dir = tempfile.gettempdir()
-#     try:
-#         if 'SYSTEM_TMP_DIR' in os.environ:
-#             run_dir = Path(os.environ['SYSTEM_TMP_DIR']).resolve() / get_collection_id(DATA_DIR=DATA_DIR)
-#             with SudoPermission(uid=0, fallback=True):
-#                 run_dir.mkdir(parents=True, exist_ok=True)
-#             if not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
-#                 if IS_ROOT:
-#                     with SudoPermission(uid=0, fallback=False):
-#                         if ARCHIVEBOX_USER == 0:
-#                             # print(f'[yellow]:warning:  Waring: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
-#                             os.system(f'chmod -R 777 "{run_dir}"')
-#                         else:
-#                             os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
-#                 else:
-#                     raise PermissionError()
-#             assert len(str(run_dir / 'supervisord.conf')) < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
-#             return run_dir
-        
-#         run_dir = (HOST_DIRS.site_runtime_path / get_collection_id(DATA_DIR=DATA_DIR)).resolve()
-#         try:
-#             assert len(str(run_dir)) + len('/supervisord.sock') < 95
-#         except AssertionError:
-#             run_dir = Path(tempfile.gettempdir()).resolve() / 'archivebox' / get_collection_id(DATA_DIR=DATA_DIR)
-#             assert len(str(run_dir)) + len('/supervisord.sock') < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
-        
-#         with SudoPermission(uid=0, fallback=True):
-#             run_dir.mkdir(parents=True, exist_ok=True)
-            
-#         if IS_ROOT or not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
-#             if IS_ROOT:
-#                 with SudoPermission(uid=0):
-#                     if ARCHIVEBOX_USER == 0:
-#                         # print(f'[yellow]:warning:  Waring: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
-#                         os.system(f'chmod -R 777 "{run_dir}"')
-#                     else:
-#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
-#             else:
-#                 raise PermissionError()
-            
-#     except (PermissionError, AssertionError):
-#         # raise PermissionError(f'SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
-#         print(f'[red]:cross_mark:  ERROR: SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
-        
-#     return run_dir
-

archivebox/config/permissions.py

@@ -1,133 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import pwd
-import sys
-
-from rich import print
-
-from pathlib import Path
-from contextlib import contextmanager
-
-#############################################################################################
-
-DATA_DIR = Path(os.getcwd())
-
-try:
-    DATA_DIR_STAT           = DATA_DIR.stat()
-    DATA_DIR_UID            = DATA_DIR_STAT.st_uid
-    DATA_DIR_GID            = DATA_DIR_STAT.st_gid
-except PermissionError:
-    DATA_DIR_UID            = 0
-    DATA_DIR_GID            = 0
-
-DEFAULT_PUID            = 911
-DEFAULT_PGID            = 911
-RUNNING_AS_UID          = os.getuid()
-RUNNING_AS_GID          = os.getgid()
-EUID                    = os.geteuid()
-EGID                    = os.getegid()
-SUDO_UID                = int(os.environ.get('SUDO_UID', 0))
-SUDO_GID                = int(os.environ.get('SUDO_GID', 0))
-USER: str               = Path('~').expanduser().resolve().name
-
-IS_ROOT = RUNNING_AS_UID == 0
-IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
-
-FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
-FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        import pwd
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == 'archivebox':
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
-
-
-os.environ.setdefault('PUID', str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault('PGID', str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ['PUID'])
-ARCHIVEBOX_GROUP = int(os.environ['PGID'])
-if not USER:
-    try:
-        # alternative method 1 to get username
-        USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name
-    except Exception:
-        pass
-        
-if not USER:
-    try:
-        # alternative method 2 to get username
-        import getpass
-        USER = getpass.getuser()
-    except Exception:
-        pass
-    
-if not USER:
-    try:
-        # alternative method 3 to get username
-        USER = os.getlogin() or 'archivebox'
-    except Exception:
-        USER = 'archivebox'
-        
-ARCHIVEBOX_USER_EXISTS = False
-try:
-    pwd.getpwuid(ARCHIVEBOX_USER)
-    ARCHIVEBOX_USER_EXISTS = True
-except Exception:
-    ARCHIVEBOX_USER_EXISTS = False
-    
-
-#############################################################################################
-
-def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
-    
-    # always run archivebox as the user that owns the data dir, never as root
-    if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
-        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
-            os.seteuid(ARCHIVEBOX_USER)
-            
-            # update environment variables so that subprocesses dont try to write to /root
-            pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
-            os.environ['HOME']     = pw_record.pw_dir
-            os.environ['LOGNAME']  = pw_record.pw_name
-            os.environ['USER']     = pw_record.pw_name
-
-    if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS:
-        print('[yellow]:warning:  Running as [red]root[/red] is not recommended and may make your [blue]DATA_DIR[/blue] inaccessible to other users on your system.[/yellow] (use [blue]sudo[/blue] instead)', file=sys.stderr)
-
-
-@contextmanager
-def SudoPermission(uid=0, fallback=False):
-    """Attempt to run code with sudo permissions for a given user (or root)"""
-    
-    if os.geteuid() == uid:
-        # no need to change effective UID, we are already that user
-        yield
-        return
-
-    try:
-        # change our effective UID to the given UID
-        os.seteuid(uid)
-    except PermissionError as err:
-        if not fallback:
-            raise PermissionError(f'Not enough permissions to run code as uid={uid}, please retry with sudo') from err
-    try:
-        # yield back to the caller so they can run code inside context as root
-        yield
-    finally:
-        # then set effective UID back to DATA_DIR owner
-        try:
-            os.seteuid(ARCHIVEBOX_USER)
-        except PermissionError as err:
-            if not fallback:
-                raise PermissionError(f'Failed to revert uid={uid} back to {ARCHIVEBOX_USER} after running code with sudo') from err

archivebox/config/version.py

@@ -1,121 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import importlib.metadata
-
-from pathlib import Path
-from functools import cache
-from datetime import datetime
-from typing import Optional
-
-#############################################################################################
-
-IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
-
-PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
-DATA_DIR: Path = Path(os.getcwd()).resolve()                  # archivebox user data dir
-ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
-
-#############################################################################################
-
-
-@cache
-def detect_installed_version(PACKAGE_DIR: Path=PACKAGE_DIR):
-    """Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file"""
-    try:
-        # if in production install, use pip-installed package metadata
-        return importlib.metadata.version('archivebox').strip()
-    except importlib.metadata.PackageNotFoundError:
-        pass
-
-    try:
-        # if in dev Git repo dir, use pyproject.toml file
-        pyproject_config = (PACKAGE_DIR.parent / 'pyproject.toml').read_text().split('\n')
-        for line in pyproject_config:
-            if line.startswith('version = '):
-                return line.split(' = ', 1)[-1].strip('"').strip()
-    except FileNotFoundError:
-        # building docs, pyproject.toml is not available
-        pass
-
-    # raise Exception('Failed to detect installed archivebox version!')
-    return 'dev'
-
-
-@cache
-def get_COMMIT_HASH() -> Optional[str]:
-    try:
-        git_dir = PACKAGE_DIR / '../.git'
-        ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
-    except Exception:
-        pass
-
-    try:
-        return list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
-    except Exception:
-        pass
-    
-    return None
-    
-@cache
-def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
-        return docker_build_end_time
-
-    src_last_modified_unix_timestamp = (PACKAGE_DIR / 'README.md').stat().st_mtime
-    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
-
-
-# def get_versions_available_on_github(config):
-#     """
-#     returns a dictionary containing the ArchiveBox GitHub release info for
-#     the recommended upgrade version and the currently installed version
-#     """
-    
-#     # we only want to perform the (relatively expensive) check for new versions
-#     # when its most relevant, e.g. when the user runs a long-running command
-#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
-#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
-#     if subcommand_run_by_user not in long_running_commands:
-#         return None
-    
-#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
-#     response = requests.get(github_releases_api)
-#     if response.status_code != 200:
-#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
-#         return None
-#     all_releases = response.json()
-
-#     installed_version = parse_version_string(config['VERSION'])
-
-#     # find current version or nearest older version (to link to)
-#     current_version = None
-#     for idx, release in enumerate(all_releases):
-#         release_version = parse_version_string(release['tag_name'])
-#         if release_version <= installed_version:
-#             current_version = release
-#             break
-
-#     current_version = current_version or all_releases[-1]
-    
-#     # recommended version is whatever comes after current_version in the release list
-#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
-#     try:
-#         recommended_version = all_releases[idx+1]
-#     except IndexError:
-#         recommended_version = None
-
-#     return {'recommended_version': recommended_version, 'current_version': current_version}
-
-# def can_upgrade(config):
-#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
-#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
-#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
-#         return recommended_version > current_version
-#     return False
-
-
-VERSION: str = detect_installed_version()

archivebox/config/views.py

@@ -1,503 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import os
-import inspect
-from pathlib import Path
-from typing import Any, List, Dict, cast
-from benedict import benedict
-
-from django.http import HttpRequest
-from django.conf import settings
-from django.utils import timezone
-from django.utils.html import format_html, mark_safe
-
-from admin_data_views.typing import TableContext, ItemContext
-from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
-
-import abx.archivebox.reads
-
-from archivebox.config import CONSTANTS
-from archivebox.misc.util import parse_date
-
-from machine.models import InstalledBinary
-
-
-def obj_to_yaml(obj: Any, indent: int=0) -> str:
-    indent_str = "  " * indent
-    if indent == 0:
-        indent_str = '\n'  # put extra newline between top-level entries
-    
-    if isinstance(obj, dict):
-        if not obj:
-            return "{}"
-        result = "\n"
-        for key, value in obj.items():
-            result += f"{indent_str}{key}:{obj_to_yaml(value, indent + 1)}\n"
-        return result
-    
-    elif isinstance(obj, list):
-        if not obj:
-            return "[]"
-        result = "\n"
-        for item in obj:
-            result += f"{indent_str}- {obj_to_yaml(item, indent + 1).lstrip()}\n"
-        return result.rstrip()
-    
-    elif isinstance(obj, str):
-        if "\n" in obj:
-            return f" |\n{indent_str}  " + obj.replace("\n", f"\n{indent_str}  ")
-        else:
-            return f" {obj}"
-    
-    elif isinstance(obj, (int, float, bool)):
-        return f" {str(obj)}"
-    
-    elif callable(obj):
-        source = '\n'.join(
-            '' if 'def ' in line else line
-            for line in inspect.getsource(obj).split('\n')
-            if line.strip()
-        ).split('lambda: ')[-1].rstrip(',')
-        return f" {indent_str}  " + source.replace("\n", f"\n{indent_str}  ")
-    
-    else:
-        return f" {str(obj)}"
-
-@render_with_table_view
-def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
-
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    rows = {
-        "Binary Name": [],
-        "Found Version": [],
-        "From Plugin": [],
-        "Provided By": [],
-        "Found Abspath": [],
-        "Related Configuration": [],
-        # "Overrides": [],
-        # "Description": [],
-    }
-
-    relevant_configs = {
-        key: val
-        for key, val in settings.FLAT_CONFIG.items()
-        if '_BINARY' in key or '_VERSION' in key
-    }
-
-    for plugin_id, plugin in abx.archivebox.reads.get_PLUGINS().items():
-        plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
-        if not plugin.hooks.get('get_BINARIES'):
-            continue
-        
-        for binary in plugin.hooks.get_BINARIES().values():
-            try:
-                installed_binary = InstalledBinary.objects.get_from_db_or_cache(binary)
-                binary = installed_binary.load_from_db()
-            except Exception as e:
-                print(e)
-
-            rows['Binary Name'].append(ItemLink(binary.name, key=binary.name))
-            rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing')
-            rows['From Plugin'].append(plugin.package)
-            rows['Provided By'].append(
-                ', '.join(
-                    f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name
-                    for binprovider in binary.binproviders_supported
-                    if binprovider
-                )
-                # binary.loaded_binprovider.name
-                # if binary.loaded_binprovider else
-                # ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported)
-            )
-            rows['Found Abspath'].append(str(binary.loaded_abspath or '❌ missing'))
-            rows['Related Configuration'].append(mark_safe(', '.join(
-                f'<a href="/admin/environment/config/{config_key}/">{config_key}</a>'
-                for config_key, config_value in relevant_configs.items()
-                    if str(binary.name).lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower()
-                    or config_value.lower().endswith(binary.name.lower())
-                    # or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower()
-            )))
-            # if not binary.overrides:
-                # import ipdb; ipdb.set_trace()
-            # rows['Overrides'].append(str(obj_to_yaml(binary.overrides) or str(binary.overrides))[:200])
-            # rows['Description'].append(binary.description)
-
-    return TableContext(
-        title="Binaries",
-        table=rows,
-    )
-
-@render_with_item_view
-def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    binary = None
-    plugin = None
-    for plugin_id in abx.archivebox.reads.get_PLUGINS().keys():
-        loaded_plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
-        try:
-            for loaded_binary in loaded_plugin.hooks.get_BINARIES().values():
-                if loaded_binary.name == key:
-                    binary = loaded_binary
-                    plugin = loaded_plugin
-                    # break  # last write wins
-        except Exception as e:
-            print(e)
-
-    assert plugin and binary, f'Could not find a binary matching the specified name: {key}'
-
-    try:
-        binary = binary.load()
-    except Exception as e:
-        print(e)
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": binary.name,
-                "description": binary.abspath,
-                "fields": {
-                    'plugin': plugin.package,
-                    'binprovider': binary.loaded_binprovider,
-                    'abspath': binary.loaded_abspath,
-                    'version': binary.loaded_version,
-                    'overrides': obj_to_yaml(binary.overrides),
-                    'providers': obj_to_yaml(binary.binproviders_supported),
-                },
-                "help_texts": {
-                    # TODO
-                },
-            },
-        ],
-    )
-
-
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    rows = {
-        "Label": [],
-        "Version": [],
-        "Author": [],
-        "Package": [],
-        "Source Code": [],
-        "Config": [],
-        "Binaries": [],
-        "Package Managers": [],
-        # "Search Backends": [],
-    }
-
-    config_colors = {
-        '_BINARY': '#339',
-        'USE_': 'green',
-        'SAVE_': 'green',
-        '_ARGS': '#33e',
-        'KEY': 'red',
-        'COOKIES': 'red',
-        'AUTH': 'red',
-        'SECRET': 'red',
-        'TOKEN': 'red',
-        'PASSWORD': 'red',
-        'TIMEOUT': '#533',
-        'RETRIES': '#533',
-        'MAX': '#533',
-        'MIN': '#533',
-    }
-    def get_color(key):
-        for pattern, color in config_colors.items():
-            if pattern in key:
-                return color
-        return 'black'
-
-    for plugin_id in settings.PLUGINS.keys():
-        
-        plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
-        plugin.hooks.get_BINPROVIDERS = plugin.hooks.get('get_BINPROVIDERS', lambda: {})
-        plugin.hooks.get_BINARIES = plugin.hooks.get('get_BINARIES', lambda: {})
-        plugin.hooks.get_CONFIG = plugin.hooks.get('get_CONFIG', lambda: {})
-        
-        rows['Label'].append(ItemLink(plugin.label, key=plugin.package))
-        rows['Version'].append(str(plugin.version))
-        rows['Author'].append(mark_safe(f'<a href="{plugin.homepage}" target="_blank">{plugin.author}</a>'))
-        rows['Package'].append(ItemLink(plugin.package, key=plugin.package))
-        rows['Source Code'].append(format_html('<code>{}</code>', str(plugin.source_code).replace(str(Path('~').expanduser()), '~')))
-        rows['Config'].append(mark_safe(''.join(
-            f'<a href="/admin/environment/config/{key}/"><b><code style="color: {get_color(key)};">{key}</code></b>=<code>{value}</code></a><br/>'
-            for configdict in plugin.hooks.get_CONFIG().values()
-                for key, value in benedict(configdict).items()
-        )))
-        rows['Binaries'].append(mark_safe(', '.join(
-            f'<a href="/admin/environment/binaries/{binary.name}/"><code>{binary.name}</code></a>'
-            for binary in plugin.hooks.get_BINARIES().values()
-        )))
-        rows['Package Managers'].append(mark_safe(', '.join(
-            f'<a href="/admin/environment/binproviders/{binprovider.name}/"><code>{binprovider.name}</code></a>'
-            for binprovider in plugin.hooks.get_BINPROVIDERS().values()
-        )))
-        # rows['Search Backends'].append(mark_safe(', '.join(
-        #     f'<a href="/admin/environment/searchbackends/{searchbackend.name}/"><code>{searchbackend.name}</code></a>'
-        #     for searchbackend in plugin.SEARCHBACKENDS.values()
-        # )))
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    plugin_id = None
-    for check_plugin_id, loaded_plugin in settings.PLUGINS.items():
-        if check_plugin_id.split('.')[-1] == key.split('.')[-1]:
-            plugin_id = check_plugin_id
-            break
-
-    assert plugin_id, f'Could not find a plugin matching the specified name: {key}'
-
-    plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": plugin.package,
-                "description": plugin.label,
-                "fields": {
-                    "id": plugin.id,
-                    "package": plugin.package,
-                    "label": plugin.label,
-                    "version": plugin.version,
-                    "author": plugin.author,
-                    "homepage": plugin.homepage,
-                    "dependencies": getattr(plugin, 'DEPENDENCIES', []),
-                    "source_code": plugin.source_code,
-                    "hooks": plugin.hooks,
-                },
-                "help_texts": {
-                    # TODO
-                },
-            },
-        ],
-    )
-
-
-@render_with_table_view
-def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "State": [],
-        "PID": [],
-        "Started": [],
-        "Command": [],
-        "Logfile": [],
-        "Exit Status": [],
-    }
-    
-    from queues.supervisor_util import get_existing_supervisord_process
-    
-    supervisor = get_existing_supervisord_process()
-    if supervisor is None:
-        return TableContext(
-            title="No running worker processes",
-            table=rows,
-        )
-        
-    all_config_entries = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
-    all_config = {config["name"]: benedict(config) for config in all_config_entries}
-
-    # Add top row for supervisord process manager
-    rows["Name"].append(ItemLink('supervisord', key='supervisord'))
-    rows["State"].append(supervisor.getState()['statename'])
-    rows['PID'].append(str(supervisor.getPID()))
-    rows["Started"].append('-')
-    rows["Command"].append('supervisord --configuration=tmp/supervisord.conf')
-    rows["Logfile"].append(
-        format_html(
-            '<a href="/admin/environment/logs/{}/">{}</a>',
-            'supervisord',
-            'logs/supervisord.log',
-        )
-    )
-    rows['Exit Status'].append('0')
-
-    # Add a row for each worker process managed by supervisord
-    for proc in cast(List[Dict[str, Any]], supervisor.getAllProcessInfo()):
-        proc = benedict(proc)
-        # {
-        #     "name": "daphne",
-        #     "group": "daphne",
-        #     "start": 1725933056,
-        #     "stop": 0,
-        #     "now": 1725933438,
-        #     "state": 20,
-        #     "statename": "RUNNING",
-        #     "spawnerr": "",
-        #     "exitstatus": 0,
-        #     "logfile": "logs/server.log",
-        #     "stdout_logfile": "logs/server.log",
-        #     "stderr_logfile": "",
-        #     "pid": 33283,
-        #     "description": "pid 33283, uptime 0:06:22",
-        # }
-        rows["Name"].append(ItemLink(proc.name, key=proc.name))
-        rows["State"].append(proc.statename)
-        rows['PID'].append(proc.description.replace('pid ', ''))
-        rows["Started"].append(parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else '')
-        rows["Command"].append(all_config[proc.name].command)
-        rows["Logfile"].append(
-            format_html(
-                '<a href="/admin/environment/logs/{}/">{}</a>',
-                proc.stdout_logfile.split("/")[-1].split('.')[0],
-                proc.stdout_logfile,
-            )
-        )
-        rows["Exit Status"].append(str(proc.exitstatus))
-
-    return TableContext(
-        title="Running worker processes",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    from queues.supervisor_util import get_existing_supervisord_process, get_worker
-    from queues.settings import SUPERVISORD_CONFIG_FILE
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor is None:
-        return ItemContext(
-            slug='none',
-            title='error: No running supervisord process.',
-            data=[],
-        )
-
-    all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
-
-    if key == 'supervisord':
-        relevant_config = SUPERVISORD_CONFIG_FILE.read_text()
-        relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
-        start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
-        uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]
-
-        proc = benedict(
-            {
-                "name": "supervisord",
-                "pid": supervisor.getPID(),
-                "statename": supervisor.getState()["statename"],
-                "start": start_ts,
-                "stop": None,
-                "exitstatus": "",
-                "stdout_logfile": "logs/supervisord.log",
-                "description": f'pid 000, uptime {uptime}',
-            }
-        )
-    else:
-        proc = benedict(get_worker(supervisor, key) or {})
-        relevant_config = [config for config in all_config if config['name'] == key][0]
-        relevant_logs = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0]
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": key,
-                "description": key,
-                "fields": {
-                    "Command": proc.name,
-                    "PID": proc.pid,
-                    "State": proc.statename,
-                    "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
-                    "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
-                    "Exit Status": str(proc.exitstatus),
-                    "Logfile": proc.stdout_logfile,
-                    "Uptime": (proc.description or "").split("uptime ", 1)[-1],
-                    "Config": relevant_config,
-                    "Logs": relevant_logs,
-                },
-                "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
-            },
-        ],
-    )
-
-
-@render_with_table_view
-def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-
-    log_files = CONSTANTS.LOGS_DIR.glob("*.log")
-    log_files = sorted(log_files, key=os.path.getmtime)[::-1]
-
-    rows = {
-        "Name": [],
-        "Last Updated": [],
-        "Size": [],
-        "Most Recent Lines": [],
-    }
-
-    # Add a row for each worker process managed by supervisord
-    for logfile in log_files:
-        st = logfile.stat()
-        rows["Name"].append(ItemLink("logs" + str(logfile).rsplit("/logs", 1)[-1], key=logfile.name))
-        rows["Last Updated"].append(parse_date(st.st_mtime).strftime("%Y-%m-%d %H:%M:%S"))
-        rows["Size"].append(f'{st.st_size//1000} kb')
-
-        with open(logfile, 'rb') as f:
-            try:
-                f.seek(-1024, os.SEEK_END)
-            except OSError:
-                f.seek(0)
-            last_lines = f.read().decode('utf-8', errors='replace').split("\n")
-            non_empty_lines = [line for line in last_lines if line.strip()]
-            rows["Most Recent Lines"].append(non_empty_lines[-1])
-
-    return TableContext(
-        title="Debug Log files",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    from django.conf import settings
-    
-    log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0]
-
-    log_text = log_file.read_text()
-    log_stat = log_file.stat()
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": key,
-                "description": key,
-                "fields": {
-                    "Path": str(log_file),
-                    "Size": f"{log_stat.st_size//1000} kb",
-                    "Last Updated": parse_date(log_stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S"),
-                    "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]),
-                    "Full Log": log_text,
-                },
-            },
-        ],
-    )

archivebox/config_stubs.py

@@ -0,0 +1,169 @@
+from pathlib import Path
+from typing import Optional, Dict, Union, Tuple, Callable, Pattern, Type, Any, List
+from mypy_extensions import TypedDict
+
+
+
+SimpleConfigValue = Union[str, bool, int, None, Pattern, Dict[str, Any]]
+SimpleConfigValueDict = Dict[str, SimpleConfigValue]
+SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
+ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
+
+SHArgs = List[str]   # shell command args list e.g. ["--something=1", "--someotherarg"]
+
+
+class BaseConfig(TypedDict):
+    pass
+
+class ConfigDict(BaseConfig, total=False):
+    """
+    # Regenerate by pasting this quine into `archivebox shell` 🥚
+    from archivebox.config import ConfigDict, CONFIG_SCHEMA
+    print('class ConfigDict(BaseConfig, total=False):')
+    print('    ' + '"'*3 + ConfigDict.__doc__ + '"'*3)
+    for section, configs in CONFIG_SCHEMA.items():
+        for key, attrs in configs.items():
+            Type, default = attrs['type'], attrs['default']
+            if default is None:
+                print(f'    {key}: Optional[{Type.__name__}]')
+            else:
+                print(f'    {key}: {Type.__name__}')
+        print()
+    """
+    IS_TTY: bool
+    USE_COLOR: bool
+    SHOW_PROGRESS: bool
+    IN_DOCKER: bool
+    IN_QEMU: bool
+    PUID: int
+    PGID: int
+
+    OUTPUT_DIR: Optional[str]
+    CONFIG_FILE: Optional[str]
+    ONLY_NEW: bool
+    TIMEOUT: int
+    MEDIA_TIMEOUT: int
+    OUTPUT_PERMISSIONS: str
+    RESTRICT_FILE_NAMES: str
+    URL_DENYLIST: str
+    URL_ALLOWLIST: Optional[str]
+    ADMIN_USERNAME: Optional[str]
+    ADMIN_PASSWORD: Optional[str]
+    ENFORCE_ATOMIC_WRITES: bool
+    TAG_SEPARATOR_PATTERN: str
+
+    SECRET_KEY: Optional[str]
+    BIND_ADDR: str
+    ALLOWED_HOSTS: str
+    DEBUG: bool
+    PUBLIC_INDEX: bool
+    PUBLIC_SNAPSHOTS: bool
+    PUBLIC_ADD_VIEW: bool
+    FOOTER_INFO: str
+    SNAPSHOTS_PER_PAGE: int
+    CUSTOM_TEMPLATES_DIR: Optional[str]
+    TIME_ZONE: str
+    TIMEZONE: str
+    REVERSE_PROXY_USER_HEADER: str
+    REVERSE_PROXY_WHITELIST: str
+    LOGOUT_REDIRECT_URL: str
+    PREVIEW_ORIGINALS: bool
+    LDAP: bool
+    LDAP_SERVER_URI: Optional[str]
+    LDAP_BIND_DN: Optional[str]
+    LDAP_BIND_PASSWORD: Optional[str]
+    LDAP_USER_BASE: Optional[str]
+    LDAP_USER_FILTER: Optional[str]
+    LDAP_USERNAME_ATTR: Optional[str]
+    LDAP_FIRSTNAME_ATTR: Optional[str]
+    LDAP_LASTNAME_ATTR: Optional[str]
+    LDAP_EMAIL_ATTR: Optional[str]
+    LDAP_CREATE_SUPERUSER: bool
+
+    SAVE_TITLE: bool
+    SAVE_FAVICON: bool
+    SAVE_WGET: bool
+    SAVE_WGET_REQUISITES: bool
+    SAVE_SINGLEFILE: bool
+    SAVE_READABILITY: bool
+    SAVE_MERCURY: bool
+    SAVE_HTMLTOTEXT: bool
+    SAVE_PDF: bool
+    SAVE_SCREENSHOT: bool
+    SAVE_DOM: bool
+    SAVE_HEADERS: bool
+    SAVE_WARC: bool
+    SAVE_GIT: bool
+    SAVE_MEDIA: bool
+    SAVE_ARCHIVE_DOT_ORG: bool
+    SAVE_ALLOWLIST: dict
+    SAVE_DENYLIST: dict
+
+    RESOLUTION: str
+    GIT_DOMAINS: str
+    CHECK_SSL_VALIDITY: bool
+    MEDIA_MAX_SIZE: str
+    CURL_USER_AGENT: str
+    WGET_USER_AGENT: str
+    CHROME_USER_AGENT: str
+    COOKIES_FILE: Optional[str]
+    CHROME_USER_DATA_DIR: Optional[str]
+    CHROME_TIMEOUT: int
+    CHROME_HEADLESS: bool
+    CHROME_SANDBOX: bool
+    YOUTUBEDL_ARGS: list
+    WGET_ARGS: list
+    CURL_ARGS: list
+    GIT_ARGS: list
+    SINGLEFILE_ARGS: Optional[list]
+    FAVICON_PROVIDER: str
+
+    USE_INDEXING_BACKEND: bool
+    USE_SEARCHING_BACKEND: bool
+    SEARCH_BACKEND_ENGINE: str
+    SEARCH_BACKEND_HOST_NAME: str
+    SEARCH_BACKEND_PORT: int
+    SEARCH_BACKEND_PASSWORD: str
+    SEARCH_PROCESS_HTML: bool
+    SONIC_COLLECTION: str
+    SONIC_BUCKET: str
+    SEARCH_BACKEND_TIMEOUT: int
+    FTS_SEPARATE_DATABASE: bool
+    FTS_TOKENIZERS: str
+    FTS_SQLITE_MAX_LENGTH: int
+
+    USE_CURL: bool
+    USE_WGET: bool
+    USE_SINGLEFILE: bool
+    USE_READABILITY: bool
+    USE_MERCURY: bool
+    USE_GIT: bool
+    USE_CHROME: bool
+    USE_NODE: bool
+    USE_YOUTUBEDL: bool
+    USE_RIPGREP: bool
+    CURL_BINARY: str
+    GIT_BINARY: str
+    WGET_BINARY: str
+    SINGLEFILE_BINARY: str
+    READABILITY_BINARY: str
+    MERCURY_BINARY: str
+    YOUTUBEDL_BINARY: str
+    NODE_BINARY: str
+    RIPGREP_BINARY: str
+    CHROME_BINARY: Optional[str]
+    POCKET_CONSUMER_KEY: Optional[str]
+    POCKET_ACCESS_TOKENS: dict
+    READWISE_READER_TOKENS: dict
+
+
+ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]
+ConfigDefaultValue = Union[ConfigValue, ConfigDefaultValueGetter]
+
+ConfigDefault = TypedDict('ConfigDefault', {
+    'default': ConfigDefaultValue,
+    'type': Optional[Type],
+    'aliases': Optional[Tuple[str, ...]],
+}, total=False)
+
+ConfigDefaultDict = Dict[str, ConfigDefault]

archivebox/core/__init__.py

@@ -1,2 +1,3 @@
 __package__ = 'archivebox.core'
 
+# default_app_config = 'core.apps.CoreAppConfig'

archivebox/core/admin.py

@@ -1,20 +1,469 @@
 __package__ = 'archivebox.core'
 
-from django.contrib.auth import get_user_model
+from io import StringIO
+from pathlib import Path
+from contextlib import redirect_stdout
+from datetime import datetime, timezone
 
+from django.contrib import admin
+from django.db.models import Count
+from django.urls import path
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+from django.shortcuts import render, redirect
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Group, Permission
+from django import forms
+
+from ..util import htmldecode, urldecode, ansi_to_html
 
 from core.models import Snapshot, ArchiveResult, Tag
-from core.admin_tags import TagAdmin
-from core.admin_snapshots import SnapshotAdmin
-from core.admin_archiveresults import ArchiveResultAdmin
-from core.admin_users import UserAdmin
+from core.forms import AddLinkForm
 
-import abx
+from core.mixins import SearchResultsAdminMixin
+
+from index.html import snapshot_icons
+from logging_util import printable_filesize
+from main import add, remove
+from extractors import archive_links
+from config import (
+    OUTPUT_DIR,
+    SNAPSHOTS_PER_PAGE,
+    VERSION,
+    VERSIONS_AVAILABLE,
+    CAN_UPGRADE
+)
+
+GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
+
+# Admin URLs
+# /admin/
+# /admin/login/
+# /admin/core/
+# /admin/core/snapshot/
+# /admin/core/snapshot/:uuid/
+# /admin/core/tag/
+# /admin/core/tag/:uuid/
 
 
-@abx.hookimpl
-def register_admin(admin_site):
-    admin_site.register(get_user_model(), UserAdmin)
-    admin_site.register(ArchiveResult, ArchiveResultAdmin)
-    admin_site.register(Snapshot, SnapshotAdmin)
-    admin_site.register(Tag, TagAdmin)
+# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
+
+
+class ArchiveBoxAdmin(admin.AdminSite):
+    site_header = 'ArchiveBox'
+    index_title = 'Links'
+    site_title = 'Index'
+    namespace = 'admin'
+
+    def get_urls(self):
+        return [
+            path('core/snapshot/add/', self.add_view, name='Add'),
+        ] + super().get_urls()
+
+    def add_view(self, request):
+        if not request.user.is_authenticated:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        request.current_app = self.name
+        context = {
+            **self.each_context(request),
+            'title': 'Add URLs',
+        }
+
+        if request.method == 'GET':
+            context['form'] = AddLinkForm()
+
+        elif request.method == 'POST':
+            form = AddLinkForm(request.POST)
+            if form.is_valid():
+                url = form.cleaned_data["url"]
+                print(f'[+] Adding URL: {url}')
+                depth = 0 if form.cleaned_data["depth"] == "0" else 1
+                input_kwargs = {
+                    "urls": url,
+                    "depth": depth,
+                    "update_all": False,
+                    "out_dir": OUTPUT_DIR,
+                }
+                add_stdout = StringIO()
+                with redirect_stdout(add_stdout):
+                   add(**input_kwargs)
+                print(add_stdout.getvalue())
+
+                context.update({
+                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
+                    "form": AddLinkForm()
+                })
+            else:
+                context["form"] = form
+
+        return render(template_name='add.html', request=request, context=context)
+
+archivebox_admin = ArchiveBoxAdmin()
+archivebox_admin.register(get_user_model())
+archivebox_admin.disable_action('delete_selected')
+
+class ArchiveResultInline(admin.TabularInline):
+    model = ArchiveResult
+
+class TagInline(admin.TabularInline):
+    model = Snapshot.tags.through
+
+from django.contrib.admin.helpers import ActionForm
+from django.contrib.admin.widgets import AutocompleteSelectMultiple
+
+class AutocompleteTags:
+    model = Tag
+    search_fields = ['name']
+    name = 'tags'
+    remote_field = TagInline
+
+class AutocompleteTagsAdminStub:
+    name = 'admin'
+
+
+class SnapshotActionForm(ActionForm):
+    tags = forms.ModelMultipleChoiceField(
+        queryset=Tag.objects.all(),
+        required=False,
+        widget=AutocompleteSelectMultiple(
+            AutocompleteTags(),
+            AutocompleteTagsAdminStub(),
+        ),
+    )
+
+    # TODO: allow selecting actions for specific extractors? is this useful?
+    # EXTRACTOR_CHOICES = [
+    #     (name, name.title())
+    #     for name, _, _ in get_default_archive_methods()
+    # ]
+    # extractor = forms.ChoiceField(
+    #     choices=EXTRACTOR_CHOICES,
+    #     required=False,
+    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
+    # )
+
+
+@admin.register(Snapshot, site=archivebox_admin)
+class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
+    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'added', 'files')
+    readonly_fields = ('info', 'bookmarked', 'added', 'updated')
+    search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
+    fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
+    list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
+    ordering = ['-added']
+    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
+    autocomplete_fields = ['tags']
+    inlines = [ArchiveResultInline]
+    list_per_page = SNAPSHOTS_PER_PAGE
+
+    action_form = SnapshotActionForm
+
+
+    def changelist_view(self, request, extra_context=None):
+        extra_context = extra_context or {}
+        return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
+        ]
+        return custom_urls + urls
+
+    def get_queryset(self, request):
+        self.request = request
+        return super().get_queryset(request).prefetch_related('tags').annotate(archiveresult_count=Count('archiveresult'))
+
+    def tag_list(self, obj):
+        return ', '.join(obj.tags.values_list('name', flat=True))
+
+    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
+    # def action(self, obj):
+    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
+    #     # action: update_snapshots
+    #     # select_across: 0
+    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
+    #     return format_html(
+    #         '''
+    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
+    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
+    #                 <input type="hidden" name="_selected_action" value="{}">
+    #                 <button name="update_snapshots">Check</button>
+    #                 <button name="update_titles">Pull title + favicon</button>
+    #                 <button name="update_snapshots">Update</button>
+    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
+    #                 <button name="delete_snapshots">Permanently delete</button>
+    #             </form>
+    #         ''',
+    #         csrf.get_token(self.request),
+    #         obj.id,
+    #     )
+
+    def info(self, obj):
+        return format_html(
+            '''
+            UUID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
+            Timestamp: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
+            URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            Archived: {} ({} files {}) &nbsp; &nbsp;
+            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
+            Status code: {} &nbsp; &nbsp;
+            Server: {} &nbsp; &nbsp;
+            Content type: {} &nbsp; &nbsp;
+            Extension: {} &nbsp; &nbsp;
+            <br/><br/>
+            <a href="/archive/{}">View Snapshot index ➡️</a> &nbsp; &nbsp;
+            <a href="/admin/core/snapshot/?id__exact={}">View actions ⚙️</a>
+            ''',
+            obj.id,
+            obj.timestamp,
+            obj.url_hash,
+            '✅' if obj.is_archived else '❌',
+            obj.num_outputs,
+            self.size(obj),
+            f'/archive/{obj.timestamp}/favicon.ico',
+            obj.status_code or '?',
+            obj.headers and obj.headers.get('Server') or '?',
+            obj.headers and obj.headers.get('Content-Type') or '?',
+            obj.extension or '?',
+            obj.timestamp,
+            obj.id,
+        )
+
+    @admin.display(
+        description='Title',
+        ordering='title',
+    )
+    def title_str(self, obj):
+        canon = obj.as_link().canonical_outputs()
+        tags = ''.join(
+            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.id, tag)
+            for tag in obj.tags.all()
+            if str(tag).strip()
+        )
+        return format_html(
+            '<a href="/{}">'
+                '<img src="/{}/{}" class="favicon" onerror="this.remove()">'
+            '</a>'
+            '<a href="/{}/index.html">'
+                '<b class="status-{}">{}</b>'
+            '</a>',
+            obj.archive_path,
+            obj.archive_path, canon['favicon_path'],
+            obj.archive_path,
+            'fetched' if obj.latest_title or obj.title else 'pending',
+            urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
+        ) + mark_safe(f' <span class="tags">{tags}</span>')
+
+    @admin.display(
+        description='Files Saved',
+        ordering='archiveresult_count',
+    )
+    def files(self, obj):
+        return snapshot_icons(obj)
+
+
+    @admin.display(
+        ordering='archiveresult_count'
+    )
+    def size(self, obj):
+        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
+        if archive_size:
+            size_txt = printable_filesize(archive_size)
+            if archive_size > 52428800:
+                size_txt = mark_safe(f'<b>{size_txt}</b>')
+        else:
+            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
+        return format_html(
+            '<a href="/{}" title="View all files">{}</a>',
+            obj.archive_path,
+            size_txt,
+        )
+
+
+    @admin.display(
+        description='Original URL',
+        ordering='url',
+    )
+    def url_str(self, obj):
+        return format_html(
+            '<a href="{}"><code style="user-select: all;">{}</code></a>',
+            obj.url,
+            obj.url,
+        )
+
+    def grid_view(self, request, extra_context=None):
+
+        # cl = self.get_changelist_instance(request)
+
+        # Save before monkey patching to restore for changelist list view
+        saved_change_list_template = self.change_list_template
+        saved_list_per_page = self.list_per_page
+        saved_list_max_show_all = self.list_max_show_all
+
+        # Monkey patch here plus core_tags.py
+        self.change_list_template = 'private_index_grid.html'
+        self.list_per_page = SNAPSHOTS_PER_PAGE
+        self.list_max_show_all = self.list_per_page
+
+        # Call monkey patched view
+        rendered_response = self.changelist_view(request, extra_context=extra_context)
+
+        # Restore values
+        self.change_list_template = saved_change_list_template
+        self.list_per_page = saved_list_per_page
+        self.list_max_show_all = saved_list_max_show_all
+
+        return rendered_response
+
+    # for debugging, uncomment this to print all requests:
+    # def changelist_view(self, request, extra_context=None):
+    #     print('[*] Got request', request.method, request.POST)
+    #     return super().changelist_view(request, extra_context=None)
+
+    @admin.action(
+        description="Pull"
+    )
+    def update_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], out_dir=OUTPUT_DIR)
+
+    @admin.action(
+        description="⬇️ Title"
+    )
+    def update_titles(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
+
+    @admin.action(
+        description="Re-Snapshot"
+    )
+    def resnapshot_snapshot(self, request, queryset):
+        for snapshot in queryset:
+            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
+            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
+            add(new_url, tag=snapshot.tags_str())
+
+    @admin.action(
+        description="Reset"
+    )
+    def overwrite_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, out_dir=OUTPUT_DIR)
+
+    @admin.action(
+        description="Delete"
+    )
+    def delete_snapshots(self, request, queryset):
+        remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
+
+
+    @admin.action(
+        description="+"
+    )
+    def add_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[+] Adding tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.add(*tags)
+
+
+    @admin.action(
+        description="–"
+    )
+    def remove_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[-] Removing tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.remove(*tags)
+
+
+        
+
+
+
+
+
+@admin.register(Tag, site=archivebox_admin)
+class TagAdmin(admin.ModelAdmin):
+    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
+    sort_fields = ('id', 'name', 'slug')
+    readonly_fields = ('id', 'num_snapshots', 'snapshots')
+    search_fields = ('id', 'name', 'slug')
+    fields = (*readonly_fields, 'name', 'slug')
+    actions = ['delete_selected']
+    ordering = ['-id']
+
+    def num_snapshots(self, obj):
+        return format_html(
+            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
+            obj.id,
+            obj.snapshot_set.count(),
+        )
+
+    def snapshots(self, obj):
+        total_count = obj.snapshot_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
+                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.id,
+                snap.timestamp,
+                snap.url,
+            )
+            for snap in obj.snapshot_set.order_by('-updated')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={obj.id}">and {total_count-10} more...<a>' if obj.snapshot_set.count() > 10 else ''))
+
+
+@admin.register(ArchiveResult, site=archivebox_admin)
+class ArchiveResultAdmin(admin.ModelAdmin):
+    list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
+    sort_fields = ('start_ts', 'extractor', 'status')
+    readonly_fields = ('id', 'uuid', 'snapshot_str', 'tags_str')
+    search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
+    autocomplete_fields = ['snapshot']
+
+    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
+    ordering = ['-start_ts']
+    list_per_page = SNAPSHOTS_PER_PAGE
+
+    @admin.display(
+        description='snapshot'
+    )
+    def snapshot_str(self, obj):
+        return format_html(
+            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
+            '<small>{}</small>',
+            obj.snapshot.timestamp,
+            obj.snapshot.timestamp,
+            obj.snapshot.url[:128],
+        )
+
+    @admin.display(
+        description='tags'
+    )
+    def tags_str(self, obj):
+        return obj.snapshot.tags_str()
+
+    def cmd_str(self, obj):
+        return format_html(
+            '<pre>{}</pre>',
+            ' '.join(obj.cmd) if isinstance(obj.cmd, list) else str(obj.cmd),
+        )
+
+    def output_str(self, obj):
+        return format_html(
+            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
+            obj.snapshot.timestamp,
+            obj.output if (obj.status == 'succeeded') and obj.extractor not in ('title', 'archive_org') else 'index.html',
+            obj.output,
+        )

archivebox/core/admin_archiveresults.py

@@ -1,199 +0,0 @@
-__package__ = 'archivebox.core'
-
-import os
-from pathlib import Path
-
-from django.contrib import admin
-from django.utils.html import format_html, mark_safe
-from django.core.exceptions import ValidationError
-from django.urls import reverse, resolve
-from django.utils import timezone
-from django.forms import forms
-
-from huey_monitor.admin import TaskModel
-
-import abx
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import SERVER_CONFIG
-from archivebox.misc.paginators import AccelleratedPaginator
-
-from abid_utils.admin import ABIDModelAdmin
-
-from core.models import ArchiveResult, Snapshot
-
-
-
-
-def result_url(result: TaskModel) -> str:
-    url = reverse("admin:huey_monitor_taskmodel_change", args=[str(result.id)])
-    return format_html('<a href="{url}" class="fade-in-progress-url">See progress...</a>'.format(url=url))
-
-
-
-class ArchiveResultInline(admin.TabularInline):
-    name = 'Archive Results Log'
-    model = ArchiveResult
-    parent_model = Snapshot
-    # fk_name = 'snapshot'
-    extra = 0
-    sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
-    readonly_fields = ('id', 'result_id', 'completed', 'command', 'version')
-    fields = ('start_ts', 'end_ts', *readonly_fields, 'extractor', 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
-    # exclude = ('id',)
-    ordering = ('end_ts',)
-    show_change_link = True
-    # # classes = ['collapse']
-    # # list_display_links = ['abid']
-
-    def get_parent_object_from_request(self, request):
-        resolved = resolve(request.path_info)
-        try:
-            return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
-        except (self.parent_model.DoesNotExist, ValidationError):
-            return self.parent_model.objects.get(pk=self.parent_model.id_from_abid(resolved.kwargs['object_id']))
-
-    @admin.display(
-        description='Completed',
-        ordering='end_ts',
-    )
-    def completed(self, obj):
-        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime('%Y-%m-%d %H:%M:%S'))
-
-    def result_id(self, obj):
-        return format_html('<a href="{}"><code style="font-size: 10px">[{}]</code></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
-    
-    def command(self, obj):
-        return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
-    
-    def version(self, obj):
-        return format_html('<small><code>{}</code></small>', obj.cmd_version or '-')
-    
-    def get_formset(self, request, obj=None, **kwargs):
-        formset = super().get_formset(request, obj, **kwargs)
-        snapshot = self.get_parent_object_from_request(request)
-
-        # import ipdb; ipdb.set_trace()
-        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
-        
-        # default values for new entries
-        formset.form.base_fields['status'].initial = 'succeeded'
-        formset.form.base_fields['start_ts'].initial = timezone.now()
-        formset.form.base_fields['end_ts'].initial = timezone.now()
-        formset.form.base_fields['cmd_version'].initial = '-'
-        formset.form.base_fields['pwd'].initial = str(snapshot.link_dir)
-        formset.form.base_fields['created_by'].initial = request.user
-        formset.form.base_fields['cmd'] = forms.JSONField(initial=['-'])
-        formset.form.base_fields['output'].initial = 'Manually recorded cmd output...'
-        
-        if obj is not None:
-            # hidden values for existing entries and new entries
-            formset.form.base_fields['start_ts'].widget = formset.form.base_fields['start_ts'].hidden_widget()
-            formset.form.base_fields['end_ts'].widget = formset.form.base_fields['end_ts'].hidden_widget()
-            formset.form.base_fields['cmd'].widget = formset.form.base_fields['cmd'].hidden_widget()
-            formset.form.base_fields['pwd'].widget = formset.form.base_fields['pwd'].hidden_widget()
-            formset.form.base_fields['created_by'].widget = formset.form.base_fields['created_by'].hidden_widget()
-            formset.form.base_fields['cmd_version'].widget = formset.form.base_fields['cmd_version'].hidden_widget()
-        return formset
-    
-    def get_readonly_fields(self, request, obj=None):
-        if obj is not None:
-            return self.readonly_fields
-        else:
-            return []
-
-
-
-class ArchiveResultAdmin(ABIDModelAdmin):
-    list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
-    sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created_at', 'modified_at', 'abid_info', 'output_summary')
-    search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
-    autocomplete_fields = ['snapshot']
-
-    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
-    ordering = ['-start_ts']
-    list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
-    
-    paginator = AccelleratedPaginator
-    save_on_top = True
-    
-    actions = ['delete_selected']
-    
-    class Meta:
-        verbose_name = 'Archive Result'
-        verbose_name_plural = 'Archive Results'
-
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
-
-    @admin.display(
-        description='Snapshot Info'
-    )
-    def snapshot_info(self, result):
-        return format_html(
-            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            result.snapshot.timestamp,
-            result.snapshot.abid,
-            result.snapshot.bookmarked_at.strftime('%Y-%m-%d %H:%M'),
-            result.snapshot.url[:128],
-        )
-
-
-    @admin.display(
-        description='Snapshot Tags'
-    )
-    def tags_str(self, result):
-        return result.snapshot.tags_str()
-
-    def cmd_str(self, result):
-        return format_html(
-            '<pre>{}</pre>',
-            ' '.join(result.cmd) if isinstance(result.cmd, list) else str(result.cmd),
-        )
-    
-    def output_str(self, result):
-        return format_html(
-            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
-            result.snapshot.timestamp,
-            result.output if (result.status == 'succeeded') and result.extractor not in ('title', 'archive_org') else 'index.html',
-            result.output,
-        )
-
-    def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split('data/', 1)[-1]
-        output_str = format_html(
-            '<pre style="display: inline-block">{}</pre><br/>',
-            result.output,
-        )
-        output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
-        path_from_output_str = (snapshot_dir / result.output)
-        output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
-        if os.access(path_from_output_str, os.R_OK):
-            root_dir = str(path_from_output_str)
-        else:
-            root_dir = str(snapshot_dir)
-
-        # print(root_dir, str(list(os.walk(root_dir))))
-
-        for root, dirs, files in os.walk(root_dir):
-            depth = root.replace(root_dir, '').count(os.sep) + 1
-            if depth > 2:
-                continue
-            indent = ' ' * 4 * (depth)
-            output_str += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
-            indentation_str = ' ' * 4 * (depth + 1)
-            for filename in sorted(files):
-                is_hidden = filename.startswith('.')
-                output_str += format_html('<span style="opacity: {}.2">{}{}</span><br/>', int(not is_hidden), indentation_str, filename.strip())
-
-        return output_str + format_html('</code></pre>')
-
-
-
-
-@abx.hookimpl
-def register_admin(admin_site):
-    admin_site.register(ArchiveResult, ArchiveResultAdmin)

archivebox/core/admin_site.py

@@ -1,42 +0,0 @@
-__package__ = 'archivebox.core'
-
-from django.contrib import admin
-
-import abx.django.use
-
-class ArchiveBoxAdmin(admin.AdminSite):
-    site_header = 'ArchiveBox'
-    index_title = 'Admin Views'
-    site_title = 'Admin'
-    namespace = 'admin'
-
-
-archivebox_admin = ArchiveBoxAdmin()
-archivebox_admin.disable_action('delete_selected')
-# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
-
-
-
-# patch admin with methods to add data views (implemented by admin_data_views package)
-# https://github.com/MrThearMan/django-admin-data-views
-# https://mrthearman.github.io/django-admin-data-views/setup/
-from admin_data_views.admin import get_app_list, admin_data_index_view, get_admin_data_urls, get_urls
-archivebox_admin.get_app_list = get_app_list.__get__(archivebox_admin, ArchiveBoxAdmin)
-archivebox_admin.admin_data_index_view = admin_data_index_view.__get__(archivebox_admin, ArchiveBoxAdmin)       # type: ignore
-archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_admin, ArchiveBoxAdmin)           # type: ignore
-archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
-############### Admin Data View sections are defined in settings.ADMIN_DATA_VIEWS #########
-
-
-def register_admin_site():
-    """Replace the default admin site with our custom ArchiveBox admin site."""
-    from django.contrib import admin
-    from django.contrib.admin import sites
-
-    admin.site = archivebox_admin
-    sites.site = archivebox_admin
-    
-    # register all plugins admin classes
-    abx.django.use.register_admin(archivebox_admin)
-    
-    return archivebox_admin