clean previous build before releasing

Fix tcp port 8080 being used twice and move novnc web interface to 127.0.0.1:8081

.github/workflows/docker.yml

@@ -81,6 +81,7 @@ jobs:
           builder: ${{ steps.buildx.outputs.name }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
           platforms: linux/amd64,linux/arm64

.gitignore

@@ -11,6 +11,7 @@ venv/
 .venv/
 .docker-venv/
 node_modules/
+typings/
 
 # Ignore dev lockfiles (should always be built fresh)
 pdm.dev.lock
@@ -25,6 +26,8 @@ build/
 dist/
 
 # Data folders
+lib/
+tmp/
 data/
 data*/
 output/

.gitmodules

@@ -1,31 +1,6 @@
 [submodule "docs"]
     path = docs
     url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
-
-[submodule "deb_dist"]
-    path = deb_dist
-    url = https://github.com/ArchiveBox/debian-archivebox.git
-[submodule "brew_dist"]
-	path = brew_dist
-	url = https://github.com/ArchiveBox/homebrew-archivebox.git
-[submodule "pip_dist"]
-	path = pip_dist
-	url = https://github.com/ArchiveBox/pip-archivebox.git
-[submodule "docker"]
-	path = docker
-	url = https://github.com/ArchiveBox/docker-archivebox.git
-[submodule "archivebox/vendor/base32-crockford"]
-	path = archivebox/vendor/base32-crockford
-	url = https://github.com/jbittel/base32-crockford
 [submodule "archivebox/vendor/pocket"]
 	path = archivebox/vendor/pocket
 	url = https://github.com/tapanpandita/pocket
-[submodule "archivebox/vendor/django-taggit"]
-	path = archivebox/vendor/django-taggit
-	url = https://github.com/jazzband/django-taggit
-[submodule "archivebox/vendor/python-atomicwrites"]
-	path = archivebox/vendor/python-atomicwrites
-	url = https://github.com/untitaker/python-atomicwrites
-[submodule "archivebox/vendor/pydantic-pkgr"]
-	path = archivebox/vendor/pydantic-pkgr
-	url = https://github.com/ArchiveBox/pydantic-pkgr

Dockerfile

@@ -10,13 +10,14 @@
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:0.7.5 -t archivebox/archivebox:dev
 #
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
+#########################################################################################
 
 # Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
-FROM python:3.11-slim-bookworm
+FROM python:3.12-slim-bookworm
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -58,8 +59,8 @@ ENV TZ=UTC \
     npm_config_loglevel=error
 
 # Version config
-ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=20
+ENV PYTHON_VERSION=3.12 \
+    NODE_VERSION=22
 
 # User config
 ENV ARCHIVEBOX_USER="archivebox" \
@@ -71,13 +72,9 @@ ENV CODE_DIR=/app \
     DATA_DIR=/data \
     GLOBAL_VENV=/venv \
     PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Application-level paths
-ENV APP_VENV=/app/.venv \
-    NODE_MODULES=/app/node_modules
+    # TODO: add TMP_DIR and LIB_DIR?
 
 # Build shell config
-ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
 
 ######### System Environment ####################################
@@ -97,7 +94,6 @@ RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
     && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && echo \
-    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
     && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
     && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
     && echo \
@@ -129,16 +125,23 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep \
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
         # 3. frivolous CLI helpers to make debugging failed archiving easier
+        tree nano iputils-ping \
         # nano iputils-ping dnsutils htop procps jq yq
     && rm -rf /var/lib/apt/lists/*
 
+# Install sonic search backend
+COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+RUN (which sonic && sonic --version) | tee -a /VERSION.txt
+
 ######### Language Environments ####################################
 
 # Install Python environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
     # && apt-get update -qq \
     # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
     #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
@@ -162,7 +165,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 
 # Install Node environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    echo "[+] Installing Node $NODE_VERSION environment..." \
     && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
     && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
     && apt-get update -qq \
@@ -201,6 +204,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
+
 # Install chromium browser using playwright
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
@@ -210,7 +214,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
         libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
         libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
         # xfonts-scalable xfonts-utils xserver-common xvfb \
         # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
         # libxss1 dbus dbus-x11 upower \
@@ -223,7 +227,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && rm -rf /var/lib/apt/lists/* \
     && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
     && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
     && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
     && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
     # Save version info
@@ -233,11 +237,12 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     ) | tee -a /VERSION.txt
 
 # Install Node dependencies
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+WORKDIR "$CODE_DIR/lib/npm"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/lib/npm"
 RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
-    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    echo "[+] Installing NPM extractor dependencies from package.json..." \
+    && npm ci --prefix="$CODE_DIR/lib/npm" --prefer-offline --no-audit --cache /root/.npm \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$CODE_DIR/lib" \
     && ( \
         which node && node --version \
         && which npm && npm version \
@@ -253,7 +258,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
     && apt-get update -qq \
     && apt-get install -qq -y -t bookworm-backports \
-        build-essential \
+        build-essential gcc \
         libssl-dev libldap2-dev libsasl2-dev \
         python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
     # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
@@ -264,32 +269,25 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     # && source $GLOBAL_VENV/bin/activate \
     && pip install -r requirements.txt \
     && apt-get purge -y \
-        build-essential \
+        build-essential gcc \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
 # Install ArchiveBox Python package from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    # && apt-get update -qq \
-    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
-    # && apt-get install -qq -y -t bookworm-backports \
-        # build-essential  \
-    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
-    && pip install -e "$CODE_DIR"[sonic,ldap] \
-    # save docker image size and always remove compilers / build tools after building is complete
-    # && apt-get purge -y build-essential \
-    # && apt-get autoremove -y \
+    && pip install -e "${CODE_DIR}[sonic,ldap]" \
     && rm -rf /var/lib/apt/lists/*
 
 ####################################################
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
+RUN openssl rand -hex 16 > /etc/machine-id
 ENV IN_DOCKER=True \
-    DISPLAY=novnc:0.0 \
-    CUSTOM_TEMPLATES_DIR=/data/templates \
+    SYSTEM_LIB_DIR=/app/lib \
+    SYSTEM_TMP_DIR=/tmp \
     GOOGLE_API_KEY=no \
     GOOGLE_DEFAULT_CLIENT_ID=no \
     GOOGLE_DEFAULT_CLIENT_SECRET=no \

README.md

@@ -22,7 +22,7 @@ curl -fsSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instruc
 
 **ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**
 
-Without active preservation effort, everything on the internet eventually dissapears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
+Without active preservation effort, everything on the internet eventually disappears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
 
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
@@ -790,7 +790,19 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
 </blockquote>
 
-These optional dependencies used for archiving sites include:
+<ul>
+<li>Language: Python <code>&gt;=3.10</code></li>
+<li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
+<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
+<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
+<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
+<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
+<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+</ul>
+
+
+These optional subdependencies used for archiving sites include:
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right" style="max-width: 100%;">
 
@@ -1614,8 +1626,7 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <br/><br/>
 <a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>&nbsp;
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>&nbsp;
-<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/>
-<sup>ArchiveBox was started by <a href="https://docs.sweeting.me/s/blog#About">Nick Sweeting</a> in 2017, and has <a href="https://star-history.com/#archivebox/archivebox&Date">grown steadily</a> with help from our <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">amazing contributors</a>.</sup>
+<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/><br/>
 <hr/>
 <i>✨ Have spare CPU/disk/bandwidth after all your 网站存档爬 and want to help the world?<br/>Check out our <a href="https://github.com/ArchiveBox/good-karma-kit">Good Karma Kit</a>...</i>
 </div>

archivebox/__init__.py

@@ -1,4 +1,105 @@
+#!/usr/bin/env python3
+
+# Welcome to the ArchiveBox source code! Thanks for checking it out!
+#
+# "We are swimming upstream against a great torrent of disorganization.
+# In this, our main obligation is to establish arbitrary enclaves of order and system.
+# It is the greatest possible victory to be, to continue to be, and to have been.
+# No defeat can deprive us of the success of having existed for some moment of time
+# in a universe that seems indifferent to us."
+# --Norber Weiner
+
 __package__ = 'archivebox'
 
+import os
+import sys
+import tempfile
+from pathlib import Path
 
-from .monkey_patches import *
+ASCII_LOGO = """
+ █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
+██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
+███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
+██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
+██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
+╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
+"""
+
+SYSTEM_TMP_DIR = Path(tempfile.gettempdir()) / 'archivebox'
+SYSTEM_TMP_DIR.mkdir(parents=True, exist_ok=True)
+os.environ['SYSTEM_TMP_DIR'] = str(SYSTEM_TMP_DIR)
+
+# if we are outside a data dir, cd into an ephemeral tmp dir so that
+# we can run version/help without polluting cwd with an index.sqlite3
+if len(sys.argv) > 1 and sys.argv[1] in ('version', 'help'):
+    current_dir = Path(os.getcwd()).resolve()
+    if not (current_dir / 'index.sqlite3').exists():
+        os.chdir(SYSTEM_TMP_DIR)
+
+# make sure PACKAGE_DIR is in sys.path so we can import all subfolders
+# without necessarily waiting for django to load them thorugh INSTALLED_APPS
+PACKAGE_DIR = Path(__file__).resolve().parent
+if str(PACKAGE_DIR) not in sys.path:
+    sys.path.append(str(PACKAGE_DIR))
+
+from .config.constants import CONSTANTS, DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, VERSION  # noqa
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
+
+# print('INSTALLING MONKEY PATCHES')
+from .monkey_patches import *                    # noqa
+# print('DONE INSTALLING MONKEY PATCHES')
+
+# print('LOADING VENDORED LIBRARIES')
+from .vendor import load_vendored_libs           # noqa
+load_vendored_libs()
+# print('DONE LOADING VENDORED LIBRARIES')
+
+__version__ = VERSION
+__author__ = 'Nick Sweeting'
+__license__ = 'MIT'
+
+ASCII_ICON = """
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+         ██                                                                            ██          
+         ██                                                                            ██        
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                    ████████████████████████████████████                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ████████████████████████████████████                    ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                 ██████████████████████████████████████████                 ██          
+         ██                 ██████████████████████████████████████████                 ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██        
+         ████████████████████████████████████████████████████████████████████████████████          
+"""

archivebox/__main__.py

@@ -1,11 +1,19 @@
 #!/usr/bin/env python3
-
+"""This is the main entry point for the ArchiveBox CLI."""
 __package__ = 'archivebox'
 
+import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys
 
 from .cli import main
 
+ASCII_LOGO_MINI = r"""
+     _             _     _           ____            
+    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
+   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
+  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  < 
+ /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
+"""
 
 if __name__ == '__main__':
     main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/abid_utils/abid.py

@@ -1,4 +1,6 @@
-from typing import NamedTuple, Any, Union, Optional
+__package__ = 'archivebox.abid_utils'
+
+from typing import NamedTuple, Any, Union, Dict
 
 import ulid
 import uuid6
@@ -9,6 +11,7 @@ from uuid import UUID
 from typeid import TypeID            # type: ignore[import-untyped]
 from datetime import datetime
 
+from archivebox.misc.util import enforce_types
 
 
 ABID_PREFIX_LEN = 4
@@ -108,9 +111,10 @@ class ABID(NamedTuple):
 ####################################################
 
 
+@enforce_types
 def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
     """
-    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
+    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
     """
     if isinstance(uri, bytes):
         uri_str: str = uri.decode()
@@ -126,35 +130,45 @@ def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
         except AttributeError:
             pass
     
+    # the uri hash is the sha256 of the domain + salt
     uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
 
     return hashlib.sha256(uri_bytes).hexdigest().upper()
 
-def abid_part_from_prefix(prefix: Optional[str]) -> str:
+@enforce_types
+def abid_part_from_prefix(prefix: str) -> str:
     """
     'snp_'
     """
-    if prefix is None:
-        return 'obj_'
+    # if prefix is None:
+    #     return 'obj_'
 
     prefix = prefix.strip('_').lower()
     assert len(prefix) == 3
     return prefix + '_'
 
-def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+@enforce_types
+def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
     """
     'E4A5CCD9'     # takes first 8 characters of sha256(url)
     """
-    uri = str(uri)
+    uri = str(uri).strip()
+    assert uri not in ('None', '')
     return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
 
-def abid_part_from_ts(ts: Optional[datetime]) -> str:
+@enforce_types
+def abid_part_from_ts(ts: datetime) -> str:
     """
     '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
     """
-    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
+    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
 
-def abid_part_from_subtype(subtype: str) -> str:
+@enforce_types
+def ts_from_abid(abid: str) -> datetime:
+    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
+
+@enforce_types
+def abid_part_from_subtype(subtype: str | int) -> str:
     """
     Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
     Also allows us to change the ulid spec later by putting special sigil values here.
@@ -165,6 +179,7 @@ def abid_part_from_subtype(subtype: str) -> str:
 
     return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
 
+@enforce_types
 def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
     """
     'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
@@ -186,17 +201,23 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
     return str(rand)[-ABID_RAND_LEN:].upper()
 
 
-def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
+@enforce_types
+def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
+    return {
+        'prefix': abid_part_from_prefix(prefix),
+        'ts': abid_part_from_ts(ts),
+        'uri': abid_part_from_uri(uri, salt=salt),
+        'subtype': abid_part_from_subtype(subtype),
+        'rand': abid_part_from_rand(rand),
+        # 'salt': don't add this, salt combined with uri above to form a single hash
+    }
+
+@enforce_types
+def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
     """
     Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
     """
 
-    abid = ABID(
-        prefix=abid_part_from_prefix(prefix),
-        ts=abid_part_from_ts(ts),
-        uri=abid_part_from_uri(uri, salt=salt),
-        subtype=abid_part_from_subtype(subtype),
-        rand=abid_part_from_rand(rand),
-    )
+    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
     assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
     return abid

archivebox/abid_utils/admin.py

@@ -1,18 +1,173 @@
-from django.contrib import admin
+__package__ = 'archivebox.abid_utils'
+
+from typing import Any
+
+from django.contrib import admin, messages
+from django.core.exceptions import ValidationError
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+from django.shortcuts import redirect
+
+from django_object_actions import DjangoObjectActions, action
 
 
-class ABIDModelAdmin(admin.ModelAdmin):
-    list_display = ('created', 'created_by', 'abid', '__str__')
-    sort_fields = ('created', 'created_by', 'abid', '__str__')
-    readonly_fields = ('abid', 'created', '__str__')
+from api.auth import get_or_create_api_token
+
+from archivebox.misc.util import parse_date
+from .abid import ABID
+
+def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
+    """highlight each character in red that differs with the char at the same index in compare_val"""
+
+    display_val = str(display_val)
+    compare_val = str(compare_val)
+
+    if len(compare_val) < len(display_val):
+        compare_val += ' ' * (len(display_val) - len(compare_val))
+
+    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
+    if invert:
+        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
+
+    return mark_safe(''.join(
+        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
+        if display_val[i] != compare_val[i] else
+        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
+        for i in range(len(display_val))
+    ))
+
+def get_abid_info(self, obj, request=None):
+    try:
+        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
+
+        fresh_values = obj.ABID_FRESH_VALUES
+        fresh_hashes = obj.ABID_FRESH_HASHES
+        fresh_diffs = obj.ABID_FRESH_DIFFS
+        fresh_abid = ABID(**fresh_hashes)
+        
+        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
+        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
+
+        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
+
+        fresh_ts = parse_date(fresh_values['ts']) or None
+        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
+
+        derived_uri = fresh_hashes['uri']
+        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
+
+        derived_subtype = fresh_hashes['subtype']
+        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
+
+        derived_rand = fresh_hashes['rand']
+        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
+
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
+            <br/><hr/>
+            <div style="opacity: 0.8">
+            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
+            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
+            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
+            <hr/>
+            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
+            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
+            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
+            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
+            <br/><hr/>
+            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
+            </div>
+            ''',
+            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
+            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
+            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
+            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
+            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
+            # str(fresh_abid), mark_safe(fresh_abid_diff),
+            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
+            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
+            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
+            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
+            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
+            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
+            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
+        )
+    except Exception as e:
+        # import ipdb; ipdb.set_trace()
+        return str(e)
+
+
+class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
+    list_display = ('created_at', 'created_by', 'abid')
+    sort_fields = ('created_at', 'created_by', 'abid')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
+    # fields = [*readonly_fields]
+    
+    change_actions = ("regenerate_abid",)
+    # changelist_actions = ("regenerate_abid",)
+
+    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
+        try:
+            object_pk = self.model.id_from_abid(object_id)
+            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
+        except (self.model.DoesNotExist, ValidationError):
+            pass
+        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
+    
+    def queryset(self, request):
+        self.request = request
+        return super().queryset(request)                                                # type: ignore
+    
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        return super().change_view(request, object_id, form_url, extra_context)
 
     def get_form(self, request, obj=None, **kwargs):
+        self.request = request
         form = super().get_form(request, obj, **kwargs)
         if 'created_by' in form.base_fields:
             form.base_fields['created_by'].initial = request.user
+            
+        if obj:
+            if obj.ABID_FRESH_DIFFS:
+                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
+
         return form
 
-    # def save_model(self, request, obj, form, change):
-    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
-    #         obj.created_by = request.user
-    #     obj.save()
+    def get_formset(self, request, formset=None, obj=None, **kwargs):
+        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
+        formset.form.base_fields['created_at'].disabled = True
+        
+        return formset
+
+    def save_model(self, request, obj, form, change):
+        self.request = request
+
+        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
+
+        super().save_model(request, obj, form, change)
+        obj.refresh_from_db()
+
+        new_abid = obj.abid
+        if new_abid != old_abid:
+            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
+        # import ipdb; ipdb.set_trace()
+
+    @admin.display(description='API Identifiers')
+    def abid_info(self, obj):
+        return get_abid_info(self, obj, request=self.request)
+
+    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
+    def regenerate_abid(self, request, obj):
+        old_abid = str(obj.abid)
+        obj.abid = obj.issue_new_abid(overwrite=True)
+        obj.save()
+        obj.refresh_from_db()
+        new_abid = str(obj.abid)
+
+        if new_abid != old_abid:
+            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
+        else:
+            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")

archivebox/abid_utils/models.py

@@ -1,23 +1,21 @@
 """
 This file provides the Django ABIDField and ABIDModel base model to inherit from.
-
-It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
 """
 
-from typing import Any, Dict, Union, List, Set, NamedTuple, cast
 
-from ulid import ULID
-from uuid import uuid4, UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
+from typing import Any, Dict, Union, List, Set, cast
+
+from uuid import uuid4
 from functools import partial
 from charidfield import CharIDField  # type: ignore[import-untyped]
 
-from django.conf import settings
+from django.contrib import admin
+from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
 from django.db import models
 from django.utils import timezone
 from django.db.utils import OperationalError
 from django.contrib.auth import get_user_model
+from django.urls import reverse_lazy
 
 from django_stubs_ext.db.models import TypedModelMeta
 
@@ -29,7 +27,9 @@ from .abid import (
     DEFAULT_ABID_PREFIX,
     DEFAULT_ABID_URI_SALT,
     abid_part_from_prefix,
-    abid_from_values
+    abid_hashes_from_values,
+    ts_from_abid,
+    abid_part_from_ts,
 )
 
 ####################################################
@@ -57,144 +57,269 @@ def get_or_create_system_user_pk(username='system'):
         return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
 
     # otherwise, create a dedicated "system" user
-    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
     return user.pk
 
 
+class AutoDateTimeField(models.DateTimeField):
+    # def pre_save(self, model_instance, add):
+    #     return timezone.now()
+    pass
+
+class ABIDError(Exception):
+    pass
+
+
 class ABIDModel(models.Model):
     """
     Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
     """
-    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
-    abid_ts_src = 'None'                    # e.g. 'self.created'
-    abid_uri_src = 'None'                   # e.g. 'self.uri'
-    abid_subtype_src = 'None'               # e.g. 'self.extractor'
-    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
+    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
+    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
+    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
+    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
+    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
+    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
+    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
 
-    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
-    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
-    abid = ABIDField(prefix=abid_prefix)
+    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    # abid = ABIDField(prefix=abid_prefix)
 
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
+    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    # modified_at = models.DateTimeField(auto_now=True)
+
+    _prefetched_objects_cache: Dict[str, Any]
 
     class Meta(TypedModelMeta):
         abstract = True
 
-    def save(self, *args: Any, **kwargs: Any) -> None:
-        # when first creating a row, self.ABID is the source of truth
-        # overwrite default prefilled self.id & self.abid with generated self.ABID value
-        if self._state.adding or not self.id:
-            self.id = self.ABID.uuid
-        if self._state.adding or not self.abid:
-            self.abid = str(self.ABID)
+    @admin.display(description='Summary')
+    def __str__(self) -> str:
+        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
 
-        super().save(*args, **kwargs)
-        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
-        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
+        super().__init__(*args, **kwargs)
+        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
+        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
+        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
+        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
+        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
+
+    def clean(self, abid_drift_allowed: bool | None=None) -> None:
+        if self._state.adding:
+            # only runs once when a new object is first saved to the DB
+            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
+            self._previous_abid = None
+            self.abid = str(self.issue_new_abid())
+
+        else:
+            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
+            abid_diffs = self.ABID_FRESH_DIFFS
+            if abid_diffs:
+                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
+
+                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
+                full_summary = (
+                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
+                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
+                )
+
+                change_error = ValidationError({
+                    **{
+                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
+                        diff['abid_src'].replace('self.', '')
+                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
+                            else NON_FIELD_ERRORS
+                        : ValidationError(
+                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
+                            code='ABIDConflict',
+                            params=diff,
+                        )
+                        for diff in abid_diffs.values()
+                    },
+                    NON_FIELD_ERRORS: ValidationError(full_summary),
+                })
+
+                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
+                if allowed_to_invalidate_abid:
+                    print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
+                    print(change_error)
+                    print('--------------------------------------------------------------------------------------------------')
+                else:
+                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
+                    print(change_error)
+                    print('--------------------------------------------------------------------------------------------------')
+                    raise change_error
+
+    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
+        """Overriden save method ensures new ABID is generated while a new object is first saving."""
+
+        self.clean(abid_drift_allowed=abid_drift_allowed)
+
+        return super().save(*args, **kwargs)
+    
+    @classmethod
+    def id_from_abid(cls, abid: str) -> str:
+        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
 
     @property
-    def abid_values(self) -> Dict[str, Any]:
+    def ABID_SOURCES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        assert self.abid_prefix
         return {
-            'prefix': self.abid_prefix,
-            'ts': eval(self.abid_ts_src),
-            'uri': eval(self.abid_uri_src),
-            'subtype': eval(self.abid_subtype_src),
-            'rand': eval(self.abid_rand_src),
+            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
+            'ts': self.abid_ts_src,
+            'uri': self.abid_uri_src,
+            'subtype': self.abid_subtype_src,
+            'rand': self.abid_rand_src,
+            'salt': 'self.abid_salt',               # defined as static class vars at build time
         }
 
-    def generate_abid(self) -> ABID:
-        """
-        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-        """
-        prefix, ts, uri, subtype, rand = self.abid_values.values()
-
-        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
-            suggested_abid = self.__class__.__name__[:3].lower()
-            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
-
-        if not ts:
-            # default to unix epoch with 00:00:00 UTC
-            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
-            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
-
-        if not uri:
-            uri = str(self)
-            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
-
-        if not subtype:
-            subtype = self.__class__.__name__
-            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
-
-        if not rand:
-            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
-            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
-
-        abid = abid_from_values(
-            prefix=prefix,
-            ts=ts,
-            uri=uri,
-            subtype=subtype,
-            rand=rand,
-            salt=DEFAULT_ABID_URI_SALT,
+    @property
+    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        abid_sources = self.ABID_SOURCES
+        assert all(src != 'None' for src in abid_sources.values())
+        return {
+            'prefix': eval(abid_sources['prefix']),
+            'ts': eval(abid_sources['ts']),
+            'uri': eval(abid_sources['uri']),
+            'subtype': eval(abid_sources['subtype']),
+            'rand': eval(abid_sources['rand']),
+            'salt': eval(abid_sources['salt']),
+        }
+    
+    @property
+    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
+        abid_values = self.ABID_FRESH_VALUES
+        assert all(val for val in abid_values.values())
+        return abid_hashes_from_values(
+            prefix=abid_values['prefix'],
+            ts=abid_values['ts'],
+            uri=abid_values['uri'],
+            subtype=abid_values['subtype'],
+            rand=abid_values['rand'],
+            salt=abid_values['salt'],
         )
-        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
-        return abid
+    
+    @property
+    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
+        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
+        existing_abid = self.ABID
+        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
+        abid_sources = self.ABID_SOURCES
+        fresh_values = self.ABID_FRESH_VALUES
+        fresh_hashes = self.ABID_FRESH_HASHES
+        return {
+            key: {
+                'key': key,
+                'model': self.__class__.__name__,
+                'pk': self.pk,
+                'abid_src': abid_sources[key],
+                'old_val': existing_values.get(key, None),
+                'old_hash': getattr(existing_abid, key),
+                'new_val': fresh_values[key],
+                'new_hash': new_hash,
+                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
+            }
+            for key, new_hash in fresh_hashes.items()
+            if getattr(existing_abid, key) != new_hash
+        }
+
+    def issue_new_abid(self, overwrite=False) -> ABID:
+        """
+        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
+        """
+        if not overwrite:
+            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
+        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
+
+        # Setup Field defaults to be ready for ABID generation
+        self.abid = None
+        self.id = self.id or uuid4()
+        self.pk = self.id
+        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
+        self.modified_at = self.modified_at or self.created_at
+        self.created_by_id = (hasattr(self, 'created_by_id') and self.created_by_id) or get_or_create_system_user_pk()
+        
+        # Compute fresh ABID values & hashes based on object's live properties
+        abid_fresh_values = self.ABID_FRESH_VALUES
+        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
+        abid_fresh_hashes = self.ABID_FRESH_HASHES
+        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
+        
+        new_abid = ABID(**abid_fresh_hashes)
+        
+        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
+
+        return new_abid
 
     @property
     def ABID(self) -> ABID:
         """
-        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
+        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
+        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
         """
+
+        if self.abid:
+            return ABID.parse(cast(str, self.abid))
         
-        # if object is not yet saved to DB, always generate fresh ABID from values
-        if self._state.adding:
-            return self.generate_abid()
-        
-        # otherwise DB is single source of truth, load ABID from existing db pk
-        abid: ABID | None = None
-        try:
-            abid = abid or ABID.parse(self.pk)
-        except Exception:
-            pass
+        return self.issue_new_abid()
 
-        try:
-            abid = abid or ABID.parse(self.id)
-        except Exception:
-            pass
+    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
+    # @property
+    # def UUID(self) -> UUID:
+    #     """
+    #     Get a uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return self.ABID.uuid
+    
+    # @property
+    # def uuid(self) -> str:
+    #     """
+    #     Get a str uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return str(self.ABID.uuid)
+    
+    # @property
+    # def ULID(self) -> ULID:
+    #     """
+    #     Get a ulid.ULID representation of the object's ABID.
+    #     """
+    #     return self.ABID.ulid
 
-        try:
-            abid = abid or ABID.parse(cast(str, self.abid))
-        except Exception:
-            pass
-
-        abid = abid or self.generate_abid()
-
-        return abid
+    # @property
+    # def TypeID(self) -> TypeID:
+    #     """
+    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+    #     """
+    #     return self.ABID.typeid
+    
+    @property
+    def api_url(self) -> str:
+        """
+        Compute the REST API URL to access this object.
+        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
+        """
+        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
 
     @property
-    def ULID(self) -> ULID:
+    def api_docs_url(self) -> str:
         """
-        Get a ulid.ULID representation of the object's ABID.
+        Compute the REST API Documentation URL to learn about accessing this object.
+        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
         """
-        return self.ABID.ulid
+        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
 
     @property
-    def UUID(self) -> UUID:
-        """
-        Get a uuid.UUID (v4) representation of the object's ABID.
-        """
-        return self.ABID.uuid
-
-    @property
-    def TypeID(self) -> TypeID:
-        """
-        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-        """
-        return self.ABID.typeid
-
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
 
+    def get_absolute_url(self):
+        return self.api_docs_url
 
 ####################################################
 
@@ -255,6 +380,7 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
     """
     Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
     e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    Honestly should only be used for debugging, no reason to expose this ability to users.
     """
 
     # convert str to ABID if necessary
@@ -304,7 +430,7 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
                 )
 
             for obj in qs:
-                if obj.generate_abid() == abid:
+                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
                     # found exact match, no need to keep iterating
                     return [obj]
                 partial_matches.append(obj)

archivebox/abx/__init__.py

@@ -0,0 +1,81 @@
+__package__ = 'abx'
+
+import importlib
+from pathlib import Path
+from typing import Dict
+
+from . import hookspec as base_spec
+from .hookspec import hookimpl, hookspec           # noqa
+from .manager import pm, PluginManager             # noqa
+
+
+pm.add_hookspecs(base_spec)
+
+
+###### PLUGIN DISCOVERY AND LOADING ########################################################
+
+def register_hookspecs(hookspecs):
+    for hookspec_import_path in hookspecs:
+        hookspec_module = importlib.import_module(hookspec_import_path)
+        pm.add_hookspecs(hookspec_module)
+
+
+def find_plugins_in_dir(plugins_dir: Path, prefix: str) -> Dict[str, Path]:
+    return {
+        f"{prefix}.{plugin_entrypoint.parent.name}": plugin_entrypoint.parent
+        for plugin_entrypoint in sorted(plugins_dir.glob("*/apps.py"))  # key=get_plugin_order  # Someday enforcing plugin import order may be required, but right now it's not needed
+    }   # "plugins_pkg.pip": "/app/archivebox/plugins_pkg/pip"
+
+
+def get_pip_installed_plugins(group='abx'):
+    """replaces pm.load_setuptools_entrypoints("abx")"""
+    import importlib.metadata
+
+    DETECTED_PLUGINS = {}   # module_name: module_dir_path
+    for dist in list(importlib.metadata.distributions()):
+        for entrypoint in dist.entry_points:
+            if entrypoint.group != group or pm.is_blocked(entrypoint.name):
+                continue
+            DETECTED_PLUGINS[entrypoint.name] = Path(entrypoint.load().__file__).parent
+            # pm.register(plugin, name=ep.name)
+            # pm._plugin_distinfo.append((plugin, DistFacade(dist)))
+    return DETECTED_PLUGINS
+
+
+def get_plugins_in_dirs(plugin_dirs: Dict[str, Path]):
+    DETECTED_PLUGINS = {}
+    for plugin_prefix, plugin_dir in plugin_dirs.items():
+        DETECTED_PLUGINS.update(find_plugins_in_dir(plugin_dir, prefix=plugin_prefix))
+    return DETECTED_PLUGINS
+
+
+# Load all plugins from pip packages, archivebox built-ins, and user plugins
+
+def load_plugins(plugins_dict: Dict[str, Path]):
+    LOADED_PLUGINS = {}
+    for plugin_module, plugin_dir in plugins_dict.items():
+        # print(f'Loading plugin: {plugin_module} from {plugin_dir}')
+        plugin_module_loaded = importlib.import_module(plugin_module)
+        pm.register(plugin_module_loaded)
+        LOADED_PLUGINS[plugin_module] = plugin_module_loaded.PLUGIN
+        # print(f'    √ Loaded plugin: {plugin_module}')
+    return LOADED_PLUGINS
+
+def get_registered_plugins():
+    plugins = {}
+    plugin_to_distinfo = dict(pm.list_plugin_distinfo())
+    for plugin in pm.get_plugins():
+        plugin_info = {
+            "name": plugin.__name__,
+            "hooks": [h.name for h in pm.get_hookcallers(plugin) or ()],
+        }
+        distinfo = plugin_to_distinfo.get(plugin)
+        if distinfo:
+            plugin_info["version"] = distinfo.version
+            plugin_info["name"] = (
+                getattr(distinfo, "name", None) or distinfo.project_name
+            )
+        plugins[plugin_info["name"]] = plugin_info
+    return plugins
+
+

archivebox/abx/archivebox/__init__.py

@@ -0,0 +1,39 @@
+__package__ = 'abx.archivebox'
+
+import importlib
+
+from typing import Dict
+from pathlib import Path
+
+
+def load_archivebox_plugins(pm, plugins_dict: Dict[str, Path]):
+    """Load archivebox plugins, very similar to abx.load_plugins but it looks for a pydantic PLUGIN model + hooks in apps.py"""
+    LOADED_PLUGINS = {}
+    for plugin_module, plugin_dir in plugins_dict.items():
+        # print(f'Loading plugin: {plugin_module} from {plugin_dir}')
+        
+        archivebox_plugins_found = []
+        
+        # 1. register the plugin module directly in case it contains any look hookimpls (e.g. in __init__.py)
+        plugin_module_loaded = importlib.import_module(plugin_module)
+        pm.register(plugin_module_loaded)
+        if hasattr(plugin_module_loaded, 'PLUGIN'):
+            archivebox_plugins_found.append(plugin_module_loaded.PLUGIN)
+        
+        # 2. then try to import plugin_module.apps as well
+        if (plugin_dir / 'apps.py').exists():
+            plugin_apps = importlib.import_module(plugin_module + '.apps')
+            pm.register(plugin_apps)                                           # register the whole .apps  in case it contains loose hookimpls (not in a class)
+            if hasattr(plugin_apps, 'PLUGIN'):
+                archivebox_plugins_found.append(plugin_apps.PLUGIN)
+        
+        # 3. then try to look for plugin_module.PLUGIN and register it + all its hooks
+        for ab_plugin in archivebox_plugins_found:
+            pm.register(ab_plugin)
+            for hook in ab_plugin.hooks:
+                hook.__signature__ = hook.__class__.__signature__              # fix to make pydantic model usable as Pluggy plugin
+                pm.register(hook)
+            LOADED_PLUGINS[plugin_module] = ab_plugin
+            
+        # print(f'    √ Loaded plugin: {LOADED_PLUGINS}')
+    return LOADED_PLUGINS

archivebox/abx/archivebox/base_admindataview.py

@@ -0,0 +1,38 @@
+__package__ = 'abx.archivebox'
+
+from typing import Dict
+
+import abx
+
+from .base_hook import BaseHook, HookType
+
+
+class BaseAdminDataView(BaseHook):
+    hook_type: HookType = "ADMINDATAVIEW"
+    
+    name: str = 'example_admin_data_view_list'
+    verbose_name: str = 'Data View'
+    route: str = '/__OVERRIDE_THIS__/'
+    view: str = 'plugins_example.example.views.example_view_list'
+    
+    items: Dict[str, str] = {
+        'route': '<str:key>/',
+        "name": 'example_admin_data_view_item',
+        'view': 'plugins_example.example.views.example_view_item',
+    }
+    
+    @abx.hookimpl
+    def get_ADMINDATAVIEWS(self):
+        return [self]
+    
+    @abx.hookimpl
+    def get_ADMIN_DATA_VIEWS_URLS(self):
+        """routes to be added to django.conf.settings.ADMIN_DATA_VIEWS['urls']"""
+        route = {
+            "route": self.route,
+            "view": self.view,
+            "name": self.verbose_name,
+            "items": self.items,
+        }
+        return [route]
+

archivebox/abx/archivebox/base_binary.py

@@ -0,0 +1,129 @@
+__package__ = "abx.archivebox"
+
+from typing import Dict, List
+from typing_extensions import Self
+
+from pydantic import Field, InstanceOf, validate_call
+from pydantic_pkgr import (
+    Binary,
+    BinProvider,
+    BinProviderName,
+    ProviderLookupDict,
+    AptProvider,
+    BrewProvider,
+    EnvProvider,
+)
+
+from archivebox.config import CONSTANTS
+
+import abx
+from .base_hook import BaseHook, HookType
+
+
+class BaseBinProvider(BaseHook, BinProvider):
+    hook_type: HookType = "BINPROVIDER"
+
+    # def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
+    #     Class = super()
+    #     get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
+    #     # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func)
+    #     return get_abspath_func()
+
+    # def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None:
+    #     Class = super()
+    #     get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context)
+    #     # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func)
+    #     return get_version_func()
+
+    
+    # TODO: add install/load/load_or_install methods as abx.hookimpl methods
+    
+    @property
+    def admin_url(self) -> str:
+        # e.g. /admin/environment/binproviders/NpmBinProvider/   TODO
+        return "/admin/environment/binaries/"
+
+    @abx.hookimpl
+    def get_BINPROVIDERS(self):
+        return [self]
+
+class BaseBinary(BaseHook, Binary):
+    hook_type: HookType = "BINARY"
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias="binproviders")
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = Field(default_factory=dict, alias="overrides")
+
+    @staticmethod
+    def symlink_to_lib(binary, bin_dir=None) -> None:
+        bin_dir = bin_dir or CONSTANTS.LIB_BIN_DIR
+        
+        if not (binary.abspath and binary.abspath.exists()):
+            return
+        
+        try:
+            bin_dir.mkdir(parents=True, exist_ok=True)
+            symlink = bin_dir / binary.name
+            symlink.unlink(missing_ok=True)
+            symlink.symlink_to(binary.abspath)
+            symlink.chmod(0o777)   # make sure its executable by everyone
+        except Exception as err:
+            # print(f'[red]:warning: Failed to symlink {symlink} -> {binary.abspath}[/red] {err}')
+            # not actually needed, we can just run without it
+            pass
+        
+    @validate_call
+    def load(self, fresh=False, **kwargs) -> Self:
+        if fresh:
+            binary = super().load(**kwargs)
+            self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
+        else:
+            # get cached binary from db
+            try:
+                from machine.models import InstalledBinary
+                installed_binary = InstalledBinary.objects.get_from_db_or_cache(self)
+                binary = InstalledBinary.load_from_db(installed_binary)
+            except Exception:
+                # maybe we are not in a DATA dir so there is no db, fallback to reading from fs
+                # (e.g. when archivebox version is run outside of a DATA dir)
+                binary = super().load(**kwargs)
+        return binary
+    
+    @validate_call
+    def install(self, **kwargs) -> Self:
+        binary = super().install(**kwargs)
+        self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
+        return binary
+    
+    @validate_call
+    def load_or_install(self, fresh=False, **kwargs) -> Self:
+        try:
+            binary = self.load(fresh=fresh)
+            if binary and binary.version:
+                self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
+                return binary
+        except Exception:
+            pass
+        return self.install(**kwargs)
+    
+    @property
+    def admin_url(self) -> str:
+        # e.g. /admin/environment/config/LdapConfig/
+        return f"/admin/environment/binaries/{self.name}/"
+
+    @abx.hookimpl
+    def get_BINARIES(self):
+        return [self]
+
+
+class AptBinProvider(AptProvider, BaseBinProvider):
+    name: BinProviderName = "apt"
+    
+class BrewBinProvider(BrewProvider, BaseBinProvider):
+    name: BinProviderName = "brew"
+    
+class EnvBinProvider(EnvProvider, BaseBinProvider):
+    name: BinProviderName = "env"
+
+apt = AptBinProvider()
+brew = BrewBinProvider()
+env = EnvBinProvider()

archivebox/abx/archivebox/base_check.py

@@ -0,0 +1,44 @@
+__package__ = "abx.archivebox"
+
+from typing import List
+
+from django.core.checks import Warning, Tags, register
+
+import abx
+
+from .base_hook import BaseHook, HookType
+
+
+class BaseCheck(BaseHook):
+    hook_type: HookType = "CHECK"
+    
+    tag: str = Tags.database
+    
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        """Override this method to implement your custom runtime check."""
+        errors = []
+        # if not hasattr(settings, 'SOME_KEY'):
+        #     errors.extend(Error(
+        #         'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
+        #         id='core.C001',
+        #         hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
+        #     ))
+        # logger.debug('[√] Loaded settings.PLUGINS succesfully.')
+        return errors
+
+    @abx.hookimpl
+    def get_CHECKS(self):
+        return [self]
+
+    @abx.hookimpl
+    def register_checks(self):
+        """Tell django that this check exists so it can be run automatically by django."""
+        def run_check(**kwargs):
+            from django.conf import settings
+            import logging
+            return self.check(settings, logging.getLogger("checks"))
+        
+        run_check.__name__ = self.id
+        run_check.tags = [self.tag]
+        register(self.tag)(run_check)

archivebox/abx/archivebox/base_configset.py

@@ -0,0 +1,229 @@
+__package__ = 'abx.archivebox'
+
+import os
+from pathlib import Path
+from typing import Type, Tuple, Callable, ClassVar
+
+from benedict import benedict
+from pydantic import model_validator, TypeAdapter
+from pydantic_settings import BaseSettings, SettingsConfigDict, PydanticBaseSettingsSource
+from pydantic_settings.sources import TomlConfigSettingsSource
+
+from pydantic_pkgr.base_types import func_takes_args_or_kwargs
+
+import abx
+
+from .base_hook import BaseHook, HookType
+from . import toml_util
+
+
+PACKAGE_DIR = Path(__file__).resolve().parent.parent
+DATA_DIR = Path(os.getcwd()).resolve()
+
+
+
+class FlatTomlConfigSettingsSource(TomlConfigSettingsSource):
+    """
+    A source class that loads variables from a TOML file
+    """
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        toml_file: Path | None=None,
+    ):
+        self.toml_file_path = toml_file or settings_cls.model_config.get("toml_file")
+        
+        self.nested_toml_data = self._read_files(self.toml_file_path)
+        self.toml_data = {}
+        for top_level_key, top_level_value in self.nested_toml_data.items():
+            if isinstance(top_level_value, dict):
+                # value is nested, flatten it
+                for key, value in top_level_value.items():
+                    self.toml_data[key] = value
+            else:
+                # value is already flat, just set it as-is
+                self.toml_data[top_level_key] = top_level_value
+                
+        # filter toml_data to only include keys that are defined on this settings_cls
+        self.toml_data = {
+            key: value
+            for key, value in self.toml_data.items()
+            if key in settings_cls.model_fields
+        }
+            
+        super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data)
+
+
+class ArchiveBoxBaseConfig(BaseSettings):
+    """
+    This is the base class for an ArchiveBox ConfigSet.
+    It handles loading values from schema defaults, ArchiveBox.conf TOML config, and environment variables.
+
+    class WgetConfig(ArchiveBoxBaseConfig):
+        WGET_BINARY: str = Field(default='wget', alias='WGET_BINARY_PATH')
+
+    c = WgetConfig()
+    print(c.WGET_BINARY)                    # outputs: wget
+
+    # you can mutate process environment variable and reload config using .__init__()
+    os.environ['WGET_BINARY_PATH'] = 'wget2'
+    c.__init__()
+
+    print(c.WGET_BINARY)                    # outputs: wget2
+
+    """
+    
+    # these pydantic config options are all VERY carefully chosen, make sure to test thoroughly before changing!!!
+    model_config = SettingsConfigDict(
+        validate_default=False,
+        case_sensitive=True,
+        extra="ignore",
+        arbitrary_types_allowed=False,
+        populate_by_name=True,
+        from_attributes=True,
+        loc_by_alias=False,
+        validate_assignment=True,
+        validate_return=True,
+        revalidate_instances="always",
+    )
+    
+    load_from_defaults: ClassVar[bool] = True
+    load_from_configfile: ClassVar[bool] = True
+    load_from_environment: ClassVar[bool] = True
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        """Defines the config precedence order: Schema defaults -> ArchiveBox.conf (TOML) -> Environment variables"""
+        
+        ARCHIVEBOX_CONFIG_FILE = DATA_DIR / "ArchiveBox.conf"
+        ARCHIVEBOX_CONFIG_FILE_BAK = ARCHIVEBOX_CONFIG_FILE.parent / ".ArchiveBox.conf.bak"
+        
+        # import ipdb; ipdb.set_trace()
+        
+        precedence_order = {}
+        
+        # if ArchiveBox.conf does not exist yet, return defaults -> env order
+        if not ARCHIVEBOX_CONFIG_FILE.is_file():
+            precedence_order = {
+                'defaults': init_settings,
+                'environment': env_settings,
+            }
+        
+        # if ArchiveBox.conf exists and is in TOML format, return default -> TOML -> env order
+        try:
+            precedence_order = precedence_order or {
+                'defaults': init_settings,
+                'configfile': FlatTomlConfigSettingsSource(settings_cls, toml_file=ARCHIVEBOX_CONFIG_FILE),
+                'environment': env_settings,
+            }
+        except Exception as err:
+            if err.__class__.__name__ != "TOMLDecodeError":
+                raise
+            # if ArchiveBox.conf exists and is in INI format, convert it then return default -> TOML -> env order
+
+            # Convert ArchiveBox.conf in INI format to TOML and save original to .ArchiveBox.bak
+            original_ini = ARCHIVEBOX_CONFIG_FILE.read_text()
+            ARCHIVEBOX_CONFIG_FILE_BAK.write_text(original_ini)
+            new_toml = toml_util.convert(original_ini)
+            ARCHIVEBOX_CONFIG_FILE.write_text(new_toml)
+
+            precedence_order = {
+                'defaults': init_settings,
+                'configfile': FlatTomlConfigSettingsSource(settings_cls, toml_file=ARCHIVEBOX_CONFIG_FILE),
+                'environment': env_settings,
+            }
+            
+        if not cls.load_from_environment:
+            precedence_order.pop('environment')
+        if not cls.load_from_configfile:
+            precedence_order.pop('configfile')
+        if not cls.load_from_defaults:
+            precedence_order.pop('defaults')
+
+        return tuple(precedence_order.values())
+
+    @model_validator(mode="after")
+    def fill_defaults(self):
+        """Populate any unset values using function provided as their default"""
+
+        for key, field in self.model_fields.items():
+            value = getattr(self, key)
+            
+            if isinstance(value, Callable):
+                # if value is a function, execute it to get the actual value, passing existing config as a dict arg if expected
+                if func_takes_args_or_kwargs(value):
+                    # assemble dict of existing field values to pass to default factory functions
+                    config_so_far = benedict(self.model_dump(include=set(self.model_fields.keys()), warnings=False))
+                    computed_default = field.default(config_so_far)
+                else:
+                    # otherwise it's a pure function with no args, just call it
+                    computed_default = field.default()
+
+                # coerce/check to make sure default factory return value matches type annotation
+                TypeAdapter(field.annotation).validate_python(computed_default)
+
+                # set generated default value as final validated value
+                setattr(self, key, computed_default)
+        return self
+    
+    def update_in_place(self, warn=True, **kwargs):
+        """
+        Update the config with new values. Use this sparingly! We should almost never be updating config at runtime.
+        Sets them in the environment so they propagate to spawned subprocesses / across future re-__init__()s and reload from environment
+
+        Example acceptable use case: user config says SEARCH_BACKEND_ENGINE=sonic but sonic_client pip library is not installed so we cannot use it.
+        SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep') can be used to reset it back to ripgrep so we can continue.
+        """
+        if warn:
+            print('[!] WARNING: Some of the provided user config values cannot be used, temporarily ignoring them:')
+        for key, value in kwargs.items():
+            os.environ[key] = str(value)
+            original_value = getattr(self, key)
+            if warn:
+                print(f'    {key}={original_value} -> {value}')
+        self.__init__()
+        return self
+    
+    def as_legacy_config_schema(self):
+        # shim for backwards compatibility with old config schema style
+        model_values = self.model_dump()
+        return benedict({
+            key: {'type': field.annotation, 'default': model_values[key]}
+            for key, field in self.model_fields.items()
+        })
+
+
+class BaseConfigSet(ArchiveBoxBaseConfig, BaseHook):      # type: ignore[type-arg]
+    hook_type: ClassVar[HookType] = 'CONFIG'
+
+    # @abx.hookimpl
+    # def ready(self, settings):
+    #    # reload config from environment, in case it's been changed by any other plugins
+    #    self.__init__()
+
+
+    @abx.hookimpl
+    def get_CONFIGS(self):
+        try:
+            return {self.id: self}
+        except Exception as e:
+            # raise Exception(f'Error computing CONFIGS for {type(self)}: {e.__class__.__name__}: {e}')
+            print(f'Error computing CONFIGS for {type(self)}: {e.__class__.__name__}: {e}')
+        return {}
+
+    @abx.hookimpl
+    def get_FLAT_CONFIG(self):
+        try:
+            return self.model_dump()
+        except Exception as e:
+            # raise Exception(f'Error computing FLAT_CONFIG for {type(self)}: {e.__class__.__name__}: {e}')
+            print(f'Error computing FLAT_CONFIG for {type(self)}: {e.__class__.__name__}: {e}')
+        return {}

archivebox/abx/archivebox/base_extractor.py

@@ -0,0 +1,222 @@
+__package__ = 'abx.archivebox'
+
+import json
+import os
+
+from typing import Optional, List, Literal, Annotated, Dict, Any, Tuple
+from typing_extensions import Self
+from pathlib import Path
+
+from pydantic import model_validator, AfterValidator
+from pydantic_pkgr import BinName
+from django.utils.functional import cached_property
+from django.utils import timezone
+
+import abx
+
+from .base_hook import BaseHook, HookType
+from .base_binary import BaseBinary
+
+
+def no_empty_args(args: List[str]) -> List[str]:
+    assert all(len(arg) for arg in args)
+    return args
+
+ExtractorName = Literal['wget', 'warc', 'media', 'singlefile'] | str
+
+HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
+CmdArgsList = Annotated[List[str] | Tuple[str, ...], AfterValidator(no_empty_args)]
+
+
+class BaseExtractor(BaseHook):
+    hook_type: HookType = 'EXTRACTOR'
+    
+    name: ExtractorName
+    binary: BinName
+
+    output_path_func: HandlerFuncStr = 'self.get_output_path'
+    should_extract_func: HandlerFuncStr = 'self.should_extract'
+    extract_func: HandlerFuncStr = 'self.extract'
+    exec_func: HandlerFuncStr = 'self.exec'
+
+    default_args: CmdArgsList = []
+    extra_args: CmdArgsList = []
+    args: Optional[CmdArgsList] = None
+
+    @model_validator(mode='after')
+    def validate_model(self) -> Self:
+        if self.args is None:
+            self.args = [*self.default_args, *self.extra_args]
+        return self
+
+
+    def get_output_path(self, snapshot) -> Path:
+        return Path(self.id.lower())
+
+    def should_extract(self, snapshot) -> bool:
+        try:
+            assert self.detect_installed_binary().version
+        except Exception:
+            raise
+            # could not load binary
+            return False
+        
+        # output_dir = self.get_output_path(snapshot)
+        # if output_dir.glob('*.*'):
+        #     return False
+        return True
+
+    @abx.hookimpl
+    def extract(self, snapshot_id: str) -> Dict[str, Any]:
+        from core.models import Snapshot
+        from archivebox import CONSTANTS
+        
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+        
+        if not self.should_extract(snapshot):
+            return {}
+        
+        status = 'failed'
+        start_ts = timezone.now()
+        uplink = self.detect_network_interface()
+        installed_binary = self.detect_installed_binary()
+        machine = installed_binary.machine
+        assert uplink.machine == installed_binary.machine  # it would be *very* weird if this wasn't true
+        
+        # output_dir = self.get_output_path(snapshot) or CONSTANTS.TMP_DIR
+        output_dir = CONSTANTS.DATA_DIR / '.tmp' / 'extractors' / self.name / str(snapshot.abid)
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        # execute the extractor binary with the given args
+        args = [snapshot.url, *self.args] if self.args is not None else [snapshot.url, *self.default_args, *self.extra_args]
+        cmd = [str(installed_binary.abspath), *args]
+        proc = self.exec(installed_binary=installed_binary, args=args, cwd=output_dir)
+
+        # collect the output
+        end_ts = timezone.now()
+        output_files = list(str(path.relative_to(output_dir)) for path in output_dir.glob('**/*.*'))
+        stdout = proc.stdout.strip()
+        stderr = proc.stderr.strip()
+        output_json = None
+        output_text = stdout
+        try:
+            output_json = json.loads(stdout.strip())
+            output_text = None
+        except json.JSONDecodeError:
+            pass
+        
+        errors = []
+        if proc.returncode == 0:
+            status = 'success'
+        else:
+            errors.append(f'{installed_binary.name} returned non-zero exit code: {proc.returncode}')   
+
+        # increment health stats counters
+        if status == 'success':
+            machine.record_health_success()
+            uplink.record_health_success()
+            installed_binary.record_health_success()
+        else:
+            machine.record_health_failure()
+            uplink.record_health_failure()
+            installed_binary.record_health_failure()
+
+        return {
+            'extractor': self.name,
+            
+            'snapshot': {
+                'id': snapshot.id,
+                'abid': snapshot.abid,
+                'url': snapshot.url,
+                'created_by_id': snapshot.created_by_id,
+            },
+            
+            'machine': {
+                'id': machine.id,
+                'abid': machine.abid,
+                'guid': machine.guid,
+                'hostname': machine.hostname,
+                'hw_in_docker': machine.hw_in_docker,
+                'hw_in_vm': machine.hw_in_vm,
+                'hw_manufacturer': machine.hw_manufacturer,
+                'hw_product': machine.hw_product,
+                'hw_uuid': machine.hw_uuid,
+                'os_arch': machine.os_arch,
+                'os_family': machine.os_family,
+                'os_platform': machine.os_platform,
+                'os_release': machine.os_release,
+                'os_kernel': machine.os_kernel,
+            },
+            
+            'uplink': { 
+                'id': uplink.id,
+                'abid': uplink.abid,
+                'mac_address': uplink.mac_address,
+                'ip_public': uplink.ip_public,
+                'ip_local': uplink.ip_local,
+                'dns_server': uplink.dns_server,
+                'hostname': uplink.hostname,
+                'iface': uplink.iface,
+                'isp': uplink.isp,
+                'city': uplink.city,
+                'region': uplink.region,
+                'country': uplink.country,
+            },
+            
+            'binary': {
+                'id': installed_binary.id,
+                'abid': installed_binary.abid,
+                'name': installed_binary.name,
+                'binprovider': installed_binary.binprovider,
+                'abspath': installed_binary.abspath,
+                'version': installed_binary.version,
+                'sha256': installed_binary.sha256,
+            },
+
+            'cmd': cmd,
+            'stdout': stdout,
+            'stderr': stderr,
+            'returncode': proc.returncode,
+            'start_ts': start_ts,
+            'end_ts': end_ts,
+            
+            'status': status,
+            'errors': errors,
+            'output_dir': str(output_dir.relative_to(CONSTANTS.DATA_DIR)),
+            'output_files': output_files,
+            'output_json': output_json or {},
+            'output_text': output_text or '',
+        }
+
+    # TODO: move this to a hookimpl
+    def exec(self, args: CmdArgsList=(), cwd: Optional[Path]=None, installed_binary=None):
+        cwd = cwd or Path(os.getcwd())
+        binary = self.load_binary(installed_binary=installed_binary)
+        
+        return binary.exec(cmd=args, cwd=cwd)
+    
+    @cached_property
+    def BINARY(self) -> BaseBinary:
+        import abx.archivebox.use
+        for binary in abx.archivebox.use.get_BINARIES().values():
+            if binary.name == self.binary:
+                return binary
+        raise ValueError(f'Binary {self.binary} not found')
+    
+    def detect_installed_binary(self):
+        from machine.models import InstalledBinary
+        # hydrates binary from DB/cache if record of installed version is recent enough
+        # otherwise it finds it from scratch by detecting installed version/abspath/sha256 on host
+        return InstalledBinary.objects.get_from_db_or_cache(self.BINARY)
+
+    def load_binary(self, installed_binary=None) -> BaseBinary:
+        installed_binary = installed_binary or self.detect_installed_binary()
+        return installed_binary.load_from_db()
+    
+    def detect_network_interface(self):
+        from machine.models import NetworkInterface
+        return NetworkInterface.objects.current()
+
+    @abx.hookimpl
+    def get_EXTRACTORS(self):
+        return [self]

archivebox/abx/archivebox/base_hook.py

@@ -0,0 +1,80 @@
+__package__ = 'abx.archivebox'
+
+import inspect
+from huey.api import TaskWrapper
+
+from pathlib import Path
+from typing import Tuple, Literal, ClassVar, get_args
+from pydantic import BaseModel, ConfigDict
+from django.utils.functional import cached_property
+
+import abx
+
+HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE', 'SEARCHBACKEND']
+hook_type_names: Tuple[HookType] = get_args(HookType)
+
+class BaseHook(BaseModel):
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+        validate_defaults=True,
+        validate_assignment=False,
+        revalidate_instances="subclass-instances",
+        ignored_types=(TaskWrapper, cached_property),
+    )
+    
+    hook_type: ClassVar[HookType]     # e.g. = 'CONFIG'
+    
+    # verbose_name: str = Field()
+    
+    _is_registered: bool = False
+    _is_ready: bool = False
+
+
+    @property
+    def id(self) -> str:
+        return self.__class__.__name__
+
+    @property
+    def hook_module(self) -> str:
+        """e.g. plugins_extractor.singlefile.apps.SinglefileConfigSet"""
+        return f'{self.__module__}.{self.__class__.__name__}'
+
+    @property
+    def hook_file(self) -> Path:
+        """e.g. plugins_extractor.singlefile.apps.SinglefileConfigSet"""
+        return Path(inspect.getfile(self.__class__))
+
+    @property
+    def plugin_module(self) -> str:
+        """e.g. plugins_extractor.singlefile"""
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
+
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
+    
+    @property
+    def admin_url(self) -> str:
+        # e.g. /admin/environment/config/LdapConfig/
+        return f"/admin/environment/{self.hook_type.lower()}/{self.id}/"
+
+
+    @abx.hookimpl
+    def register(self, settings):
+        """Called when django.apps.AppConfig.ready() is called"""
+        
+        # print("REGISTERED HOOK:", self.hook_module)
+        self._is_registered = True
+        
+
+    @abx.hookimpl
+    def ready(self):
+        """Called when django.apps.AppConfig.ready() is called"""
+        
+        assert self._is_registered, f"Tried to run {self.hook_module}.ready() but it was never registered!"
+       
+        # print("READY HOOK:", self.hook_module)
+        self._is_ready = True

archivebox/abx/archivebox/base_plugin.py

@@ -0,0 +1,154 @@
+__package__ = 'abx.archivebox'
+
+import abx
+import inspect
+from pathlib import Path
+
+from django.apps import AppConfig
+
+from typing import List, Type, Dict
+from typing_extensions import Self
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    model_validator,
+    InstanceOf,
+    computed_field,
+)
+from benedict import benedict
+
+from .base_hook import BaseHook, HookType
+
+class BasePlugin(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        arbitrary_types_allowed=True,
+        populate_by_name=True,
+        from_attributes=True,
+        validate_defaults=False,
+        validate_assignment=False,
+        revalidate_instances="always",
+        # frozen=True,
+    )
+
+    # Required by AppConfig:
+    app_label: str = Field()                      # e.g. 'singlefile'                  (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name)
+    verbose_name: str = Field()                   # e.g. 'SingleFile'                  (human-readable *short* label, for use in column names, form labels, etc.)
+    docs_url: str = Field(default=None)           # e.g. 'https://github.com/...'
+    
+    # All the hooks the plugin will install:
+    hooks: List[InstanceOf[BaseHook]] = Field(default=[])
+    
+    _is_registered: bool = False
+    _is_ready: bool = False
+    
+    @computed_field
+    @property
+    def id(self) -> str:
+        return self.__class__.__name__
+    
+    @property
+    def name(self) -> str:
+        return self.app_label
+    
+    # @computed_field
+    @property
+    def plugin_module(self) -> str:  # DottedImportPath
+        """ "
+        Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
+        e.g. 'archivebox.plugins_pkg.npm.apps.NpmPlugin' -> 'plugins_pkg.npm'
+        """
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit('.apps.', 1)[0]
+
+
+    @property
+    def plugin_module_full(self) -> str:  # DottedImportPath
+        """e.g. 'archivebox.plugins_pkg.npm.apps.NpmPlugin'"""
+        return f"{self.__module__}.{self.__class__.__name__}"
+    
+    # @computed_field
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
+    
+    @model_validator(mode='after')
+    def validate(self) -> Self:
+        """Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
+        
+        # VERY IMPORTANT:
+        # preserve references to original default objects,
+        # pydantic deepcopies them by default which breaks mutability
+        # see https://github.com/pydantic/pydantic/issues/7608
+        # if we dont do this, then plugins_extractor.SINGLEFILE_CONFIG != settings.CONFIGS.SingleFileConfig for example
+        # and calling .__init__() on one of them will not update the other
+        self.hooks = self.model_fields['hooks'].default
+        
+        assert self.app_label and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
+        
+        # assert json.dumps(self.model_json_schema(), indent=4), f"Plugin {self.plugin_module} has invalid JSON schema."
+        
+        return self
+    
+    @property
+    def AppConfig(plugin_self) -> Type[AppConfig]:
+        """Generate a Django AppConfig class for this plugin."""
+
+
+        class PluginAppConfig(AppConfig):
+            """Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS."""
+            name = plugin_self.plugin_module
+            app_label = plugin_self.app_label
+            verbose_name = plugin_self.verbose_name
+
+            default_auto_field = 'django.db.models.AutoField'
+
+            # handled by abx.hookimpl  ready()
+            # def ready(self):
+            #     from django.conf import settings
+            #     plugin_self.ready(settings)
+
+        return PluginAppConfig
+
+    @property
+    def HOOKS_BY_ID(self) -> Dict[str, InstanceOf[BaseHook]]:
+        return benedict({hook.id: hook for hook in self.hooks})
+
+    @property
+    def HOOKS_BY_TYPE(self) -> Dict[HookType, Dict[str, InstanceOf[BaseHook]]]:
+        hooks = benedict({})
+        for hook in self.hooks:
+            hooks[hook.hook_type] = hooks.get(hook.hook_type) or benedict({})
+            hooks[hook.hook_type][hook.id] = hook
+        return hooks
+
+
+
+    @abx.hookimpl
+    def register(self, settings):
+        from archivebox.config.legacy import bump_startup_progress_bar
+
+        self._is_registered = True
+        bump_startup_progress_bar()
+
+        # print('◣----------------- REGISTERED PLUGIN:', self.plugin_module, '-----------------◢')
+        # print()
+
+    @abx.hookimpl
+    def ready(self, settings=None):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        from archivebox.config.legacy import bump_startup_progress_bar
+
+        assert self._is_registered, f"Tried to run {self.plugin_module}.ready() but it was never registered!"
+        self._is_ready = True
+
+        # settings.PLUGINS[self.id]._is_ready = True
+        bump_startup_progress_bar()
+
+
+    @abx.hookimpl
+    def get_INSTALLED_APPS(self):
+        return [self.plugin_module]
+

archivebox/abx/archivebox/base_queue.py

@@ -0,0 +1,106 @@
+__package__ = 'abx.archivebox'
+
+import importlib
+
+from typing import Dict, List, TYPE_CHECKING
+from pydantic import Field, InstanceOf
+from benedict import benedict
+
+if TYPE_CHECKING:
+    from huey.api import TaskWrapper
+
+import abx
+
+from .base_hook import BaseHook, HookType
+from .base_binary import BaseBinary
+
+
+
+class BaseQueue(BaseHook):
+    hook_type: HookType = 'QUEUE'
+
+    name: str = Field()       # e.g. 'singlefile'
+
+    binaries: List[InstanceOf[BaseBinary]] = Field()
+
+    @property
+    def tasks(self) -> Dict[str, 'TaskWrapper']:
+        """Return an dict of all the background worker tasks defined in the plugin's tasks.py file."""
+        tasks = importlib.import_module(f"{self.plugin_module}.tasks")
+
+        all_tasks = {}
+
+        for task_name, task in tasks.__dict__.items():
+            # if attr is a Huey task and its queue_name matches our hook's queue name
+            if hasattr(task, "task_class") and task.huey.name == self.name:
+                all_tasks[task_name] = task
+
+        return benedict(all_tasks)
+
+    def get_django_huey_config(self, QUEUE_DATABASE_NAME) -> dict:
+        """Get the config dict to insert into django.conf.settings.DJANGO_HUEY['queues']."""
+        return {
+            "huey_class": "huey.SqliteHuey",
+            "filename": QUEUE_DATABASE_NAME,
+            "name": self.name,
+            "results": True,
+            "store_none": True,
+            "immediate": False,
+            "utc": True,
+            "consumer": {
+                "workers": 1,
+                "worker_type": "thread",
+                "initial_delay": 0.1,  # Smallest polling interval, same as -d.
+                "backoff": 1.15,  # Exponential backoff using this rate, -b.
+                "max_delay": 10.0,  # Max possible polling interval, -m.
+                "scheduler_interval": 1,  # Check schedule every second, -s.
+                "periodic": True,  # Enable crontab feature.
+                "check_worker_health": True,  # Enable worker health checks.
+                "health_check_interval": 1,  # Check worker health every second.
+            },
+        }
+        
+    def get_supervisord_config(self, settings) -> dict:
+        """Ge the config dict used to tell sueprvisord to start a huey consumer for this queue."""
+        return {
+            "name": f"worker_{self.name}",
+            "command": f"archivebox manage djangohuey --queue {self.name}",
+            "stdout_logfile": f"logs/worker_{self.name}.log",
+            "redirect_stderr": "true",
+            "autorestart": "true",
+            "autostart": "false",
+        }
+        
+    def start_supervisord_worker(self, settings, lazy=True):
+        from queues.supervisor_util import get_or_create_supervisord_process, start_worker
+        print()
+        try:
+            supervisor = get_or_create_supervisord_process(daemonize=False)
+        except Exception as e:
+            print(f"Error starting worker for queue {self.name}: {e}")
+            return None
+        print()
+        worker = start_worker(supervisor, self.get_supervisord_config(settings), lazy=lazy)
+
+        # Update settings.WORKERS to include this worker
+        settings.WORKERS = getattr(settings, "WORKERS", None) or benedict({})
+        settings.WORKERS[self.id] = self.start_supervisord_worker(settings, lazy=True)
+
+        return worker
+
+    @abx.hookimpl
+    def get_QUEUES(self):
+        return [self]
+
+    @abx.hookimpl
+    def get_DJANGO_HUEY_QUEUES(self, QUEUE_DATABASE_NAME):
+        """queue configs to be added to django.conf.settings.DJANGO_HUEY['queues']"""
+        return {
+            self.name: self.get_django_huey_config(QUEUE_DATABASE_NAME)
+        }
+        
+        
+    # @abx.hookimpl
+    # def ready(self, settings):
+    #     self.start_supervisord_worker(settings, lazy=True)
+    #     super().ready(settings)

archivebox/abx/archivebox/base_replayer.py

@@ -1,14 +1,15 @@
-__package__ = 'archivebox.plugantic'
+__package__ = 'abx.archivebox'
+
+import abx
+
+from .base_hook import BaseHook, HookType
 
 
-from pydantic import BaseModel
-
-# from .binproviders import LazyImportStr
-
-
-class Replayer(BaseModel):
+class BaseReplayer(BaseHook):
     """Describes how to render an ArchiveResult in several contexts"""
-    name: str = 'GenericReplayer'
+    
+    hook_type: HookType = 'REPLAYER'
+    
     url_pattern: str = '*'
 
     row_template: str = 'plugins/generic_replayer/templates/row.html'
@@ -21,5 +22,8 @@ class Replayer(BaseModel):
     # icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
     # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
 
+    @abx.hookimpl
+    def get_REPLAYERS(self):
+        return [self]
 
-MEDIA_REPLAYER = Replayer(name='media')
+    # TODO: add hookimpl methods for get_row_template, get_embed_template, get_fullpage_template, etc...

archivebox/abx/archivebox/base_searchbackend.py

@@ -0,0 +1,33 @@
+__package__ = 'abx.archivebox'
+
+from typing import Iterable, List
+from pydantic import Field
+
+import abx
+from .base_hook import BaseHook, HookType
+
+
+
+class BaseSearchBackend(BaseHook):
+    hook_type: HookType = 'SEARCHBACKEND'
+
+    name: str = Field()       # e.g. 'singlefile'
+
+
+    # TODO: move these to a hookimpl
+
+    @staticmethod
+    def index(snapshot_id: str, texts: List[str]):
+        return
+
+    @staticmethod
+    def flush(snapshot_ids: Iterable[str]):
+        return
+
+    @staticmethod
+    def search(text: str) -> List[str]:
+        raise NotImplementedError("search method must be implemented by subclass")
+    
+    @abx.hookimpl
+    def get_SEARCHBACKENDS(self):
+        return [self]

archivebox/abx/archivebox/hookspec.py

@@ -0,0 +1,39 @@
+__package__ = 'abx.archivebox'
+
+from typing import Dict, Any
+
+from .. import hookspec
+
+
+@hookspec
+def get_CONFIGS():
+    return {}
+
+@hookspec
+def get_EXTRACTORS():
+    return {}
+
+@hookspec
+def get_REPLAYERS():
+    return {}
+
+@hookspec
+def get_CHECKS():
+    return {}
+
+@hookspec
+def get_ADMINDATAVIEWS():
+    return {}
+
+@hookspec
+def get_QUEUES():
+    return {}
+
+@hookspec
+def get_SEARCHBACKENDS():
+    return {}
+
+
+@hookspec
+def extract(snapshot_id) -> Dict[str, Any]:
+    return {}

archivebox/abx/archivebox/toml_util.py

@@ -0,0 +1,114 @@
+from typing import Any, List, Callable
+
+import json
+import ast
+import inspect
+import toml
+import re
+import configparser
+
+from pathlib import Path, PosixPath
+
+from pydantic.json_schema import GenerateJsonSchema
+from pydantic_core import to_jsonable_python
+
+JSONValue = str | bool | int | None | List['JSONValue']
+
+TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
+
+def load_ini_value(val: str) -> JSONValue:
+    """Convert lax INI values into strict TOML-compliant (JSON) values"""
+    if val.lower() in ('true', 'yes', '1'):
+        return True
+    if val.lower() in ('false', 'no', '0'):
+        return False
+    if val.isdigit():
+        return int(val)
+
+    try:
+        return ast.literal_eval(val)
+    except Exception:
+        pass
+
+    try:
+        return json.loads(val)
+    except Exception:
+        pass
+    
+    return val
+
+
+def convert(ini_str: str) -> str:
+    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
+
+    config = configparser.ConfigParser()
+    config.optionxform = str  # capitalize key names
+    config.read_string(ini_str)
+
+    # Initialize an empty dictionary to store the TOML representation
+    toml_dict = {}
+
+    # Iterate over each section in the INI configuration
+    for section in config.sections():
+        toml_dict[section] = {}
+
+        # Iterate over each key-value pair in the section
+        for key, value in config.items(section):
+            parsed_value = load_ini_value(value)
+
+            # Convert the parsed value to its TOML-compatible JSON representation
+            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
+
+    # Build the TOML string
+    toml_str = TOML_HEADER
+    for section, items in toml_dict.items():
+        toml_str += f"[{section}]\n"
+        for key, value in items.items():
+            toml_str += f"{key} = {value}\n"
+        toml_str += "\n"
+
+    return toml_str.strip()
+
+
+
+class JSONSchemaWithLambdas(GenerateJsonSchema):
+    """
+    Encode lambda functions in default values properly.
+    Usage:
+    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
+    """
+    def encode_default(self, default: Any) -> Any:
+        config = self._config
+        if isinstance(default, Callable):
+            return '{{lambda ' + inspect.getsource(default).split('=lambda ')[-1].strip()[:-1] + '}}'
+        return to_jsonable_python(
+           default,
+           timedelta_mode=config.ser_json_timedelta,
+           bytes_mode=config.ser_json_bytes,
+           serialize_unknown=True
+        )
+
+    # for computed_field properties render them like this instead:
+    # inspect.getsource(field.wrapped_property.fget).split('def ', 1)[-1].split('\n', 1)[-1].strip().strip('return '),
+
+
+def better_toml_dump_str(val: Any) -> str:
+    try:
+        return toml.encoder._dump_str(val)     # type: ignore
+    except Exception:
+        # if we hit any of toml's numerous encoding bugs,
+        # fall back to using json representation of string
+        return json.dumps(str(val))
+
+class CustomTOMLEncoder(toml.encoder.TomlEncoder):
+    """
+    Custom TomlEncoder to work around https://github.com/uiri/toml's many encoding bugs.
+    More info: https://github.com/fabiocaccamo/python-benedict/issues/439
+    >>> toml.dumps(value, encoder=CustomTOMLEncoder())
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.dump_funcs[Path] = lambda x: json.dumps(str(x))
+        self.dump_funcs[PosixPath] = lambda x: json.dumps(str(x))
+        self.dump_funcs[str] = better_toml_dump_str
+        self.dump_funcs[re.RegexFlag] = better_toml_dump_str

archivebox/abx/archivebox/use.py

@@ -0,0 +1,138 @@
+__package__ = 'abx.archivebox'
+
+from typing import Dict, Any, TYPE_CHECKING
+
+from django.utils import timezone
+from benedict import benedict
+
+from .. import pm
+
+if TYPE_CHECKING:
+    from .base_hook import BaseHook
+    from .base_configset import BaseConfigSet
+    from .base_binary import BaseBinary, BaseBinProvider
+    from .base_extractor import BaseExtractor
+    from .base_replayer import BaseReplayer
+    from .base_check import BaseCheck
+    from .base_queue import BaseQueue
+    from .base_admindataview import BaseAdminDataView
+    from .base_searchbackend import BaseSearchBackend
+
+# API exposed to ArchiveBox code
+
+def get_PLUGINS():
+    return benedict({
+        plugin.PLUGIN.id: plugin.PLUGIN
+        for plugin in pm.get_plugins()
+    })
+
+def get_HOOKS(PLUGINS) -> Dict[str, 'BaseHook']:
+    return benedict({
+        hook.id: hook
+        for plugin in PLUGINS.values()
+            for hook in plugin.hooks
+    })
+
+def get_CONFIGS() -> Dict[str, 'BaseConfigSet']:
+    return benedict({
+        config_id: config
+        for plugin_configs in pm.hook.get_CONFIGS()
+            for config_id, config in plugin_configs.items()
+    })
+    
+def get_FLAT_CONFIG() -> Dict[str, Any]:
+    return benedict({
+        key: value
+        for plugin_config_dict in pm.hook.get_FLAT_CONFIG()
+            for key, value in plugin_config_dict.items()
+    })
+
+def get_BINPROVIDERS() -> Dict[str, 'BaseBinProvider']:
+    # TODO: move these to plugins
+    from abx.archivebox.base_binary import apt, brew, env
+    builtin_binproviders = [apt, brew, env]
+    
+    return benedict({
+        binprovider.id: binprovider
+        for plugin_binproviders in [builtin_binproviders, *pm.hook.get_BINPROVIDERS()]
+            for binprovider in plugin_binproviders
+    })
+
+def get_BINARIES() -> Dict[str, 'BaseBinary']:
+    return benedict({
+        binary.id: binary
+        for plugin_binaries in pm.hook.get_BINARIES()
+            for binary in plugin_binaries
+    })
+
+def get_EXTRACTORS() -> Dict[str, 'BaseExtractor']:
+    return benedict({
+        extractor.id: extractor
+        for plugin_extractors in pm.hook.get_EXTRACTORS()
+            for extractor in plugin_extractors
+    })
+
+def get_REPLAYERS() -> Dict[str, 'BaseReplayer']:
+    return benedict({
+        replayer.id: replayer
+        for plugin_replayers in pm.hook.get_REPLAYERS()
+            for replayer in plugin_replayers
+    })
+
+def get_CHECKS() -> Dict[str, 'BaseCheck']:
+    return benedict({
+        check.id: check
+        for plugin_checks in pm.hook.get_CHECKS()
+            for check in plugin_checks
+    })
+
+def get_ADMINDATAVIEWS() -> Dict[str, 'BaseAdminDataView']:
+    return benedict({
+        admin_dataview.id: admin_dataview
+        for plugin_admin_dataviews in pm.hook.get_ADMINDATAVIEWS()
+            for admin_dataview in plugin_admin_dataviews
+    })
+
+def get_QUEUES() -> Dict[str, 'BaseQueue']:
+    return benedict({
+        queue.id: queue
+        for plugin_queues in pm.hook.get_QUEUES()
+            for queue in plugin_queues
+    })
+
+def get_SEARCHBACKENDS() -> Dict[str, 'BaseSearchBackend']:
+    return benedict({
+        searchbackend.id: searchbackend
+        for plugin_searchbackends in pm.hook.get_SEARCHBACKENDS()
+            for searchbackend in plugin_searchbackends
+    })
+
+
+###########################
+
+
+def register_all_hooks(settings):
+    pm.hook.register(settings=settings)
+
+
+
+def extract(url_or_snapshot_id):
+    from core.models import Snapshot
+    
+    url, snapshot_abid, snapshot_id = None, None, None
+    snapshot = None
+    if '://' in url_or_snapshot_id:
+        url = url_or_snapshot_id
+        try:
+            snapshot = Snapshot.objects.get(url=url)
+        except Snapshot.DoesNotExist:
+            snapshot = Snapshot(url=url_or_snapshot_id, timestamp=str(timezone.now().timestamp()), bookmarked_at=timezone.now())
+            snapshot.save()
+    elif '-' in url_or_snapshot_id:
+        snapshot_id = url_or_snapshot_id
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+    else:
+        snapshot_abid = url_or_snapshot_id
+        snapshot = Snapshot.objects.get(abid=snapshot_abid)
+
+    return pm.hook.extract(snapshot_id=snapshot.id)

archivebox/abx/django/__init__.py

@@ -0,0 +1 @@
+__package__ = 'abx.django'

archivebox/abx/django/apps.py

@@ -0,0 +1,13 @@
+__package__ = 'abx.django'
+
+from django.apps import AppConfig
+
+
+class ABXConfig(AppConfig):
+    name = 'abx'
+
+    def ready(self):
+        import abx
+        from django.conf import settings
+        
+        abx.pm.hook.ready(settings=settings)

archivebox/abx/django/hookspec.py

@@ -0,0 +1,120 @@
+__package__ = 'abx.django'
+
+from ..hookspec import hookspec
+
+
+###########################################################################################
+
+@hookspec
+def get_INSTALLED_APPS():
+    """Return a list of apps to add to INSTALLED_APPS"""
+    # e.g. ['your_plugin_type.plugin_name']
+    return []
+
+# @hookspec
+# def register_INSTALLED_APPS(INSTALLED_APPS):
+#     """Mutate INSTALLED_APPS in place to add your app in a specific position"""
+#     # idx_of_contrib = INSTALLED_APPS.index('django.contrib.auth')
+#     # INSTALLED_APPS.insert(idx_of_contrib + 1, 'your_plugin_type.plugin_name')
+#     pass
+
+
+@hookspec
+def get_TEMPLATE_DIRS():
+    return []     # e.g. ['your_plugin_type/plugin_name/templates']
+
+# @hookspec
+# def register_TEMPLATE_DIRS(TEMPLATE_DIRS):
+#     """Install django settings"""
+#     # e.g. TEMPLATE_DIRS.insert(0, 'your_plugin_type/plugin_name/templates')
+#     pass
+
+
+@hookspec
+def get_STATICFILES_DIRS():
+    return []     # e.g. ['your_plugin_type/plugin_name/static']
+
+# @hookspec
+# def register_STATICFILES_DIRS(STATICFILES_DIRS):
+#     """Mutate STATICFILES_DIRS in place to add your static dirs in a specific position"""
+#     # e.g. STATICFILES_DIRS.insert(0, 'your_plugin_type/plugin_name/static')
+#     pass
+
+
+@hookspec
+def get_MIDDLEWARE():
+    return []     # e.g. ['your_plugin_type.plugin_name.middleware.YourMiddleware']
+
+# @hookspec
+# def register_MIDDLEWARE(MIDDLEWARE):
+#     """Mutate MIDDLEWARE in place to add your middleware in a specific position"""
+#     # e.g. MIDDLEWARE.insert(0, 'your_plugin_type.plugin_name.middleware.YourMiddleware')
+#     pass
+
+
+@hookspec
+def get_AUTHENTICATION_BACKENDS():
+    return []     # e.g. ['django_auth_ldap.backend.LDAPBackend']
+
+# @hookspec
+# def register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS):
+#     """Mutate AUTHENTICATION_BACKENDS in place to add your auth backends in a specific position"""
+#     # e.g. AUTHENTICATION_BACKENDS.insert(0, 'your_plugin_type.plugin_name.backend.YourBackend')
+#     pass
+
+@hookspec
+def get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME):
+    return []     # e.g. [{'name': 'your_plugin_type.plugin_name', 'HUEY': {...}}]
+
+# @hookspec
+# def register_DJANGO_HUEY(DJANGO_HUEY):
+#     """Mutate DJANGO_HUEY in place to add your huey queues in a specific position"""
+#     # e.g. DJANGO_HUEY['queues']['some_queue_name']['some_setting'] = 'some_value'
+#     pass
+
+
+@hookspec
+def get_ADMIN_DATA_VIEWS_URLS():
+    return []
+
+# @hookspec
+# def register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS):
+#     """Mutate ADMIN_DATA_VIEWS in place to add your admin data views in a specific position"""
+#     # e.g. ADMIN_DATA_VIEWS['URLS'].insert(0, 'your_plugin_type/plugin_name/admin_data_views.py')
+#     pass
+
+
+# @hookspec
+# def register_settings(settings):
+#     """Mutate settings in place to add your settings / modify existing settings"""
+#     # settings.SOME_KEY = 'some_value'
+#     pass
+
+
+###########################################################################################
+
+@hookspec
+def get_urlpatterns():
+    return []     # e.g. [path('your_plugin_type/plugin_name/url.py', your_view)]
+
+# @hookspec
+# def register_urlpatterns(urlpatterns):
+#     """Mutate urlpatterns in place to add your urlpatterns in a specific position"""
+#     # e.g. urlpatterns.insert(0, path('your_plugin_type/plugin_name/url.py', your_view))
+#     pass
+
+###########################################################################################
+
+@hookspec
+def register_checks():
+    """Register django checks with django system checks system"""
+    pass
+
+
+###########################################################################################
+
+
+@hookspec
+def ready():
+    """Called when Django apps app.ready() are triggered"""
+    pass

archivebox/abx/django/use.py

@@ -0,0 +1,98 @@
+__package__ = 'abx.django'
+
+import itertools
+# from benedict import benedict
+
+from .. import pm
+
+
+def get_INSTALLED_APPS():
+    return itertools.chain(*reversed(pm.hook.get_INSTALLED_APPS()))
+
+# def register_INSTALLLED_APPS(INSTALLED_APPS):
+#     pm.hook.register_INSTALLED_APPS(INSTALLED_APPS=INSTALLED_APPS)
+
+
+def get_MIDDLEWARES():
+    return itertools.chain(*reversed(pm.hook.get_MIDDLEWARE()))
+
+# def register_MIDDLEWARES(MIDDLEWARE):
+#     pm.hook.register_MIDDLEWARE(MIDDLEWARE=MIDDLEWARE)
+
+
+def get_AUTHENTICATION_BACKENDS():
+    return itertools.chain(*reversed(pm.hook.get_AUTHENTICATION_BACKENDS()))
+
+# def register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS):
+#     pm.hook.register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS=AUTHENTICATION_BACKENDS)
+
+
+def get_STATICFILES_DIRS():
+    return itertools.chain(*reversed(pm.hook.get_STATICFILES_DIRS()))
+
+# def register_STATICFILES_DIRS(STATICFILES_DIRS):
+#     pm.hook.register_STATICFILES_DIRS(STATICFILES_DIRS=STATICFILES_DIRS)
+
+
+def get_TEMPLATE_DIRS():
+    return itertools.chain(*reversed(pm.hook.get_TEMPLATE_DIRS()))
+
+# def register_TEMPLATE_DIRS(TEMPLATE_DIRS):
+#     pm.hook.register_TEMPLATE_DIRS(TEMPLATE_DIRS=TEMPLATE_DIRS)
+
+def get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME='queue.sqlite3'):
+    HUEY_QUEUES = {}
+    for plugin_result in pm.hook.get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME=QUEUE_DATABASE_NAME):
+        HUEY_QUEUES.update(plugin_result)
+    return HUEY_QUEUES
+
+# def register_DJANGO_HUEY(DJANGO_HUEY):
+#     pm.hook.register_DJANGO_HUEY(DJANGO_HUEY=DJANGO_HUEY)
+
+def get_ADMIN_DATA_VIEWS_URLS():
+    return itertools.chain(*reversed(pm.hook.get_ADMIN_DATA_VIEWS_URLS()))
+
+# def register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS):
+#     pm.hook.register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS=ADMIN_DATA_VIEWS)
+
+
+# def register_settings(settings):
+#     # convert settings dict to an benedict so we can set values using settings.attr = xyz notation
+#     settings_as_obj = benedict(settings, keypath_separator=None)
+    
+#     # set default values for settings that are used by plugins
+#     # settings_as_obj.INSTALLED_APPS = settings_as_obj.get('INSTALLED_APPS', [])
+#     # settings_as_obj.MIDDLEWARE = settings_as_obj.get('MIDDLEWARE', [])
+#     # settings_as_obj.AUTHENTICATION_BACKENDS = settings_as_obj.get('AUTHENTICATION_BACKENDS', [])
+#     # settings_as_obj.STATICFILES_DIRS = settings_as_obj.get('STATICFILES_DIRS', [])
+#     # settings_as_obj.TEMPLATE_DIRS = settings_as_obj.get('TEMPLATE_DIRS', [])
+#     # settings_as_obj.DJANGO_HUEY = settings_as_obj.get('DJANGO_HUEY', {'queues': {}})
+#     # settings_as_obj.ADMIN_DATA_VIEWS = settings_as_obj.get('ADMIN_DATA_VIEWS', {'URLS': []})
+    
+#     # # call all the hook functions to mutate the settings values in-place
+#     # register_INSTALLLED_APPS(settings_as_obj.INSTALLED_APPS)
+#     # register_MIDDLEWARES(settings_as_obj.MIDDLEWARE)
+#     # register_AUTHENTICATION_BACKENDS(settings_as_obj.AUTHENTICATION_BACKENDS)
+#     # register_STATICFILES_DIRS(settings_as_obj.STATICFILES_DIRS)
+#     # register_TEMPLATE_DIRS(settings_as_obj.TEMPLATE_DIRS)
+#     # register_DJANGO_HUEY(settings_as_obj.DJANGO_HUEY)
+#     # register_ADMIN_DATA_VIEWS(settings_as_obj.ADMIN_DATA_VIEWS)
+    
+#     # calls Plugin.settings(settings) on each registered plugin
+#     pm.hook.register_settings(settings=settings_as_obj)
+    
+#     # then finally update the settings globals() object will all the new settings
+#     # settings.update(settings_as_obj)
+
+
+def get_urlpatterns():
+    return list(itertools.chain(*pm.hook.urlpatterns()))
+
+def register_urlpatterns(urlpatterns):
+    pm.hook.register_urlpatterns(urlpatterns=urlpatterns)
+
+
+def register_checks():
+    """register any django system checks"""
+    pm.hook.register_checks()
+

archivebox/abx/hookspec.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+from pluggy import HookimplMarker
+from pluggy import HookspecMarker
+
+spec = hookspec = HookspecMarker("abx")
+impl = hookimpl = HookimplMarker("abx")
+
+
+@hookspec
+@hookimpl
+def get_system_user() -> str:
+    return Path('~').expanduser().name
+

archivebox/abx/manager.py

@@ -0,0 +1,30 @@
+import inspect
+
+import pluggy
+
+
+class PluginManager(pluggy.PluginManager):
+    """
+    Patch to fix pluggy's PluginManager to work with pydantic models.
+    See: https://github.com/pytest-dev/pluggy/pull/536
+    """
+    def parse_hookimpl_opts(self, plugin, name: str) -> pluggy.HookimplOpts | None:
+        # IMPORTANT: @property methods can have side effects, and are never hookimpl
+        # if attr is a property, skip it in advance
+        plugin_class = plugin if inspect.isclass(plugin) else type(plugin)
+        if isinstance(getattr(plugin_class, name, None), property):
+            return None
+
+        # pydantic model fields are like attrs and also can never be hookimpls
+        plugin_is_pydantic_obj = hasattr(plugin, "__pydantic_core_schema__")
+        if plugin_is_pydantic_obj and name in getattr(plugin, "model_fields", {}):
+            # pydantic models mess with the class and attr __signature__
+            # so inspect.isroutine(...) throws exceptions and cant be used
+            return None
+        
+        try:
+            return super().parse_hookimpl_opts(plugin, name)
+        except AttributeError:
+            return super().parse_hookimpl_opts(type(plugin), name)
+
+pm = PluginManager("abx")

archivebox/abx/pydantic_pkgr/__init__.py

@@ -0,0 +1 @@
+__package__ = 'abx.pydantic_pkgr'

archivebox/abx/pydantic_pkgr/hookspec.py

@@ -0,0 +1,13 @@
+
+from ..hookspec import hookspec
+
+###########################################################################################
+
+@hookspec
+def get_BINPROVIDERS():
+    return {}
+
+@hookspec
+def get_BINARIES():
+    return {}
+

archivebox/api/auth.py

@@ -1,13 +1,34 @@
 __package__ = 'archivebox.api'
 
-from typing import Optional, cast
+from typing import Any, Optional, cast
+from datetime import timedelta
 
 from django.http import HttpRequest
+from django.utils import timezone
 from django.contrib.auth import login
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import AbstractBaseUser
 
 from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
+from ninja.errors import HttpError
+
+
+def get_or_create_api_token(user):
+    from api.models import APIToken
+    
+    if user and user.is_superuser:
+        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
+        if api_tokens.exists():
+            # unexpired token exists, use it
+            api_token = api_tokens.last()
+        else:
+            # does not exist, create a new one
+            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
+        
+        assert api_token.is_valid(), f"API token is not valid {api_token}"
+
+        return api_token
+    return None
 
 
 def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
@@ -16,39 +37,35 @@ def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[Abs
     
     user = None
 
-    submitted_empty_form = token in ('string', '', None)
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
+    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
+    if not submitted_empty_form:
         try:
             token = APIToken.objects.get(token=token)
             if token.is_valid():
                 user = token.created_by
+                request._api_token = token
         except APIToken.DoesNotExist:
             pass
 
     if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
-
-    return None
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        return None
+    
+    return cast(AbstractBaseUser, user)
 
 def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user = None
     
     submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
+    if not submitted_empty_form:
         user = authenticate(
             username=username,
             password=password,
         )
 
     if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
+        # print('[❌] Failed to authenticate API user using API Key:', request)
         user = None
 
     return cast(AbstractBaseUser | None, user)
@@ -56,28 +73,41 @@ def auth_using_password(username, password, request: Optional[HttpRequest]=None)
 
 ### Base Auth Types
 
+
 class APITokenAuthCheck:
     """The base class for authentication methods that use an api.models.APIToken"""
     def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_token(
+        request.user = auth_using_token(
             token=key,
             request=request,
         )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+        return request.user
+
 
 class UserPassAuthCheck:
     """The base class for authentication methods that use a username & password"""
     def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_password(
+        request.user = auth_using_password(
             username=username,
             password=password,
             request=request,
         )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+
+        return request.user
 
 
 ### Django-Ninja-Provided Auth Methods
@@ -98,13 +128,12 @@ class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
     """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
     pass
 
-
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [
     HeaderTokenAuth(),
     BearerTokenAuth(),
     QueryParamTokenAuth(), 
-    django_auth_superuser,
+    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
     UsernameAndPasswordAuth(),
 ]

archivebox/api/migrations/0008_alter_apitoken_created_alter_apitoken_created_by_and_more.py

@@ -0,0 +1,47 @@
+# Generated by Django 5.1 on 2024-09-04 23:32
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0007_alter_apitoken_created_by'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+    ]

archivebox/api/migrations/0009_rename_created_apitoken_created_at_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 5.1 on 2024-09-05 00:26
+
+import abid_utils.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.AddField(
+            model_name='outboundwebhook',
+            name='created_at',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
+        ),
+    ]

archivebox/api/models.py

@@ -1,6 +1,5 @@
 __package__ = 'archivebox.api'
 
-import uuid
 import secrets
 from datetime import timedelta
 
@@ -12,7 +11,7 @@ from signal_webhooks.models import WebhookBase
 
 from django_stubs_ext.db.models import TypedModelMeta
 
-from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
+from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
 
 
 
@@ -27,16 +26,18 @@ class APIToken(ABIDModel):
     """
     # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
     abid_prefix = 'apt_'
-    abid_ts_src = 'self.created'
-    abid_uri_src = 'self.token'
-    abid_subtype_src = 'self.created_by_id'
+    abid_ts_src = 'self.created_at'
+    abid_uri_src = 'self.created_by_id'
+    abid_subtype_src = '"01"'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
     expires = models.DateTimeField(null=True, blank=True)
@@ -58,7 +59,7 @@ class APIToken(ABIDModel):
             "abid":             str(self.ABID),
             "created_by_id":    str(self.created_by_id),
             "token":            self.token,
-            "created":          self.created.isoformat(),
+            "created_at":       self.created_at.isoformat(),
             "expires":          self.expires_as_iso8601,
         }
 
@@ -94,17 +95,18 @@ class OutboundWebhook(ABIDModel, WebhookBase):
         settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
     """
     abid_prefix = 'whk_'
-    abid_ts_src = 'self.created'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.endpoint'
     abid_subtype_src = 'self.ref'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     # More fields here: WebhookBase...
 
@@ -120,3 +122,6 @@ class OutboundWebhook(ABIDModel, WebhookBase):
     class Meta(WebhookBase.Meta):
         verbose_name = 'API Outbound Webhook'
 
+
+    def __str__(self) -> str:
+        return f'[{self.abid}] {self.ref} -> {self.endpoint}'

archivebox/api/tests.py

@@ -1,30 +1,30 @@
 __package__ = 'archivebox.api'
 
-from django.test import TestCase
-from ninja.testing import TestClient
+# from django.test import TestCase
+# from ninja.testing import TestClient
 
-from .routes_cli import router
+# from .routes_cli import router
 
-class ArchiveBoxCLIAPITestCase(TestCase):
-    def setUp(self):
-        self.client = TestClient(router)
+# class ArchiveBoxCLIAPITestCase(TestCase):
+#     def setUp(self):
+#         self.client = TestClient(router)
 
-    def test_add_endpoint(self):
-        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_add_endpoint(self):
+#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_remove_endpoint(self):
-        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_remove_endpoint(self):
+#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_update_endpoint(self):
-        response = self.client.post("/update", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_update_endpoint(self):
+#         response = self.client.post("/update", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_list_all_endpoint(self):
-        response = self.client.post("/list_all", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_list_all_endpoint(self):
+#         response = self.client.post("/list_all", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])

archivebox/api/v1_api.py

@@ -12,11 +12,12 @@ from ninja import NinjaAPI, Swagger
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
+from archivebox.config import SHELL_CONFIG, VERSION
+
 from api.auth import API_AUTH_METHODS
-from ..config import VERSION, COMMIT_HASH
 
 
-COMMIT_HASH = COMMIT_HASH or 'unknown'
+COMMIT_HASH = SHELL_CONFIG.COMMIT_HASH or 'unknown'
 
 html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
@@ -52,7 +53,26 @@ class NinjaAPIWithIOCapture(NinjaAPI):
 
                 response = super().create_temporal_response(request)
 
-        print('RESPONDING NOW', response)
+        # Diable caching of API responses entirely
+        response['Cache-Control'] = 'no-store'
+
+        # Add debug stdout and stderr headers to response
+        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
+        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
+        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
+
+        # Add Auth Headers to response
+        api_token = getattr(request, '_api_token', None)
+        token_expiry = api_token.expires.isoformat() if api_token else 'Never'
+
+        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
+        response['X-ArchiveBox-Auth-Expires'] = token_expiry
+        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
+        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
+        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
+
+        # import ipdb; ipdb.set_trace()
+        # print('RESPONDING NOW', response)
 
         return response
 

archivebox/api/v1_auth.py

@@ -3,12 +3,14 @@ __package__ = 'archivebox.api'
 from typing import Optional
 
 from ninja import Router, Schema
+from django.utils import timezone
+from datetime import timedelta
 
 from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password
+from api.auth import auth_using_token, auth_using_password, get_or_create_api_token
 
 
-router = Router(tags=['Authentication'])
+router = Router(tags=['Authentication'], auth=None)
 
 
 class PasswordAuthSchema(Schema):
@@ -25,10 +27,9 @@ def get_api_token(request, auth_data: PasswordAuthSchema):
         request=request,
     )
 
-    if user:
-        # TODO: support multiple tokens in the future, for now we just have one per user
-        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
-
+    if user and user.is_superuser:
+        api_token = get_or_create_api_token(user)
+        assert api_token is not None, "Failed to create API token"
         return api_token.__json__()
     
     return {"success": False, "errors": ["Invalid credentials"]}

archivebox/api/v1_cli.py

@@ -12,12 +12,14 @@ from ..main import (
     list_all,
     schedule,
 )
-from ..util import ansi_to_html
-from ..config import ONLY_NEW
+from archivebox.misc.util import ansi_to_html
+from archivebox.config import ARCHIVING_CONFIG
 
 
+from .auth import API_AUTH_METHODS
+
 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'])
+router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
 
 
 # Schemas
@@ -56,7 +58,7 @@ class AddCommandSchema(Schema):
     urls: List[str]
     tag: str = ""
     depth: int = 0
-    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
     update_all: bool = False
     index_only: bool = False
     overwrite: bool = False
@@ -66,7 +68,7 @@ class AddCommandSchema(Schema):
 
 class UpdateCommandSchema(Schema):
     resume: Optional[float] = 0
-    only_new: bool = ONLY_NEW
+    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
     index_only: bool = False
     overwrite: bool = False
     after: Optional[float] = 0
@@ -83,7 +85,7 @@ class ScheduleCommandSchema(Schema):
     tag: str = ''
     depth: int = 0
     overwrite: bool = False
-    update: bool = not ONLY_NEW
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
     clear: bool = False
 
 class ListCommandSchema(Schema):
@@ -92,7 +94,7 @@ class ListCommandSchema(Schema):
     status: Optional[StatusChoices] = StatusChoices.indexed
     after: Optional[float] = 0
     before: Optional[float] = 999999999999999
-    sort: str = 'added'
+    sort: str = 'bookmarked_at'
     as_json: bool = True
     as_html: bool = False
     as_csv: str | bool = 'timestamp,url'

archivebox/api/v1_core.py

@@ -12,11 +12,15 @@ from django.contrib.auth import get_user_model
 
 from ninja import Router, Schema, FilterSchema, Field, Query
 from ninja.pagination import paginate, PaginationBase
+from ninja.errors import HttpError
 
 from core.models import Snapshot, ArchiveResult, Tag
+from api.models import APIToken, OutboundWebhook
 from abid_utils.abid import ABID
 
-router = Router(tags=['Core Models'])
+from .auth import API_AUTH_METHODS
+
+router = Router(tags=['Core Models'], auth=API_AUTH_METHODS)
 
 
 
@@ -56,23 +60,17 @@ class CustomPagination(PaginationBase):
 
 ### ArchiveResult #########################################################################
 
-class ArchiveResultSchema(Schema):
+class MinimalArchiveResultSchema(Schema):
     TYPE: str = 'core.models.ArchiveResult'
 
     id: UUID
-    old_id: int
     abid: str
 
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
 
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: str
-
     extractor: str
     cmd_version: Optional[str]
     cmd: List[str]
@@ -80,6 +78,9 @@ class ArchiveResultSchema(Schema):
     status: str
     output: str
 
+    start_ts: Optional[datetime]
+    end_ts: Optional[datetime]
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -89,20 +90,12 @@ class ArchiveResultSchema(Schema):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
 
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
     @staticmethod
     def resolve_abid(obj):
         return str(obj.ABID)
 
     @staticmethod
-    def resolve_created(obj):
+    def resolve_created_at(obj):
         return obj.start_ts
 
     @staticmethod
@@ -113,19 +106,34 @@ class ArchiveResultSchema(Schema):
     def resolve_snapshot_url(obj):
         return obj.snapshot.url
 
+    @staticmethod
+    def resolve_snapshot_id(obj):
+        return str(obj.snapshot_id)
+    
     @staticmethod
     def resolve_snapshot_abid(obj):
         return str(obj.snapshot.ABID)
 
     @staticmethod
     def resolve_snapshot_tags(obj):
-        return obj.snapshot.tags_str()
+        return sorted(tag.name for tag in obj.snapshot.tags.all())
+
+class ArchiveResultSchema(MinimalArchiveResultSchema):
+    TYPE: str = 'core.models.ArchiveResult'
+
+    # ... Extends MinimalArchiveResultSchema fields ...
+
+    snapshot_id: UUID
+    snapshot_abid: str
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: List[str]
 
 
 class ArchiveResultFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
 
-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
     snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
     snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
     snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
@@ -137,9 +145,9 @@ class ArchiveResultFilterSchema(FilterSchema):
     pwd: Optional[str] = Field(None, q='pwd__icontains')
     cmd_version: Optional[str] = Field(None, q='cmd_version')
 
-    created: Optional[datetime] = Field(None, q='updated')
-    created__gte: Optional[datetime] = Field(None, q='updated__gte')
-    created__lt: Optional[datetime] = Field(None, q='updated__lt')
+    created_at: Optional[datetime] = Field(None, q='created_at')
+    created_at__gte: Optional[datetime] = Field(None, q='created_at__gte')
+    created_at__lt: Optional[datetime] = Field(None, q='created_at__lt')
 
 
 @router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
@@ -153,8 +161,8 @@ def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...))
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by pk, abid, or old_id."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
+    """Get a specific ArchiveResult by id or abid."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id))
 
 
 # @router.post("/archiveresult", response=ArchiveResultSchema)
@@ -189,26 +197,27 @@ class SnapshotSchema(Schema):
     TYPE: str = 'core.models.Snapshot'
 
     id: UUID
-    old_id: UUID
     abid: str
 
-    modified: datetime
-    created: datetime
     created_by_id: str
     created_by_username: str
+    created_at: datetime
+    modified_at: datetime
+
+    bookmarked_at: datetime
+    downloaded_at: Optional[datetime]
 
     url: str
-    tags: str
+    tags: List[str]
     title: Optional[str]
     timestamp: str
     archive_path: str
 
-    bookmarked: datetime
-    added: datetime
-    updated: Optional[datetime]
+    # url_for_admin: str
+    # url_for_view: str
 
     num_archiveresults: int
-    archiveresults: List[ArchiveResultSchema]
+    archiveresults: List[MinimalArchiveResultSchema]
 
     @staticmethod
     def resolve_created_by_id(obj):
@@ -219,21 +228,21 @@ class SnapshotSchema(Schema):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
 
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
     @staticmethod
     def resolve_abid(obj):
         return str(obj.ABID)
 
     @staticmethod
     def resolve_tags(obj):
-        return obj.tags_str()
+        return sorted(tag.name for tag in obj.tags.all())
+
+    # @staticmethod
+    # def resolve_url_for_admin(obj):
+    #     return f"/admin/core/snapshot/{obj.id}/change/"
+    
+    # @staticmethod
+    # def resolve_url_for_view(obj):
+    #     return f"/{obj.archive_path}"
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
@@ -247,29 +256,27 @@ class SnapshotSchema(Schema):
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
-
-    old_id: Optional[str] = Field(None, q='old_id__icontains')
+    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'timestamp__startswith'])
     abid: Optional[str] = Field(None, q='abid__icontains')
 
     created_by_id: str = Field(None, q='created_by_id')
     created_by_username: str = Field(None, q='created_by__username__icontains')
 
-    created__gte: datetime = Field(None, q='created__gte')
-    created__lt: datetime = Field(None, q='created__lt')
-    created: datetime = Field(None, q='created')
-    modified: datetime = Field(None, q='modified')
-    modified__gte: datetime = Field(None, q='modified__gte')
-    modified__lt: datetime = Field(None, q='modified__lt')
+    created_at__gte: datetime = Field(None, q='created_at__gte')
+    created_at__lt: datetime = Field(None, q='created_at__lt')
+    created_at: datetime = Field(None, q='created_at')
+    modified_at: datetime = Field(None, q='modified_at')
+    modified_at__gte: datetime = Field(None, q='modified_at__gte')
+    modified_at__lt: datetime = Field(None, q='modified_at__lt')
 
-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'timestamp__startswith'])
     url: Optional[str] = Field(None, q='url')
     tag: Optional[str] = Field(None, q='tags__name')
     title: Optional[str] = Field(None, q='title__icontains')
     timestamp: Optional[str] = Field(None, q='timestamp__startswith')
     
-    added__gte: Optional[datetime] = Field(None, q='added__gte')
-    added__lt: Optional[datetime] = Field(None, q='added__lt')
+    bookmarked_at__gte: Optional[datetime] = Field(None, q='bookmarked_at__gte')
+    bookmarked_at__lt: Optional[datetime] = Field(None, q='bookmarked_at__lt')
 
 
 
@@ -285,16 +292,16 @@ def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_arch
 
 @router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
 def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid, uuid, or pk."""
+    """Get a specific Snapshot by abid or id."""
     request.with_archiveresults = with_archiveresults
     snapshot = None
     try:
-        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
         pass
 
     try:
-        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
+        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
     except Snapshot.DoesNotExist:
         pass
 
@@ -311,7 +318,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
 # def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
 #
 #     for attr, value in payload.dict().items():
 #         setattr(snapshot, attr, value)
@@ -321,7 +328,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.delete("/snapshot/{snapshot_id}")
 # def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
 #     snapshot.delete()
 #     return {"success": True}
 
@@ -334,11 +341,10 @@ class TagSchema(Schema):
     TYPE: str = 'core.models.Tag'
 
     id: UUID
-    old_id: str
     abid: str
 
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
 
@@ -347,10 +353,6 @@ class TagSchema(Schema):
     num_snapshots: int
     snapshots: List[SnapshotSchema]
 
-    @staticmethod
-    def resolve_old_id(obj):
-        return str(obj.old_id)
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -382,11 +384,6 @@ def get_tag(request, tag_id: str, with_snapshots: bool=True):
     request.with_snapshots = with_snapshots
     request.with_archiveresults = False
     tag = None
-    try:
-        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError, ValueError):
-        pass
-
     try:
         tag = Tag.objects.get(abid__icontains=tag_id)
     except (Tag.DoesNotExist, ValidationError):
@@ -421,4 +418,10 @@ def get_any(request, abid: str):
     except Exception:
         pass
 
-    return response
+    if abid.startswith(APIToken.abid_prefix):
+        raise HttpError(403, 'APIToken objects are not accessible via REST API')
+    
+    if abid.startswith(OutboundWebhook.abid_prefix):
+        raise HttpError(403, 'OutboundWebhook objects are not accessible via REST API')
+
+    raise HttpError(404, 'Object with given ABID not found')

archivebox/builtin_plugins/base/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

archivebox/builtin_plugins/base/apps.py

@@ -1,82 +0,0 @@
-import sys
-import inspect
-from typing import List, Dict, Any, Optional
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-from plugantic.replayers import Replayer
-
-
-class PythonBinary(Binary):
-    name: BinName = 'python'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides: Dict[str, Any] = {
-        'env': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(Binary):
-    name: BinName = 'sqlite'
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(sqlite3),
-            'version': \
-                lambda: sqlite3.version,
-        },
-    }
-
-class DjangoBinary(Binary):
-    name: BinName = 'django'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(Replayer):
-    name: str = 'basic'
-
-
-class BasePlugin(Plugin):
-    name: str = 'base'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = []
-    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
-
-
-PLUGINS = [BasePlugin()]
-
-
-class BaseConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'builtin_plugins.base'

archivebox/builtin_plugins/base/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/builtin_plugins/base/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/base/views.py

@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,94 +0,0 @@
-from typing import List, Optional
-from pathlib import Path
-
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-
-
-###################### Config ##########################
-
-class SinglefileToggleConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
-
-    SAVE_SINGLEFILE: bool = True
-
-
-class SinglefileDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    SINGLEFILE_BINARY: str = Field(default='wget')
-    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
-    SINGLEFILE_EXTRA_ARGS: List[str] = []
-    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class SinglefileOptionsConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
-
-    # loaded from shared config
-    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
-    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
-    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
-    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
-    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
-
-
-
-DEFAULT_CONFIG = {
-    'CHECK_SSL_VALIDITY': False,
-    'SAVE_SINGLEFILE': True,
-    'TIMEOUT': 120,
-}
-
-PLUGIN_CONFIG = [
-    SinglefileToggleConfig(**DEFAULT_CONFIG),
-    SinglefileDependencyConfig(**DEFAULT_CONFIG),
-    SinglefileOptionsConfig(**DEFAULT_CONFIG),
-]
-
-###################### Binaries ############################
-
-class SinglefileBinary(Binary):
-    name: BinName = 'single-file'
-    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
-
-
-###################### Extractors ##########################
-
-class SinglefileExtractor(Extractor):
-    name: ExtractorName = 'singlefile'
-    binary: Binary = SinglefileBinary()
-
-    def get_output_path(self, snapshot) -> Path:
-        return Path(snapshot.link_dir) / 'singlefile.html'
-
-
-###################### Plugins #############################
-
-
-class SinglefilePlugin(Plugin):
-    name: str = 'singlefile'
-    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
-
-PLUGINS = [SinglefilePlugin()]
-
-###################### Django Apps #########################
-
-class SinglefileConfig(AppConfig):
-    name = 'builtin_plugins.singlefile'
-    verbose_name = 'SingleFile'
-
-    def ready(self):
-        print('Loaded singlefile plugin')

archivebox/builtin_plugins/singlefile/config.yaml

@@ -1,66 +0,0 @@
-name: singlefile
-plugin_version: '0.0.1'
-plugin_spec: '0.0.1'
-
-binaries:
-    singlefile:
-        providers:
-            - env
-            - npm
-
-commands:
-    - singlefile.exec
-    - singlefile.extract
-    - singlefile.should_extract
-    - singlefile.get_output_path
-
-extractors:
-    singlefile:
-        binary: singlefile
-        test: singlefile.should_extract
-        extract: singlefile.extract
-        output_files:
-            - singlefile.html
-
-configs:
-    ARCHIVE_METHOD_TOGGLES:
-        SAVE_SINGLEFILE:
-            type: bool
-            default: true
-
-    DEPENDENCY_CONFIG:
-        SINGLEFILE_BINARY:
-            type: str
-            default: wget
-        SINGLEFILE_ARGS:
-            type: Optional[List[str]]
-            default: null
-        SINGLEFILE_EXTRA_ARGS:
-            type: List[str]
-            default: []
-        SINGLEFILE_DEFAULT_ARGS:
-            type: List[str]
-            default: 
-            - "--timeout={TIMEOUT-10}"
-
-    ARCHIVE_METHOD_OPTIONS:
-        SINGLEFILE_USER_AGENT:
-            type: str
-            default: ""
-            alias: USER_AGENT
-        SINGLEFILE_TIMEOUT:
-            type: int
-            default: 60
-            alias: TIMEOUT
-        SINGLEFILE_CHECK_SSL_VALIDITY:
-            type: bool
-            default: true
-            alias: CHECK_SSL_VALIDITY
-        SINGLEFILE_RESTRICT_FILE_NAMES:
-            type: str
-            default: windows
-            alias: RESTRICT_FILE_NAMES
-        SINGLEFILE_COOKIES_FILE:
-            type: Optional[Path]
-            default: null
-            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/cli/__init__.py

@@ -1,30 +1,70 @@
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox'
 
-import os
 import sys
 import argparse
 import threading
+
 from time import sleep
+from collections.abc import Mapping
 
-from typing import Optional, Dict, List, IO, Union, Iterable
+from rich import print
+
+from typing import Optional, List, IO, Union, Iterable
 from pathlib import Path
 
-from ..config import OUTPUT_DIR, check_data_folder, check_migrations, stderr
-
 from importlib import import_module
 
 BUILTIN_LIST = list
 
 CLI_DIR = Path(__file__).resolve().parent
 
-# these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
+# rewrite setup -> install for backwards compatibility
+if len(sys.argv) > 1 and sys.argv[1] == 'setup':
+    from rich import print
+    print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
+    sys.argv[1] = 'install'
 
-display_first = (*meta_cmds, *main_cmds, *archive_cmds)
+
+# def list_subcommands() -> Dict[str, str]:
+#     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
+#     COMMANDS = []
+#     for filename in os.listdir(CLI_DIR):
+#         if is_cli_module(filename):
+#             subcommand = filename.replace('archivebox_', '').replace('.py', '')
+#             module = import_module('.archivebox_{}'.format(subcommand), __package__)
+#             assert is_valid_cli_module(module, subcommand)
+#             COMMANDS.append((subcommand, module.main.__doc__))
+#             globals()[subcommand] = module.main
+#     display_order = lambda cmd: (
+#         display_first.index(cmd[0])
+#         if cmd[0] in display_first else
+#         100 + len(cmd[0])
+#     )
+#     return dict(sorted(COMMANDS, key=display_order))
+
+# just define it statically, it's much faster:
+SUBCOMMAND_MODULES = {
+    'help': 'archivebox_help',
+    'version': 'archivebox_version' ,
+    
+    'init': 'archivebox_init',
+    'install': 'archivebox_install',
+    'config': 'archivebox_config',
+    
+    'add': 'archivebox_add',
+    'remove': 'archivebox_remove',
+    'update': 'archivebox_update',
+    'list': 'archivebox_list',
+    'status': 'archivebox_status',
+    
+    'schedule': 'archivebox_schedule',
+    'server': 'archivebox_server',
+    'shell': 'archivebox_shell',
+    'manage': 'archivebox_manage',
+
+    'oneshot': 'archivebox_oneshot',
+}
 
 # every imported command module must have these properties in order to be valid
 required_attrs = ('__package__', '__command__', 'main')
@@ -36,8 +76,40 @@ is_valid_cli_module = lambda module, subcommand: (
     and module.__command__.split(' ')[-1] == subcommand
 )
 
+class LazySubcommands(Mapping):
+    def keys(self):
+        return SUBCOMMAND_MODULES.keys()
+    
+    def values(self):
+        return [self[key] for key in self.keys()]
+    
+    def items(self):
+        return [(key, self[key]) for key in self.keys()]
+    
+    def __getitem__(self, key):
+        module = import_module(f'.{SUBCOMMAND_MODULES[key]}', __package__)
+        assert is_valid_cli_module(module, key)
+        return module.main
+    
+    def __iter__(self):
+        return iter(SUBCOMMAND_MODULES.keys())
+    
+    def __len__(self):
+        return len(SUBCOMMAND_MODULES)
 
-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting
+CLI_SUBCOMMANDS = LazySubcommands()
+
+
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup', 'install')            # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
+
+display_first = (*meta_cmds, *main_cmds, *archive_cmds)
+
+
+IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
 
 
 def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
@@ -61,56 +133,38 @@ def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: It
         if blocking_threads:
             sleep(1)
             if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                stderr(
+                print(
                     f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
                     threads_summary,
+                    file=sys.stderr,
                 )
         else:
             return tries
 
-    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')
+    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
 
 
-def list_subcommands() -> Dict[str, str]:
-    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
-
-    COMMANDS = []
-    for filename in os.listdir(CLI_DIR):
-        if is_cli_module(filename):
-            subcommand = filename.replace('archivebox_', '').replace('.py', '')
-            module = import_module('.archivebox_{}'.format(subcommand), __package__)
-            assert is_valid_cli_module(module, subcommand)
-            COMMANDS.append((subcommand, module.main.__doc__))
-            globals()[subcommand] = module.main
-
-    display_order = lambda cmd: (
-        display_first.index(cmd[0])
-        if cmd[0] in display_first else
-        100 + len(cmd[0])
-    )
-
-    return dict(sorted(COMMANDS, key=display_order))
-
 
 def run_subcommand(subcommand: str,
-                   subcommand_args: List[str]=None,
+                   subcommand_args: List[str] | None = None,
                    stdin: Optional[IO]=None,
                    pwd: Union[Path, str, None]=None) -> None:
     """Run a given ArchiveBox subcommand with the given list of args"""
 
     subcommand_args = subcommand_args or []
 
+    from archivebox.misc.checks import check_migrations
+    from archivebox.config.legacy import setup_django
+    
+    # print('DATA_DIR is', DATA_DIR)
+    # print('pwd is', os.getcwd())
+
+    cmd_requires_db = subcommand in archive_cmds
+    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+
+    setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
+
     if subcommand not in meta_cmds:
-        from ..config import setup_django
-
-        cmd_requires_db = subcommand in archive_cmds
-        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
-
-        if cmd_requires_db:
-            check_data_folder(pwd)
-
-        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
-
         if cmd_requires_db:
             check_migrations()
 
@@ -121,17 +175,28 @@ def run_subcommand(subcommand: str,
     wait_for_bg_threads_to_exit(timeout=60)
 
 
-SUBCOMMANDS = list_subcommands()
+
+
 
 class NotProvided:
-    pass
+    def __len__(self):
+        return 0
+    def __bool__(self):
+        return False
+    def __repr__(self):
+        return '<not provided>'
+
+Omitted = Union[None, NotProvided]
+
+OMITTED = NotProvided()
 
 
-def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
-    args = sys.argv[1:] if args is NotProvided else args
-    stdin = sys.stdin if stdin is NotProvided else stdin
+def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: str | None=None) -> None:
+    # print('STARTING CLI MAIN ENTRYPOINT')
+    
+    args = sys.argv[1:] if args is OMITTED else args
+    stdin = sys.stdin if stdin is OMITTED else stdin
 
-    subcommands = list_subcommands()
     parser = argparse.ArgumentParser(
         prog=__command__,
         description='ArchiveBox: The self-hosted internet archive',
@@ -141,19 +206,19 @@ def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided,
     group.add_argument(
         '--help', '-h',
         action='store_true',
-        help=subcommands['help'],
+        help=CLI_SUBCOMMANDS['help'].__doc__,
     )
     group.add_argument(
         '--version',
         action='store_true',
-        help=subcommands['version'],
+        help=CLI_SUBCOMMANDS['version'].__doc__,
     )
     group.add_argument(
         "subcommand",
         type=str,
         help= "The name of the subcommand to run",
         nargs='?',
-        choices=subcommands.keys(),
+        choices=CLI_SUBCOMMANDS.keys(),
         default=None,
     )
     parser.add_argument(
@@ -168,29 +233,20 @@ def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided,
     elif command.help or command.subcommand is None:
         command.subcommand = 'help'
 
-    if command.subcommand not in ('help', 'version', 'status'):
+    if command.subcommand not in ('version',):
         from ..logging_util import log_cli_command
 
         log_cli_command(
             subcommand=command.subcommand,
             subcommand_args=command.subcommand_args,
-            stdin=stdin,
-            pwd=pwd or OUTPUT_DIR
+            stdin=stdin or None,
         )
 
-    run_subcommand(
-        subcommand=command.subcommand,
-        subcommand_args=command.subcommand_args,
-        stdin=stdin,
-        pwd=pwd or OUTPUT_DIR,
-    )
-
-
-__all__ = (
-    'SUBCOMMANDS',
-    'list_subcommands',
-    'run_subcommand',
-    *SUBCOMMANDS.keys(),
-)
-
-
+    try:
+        run_subcommand(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin or None,
+        )
+    except KeyboardInterrupt:
+        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')

archivebox/cli/archivebox_add.py

@@ -8,10 +8,11 @@ import argparse
 
 from typing import List, Optional, IO
 
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR, ARCHIVING_CONFIG
+
 from ..main import add
-from ..util import docstring
 from ..parsers import PARSERS
-from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr
 
 
@@ -32,7 +33,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--update', #'-u',
         action='store_true',
-        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        default=not ARCHIVING_CONFIG.ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
     parser.add_argument(
@@ -117,7 +118,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         init=command.init,
         extractors=command.extract,
         parser=command.parser,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=pwd or DATA_DIR,
     )
 
 

archivebox/cli/archivebox_config.py

@@ -5,12 +5,13 @@ __command__ = 'archivebox config'
 
 import sys
 import argparse
+from pathlib import Path
 
 from typing import Optional, List, IO
 
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..main import config
-from ..util import docstring
-from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, accept_stdin
 
 
@@ -56,7 +57,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         get=command.get,
         set=command.set,
         reset=command.reset,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
 
 

archivebox/cli/archivebox_help.py

@@ -5,12 +5,12 @@ __command__ = 'archivebox help'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..main import help
-from ..util import docstring
-from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, reject_stdin
 
 
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
-    help(out_dir=pwd or OUTPUT_DIR)
+    help(out_dir=Path(pwd) if pwd else DATA_DIR)
 
 
 if __name__ == '__main__':

archivebox/cli/archivebox_init.py

@@ -8,10 +8,10 @@ import argparse
 
 from typing import Optional, List, IO
 
-from ..main import init
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, reject_stdin
+from ..main import init
 
 
 @docstring(init.__doc__)
@@ -33,18 +33,23 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='Run any updates or migrations without rechecking all snapshot dirs',
     )
     parser.add_argument(
-        '--setup', #'-s',
+        '--install', #'-s',
         action='store_true',
         help='Automatically install dependencies and extras used for archiving',
     )
+    parser.add_argument(
+        '--setup', #'-s',
+        action='store_true',
+        help='DEPRECATED: equivalent to --install',
+    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
     init(
         force=command.force,
         quick=command.quick,
-        setup=command.setup,
-        out_dir=pwd or OUTPUT_DIR,
+        install=command.install or command.setup,
+        out_dir=pwd or DATA_DIR,
     )
     
 

archivebox/cli/archivebox_install.py

@@ -1,24 +1,24 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox setup'
+__command__ = 'archivebox install'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import setup
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, reject_stdin
+from ..main import install
 
 
-@docstring(setup.__doc__)
+@docstring(install.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=setup.__doc__,
+        description=install.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
@@ -30,9 +30,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     command = parser.parse_args(args or ())   # noqa
     reject_stdin(__command__, stdin)
 
-    setup(
+    install(
         # force=command.force,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
     
 

archivebox/cli/archivebox_list.py

@@ -5,12 +5,12 @@ __command__ = 'archivebox list'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..main import list_all
-from ..util import docstring
-from ..config import OUTPUT_DIR
 from ..index import (
     LINK_FILTERS,
     get_indexed_folders,
@@ -131,7 +131,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         json=command.json,
         html=command.html,
         with_headers=command.with_headers,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
     raise SystemExit(not matching_folders)
 

archivebox/cli/archivebox_manage.py

@@ -4,19 +4,19 @@ __package__ = 'archivebox.cli'
 __command__ = 'archivebox manage'
 
 import sys
-
+from pathlib import Path
 from typing import Optional, List, IO
 
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..main import manage
-from ..util import docstring
-from ..config import OUTPUT_DIR
 
 
 @docstring(manage.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     manage(
         args=args,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
 
 

archivebox/cli/archivebox_oneshot.py

@@ -9,10 +9,10 @@ import argparse
 from pathlib import Path
 from typing import List, Optional, IO
 
-from ..main import oneshot
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, accept_stdin, stderr
+from ..main import oneshot
 
 
 @docstring(oneshot.__doc__)
@@ -46,7 +46,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--out-dir',
         type=str,
-        default=OUTPUT_DIR,
+        default=DATA_DIR,
         help= "Path to save the single archive folder to, e.g. ./example.com_archive"
     )
     command = parser.parse_args(args or ())

archivebox/cli/archivebox_remove.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox remove'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import remove
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, accept_stdin
+from ..main import remove
 
 
 @docstring(remove.__doc__)
@@ -74,7 +74,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         after=command.after,
         yes=command.yes,
         delete=command.delete,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
     
 

archivebox/cli/archivebox_schedule.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox schedule'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import schedule
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, reject_stdin
+from ..main import schedule
 
 
 @docstring(schedule.__doc__)
@@ -108,7 +108,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         overwrite=command.overwrite,
         update=command.update,
         import_path=command.import_path,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
 
 

archivebox/cli/archivebox_server.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox server'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import server
-from ..util import docstring
-from ..config import OUTPUT_DIR, BIND_ADDR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR, SERVER_CONFIG
 from ..logging_util import SmartFormatter, reject_stdin
+from ..main import server
 
 @docstring(server.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         'runserver_args',
         nargs='*',
         type=str,
-        default=[BIND_ADDR],
+        default=[SERVER_CONFIG.BIND_ADDR],
         help='Arguments to pass to Django runserver'
     )
     parser.add_argument(
@@ -58,6 +58,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Run archivebox manage createsuperuser before starting the server',
     )
+    parser.add_argument(
+        '--daemonize',
+        action='store_true',
+        help='Run the server in the background as a daemon',
+    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
@@ -68,7 +73,8 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         init=command.init,
         quick_init=command.quick_init,
         createsuperuser=command.createsuperuser,
-        out_dir=pwd or OUTPUT_DIR,
+        daemonize=command.daemonize,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
 
 

archivebox/cli/archivebox_shell.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox shell'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import shell
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, reject_stdin
+from ..main import shell
 
 
 @docstring(shell.__doc__)
@@ -26,7 +26,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     reject_stdin(__command__, stdin)
     
     shell(
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
     
 

archivebox/cli/archivebox_status.py

@@ -5,13 +5,13 @@ __command__ = 'archivebox status'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import status
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..logging_util import SmartFormatter, reject_stdin
+from ..main import status
 
 
 @docstring(status.__doc__)
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
-    status(out_dir=pwd or OUTPUT_DIR)
+    status(out_dir=Path(pwd) if pwd else DATA_DIR)
 
 
 if __name__ == '__main__':

archivebox/cli/archivebox_update.py

@@ -5,12 +5,11 @@ __command__ = 'archivebox update'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import List, Optional, IO
 
-from ..main import update
-from ..util import docstring
-from ..config import OUTPUT_DIR
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
 from ..index import (
     LINK_FILTERS,
     get_indexed_folders,
@@ -25,6 +24,7 @@ from ..index import (
     get_unrecognized_folders,
 )
 from ..logging_util import SmartFormatter, accept_stdin
+from ..main import update
 
 
 @docstring(update.__doc__)
@@ -127,7 +127,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         status=command.status,
         after=command.after,
         before=command.before,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
         extractors=command.extract,
     )
     

archivebox/cli/archivebox_version.py

@@ -5,20 +5,20 @@ __command__ = 'archivebox version'
 
 import sys
 import argparse
-
+from pathlib import Path
 from typing import Optional, List, IO
 
-from ..main import version
-from ..util import docstring
-from ..config import OUTPUT_DIR
+# from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR, VERSION
 from ..logging_util import SmartFormatter, reject_stdin
 
 
-@docstring(version.__doc__)
+# @docstring(version.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    """Print the ArchiveBox version and dependency information"""
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=version.__doc__,
+        description="Print the ArchiveBox version and dependency information",   # version.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
@@ -30,9 +30,16 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
     
+    # for speed reasons, check if quiet flag was set and just return simple version immediately if so
+    if command.quiet:
+        print(VERSION)
+        return
+    
+    # otherwise do big expensive import to get the full version
+    from ..main import version
     version(
         quiet=command.quiet,
-        out_dir=pwd or OUTPUT_DIR,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
     )
 
 

archivebox/cli/tests.py

@@ -15,7 +15,7 @@ TEST_CONFIG = {
     'USE_COLOR': 'False',
     'SHOW_PROGRESS': 'False',
 
-    'OUTPUT_DIR': 'data.tests',
+    'DATA_DIR': 'data.tests',
     
     'SAVE_ARCHIVE_DOT_ORG': 'False',
     'SAVE_TITLE': 'False',
@@ -27,12 +27,12 @@ TEST_CONFIG = {
     'USE_YOUTUBEDL': 'False',
 }
 
-OUTPUT_DIR = 'data.tests'
+DATA_DIR = 'data.tests'
 os.environ.update(TEST_CONFIG)
 
 from ..main import init
 from ..index import load_main_index
-from ..config import (
+from archivebox.config.constants import (
     SQL_INDEX_FILENAME,
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,
@@ -101,22 +101,22 @@ def output_hidden(show_failing=True):
 
 class TestInit(unittest.TestCase):
     def setUp(self):
-        os.makedirs(OUTPUT_DIR, exist_ok=True)
+        os.makedirs(DATA_DIR, exist_ok=True)
 
     def tearDown(self):
-        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+        shutil.rmtree(DATA_DIR, ignore_errors=True)
 
     def test_basic_init(self):
         with output_hidden():
             archivebox_init.main([])
 
-        assert (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
-        assert (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
-        assert (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
-        assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
+        assert (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
+        assert (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
+        assert (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
+        assert len(load_main_index(out_dir=DATA_DIR)) == 0
 
     def test_conflicting_init(self):
-        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
+        with open(Path(DATA_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
             f.write('test')
 
         try:
@@ -126,11 +126,11 @@ class TestInit(unittest.TestCase):
         except SystemExit:
             pass
 
-        assert not (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
-        assert not (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
-        assert not (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
+        assert not (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
+        assert not (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
+        assert not (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
         try:
-            load_main_index(out_dir=OUTPUT_DIR)
+            load_main_index(out_dir=DATA_DIR)
             assert False, 'load_main_index should raise an exception when no index is present'
         except Exception:
             pass
@@ -138,36 +138,36 @@ class TestInit(unittest.TestCase):
     def test_no_dirty_state(self):
         with output_hidden():
             init()
-        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+        shutil.rmtree(DATA_DIR, ignore_errors=True)
         with output_hidden():
             init()
 
 
 class TestAdd(unittest.TestCase):
     def setUp(self):
-        os.makedirs(OUTPUT_DIR, exist_ok=True)
+        os.makedirs(DATA_DIR, exist_ok=True)
         with output_hidden():
             init()
 
     def tearDown(self):
-        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+        shutil.rmtree(DATA_DIR, ignore_errors=True)
 
     def test_add_arg_url(self):
         with output_hidden():
             archivebox_add.main(['https://getpocket.com/users/nikisweeting/feed/all'])
 
-        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        all_links = load_main_index(out_dir=DATA_DIR)
         assert len(all_links) == 30
 
     def test_add_arg_file(self):
-        test_file = Path(OUTPUT_DIR) / 'test.txt'
+        test_file = Path(DATA_DIR) / 'test.txt'
         with open(test_file, 'w+', encoding='utf') as f:
             f.write(test_urls)
 
         with output_hidden():
             archivebox_add.main([test_file])
 
-        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        all_links = load_main_index(out_dir=DATA_DIR)
         assert len(all_links) == 12
         os.remove(test_file)
 
@@ -175,40 +175,40 @@ class TestAdd(unittest.TestCase):
         with output_hidden():
             archivebox_add.main([], stdin=test_urls)
 
-        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        all_links = load_main_index(out_dir=DATA_DIR)
         assert len(all_links) == 12
 
 
 class TestRemove(unittest.TestCase):
     def setUp(self):
-        os.makedirs(OUTPUT_DIR, exist_ok=True)
+        os.makedirs(DATA_DIR, exist_ok=True)
         with output_hidden():
             init()
             archivebox_add.main([], stdin=test_urls)
 
     # def tearDown(self):
-        # shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+        # shutil.rmtree(DATA_DIR, ignore_errors=True)
 
 
     def test_remove_exact(self):
         with output_hidden():
             archivebox_remove.main(['--yes', '--delete', 'https://example5.com/'])
 
-        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        all_links = load_main_index(out_dir=DATA_DIR)
         assert len(all_links) == 11
 
     def test_remove_regex(self):
         with output_hidden():
             archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)'])
 
-        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        all_links = load_main_index(out_dir=DATA_DIR)
         assert len(all_links) == 4
 
     def test_remove_domain(self):
         with output_hidden():
             archivebox_remove.main(['--yes', '--delete', '--filter-type=domain', 'example5.com', 'example6.com'])
 
-        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        all_links = load_main_index(out_dir=DATA_DIR)
         assert len(all_links) == 10
 
     def test_remove_none(self):

archivebox/config/__init__.py

@@ -0,0 +1,27 @@
+__package__ = 'archivebox.config'
+
+from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR, VERSION
+from .defaults import (
+    SHELL_CONFIG,
+    STORAGE_CONFIG,
+    GENERAL_CONFIG,
+    SERVER_CONFIG,
+    ARCHIVING_CONFIG,
+    SEARCH_BACKEND_CONFIG,
+)
+
+
+__all__ = [
+    'CONSTANTS',
+    'PACKAGE_DIR',
+    'DATA_DIR',
+    'ARCHIVE_DIR',
+    'VERSION',
+    'SHELL_CONFIG',
+    'STORAGE_CONFIG',
+    'GENERAL_CONFIG',
+    'SERVER_CONFIG',
+    'ARCHIVING_CONFIG',
+    'SEARCH_BACKEND_CONFIG',
+    'CONSTANTS_CONFIG',
+]

archivebox/config/apps.py

@@ -0,0 +1,57 @@
+__package__ = 'archivebox.config'
+
+from typing import List
+from pydantic import InstanceOf
+
+from abx.archivebox.base_plugin import BasePlugin
+from abx.archivebox.base_hook import BaseHook
+
+
+from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR      # noqa
+from .defaults import (
+    ShellConfig,                    # noqa: F401
+    StorageConfig,                  # noqa: F401
+    GeneralConfig,                  # noqa: F401
+    ServerConfig,                   # noqa: F401
+    ArchivingConfig,                # noqa: F401
+    SearchBackendConfig,            # noqa: F401
+    SHELL_CONFIG,
+    STORAGE_CONFIG,
+    GENERAL_CONFIG,
+    SERVER_CONFIG,
+    ARCHIVING_CONFIG,
+    SEARCH_BACKEND_CONFIG,
+)
+
+###################### Config ##########################
+
+
+class ConfigPlugin(BasePlugin):
+    app_label: str = 'CONFIG'
+    verbose_name: str = 'Configuration'
+
+    hooks: List[InstanceOf[BaseHook]] = [
+        SHELL_CONFIG,
+        GENERAL_CONFIG,
+        STORAGE_CONFIG,
+        SERVER_CONFIG,
+        ARCHIVING_CONFIG,
+        SEARCH_BACKEND_CONFIG,
+    ]
+
+
+PLUGIN = ConfigPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+
+
+
+# # register django apps
+# @abx.hookimpl
+# def get_INSTALLED_APPS():
+#     return [DJANGO_APP.name]
+
+# # register configs
+# @abx.hookimpl
+# def register_CONFIG():
+#     return PLUGIN.HOOKS_BY_TYPE['CONFIG'].values()
+

archivebox/config/check_for_update.py

@@ -0,0 +1,47 @@
+# def get_versions_available_on_github(config):
+#     """
+#     returns a dictionary containing the ArchiveBox GitHub release info for
+#     the recommended upgrade version and the currently installed version
+#     """
+    
+#     # we only want to perform the (relatively expensive) check for new versions
+#     # when its most relevant, e.g. when the user runs a long-running command
+#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
+#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
+#     if subcommand_run_by_user not in long_running_commands:
+#         return None
+    
+#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
+#     response = requests.get(github_releases_api)
+#     if response.status_code != 200:
+#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
+#         return None
+#     all_releases = response.json()
+
+#     installed_version = parse_version_string(config['VERSION'])
+
+#     # find current version or nearest older version (to link to)
+#     current_version = None
+#     for idx, release in enumerate(all_releases):
+#         release_version = parse_version_string(release['tag_name'])
+#         if release_version <= installed_version:
+#             current_version = release
+#             break
+
+#     current_version = current_version or all_releases[-1]
+    
+#     # recommended version is whatever comes after current_version in the release list
+#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
+#     try:
+#         recommended_version = all_releases[idx+1]
+#     except IndexError:
+#         recommended_version = None
+
+#     return {'recommended_version': recommended_version, 'current_version': current_version}
+
+# def can_upgrade(config):
+#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
+#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
+#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
+#         return recommended_version > current_version
+#     return False

archivebox/config/config_stubs.py

@@ -2,22 +2,19 @@ from pathlib import Path
 from typing import Optional, Dict, Union, Tuple, Callable, Pattern, Type, Any, List
 from mypy_extensions import TypedDict
 
-
+from benedict import benedict
 
 SimpleConfigValue = Union[str, bool, int, None, Pattern, Dict[str, Any]]
 SimpleConfigValueDict = Dict[str, SimpleConfigValue]
 SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
 ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
 
-class AttrDict(dict):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.__dict__ = self
+
 
 class BaseConfig(TypedDict):
     pass
 
-class ConfigDict(BaseConfig, AttrDict, total=False):
+class ConfigDict(BaseConfig, benedict, total=False):
     """
     # Regenerate by pasting this quine into `archivebox shell` 🥚
     from archivebox.config import ConfigDict, CONFIG_DEFAULTS
@@ -39,7 +36,6 @@ class ConfigDict(BaseConfig, AttrDict, total=False):
     IN_DOCKER: bool
 
     PACKAGE_DIR: Path
-    OUTPUT_DIR: Path
     CONFIG_FILE: Path
     ONLY_NEW: bool
     TIMEOUT: int

archivebox/config/constants.py

@@ -0,0 +1,328 @@
+__package__ = 'archivebox.config'
+
+
+import os
+import re
+import platform
+import machineid
+import tempfile
+
+from typing import Dict
+from pathlib import Path
+import importlib.metadata
+from collections.abc import Mapping
+
+from benedict import benedict
+
+from ..misc.logging import DEFAULT_CLI_COLORS
+
+###################### Config ##########################
+
+PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()                    # archivebox user data dir
+ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
+
+def _detect_installed_version(PACKAGE_DIR: Path):
+    """Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file"""
+    try:
+        # if in production install, use pip-installed package metadata
+        return importlib.metadata.version(__package__ or 'archivebox').strip()
+    except importlib.metadata.PackageNotFoundError:
+        pass
+
+    try:
+        # if in dev Git repo dir, use pyproject.toml file
+        pyproject_config = (PACKAGE_DIR.parent / 'pyproject.toml').read_text().split('\n')
+        for line in pyproject_config:
+            if line.startswith('version = '):
+                return line.split(' = ', 1)[-1].strip('"').strip()
+    except FileNotFoundError:
+        # building docs, pyproject.toml is not available
+        pass
+    
+    try:
+        # if in dev but not in Git repo dir, fallback to using package.json file
+        package_json = (PACKAGE_DIR / 'package.json').read_text().split('\n')
+        for line in package_json:
+            if '"version": "' in line:
+                return line.replace('"', '').split(':')[-1].strip(',').strip()
+    except FileNotFoundError:
+        pass
+
+    # raise Exception('Failed to detect installed archivebox version!')
+    return 'dev'
+
+VERSION: str = _detect_installed_version(PACKAGE_DIR)
+
+
+
+
+class ConstantsDict(Mapping):
+    IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'yes')
+    OS = platform.system().lower()      # darwin, linux, etc.
+    ARCH = platform.machine().lower()   # arm64, x86_64, etc.
+    LIB_DIR_SCOPE = f'{ARCH}-{OS}' + ('-docker' if IN_DOCKER else '')
+
+    PACKAGE_DIR: Path = PACKAGE_DIR     # archivebox source code dir
+    DATA_DIR: Path = DATA_DIR           # archivebox user data dir
+    ARCHIVE_DIR: Path = ARCHIVE_DIR     # archivebox snapshot data dir
+    VERSION: str = VERSION
+    
+    PACKAGE_DIR_NAME: str               = PACKAGE_DIR.name
+    TEMPLATES_DIR_NAME: str             = 'templates'
+    TEMPLATES_DIR: Path                 = PACKAGE_DIR / TEMPLATES_DIR_NAME
+    STATIC_DIR: Path                    = TEMPLATES_DIR / 'static'
+    USER_PLUGINS_DIR_NAME: str          = 'user_plugins'
+    CUSTOM_TEMPLATES_DIR_NAME: str      = 'user_templates'
+
+    ARCHIVE_DIR_NAME: str = 'archive'
+    SOURCES_DIR_NAME: str = 'sources'
+    PERSONAS_DIR_NAME: str = 'personas'
+    CRONTABS_DIR_NAME: str = 'crontabs'
+    CACHE_DIR_NAME: str = 'cache'
+    LOGS_DIR_NAME: str = 'logs'
+    LIB_DIR_NAME: str = 'lib'
+    TMP_DIR_NAME: str = 'tmp'
+
+    SYSTEM_TMP_DIR: Path                = Path(os.environ['SYSTEM_TMP_DIR']) if 'SYSTEM_TMP_DIR' in os.environ else (Path(tempfile.gettempdir()) / 'archivebox')
+    # DATA_DIR_TMP_DIR: Path              = DATA_DIR / TMP_DIR_NAME / machineid.hashed_id('archivebox')[:16]   # cant be used because of socket path length restrictions break too often if data dir is in some deep subdir: ocket.error reported AF_UNIX path too long
+    SYSTEM_LIB_DIR: Path                = Path(os.environ['SYSTEM_LIB_DIR']) if 'SYSTEM_LIB_DIR' in os.environ else (PACKAGE_DIR / LIB_DIR_NAME)
+    DATA_DIR_LIB_DIR: Path              = DATA_DIR / LIB_DIR_NAME / LIB_DIR_SCOPE
+
+    ARCHIVE_DIR: Path                   = DATA_DIR / ARCHIVE_DIR_NAME
+    SOURCES_DIR: Path                   = DATA_DIR / SOURCES_DIR_NAME
+    PERSONAS_DIR: Path                  = DATA_DIR / PERSONAS_DIR_NAME
+    CACHE_DIR: Path                     = DATA_DIR / CACHE_DIR_NAME
+    LOGS_DIR: Path                      = DATA_DIR / LOGS_DIR_NAME
+    LIB_DIR: Path                       = SYSTEM_LIB_DIR if IN_DOCKER else DATA_DIR_LIB_DIR  # e.g. /app/lib or ./data/lib/arm64-darwin-docker
+    TMP_DIR: Path                       = SYSTEM_TMP_DIR
+    CUSTOM_TEMPLATES_DIR: Path          = DATA_DIR / CUSTOM_TEMPLATES_DIR_NAME
+    USER_PLUGINS_DIR: Path              = DATA_DIR / USER_PLUGINS_DIR_NAME
+
+    LIB_PIP_DIR: Path                   = LIB_DIR / 'pip'
+    LIB_NPM_DIR: Path                   = LIB_DIR / 'npm'
+    LIB_BROWSERS_DIR: Path              = LIB_DIR / 'browsers'
+    LIB_BIN_DIR: Path                   = LIB_DIR / 'bin'
+    BIN_DIR: Path                       = LIB_BIN_DIR
+
+    CONFIG_FILENAME: str = 'ArchiveBox.conf'
+    SQL_INDEX_FILENAME: str = 'index.sqlite3'
+    QUEUE_DATABASE_FILENAME: str = 'queue.sqlite3'
+
+    CONFIG_FILE: Path                   = DATA_DIR / CONFIG_FILENAME
+    DATABASE_FILE: Path                 = DATA_DIR / SQL_INDEX_FILENAME
+    QUEUE_DATABASE_FILE: Path           = DATA_DIR / QUEUE_DATABASE_FILENAME
+
+    JSON_INDEX_FILENAME: str = 'index.json'
+    HTML_INDEX_FILENAME: str = 'index.html'
+    ROBOTS_TXT_FILENAME: str = 'robots.txt'
+    FAVICON_FILENAME: str = 'favicon.ico'
+
+    TIMEZONE: str                             = 'UTC'
+    DEFAULT_CLI_COLORS: Dict[str, str]        = DEFAULT_CLI_COLORS
+    DISABLED_CLI_COLORS: Dict[str, str]       = benedict({k: '' for k in DEFAULT_CLI_COLORS})
+
+    ALLOWDENYLIST_REGEX_FLAGS: int = re.IGNORECASE | re.UNICODE | re.MULTILINE
+
+    STATICFILE_EXTENSIONS: frozenset[str] = frozenset((
+        # 99.999% of the time, URLs ending in these extensions are static files
+        # that can be downloaded as-is, not html pages that need to be rendered
+        'gif', 'jpeg', 'jpg', 'png', 'tif', 'tiff', 'wbmp', 'ico', 'jng', 'bmp',
+        'svg', 'svgz', 'webp', 'ps', 'eps', 'ai',
+        'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v',
+        'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8',
+        'pdf', 'txt', 'rtf', 'rtfd', 'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx',
+        'atom', 'rss', 'css', 'js', 'json',
+        'dmg', 'iso', 'img',
+        'rar', 'war', 'hqx', 'zip', 'gz', 'bz2', '7z',
+
+        # Less common extensions to consider adding later
+        # jar, swf, bin, com, exe, dll, deb
+        # ear, hqx, eot, wmlc, kml, kmz, cco, jardiff, jnlp, run, msi, msp, msm,
+        # pl pm, prc pdb, rar, rpm, sea, sit, tcl tk, der, pem, crt, xpi, xspf,
+        # ra, mng, asx, asf, 3gpp, 3gp, mid, midi, kar, jad, wml, htc, mml
+
+        # These are always treated as pages, not as static files, never add them:
+        # html, htm, shtml, xhtml, xml, aspx, php, cgi
+    ))
+
+    INGORED_PATHS: frozenset[str] = frozenset((
+        ".git",
+        ".svn",
+        ".DS_Store",
+        ".gitignore",
+        "lost+found",
+        ".DS_Store",
+        ".env",
+        "Dockerfile",
+        ".ArchiveBox.conf.bak",
+    ))
+    PIP_RELATED_NAMES: frozenset[str] = frozenset((
+        ".venv",
+        "venv",
+        "virtualenv",
+        ".virtualenv",
+    ))
+    NPM_RELATED_NAMES: frozenset[str] = frozenset((
+        "node_modules",
+        "package.json",
+        "package-lock.json",
+        "yarn.lock",
+    ))
+
+    DATA_DIR_NAMES: frozenset[str] = frozenset((
+        ARCHIVE_DIR_NAME,
+        SOURCES_DIR_NAME,
+        LOGS_DIR_NAME,
+        CACHE_DIR_NAME,
+        LIB_DIR_NAME,
+        TMP_DIR_NAME,
+        PERSONAS_DIR_NAME,
+        CUSTOM_TEMPLATES_DIR_NAME,
+        USER_PLUGINS_DIR_NAME,
+        CRONTABS_DIR_NAME,
+    ))
+    DATA_DIRS: frozenset[Path] = frozenset(DATA_DIR / dirname for dirname in DATA_DIR_NAMES)
+    DATA_FILE_NAMES: frozenset[str] = frozenset((
+        CONFIG_FILENAME,
+        SQL_INDEX_FILENAME,
+        f"{SQL_INDEX_FILENAME}-wal",
+        f"{SQL_INDEX_FILENAME}-shm",
+        QUEUE_DATABASE_FILENAME,
+        f"{QUEUE_DATABASE_FILENAME}-wal",
+        f"{QUEUE_DATABASE_FILENAME}-shm",
+        "search.sqlite3",
+        JSON_INDEX_FILENAME,
+        HTML_INDEX_FILENAME,
+        ROBOTS_TXT_FILENAME,
+        FAVICON_FILENAME,
+        CONFIG_FILENAME,
+        f"{CONFIG_FILENAME}.bak",
+        "static_index.json",
+    ))
+
+    # When initializing archivebox in a new directory, we check to make sure the dir is
+    # actually empty so that we dont clobber someone's home directory or desktop by accident.
+    # These files are exceptions to the is_empty check when we're trying to init a new dir,
+    # as they could be from a previous archivebox version, system artifacts, dependencies, etc.
+    ALLOWED_IN_DATA_DIR: frozenset[str] = frozenset((
+        *INGORED_PATHS,
+        *PIP_RELATED_NAMES,
+        *NPM_RELATED_NAMES,
+        *DATA_DIR_NAMES,
+        *DATA_FILE_NAMES,
+        "static",                # created by old static exports <v0.6.0
+        "sonic",                 # created by docker bind mount
+    ))
+
+    CODE_LOCATIONS = benedict({
+        'PACKAGE_DIR': {
+            'path': (PACKAGE_DIR).resolve(),
+            'enabled': True,
+            'is_valid': (PACKAGE_DIR / '__main__.py').exists(),
+        },
+        'TEMPLATES_DIR': {
+            'path': TEMPLATES_DIR.resolve(),
+            'enabled': True,
+            'is_valid': STATIC_DIR.exists(),
+        },
+        'LIB_DIR': {
+            'path': LIB_DIR.resolve(),
+            'enabled': True,
+            'is_valid': LIB_DIR.is_dir(),
+        },
+        'TMP_DIR': {
+            'path': TMP_DIR.resolve(),
+            'enabled': True,
+            'is_valid': TMP_DIR.is_dir(),
+        },
+    })
+        
+    DATA_LOCATIONS = benedict({
+        "DATA_DIR": {
+            "path": DATA_DIR.resolve(),
+            "enabled": True,
+            "is_valid": DATABASE_FILE.exists(),
+            "is_mount": os.path.ismount(DATA_DIR.resolve()),
+        },
+        "CONFIG_FILE": {
+            "path": CONFIG_FILE.resolve(),
+            "enabled": True,
+            "is_valid": CONFIG_FILE.exists(),
+        },
+        "SQL_INDEX": {
+            "path": DATABASE_FILE.resolve(),
+            "enabled": True,
+            "is_valid": DATABASE_FILE.exists(),
+            "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
+        },
+        "QUEUE_DATABASE": {
+            "path": QUEUE_DATABASE_FILE.resolve(),
+            "enabled": True,
+            "is_valid": QUEUE_DATABASE_FILE.exists(),
+            "is_mount": os.path.ismount(QUEUE_DATABASE_FILE.resolve()),
+        },
+        "ARCHIVE_DIR": {
+            "path": ARCHIVE_DIR.resolve(),
+            "enabled": True,
+            "is_valid": ARCHIVE_DIR.exists(),
+            "is_mount": os.path.ismount(ARCHIVE_DIR.resolve()),
+        },
+        "SOURCES_DIR": {
+            "path": SOURCES_DIR.resolve(),
+            "enabled": True,
+            "is_valid": SOURCES_DIR.exists(),
+        },
+        "LOGS_DIR": {
+            "path": LOGS_DIR.resolve(),
+            "enabled": True,
+            "is_valid": LOGS_DIR.is_dir(),
+        },
+        # "CACHE_DIR": {
+        #     "path": CACHE_DIR.resolve(),
+        #     "enabled": True,
+        #     "is_valid": CACHE_DIR.is_dir(),
+        # },
+        "PERSONAS_DIR": {
+            "path": PERSONAS_DIR.resolve(),
+            "enabled": PERSONAS_DIR.exists(),
+            "is_valid": PERSONAS_DIR.is_dir(),
+        },
+        'CUSTOM_TEMPLATES_DIR': {
+            'path': CUSTOM_TEMPLATES_DIR.resolve(),
+            'enabled': CUSTOM_TEMPLATES_DIR.exists(),
+            'is_valid': CUSTOM_TEMPLATES_DIR.is_dir(),
+        },
+        'USER_PLUGINS_DIR': {
+            'path': USER_PLUGINS_DIR.resolve(),
+            'enabled': USER_PLUGINS_DIR.exists(),
+            'is_valid': USER_PLUGINS_DIR.is_dir(),
+        },
+    })
+
+    @classmethod
+    def __getitem__(cls, key: str):
+        return getattr(cls, key)
+    
+    @classmethod
+    def __benedict__(cls):
+        return benedict({key: value for key, value in cls.__dict__.items() if key.isupper() and not key.startswith('_')})
+    
+    @classmethod
+    def __len__(cls):
+        return len(cls.__benedict__())
+
+    @classmethod
+    def __iter__(cls):
+        return iter(cls.__benedict__())
+
+CONSTANTS = ConstantsDict()
+CONSTANTS_CONFIG = CONSTANTS.__benedict__()
+
+# add all key: values to globals() for easier importing
+globals().update(CONSTANTS)
+
+
+# these need to always exist as we need them to run almost everything
+CONSTANTS.LIB_DIR.mkdir(parents=True, exist_ok=True)
+CONSTANTS.TMP_DIR.mkdir(parents=True, exist_ok=True)

archivebox/config/defaults.py

@@ -0,0 +1,216 @@
+__package__ = 'archivebox.config'
+
+import os
+import sys
+import shutil
+
+from typing import Dict, Optional
+from datetime import datetime
+from pathlib import Path
+
+from rich import print
+from pydantic import Field, field_validator, model_validator, computed_field
+from django.utils.crypto import get_random_string
+
+from abx.archivebox.base_configset import BaseConfigSet
+
+
+from .constants import CONSTANTS, PACKAGE_DIR
+
+###################### Config ##########################
+
+
+class ShellConfig(BaseConfigSet):
+    DEBUG: bool                         = Field(default=lambda: '--debug' in sys.argv)
+    
+    IS_TTY: bool                        = Field(default=sys.stdout.isatty())
+    USE_COLOR: bool                     = Field(default=lambda c: c.IS_TTY)
+    SHOW_PROGRESS: bool                 = Field(default=lambda c: c.IS_TTY)
+    
+    IN_DOCKER: bool                     = Field(default=False)
+    IN_QEMU: bool                       = Field(default=False)
+    
+    USER: str                           = Field(default=Path('~').expanduser().resolve().name)
+    PUID: int                           = Field(default=os.getuid())
+    PGID: int                           = Field(default=os.getgid())
+    
+    PYTHON_ENCODING: str                = Field(default=(sys.__stdout__ or sys.stdout or sys.__stderr__ or sys.stderr).encoding.upper().replace('UTF8', 'UTF-8'))
+
+    ANSI: Dict[str, str]                = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
+
+    VERSIONS_AVAILABLE: bool = False             # .check_for_update.get_versions_available_on_github(c)},
+    CAN_UPGRADE: bool = False                    # .check_for_update.can_upgrade(c)},
+
+    
+    @computed_field
+    @property
+    def TERM_WIDTH(self) -> int:
+        if not self.IS_TTY:
+            return 200
+        return shutil.get_terminal_size((140, 10)).columns
+    
+    @computed_field
+    @property
+    def COMMIT_HASH(self) -> Optional[str]:
+        try:
+            git_dir = PACKAGE_DIR / '../.git'
+            ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
+            commit_hash = git_dir.joinpath(ref).read_text().strip()
+            return commit_hash
+        except Exception:
+            pass
+    
+        try:
+            return list((PACKAGE_DIR / '../.git/refs/heads/').glob('*'))[0].read_text().strip()
+        except Exception:
+            pass
+        
+        return None
+    
+    @computed_field
+    @property
+    def BUILD_TIME(self) -> str:
+        if self.IN_DOCKER:
+            docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
+            return docker_build_end_time
+    
+        src_last_modified_unix_timestamp = (PACKAGE_DIR / 'package.json').stat().st_mtime
+        return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
+    
+
+    @model_validator(mode='after')
+    def validate_not_running_as_root(self):
+        attempted_command = ' '.join(sys.argv[:3])
+        if self.PUID == 0 and attempted_command not in ('setup', 'install'):
+            # stderr('[!] ArchiveBox should never be run as root!', color='red')
+            # stderr('    For more information, see the security overview documentation:')
+            # stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root')
+            print('[red][!] ArchiveBox should never be run as root![/red]', file=sys.stderr)
+            print('    For more information, see the security overview documentation:', file=sys.stderr)
+            print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root', file=sys.stderr)
+            
+            if self.IN_DOCKER:
+                print('[red][!] When using Docker, you must run commands with [green]docker run[/green] instead of [yellow3]docker exec[/yellow3], e.g.:', file=sys.stderr)
+                print('        docker compose run archivebox {attempted_command}', file=sys.stderr)
+                print(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}', file=sys.stderr)
+                print('        or:', file=sys.stderr)
+                print(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
+                print(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
+            raise SystemExit(2)
+        
+        # check python locale
+        if self.PYTHON_ENCODING != 'UTF-8':
+            print(f'[red][X] Your system is running python3 scripts with a bad locale setting: {self.PYTHON_ENCODING} (it should be UTF-8).[/red]', file=sys.stderr)
+            print('    To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)', file=sys.stderr)
+            print('    Or if you\'re using ubuntu/debian, run "dpkg-reconfigure locales"', file=sys.stderr)
+            print('')
+            print('    Confirm that it\'s fixed by opening a new shell and running:', file=sys.stderr)
+            print('        python3 -c "import sys; print(sys.stdout.encoding)"   # should output UTF-8', file=sys.stderr)
+            raise SystemExit(2)
+        
+        return self
+
+SHELL_CONFIG = ShellConfig()
+
+
+class StorageConfig(BaseConfigSet):
+    OUTPUT_PERMISSIONS: str             = Field(default='644')
+    RESTRICT_FILE_NAMES: str            = Field(default='windows')
+    ENFORCE_ATOMIC_WRITES: bool         = Field(default=True)
+    
+    # not supposed to be user settable:
+    DIR_OUTPUT_PERMISSIONS: str         = Field(default=lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5'))
+
+
+STORAGE_CONFIG = StorageConfig()
+
+
+class GeneralConfig(BaseConfigSet):
+    TAG_SEPARATOR_PATTERN: str          = Field(default=r'[,]')
+
+
+GENERAL_CONFIG = GeneralConfig()
+
+
+class ServerConfig(BaseConfigSet):
+    SECRET_KEY: str                     = Field(default=lambda: get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_'))
+    BIND_ADDR: str                      = Field(default=lambda: ['127.0.0.1:8000', '0.0.0.0:8000'][SHELL_CONFIG.IN_DOCKER])
+    ALLOWED_HOSTS: str                  = Field(default='*')
+    CSRF_TRUSTED_ORIGINS: str           = Field(default=lambda c: 'http://localhost:8000,http://127.0.0.1:8000,http://0.0.0.0:8000,http://{}'.format(c.BIND_ADDR))
+    
+    SNAPSHOTS_PER_PAGE: int             = Field(default=40)
+    FOOTER_INFO: str                    = Field(default='Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.')
+    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
+
+    PUBLIC_INDEX: bool                  = Field(default=True)
+    PUBLIC_SNAPSHOTS: bool              = Field(default=True)
+    PUBLIC_ADD_VIEW: bool               = Field(default=False)
+    
+    ADMIN_USERNAME: str                 = Field(default=None)
+    ADMIN_PASSWORD: str                 = Field(default=None)
+    REVERSE_PROXY_USER_HEADER: str      = Field(default='Remote-User')
+    REVERSE_PROXY_WHITELIST: str        = Field(default='')
+    LOGOUT_REDIRECT_URL: str            = Field(default='/')
+    PREVIEW_ORIGINALS: bool             = Field(default=True)
+    
+SERVER_CONFIG = ServerConfig()
+
+
+class ArchivingConfig(BaseConfigSet):
+    ONLY_NEW: bool                      = Field(default=True)
+    
+    TIMEOUT: int                        = Field(default=60)
+    MEDIA_TIMEOUT: int                  = Field(default=3600)
+
+    MEDIA_MAX_SIZE: str                 = Field(default='750m')
+    RESOLUTION: str                     = Field(default='1440,2000')
+    CHECK_SSL_VALIDITY: bool            = Field(default=True)
+    USER_AGENT: str                     = Field(default='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
+    COOKIES_FILE: Path | None           = Field(default=None)
+    
+    URL_DENYLIST: str                   = Field(default=r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', alias='URL_BLACKLIST')
+    URL_ALLOWLIST: str | None           = Field(default=None, alias='URL_WHITELIST')
+    
+    # GIT_DOMAINS: str                    = Field(default='github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht')
+    # WGET_USER_AGENT: str                = Field(default=lambda c: c['USER_AGENT'] + ' wget/{WGET_VERSION}')
+    # CURL_USER_AGENT: str                = Field(default=lambda c: c['USER_AGENT'] + ' curl/{CURL_VERSION}')
+    # CHROME_USER_AGENT: str              = Field(default=lambda c: c['USER_AGENT'])
+    # CHROME_USER_DATA_DIR: str | None    = Field(default=None)
+    # CHROME_TIMEOUT: int                 = Field(default=0)
+    # CHROME_HEADLESS: bool               = Field(default=True)
+    # CHROME_SANDBOX: bool                = Field(default=lambda: not SHELL_CONFIG.IN_DOCKER)
+
+    @field_validator('TIMEOUT', mode='after')
+    def validate_timeout(cls, v):
+        print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={v} seconds)[/red]', file=sys.stderr)
+        print('    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
+        print('    (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
+        print(file=sys.stderr)
+        print('    If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
+        print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
+        print(file=sys.stderr)
+        return v
+    
+    @field_validator('CHECK_SSL_VALIDITY', mode='after')
+    def validate_check_ssl_validity(cls, v):
+        """SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests"""
+        if not v:
+            import requests
+            import urllib3
+            requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+        return v
+
+ARCHIVING_CONFIG = ArchivingConfig()
+
+
+class SearchBackendConfig(BaseConfigSet):
+    USE_INDEXING_BACKEND: bool          = Field(default=True)
+    USE_SEARCHING_BACKEND: bool         = Field(default=True)
+    
+    SEARCH_BACKEND_ENGINE: str          = Field(default='ripgrep')
+    SEARCH_PROCESS_HTML: bool           = Field(default=True)
+    SEARCH_BACKEND_TIMEOUT: int         = Field(default=10)
+
+SEARCH_BACKEND_CONFIG = SearchBackendConfig()
+

archivebox/config/legacy.py

@@ -0,0 +1,650 @@
+"""
+ArchiveBox config definitons (including defaults and dynamic config options).
+
+Config Usage Example:
+
+    archivebox config --set MEDIA_TIMEOUT=600
+    env MEDIA_TIMEOUT=600 USE_COLOR=False ... archivebox [subcommand] ...
+
+Config Precedence Order:
+
+  1. cli args                 (--update-all / --index-only / etc.)
+  2. shell environment vars   (env USE_COLOR=False archivebox add '...')
+  3. config file              (echo "SAVE_FAVICON=False" >> ArchiveBox.conf)
+  4. defaults                 (defined below in Python)
+
+Documentation:
+
+  https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
+
+"""
+
+__package__ = 'archivebox.config'
+
+import os
+import io
+import re
+import sys
+import json
+import shutil
+
+from hashlib import md5
+from pathlib import Path
+from datetime import datetime, timezone
+from typing import Optional, Type, Tuple, Dict
+from subprocess import run, PIPE, DEVNULL, STDOUT, TimeoutExpired
+from configparser import ConfigParser
+
+from rich.progress import Progress
+from rich.console import Console
+from benedict import benedict
+from pydantic_pkgr import SemVer
+
+import django
+from django.db.backends.sqlite3.base import Database as sqlite3
+
+
+from .constants import CONSTANTS, TIMEZONE
+from .constants import *
+from .config_stubs import (
+    ConfigValue,
+    ConfigDefaultValue,
+    ConfigDefaultDict,
+)
+from ..misc.logging import (
+    stderr,
+    hint,      # noqa
+)
+
+from .defaults import SHELL_CONFIG, GENERAL_CONFIG, ARCHIVING_CONFIG, SERVER_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG
+from archivebox.plugins_auth.ldap.apps import LDAP_CONFIG
+from archivebox.plugins_extractor.favicon.apps import FAVICON_CONFIG
+from archivebox.plugins_extractor.wget.apps import WGET_CONFIG
+from archivebox.plugins_extractor.curl.apps import CURL_CONFIG
+
+ANSI = SHELL_CONFIG.ANSI
+LDAP = LDAP_CONFIG.LDAP_ENABLED
+
+############################### Config Schema ##################################
+
+CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
+    'SHELL_CONFIG': SHELL_CONFIG.as_legacy_config_schema(),
+
+    'SERVER_CONFIG': SERVER_CONFIG.as_legacy_config_schema(),
+    
+    'GENERAL_CONFIG': GENERAL_CONFIG.as_legacy_config_schema(),
+
+    'ARCHIVING_CONFIG': ARCHIVING_CONFIG.as_legacy_config_schema(),
+
+    'SEARCH_BACKEND_CONFIG': SEARCH_BACKEND_CONFIG.as_legacy_config_schema(),
+
+    'STORAGE_CONFIG': STORAGE_CONFIG.as_legacy_config_schema(),
+    
+    'LDAP_CONFIG': LDAP_CONFIG.as_legacy_config_schema(),
+    
+    # 'FAVICON_CONFIG': FAVICON_CONFIG.as_legacy_config_schema(),
+    
+    # 'WGET_CONFIG': WGET_CONFIG.as_legacy_config_schema(),
+    
+    # 'CURL_CONFIG': CURL_CONFIG.as_legacy_config_schema(),
+
+
+    'ARCHIVE_METHOD_TOGGLES': {
+        'SAVE_TITLE':               {'type': bool,  'default': True, 'aliases': ('FETCH_TITLE',)},
+        'SAVE_FAVICON':             {'type': bool,  'default': True, 'aliases': ('FETCH_FAVICON',)},
+        'SAVE_WGET':                {'type': bool,  'default': True, 'aliases': ('FETCH_WGET',)},
+        'SAVE_WGET_REQUISITES':     {'type': bool,  'default': True, 'aliases': ('FETCH_WGET_REQUISITES',)},
+        'SAVE_SINGLEFILE':          {'type': bool,  'default': True, 'aliases': ('FETCH_SINGLEFILE',)},
+        'SAVE_READABILITY':         {'type': bool,  'default': True, 'aliases': ('FETCH_READABILITY',)},
+        'SAVE_MERCURY':             {'type': bool,  'default': True, 'aliases': ('FETCH_MERCURY',)},
+        'SAVE_HTMLTOTEXT':          {'type': bool,  'default': True, 'aliases': ('FETCH_HTMLTOTEXT',)},
+        'SAVE_PDF':                 {'type': bool,  'default': True, 'aliases': ('FETCH_PDF',)},
+        'SAVE_SCREENSHOT':          {'type': bool,  'default': True, 'aliases': ('FETCH_SCREENSHOT',)},
+        'SAVE_DOM':                 {'type': bool,  'default': True, 'aliases': ('FETCH_DOM',)},
+        'SAVE_HEADERS':             {'type': bool,  'default': True, 'aliases': ('FETCH_HEADERS',)},
+        'SAVE_WARC':                {'type': bool,  'default': True, 'aliases': ('FETCH_WARC',)},
+        'SAVE_GIT':                 {'type': bool,  'default': True, 'aliases': ('FETCH_GIT',)},
+        'SAVE_MEDIA':               {'type': bool,  'default': True, 'aliases': ('FETCH_MEDIA',)},
+        'SAVE_ARCHIVE_DOT_ORG':     {'type': bool,  'default': True, 'aliases': ('SUBMIT_ARCHIVE_DOT_ORG',)},
+        'SAVE_ALLOWLIST':           {'type': dict,  'default': {},},
+        'SAVE_DENYLIST':            {'type': dict,  'default': {},},
+    },
+
+    'ARCHIVE_METHOD_OPTIONS': {
+        'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION','WINDOW_SIZE')},
+        # 'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht'},
+        'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
+        'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
+
+        'USER_AGENT':               {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'CURL_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT']}, # + ' curl/{CURL_VERSION}'},
+
+        'COOKIES_FILE':             {'type': str,   'default': None},
+
+        'YOUTUBEDL_ARGS':           {'type': list,  'default': lambda c: [
+                                                                '--restrict-filenames',
+                                                                '--trim-filenames', '128',
+                                                                '--write-description',
+                                                                '--write-info-json',
+                                                                '--write-annotations',
+                                                                '--write-thumbnail',
+                                                                '--no-call-home',
+                                                                '--write-sub',
+                                                                '--write-auto-subs',
+                                                                '--convert-subs=srt',
+                                                                '--yes-playlist',
+                                                                '--continue',
+                                                                # This flag doesn't exist in youtube-dl
+                                                                # only in yt-dlp
+                                                                '--no-abort-on-error',
+                                                                # --ignore-errors must come AFTER
+                                                                # --no-abort-on-error
+                                                                # https://github.com/yt-dlp/yt-dlp/issues/4914
+                                                                '--ignore-errors',
+                                                                '--geo-bypass',
+                                                                '--add-metadata',
+                                                                '--format=(bv*+ba/b)[filesize<={}][filesize_approx<=?{}]/(bv*+ba/b)'.format(c['MEDIA_MAX_SIZE'], c['MEDIA_MAX_SIZE']),
+                                                                ]},
+        'YOUTUBEDL_EXTRA_ARGS':     {'type': list,  'default': None},
+
+    },
+
+    'DEPENDENCY_CONFIG': {
+        'USE_CURL':                 {'type': bool,  'default': True},
+        'USE_SINGLEFILE':           {'type': bool,  'default': True},
+        'USE_READABILITY':          {'type': bool,  'default': True},
+        'USE_GIT':                  {'type': bool,  'default': True},
+        'USE_CHROME':               {'type': bool,  'default': True},
+        'USE_YOUTUBEDL':            {'type': bool,  'default': True},
+        'USE_RIPGREP':              {'type': bool,  'default': True},
+
+        # 'GIT_BINARY':               {'type': str,   'default': 'git'},
+        # 'CURL_BINARY':              {'type': str,   'default': 'curl'},
+        # 'NODE_BINARY':              {'type': str,   'default': 'node'},
+        # 'YOUTUBEDL_BINARY':         {'type': str,   'default': 'yt-dlp'},   # also can accept youtube-dl
+        # 'SINGLEFILE_BINARY':        {'type': str,   'default': lambda c: bin_path('single-file')},
+        # 'READABILITY_BINARY':       {'type': str,   'default': lambda c: bin_path('readability-extractor')},
+        # 'RIPGREP_BINARY':           {'type': str,   'default': 'rg'},
+
+        'POCKET_CONSUMER_KEY':      {'type': str,   'default': None},
+        'POCKET_ACCESS_TOKENS':     {'type': dict,  'default': {}},
+
+        'READWISE_READER_TOKENS':   {'type': dict,  'default': {}},
+    },
+}
+
+
+########################## Backwards-Compatibility #############################
+
+
+# for backwards compatibility with old config files, check old/deprecated names for each key
+CONFIG_ALIASES = {
+    alias: key
+    for section in CONFIG_SCHEMA.values()
+        for key, default in section.items()
+            for alias in default.get('aliases', ())
+}
+USER_CONFIG = {key: section[key] for section in CONFIG_SCHEMA.values() for key in section.keys()}
+
+def get_real_name(key: str) -> str:
+    """get the current canonical name for a given deprecated config key"""
+    return CONFIG_ALIASES.get(key.upper().strip(), key.upper().strip())
+
+
+
+# These are derived/computed values calculated *after* all user-provided config values are ingested
+# they appear in `archivebox config` output and are intended to be read-only for the user
+DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
+    'URL_DENYLIST_PTN':         {'default': lambda c: c['URL_DENYLIST'] and re.compile(c['URL_DENYLIST'] or '', CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)},
+    'URL_ALLOWLIST_PTN':        {'default': lambda c: c['URL_ALLOWLIST'] and re.compile(c['URL_ALLOWLIST'] or '', CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)},
+
+    'SAVE_ALLOWLIST_PTN':       {'default': lambda c: c['SAVE_ALLOWLIST'] and {re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_ALLOWLIST'].items()}},
+    'SAVE_DENYLIST_PTN':        {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
+}
+
+
+# print("FINISHED DEFINING SCHEMAS")
+
+################################### Helpers ####################################
+
+
+def load_config_val(key: str,
+                    default: ConfigDefaultValue=None,
+                    type: Optional[Type]=None,
+                    aliases: Optional[Tuple[str, ...]]=None,
+                    config: Optional[benedict]=None,
+                    env_vars: Optional[os._Environ]=None,
+                    config_file_vars: Optional[Dict[str, str]]=None) -> ConfigValue:
+    """parse bool, int, and str key=value pairs from env"""
+
+    assert isinstance(config, dict)
+
+    is_read_only = type is None
+    if is_read_only:
+        if callable(default):
+            return default(config)
+        return default
+
+    # get value from environment variables or config files
+    config_keys_to_check = (key, *(aliases or ()))
+    val = None
+    for key in config_keys_to_check:
+        if env_vars:
+            val = env_vars.get(key)
+            if val:
+                break
+
+        if config_file_vars:
+            val = config_file_vars.get(key)
+            if val:
+                break
+
+    is_unset = val is None
+    if is_unset:
+        if callable(default):
+            return default(config)
+        return default
+
+    # calculate value based on expected type
+    BOOL_TRUEIES = ('true', 'yes', '1')
+    BOOL_FALSEIES = ('false', 'no', '0')
+
+    if type is bool:
+        if val.lower() in BOOL_TRUEIES:
+            return True
+        elif val.lower() in BOOL_FALSEIES:
+            return False
+        else:
+            raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)')
+
+    elif type is str:
+        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
+            raise ValueError(f'Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)')
+        return val.strip()
+
+    elif type is int:
+        if not val.strip().isdigit():
+            raise ValueError(f'Invalid configuration option {key}={val} (expected an integer)')
+        return int(val.strip())
+
+    elif type is list or type is dict:
+        return json.loads(val)
+
+    raise Exception('Config values can only be str, bool, int, or json')
+
+
+def load_config_file(out_dir: str | None=CONSTANTS.DATA_DIR) -> Optional[benedict]:
+    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
+
+    config_path = CONSTANTS.CONFIG_FILE
+    if config_path.exists():
+        config_file = ConfigParser()
+        config_file.optionxform = str
+        config_file.read(config_path)
+        # flatten into one namespace
+        config_file_vars = benedict({
+            key.upper(): val
+            for section, options in config_file.items()
+                for key, val in options.items()
+        })
+        # print('[i] Loaded config file', os.path.abspath(config_path))
+        # print(config_file_vars)
+        return config_file_vars
+    return None
+
+
+def write_config_file(config: Dict[str, str], out_dir: str | None=CONSTANTS.DATA_DIR) -> benedict:
+    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
+
+    from archivebox.misc.system import atomic_write
+
+    CONFIG_HEADER = (
+    """# This is the config file for your ArchiveBox collection.
+    #
+    # You can add options here manually in INI format, or automatically by running:
+    #    archivebox config --set KEY=VALUE
+    #
+    # If you modify this file manually, make sure to update your archive after by running:
+    #    archivebox init
+    #
+    # A list of all possible config with documentation and examples can be found here:
+    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
+
+    """)
+
+    config_path = CONSTANTS.CONFIG_FILE
+
+    if not config_path.exists():
+        atomic_write(config_path, CONFIG_HEADER)
+
+    config_file = ConfigParser()
+    config_file.optionxform = str
+    config_file.read(config_path)
+
+    with open(config_path, 'r', encoding='utf-8') as old:
+        atomic_write(f'{config_path}.bak', old.read())
+
+    find_section = lambda key: [name for name, opts in CONFIG_SCHEMA.items() if key in opts][0]
+
+    # Set up sections in empty config file
+    for key, val in config.items():
+        section = find_section(key)
+        if section in config_file:
+            existing_config = dict(config_file[section])
+        else:
+            existing_config = {}
+        config_file[section] = benedict({**existing_config, key: val})
+
+    # always make sure there's a SECRET_KEY defined for Django
+    existing_secret_key = None
+    if 'SERVER_CONFIG' in config_file and 'SECRET_KEY' in config_file['SERVER_CONFIG']:
+        existing_secret_key = config_file['SERVER_CONFIG']['SECRET_KEY']
+
+    if (not existing_secret_key) or ('not a valid secret' in existing_secret_key):
+        from django.utils.crypto import get_random_string
+        chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
+        random_secret_key = get_random_string(50, chars)
+        if 'SERVER_CONFIG' in config_file:
+            config_file['SERVER_CONFIG']['SECRET_KEY'] = random_secret_key
+        else:
+            config_file['SERVER_CONFIG'] = {'SECRET_KEY': random_secret_key}
+
+    with open(config_path, 'w+', encoding='utf-8') as new:
+        config_file.write(new)
+
+    try:
+        # validate the config by attempting to re-parse it
+        CONFIG = load_all_config()
+    except BaseException:                                                       # lgtm [py/catch-base-exception]
+        # something went horribly wrong, rever to the previous version
+        with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
+            atomic_write(config_path, old.read())
+
+        raise
+
+    if Path(f'{config_path}.bak').exists():
+        os.remove(f'{config_path}.bak')
+
+    return benedict({
+        key.upper(): CONFIG.get(key.upper())
+        for key in config.keys()
+    })
+
+
+
+def load_config(defaults: ConfigDefaultDict,
+                config: Optional[benedict]=None,
+                out_dir: Optional[str]=None,
+                env_vars: Optional[os._Environ]=None,
+                config_file_vars: Optional[Dict[str, str]]=None) -> benedict:
+
+    env_vars = env_vars or os.environ
+    config_file_vars = config_file_vars or load_config_file(out_dir=out_dir)
+
+    extended_config = benedict(config.copy() if config else {})
+    for key, default in defaults.items():
+        try:
+            # print('LOADING CONFIG KEY:', key, 'DEFAULT=', default)
+            extended_config[key] = load_config_val(
+                key,
+                default=default['default'],
+                type=default.get('type'),
+                aliases=default.get('aliases'),
+                config=extended_config,
+                env_vars=env_vars,
+                config_file_vars=config_file_vars,
+            )
+        except KeyboardInterrupt:
+            raise SystemExit(0)
+        except Exception as e:
+            stderr()
+            stderr(f'[X] Error while loading configuration value: {key}', color='red', config=extended_config)
+            stderr('    {}: {}'.format(e.__class__.__name__, e))
+            stderr()
+            stderr('    Check your config for mistakes and try again (your archive data is unaffected).')
+            stderr()
+            stderr('    For config documentation and examples see:')
+            stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration')
+            stderr()
+            # raise
+            # raise SystemExit(2)
+
+    return benedict(extended_config)
+
+
+
+def find_chrome_binary() -> Optional[str]:
+    """find any installed chrome binaries in the default locations"""
+    # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
+    # make sure data dir finding precedence order always matches binary finding order
+    default_executable_paths = (
+        # '~/Library/Caches/ms-playwright/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium',
+        'chromium-browser',
+        'chromium',
+        '/Applications/Chromium.app/Contents/MacOS/Chromium',
+        'chrome',
+        'google-chrome',
+        '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+        'google-chrome-stable',
+        'google-chrome-beta',
+        'google-chrome-canary',
+        '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+        'google-chrome-unstable',
+        'google-chrome-dev',
+    )
+    for name in default_executable_paths:
+        full_path_exists = shutil.which(name)
+        if full_path_exists:
+            return name
+
+    return None
+
+def find_chrome_data_dir() -> Optional[str]:
+    """find any installed chrome user data directories in the default locations"""
+    # deprecated because this is DANGEROUS, do not re-implement/uncomment this behavior.
+
+    # Going forward we want to discourage people from using their main chrome profile for archiving.
+    # Session tokens, personal data, and cookies are often returned in server responses,
+    # when they get archived, they are essentially burned as anyone who can view the archive
+    # can use that data to masquerade as the logged-in user that did the archiving.
+    # For this reason users should always create dedicated burner profiles for archiving and not use
+    # their daily driver main accounts.
+
+    # # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
+    # # make sure data dir finding precedence order always matches binary finding order
+    # default_profile_paths = (
+    #     '~/.config/chromium',
+    #     '~/Library/Application Support/Chromium',
+    #     '~/AppData/Local/Chromium/User Data',
+    #     '~/.config/chrome',
+    #     '~/.config/google-chrome',
+    #     '~/Library/Application Support/Google/Chrome',
+    #     '~/AppData/Local/Google/Chrome/User Data',
+    #     '~/.config/google-chrome-stable',
+    #     '~/.config/google-chrome-beta',
+    #     '~/Library/Application Support/Google/Chrome Canary',
+    #     '~/AppData/Local/Google/Chrome SxS/User Data',
+    #     '~/.config/google-chrome-unstable',
+    #     '~/.config/google-chrome-dev',
+    # )
+    # for path in default_profile_paths:
+    #     full_path = Path(path).resolve()
+    #     if full_path.exists():
+    #         return full_path
+    return None
+
+def wget_supports_compression(config):
+    try:
+        cmd = [
+            config['WGET_BINARY'],
+            "--compression=auto",
+            "--help",
+        ]
+        return not run(cmd, stdout=DEVNULL, stderr=DEVNULL).returncode
+    except (FileNotFoundError, OSError):
+        return False
+
+
+# ******************************************************************************
+# ******************************************************************************
+# ******************************** Load Config *********************************
+# ******* (compile the defaults, configs, and metadata all into CONFIG) ********
+# ******************************************************************************
+# ******************************************************************************
+
+
+def load_all_config():
+    CONFIG = benedict()
+    for section_name, section_config in CONFIG_SCHEMA.items():
+        # print('LOADING CONFIG SECTION:', section_name)
+        CONFIG = load_config(section_config, CONFIG)
+
+    # print("LOADING CONFIG SECTION:", 'DYNAMIC')
+    return load_config(DYNAMIC_CONFIG_SCHEMA, CONFIG)
+
+# add all final config values in CONFIG to globals in this file
+CONFIG: benedict = load_all_config()
+globals().update(CONFIG)
+# this lets us do:  from .config import DEBUG, MEDIA_TIMEOUT, ...
+
+# print("FINISHED LOADING CONFIG USING SCHEMAS + FILE + ENV")
+
+# ******************************************************************************
+# ******************************************************************************
+# ******************************************************************************
+# ******************************************************************************
+# ******************************************************************************
+
+
+
+########################### System Environment Setup ###########################
+
+
+# Set timezone to UTC and umask to OUTPUT_PERMISSIONS
+assert TIMEZONE == 'UTC', f'The server timezone should always be set to UTC (got {TIMEZONE})'  # noqa: F821
+os.environ["TZ"] = TIMEZONE                                                  # noqa: F821
+os.umask(0o777 - int(STORAGE_CONFIG.DIR_OUTPUT_PERMISSIONS, base=8))                        # noqa: F821
+
+########################### Config Validity Checkers ###########################
+
+if not SHELL_CONFIG.USE_COLOR:
+    os.environ['NO_COLOR'] = '1'
+if not SHELL_CONFIG.SHOW_PROGRESS:
+    os.environ['TERM'] = 'dumb'
+
+# recreate rich console obj based on new config values
+CONSOLE = Console()
+from ..misc import logging
+logging.CONSOLE = CONSOLE
+
+
+INITIAL_STARTUP_PROGRESS = None
+INITIAL_STARTUP_PROGRESS_TASK = 0
+
+def bump_startup_progress_bar():
+    global INITIAL_STARTUP_PROGRESS
+    global INITIAL_STARTUP_PROGRESS_TASK
+    if INITIAL_STARTUP_PROGRESS:
+        INITIAL_STARTUP_PROGRESS.update(INITIAL_STARTUP_PROGRESS_TASK, advance=1)   # type: ignore
+
+
+def setup_django_minimal():
+    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
+    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
+    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+    # django.setup()
+    raise Exception('dont use this anymore')
+
+DJANGO_SET_UP = False
+
+
+def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CONFIG, in_memory_db=False) -> None:
+    global INITIAL_STARTUP_PROGRESS
+    global INITIAL_STARTUP_PROGRESS_TASK
+    global DJANGO_SET_UP
+
+    if DJANGO_SET_UP:
+        # raise Exception('django is already set up!')
+        # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
+        return
+
+    with Progress(transient=True, expand=True, console=CONSOLE) as INITIAL_STARTUP_PROGRESS:
+        INITIAL_STARTUP_PROGRESS_TASK = INITIAL_STARTUP_PROGRESS.add_task("[green]Loading modules...", total=25)
+
+        output_dir = out_dir or CONSTANTS.DATA_DIR
+
+        assert isinstance(output_dir, Path) and isinstance(CONSTANTS.PACKAGE_DIR, Path)
+
+        bump_startup_progress_bar()
+        try:
+            from django.core.management import call_command
+                
+            bump_startup_progress_bar()
+
+            if in_memory_db:
+                raise Exception('dont use this anymore')
+            
+                # some commands (e.g. oneshot) dont store a long-lived sqlite3 db file on disk.
+                # in those cases we create a temporary in-memory db and run the migrations
+                # immediately to get a usable in-memory-database at startup
+                os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
+                django.setup()
+                
+                bump_startup_progress_bar()
+                call_command("migrate", interactive=False, verbosity=0)
+            else:
+                # Otherwise use default sqlite3 file-based database and initialize django
+                # without running migrations automatically (user runs them manually by calling init)
+                django.setup()
+            
+            bump_startup_progress_bar()
+
+            from django.conf import settings
+            
+            # log startup message to the error log
+            with open(settings.ERROR_LOG, "a", encoding='utf-8') as f:
+                command = ' '.join(sys.argv)
+                ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
+                f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n")
+
+            if check_db:
+                # Create cache table in DB if needed
+                try:
+                    from django.core.cache import cache
+                    cache.get('test', None)
+                except django.db.utils.OperationalError:
+                    call_command("createcachetable", verbosity=0)
+
+                bump_startup_progress_bar()
+
+                # if archivebox gets imported multiple times, we have to close
+                # the sqlite3 whenever we init from scratch to avoid multiple threads
+                # sharing the same connection by accident
+                from django.db import connections
+                for conn in connections.all():
+                    conn.close_if_unusable_or_obsolete()
+
+                sql_index_path = CONSTANTS.DATABASE_FILE
+                assert sql_index_path.exists(), (
+                    f'No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)')
+
+                bump_startup_progress_bar()
+
+                # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
+                # if settings.DEBUG_LOGFIRE:
+                #     from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
+                #     SQLite3Instrumentor().instrument()
+
+                #     import logfire
+
+                #     logfire.configure()
+                #     logfire.instrument_django(is_sql_commentor_enabled=True)
+                #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
+
+        except KeyboardInterrupt:
+            raise SystemExit(2)
+        
+    DJANGO_SET_UP = True
+
+    INITIAL_STARTUP_PROGRESS = None
+    INITIAL_STARTUP_PROGRESS_TASK = None

archivebox/config/views.py

@@ -0,0 +1,446 @@
+__package__ = 'abx.archivebox'
+
+import os
+import inspect
+from typing import Any, List, Dict, cast
+from benedict import benedict
+
+from django.http import HttpRequest
+from django.conf import settings
+from django.utils import timezone
+from django.utils.html import format_html, mark_safe
+
+from admin_data_views.typing import TableContext, ItemContext
+from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
+
+from archivebox.config import CONSTANTS
+from archivebox.misc.util import parse_date
+
+from machine.models import InstalledBinary
+
+
+def obj_to_yaml(obj: Any, indent: int=0) -> str:
+    indent_str = "  " * indent
+    if indent == 0:
+        indent_str = '\n'  # put extra newline between top-level entries
+    
+    if isinstance(obj, dict):
+        if not obj:
+            return "{}"
+        result = "\n"
+        for key, value in obj.items():
+            result += f"{indent_str}{key}:{obj_to_yaml(value, indent + 1)}\n"
+        return result
+    
+    elif isinstance(obj, list):
+        if not obj:
+            return "[]"
+        result = "\n"
+        for item in obj:
+            result += f"{indent_str}- {obj_to_yaml(item, indent + 1).lstrip()}\n"
+        return result.rstrip()
+    
+    elif isinstance(obj, str):
+        if "\n" in obj:
+            return f" |\n{indent_str}  " + obj.replace("\n", f"\n{indent_str}  ")
+        else:
+            return f" {obj}"
+    
+    elif isinstance(obj, (int, float, bool)):
+        return f" {str(obj)}"
+    
+    elif callable(obj):
+        source = '\n'.join(
+            '' if 'def ' in line else line
+            for line in inspect.getsource(obj).split('\n')
+            if line.strip()
+        ).split('lambda: ')[-1].rstrip(',')
+        return f" {indent_str}  " + source.replace("\n", f"\n{indent_str}  ")
+    
+    else:
+        return f" {str(obj)}"
+
+@render_with_table_view
+def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
+
+    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+
+    rows = {
+        "Binary Name": [],
+        "Found Version": [],
+        "From Plugin": [],
+        "Provided By": [],
+        "Found Abspath": [],
+        "Related Configuration": [],
+        # "Overrides": [],
+        # "Description": [],
+    }
+
+    relevant_configs = {
+        key: val
+        for key, val in settings.FLAT_CONFIG.items()
+        if '_BINARY' in key or '_VERSION' in key
+    }
+
+    for plugin in settings.PLUGINS.values():
+        for binary in plugin.HOOKS_BY_TYPE.get('BINARY', {}).values():
+            try:
+                installed_binary = InstalledBinary.objects.get_from_db_or_cache(binary)
+                binary = installed_binary.load_from_db()
+            except Exception as e:
+                print(e)
+
+            rows['Binary Name'].append(ItemLink(binary.name, key=binary.name))
+            rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing')
+            rows['From Plugin'].append(plugin.plugin_module)
+            rows['Provided By'].append(
+                ', '.join(
+                    f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name
+                    for binprovider in binary.binproviders_supported
+                    if binprovider
+                )
+                # binary.loaded_binprovider.name
+                # if binary.loaded_binprovider else
+                # ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported)
+            )
+            rows['Found Abspath'].append(str(binary.loaded_abspath or '❌ missing'))
+            rows['Related Configuration'].append(mark_safe(', '.join(
+                f'<a href="/admin/environment/config/{config_key}/">{config_key}</a>'
+                for config_key, config_value in relevant_configs.items()
+                    if str(binary.name).lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower()
+                    or config_value.lower().endswith(binary.name.lower())
+                    # or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower()
+            )))
+            # if not binary.provider_overrides:
+                # import ipdb; ipdb.set_trace()
+            # rows['Overrides'].append(str(obj_to_yaml(binary.provider_overrides) or str(binary.provider_overrides))[:200])
+            # rows['Description'].append(binary.description)
+
+    return TableContext(
+        title="Binaries",
+        table=rows,
+    )
+
+@render_with_item_view
+def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+
+    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+
+    binary = None
+    plugin = None
+    for loaded_plugin in settings.PLUGINS.values():
+        for loaded_binary in loaded_plugin.HOOKS_BY_TYPE.get('BINARY', {}).values():
+            if loaded_binary.name == key:
+                binary = loaded_binary
+                plugin = loaded_plugin
+
+    assert plugin and binary, f'Could not find a binary matching the specified name: {key}'
+
+    try:
+        binary = binary.load()
+    except Exception as e:
+        print(e)
+
+    return ItemContext(
+        slug=key,
+        title=key,
+        data=[
+            {
+                "name": binary.name,
+                "description": binary.abspath,
+                "fields": {
+                    'plugin': plugin.name,
+                    'binprovider': binary.loaded_binprovider,
+                    'abspath': binary.loaded_abspath,
+                    'version': binary.loaded_version,
+                    'overrides': obj_to_yaml(binary.provider_overrides),
+                    'providers': obj_to_yaml(binary.binproviders_supported),
+                },
+                "help_texts": {
+                    # TODO
+                },
+            },
+        ],
+    )
+
+
+@render_with_table_view
+def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
+
+    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+
+    rows = {
+        "Name": [],
+        "verbose_name": [],
+        "module": [],
+        "source_code": [],
+        "hooks": [],
+    }
+
+
+    for plugin in settings.PLUGINS.values():
+        # try:
+        #     plugin.load_binaries()
+        # except Exception as e:
+        #     print(e)
+
+        rows['Name'].append(ItemLink(plugin.id, key=plugin.id))
+        rows['verbose_name'].append(mark_safe(f'<a href="{plugin.docs_url}" target="_blank">{plugin.verbose_name}</a>'))
+        rows['module'].append(str(plugin.plugin_module))
+        rows['source_code'].append(str(plugin.plugin_dir))
+        rows['hooks'].append(mark_safe(', '.join(
+            f'<a href="{hook.admin_url}">{hook.id}</a>'
+            for hook in plugin.hooks
+        )))
+
+    return TableContext(
+        title="Installed plugins",
+        table=rows,
+    )
+
+@render_with_item_view
+def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+
+    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+
+    plugin = None
+    for loaded_plugin in settings.PLUGINS.values():
+        if loaded_plugin.id == key:
+            plugin = loaded_plugin
+
+    assert plugin, f'Could not find a plugin matching the specified name: {key}'
+
+    try:
+        plugin = plugin.load_binaries()
+    except Exception as e:
+        print(e)
+
+    return ItemContext(
+        slug=key,
+        title=key,
+        data=[
+            {
+                "name": plugin.id,
+                "description": plugin.verbose_name,
+                "fields": {
+                    "hooks": plugin.hooks,
+                    "schema": obj_to_yaml(plugin.model_dump(include=("name", "verbose_name", "app_label", "hooks"))),
+                },
+                "help_texts": {
+                    # TODO
+                },
+            },
+        ],
+    )
+
+
+@render_with_table_view
+def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+
+    rows = {
+        "Name": [],
+        "State": [],
+        "PID": [],
+        "Started": [],
+        "Command": [],
+        "Logfile": [],
+        "Exit Status": [],
+    }
+    
+    from queues.supervisor_util import get_existing_supervisord_process
+    
+    supervisor = get_existing_supervisord_process()
+    if supervisor is None:
+        return TableContext(
+            title="No running worker processes",
+            table=rows,
+        )
+        
+    all_config_entries = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
+    all_config = {config["name"]: benedict(config) for config in all_config_entries}
+
+    # Add top row for supervisord process manager
+    rows["Name"].append(ItemLink('supervisord', key='supervisord'))
+    rows["State"].append(supervisor.getState()['statename'])
+    rows['PID'].append(str(supervisor.getPID()))
+    rows["Started"].append('-')
+    rows["Command"].append('supervisord --configuration=tmp/supervisord.conf')
+    rows["Logfile"].append(
+        format_html(
+            '<a href="/admin/environment/logs/{}/">{}</a>',
+            'supervisord',
+            'logs/supervisord.log',
+        )
+    )
+    rows['Exit Status'].append('0')
+
+    # Add a row for each worker process managed by supervisord
+    for proc in cast(List[Dict[str, Any]], supervisor.getAllProcessInfo()):
+        proc = benedict(proc)
+        # {
+        #     "name": "daphne",
+        #     "group": "daphne",
+        #     "start": 1725933056,
+        #     "stop": 0,
+        #     "now": 1725933438,
+        #     "state": 20,
+        #     "statename": "RUNNING",
+        #     "spawnerr": "",
+        #     "exitstatus": 0,
+        #     "logfile": "logs/server.log",
+        #     "stdout_logfile": "logs/server.log",
+        #     "stderr_logfile": "",
+        #     "pid": 33283,
+        #     "description": "pid 33283, uptime 0:06:22",
+        # }
+        rows["Name"].append(ItemLink(proc.name, key=proc.name))
+        rows["State"].append(proc.statename)
+        rows['PID'].append(proc.description.replace('pid ', ''))
+        rows["Started"].append(parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else '')
+        rows["Command"].append(all_config[proc.name].command)
+        rows["Logfile"].append(
+            format_html(
+                '<a href="/admin/environment/logs/{}/">{}</a>',
+                proc.stdout_logfile.split("/")[-1].split('.')[0],
+                proc.stdout_logfile,
+            )
+        )
+        rows["Exit Status"].append(str(proc.exitstatus))
+
+    return TableContext(
+        title="Running worker processes",
+        table=rows,
+    )
+
+
+@render_with_item_view
+def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+
+    from queues.supervisor_util import get_existing_supervisord_process, get_worker
+    from queues.settings import CONFIG_FILE
+
+    supervisor = get_existing_supervisord_process()
+    if supervisor is None:
+        return ItemContext(
+            slug='none',
+            title='error: No running supervisord process.',
+            data=[],
+        )
+
+    all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
+
+    if key == 'supervisord':
+        relevant_config = CONFIG_FILE.read_text()
+        relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
+        start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
+        uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]
+
+        proc = benedict(
+            {
+                "name": "supervisord",
+                "pid": supervisor.getPID(),
+                "statename": supervisor.getState()["statename"],
+                "start": start_ts,
+                "stop": None,
+                "exitstatus": "",
+                "stdout_logfile": "logs/supervisord.log",
+                "description": f'pid 000, uptime {uptime}',
+            }
+        )
+    else:
+        proc = benedict(get_worker(supervisor, key) or {})
+        relevant_config = [config for config in all_config if config['name'] == key][0]
+        relevant_logs = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0]
+
+    return ItemContext(
+        slug=key,
+        title=key,
+        data=[
+            {
+                "name": key,
+                "description": key,
+                "fields": {
+                    "Command": proc.name,
+                    "PID": proc.pid,
+                    "State": proc.statename,
+                    "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
+                    "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
+                    "Exit Status": str(proc.exitstatus),
+                    "Logfile": proc.stdout_logfile,
+                    "Uptime": (proc.description or "").split("uptime ", 1)[-1],
+                    "Config": relevant_config,
+                    "Logs": relevant_logs,
+                },
+                "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
+            },
+        ],
+    )
+
+
+@render_with_table_view
+def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+
+
+    log_files = CONSTANTS.LOGS_DIR.glob("*.log")
+    log_files = sorted(log_files, key=os.path.getmtime)[::-1]
+
+    rows = {
+        "Name": [],
+        "Last Updated": [],
+        "Size": [],
+        "Most Recent Lines": [],
+    }
+
+    # Add a row for each worker process managed by supervisord
+    for logfile in log_files:
+        st = logfile.stat()
+        rows["Name"].append(ItemLink("logs" + str(logfile).rsplit("/logs", 1)[-1], key=logfile.name))
+        rows["Last Updated"].append(parse_date(st.st_mtime).strftime("%Y-%m-%d %H:%M:%S"))
+        rows["Size"].append(f'{st.st_size//1000} kb')
+
+        with open(logfile, 'rb') as f:
+            try:
+                f.seek(-1024, os.SEEK_END)
+            except OSError:
+                f.seek(0)
+            last_lines = f.read().decode('utf-8', errors='replace').split("\n")
+            non_empty_lines = [line for line in last_lines if line.strip()]
+            rows["Most Recent Lines"].append(non_empty_lines[-1])
+
+    return TableContext(
+        title="Debug Log files",
+        table=rows,
+    )
+
+
+@render_with_item_view
+def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+
+    from django.conf import settings
+    
+    log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0]
+
+    log_text = log_file.read_text()
+    log_stat = log_file.stat()
+
+    return ItemContext(
+        slug=key,
+        title=key,
+        data=[
+            {
+                "name": key,
+                "description": key,
+                "fields": {
+                    "Path": str(log_file),
+                    "Size": f"{log_stat.st_size//1000} kb",
+                    "Last Updated": parse_date(log_stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S"),
+                    "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]),
+                    "Full Log": log_text,
+                },
+            },
+        ],
+    )

archivebox/core/admin.py

@@ -1,51 +1,45 @@
 __package__ = 'archivebox.core'
 
 import os
-import json
 
-from io import StringIO
 from pathlib import Path
-from contextlib import redirect_stdout
-from datetime import datetime, timezone
-from typing import Dict, Any
 
-from django.contrib import admin
-from django.db.models import Count, Q, Prefetch
+from django.contrib import admin, messages
 from django.urls import path, reverse, resolve
 from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
-from django.shortcuts import render, redirect
-from django.contrib.auth import get_user_model
+from django.contrib.auth import get_user_model, get_permission_codename
+from django.contrib.auth.admin import UserAdmin
 from django.core.paginator import Paginator
 from django.core.exceptions import ValidationError
+from django.template import Template, RequestContext
 from django.conf import settings
 from django import forms
 
-
 from signal_webhooks.admin import WebhookAdmin
 from signal_webhooks.utils import get_webhook_model
-# from plugantic.admin import CustomPlugin
 
-from ..util import htmldecode, urldecode, ansi_to_html
+from archivebox.config import VERSION, DATA_DIR
+from archivebox.misc.util import htmldecode, urldecode
 
-from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
-from core.forms import AddLinkForm
+from core.models import Snapshot, ArchiveResult, Tag
 from core.mixins import SearchResultsAdminMixin
 from api.models import APIToken
-from abid_utils.models import get_or_create_system_user_pk
 from abid_utils.admin import ABIDModelAdmin
+from queues.tasks import bg_archive_links, bg_add
+from machine.models import Machine, NetworkInterface, InstalledBinary
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
-from main import add, remove
+from main import remove
 from extractors import archive_links
 
 
-CONFIG = settings.CONFIG
+CONFIG = settings.FLAT_CONFIG
 
-GLOBAL_CONTEXT = {'VERSION': CONFIG.VERSION, 'VERSIONS_AVAILABLE': CONFIG.VERSIONS_AVAILABLE, 'CAN_UPGRADE': CONFIG.CAN_UPGRADE}
+GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
 
 # Admin URLs
 # /admin/
@@ -66,53 +60,87 @@ class ArchiveBoxAdmin(admin.AdminSite):
     site_title = 'Index'
     namespace = 'admin'
 
-    def get_urls(self):
-        return [
-            path('core/snapshot/add/', self.add_view, name='Add'),
-        ] + super().get_urls()
 
-    def add_view(self, request):
-        if not request.user.is_authenticated:
-            return redirect(f'/admin/login/?next={request.path}')
+class CustomUserAdmin(UserAdmin):
+    sort_fields = ['id', 'email', 'username', 'is_superuser', 'last_login', 'date_joined']
+    list_display = ['username', 'id', 'email', 'is_superuser', 'last_login', 'date_joined']
+    readonly_fields = ('snapshot_set', 'archiveresult_set', 'tag_set', 'apitoken_set', 'outboundwebhook_set')
+    fieldsets = [*UserAdmin.fieldsets, ('Data', {'fields': readonly_fields})]
 
-        request.current_app = self.name
-        context: Dict[str, Any] = {
-            **self.each_context(request),
-            'title': 'Add URLs',
-        }
+    @admin.display(description='Snapshots')
+    def snapshot_set(self, obj):
+        total_count = obj.snapshot_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> {}',
+                snap.pk,
+                snap.abid,
+                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
+                snap.url[:64],
+            )
+            for snap in obj.snapshot_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
-        if request.method == 'GET':
-            context['form'] = AddLinkForm()
+    @admin.display(description='Archive Result Logs')
+    def archiveresult_set(self, obj):
+        total_count = obj.archiveresult_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/core/archiveresult/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> <b>📄 {}</b> {}',
+                result.pk,
+                result.abid,
+                result.snapshot.downloaded_at.strftime('%Y-%m-%d %H:%M') if result.snapshot.downloaded_at else 'pending...',
+                result.extractor,
+                result.snapshot.url[:64],
+            )
+            for result in obj.archiveresult_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/core/archiveresult/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
-        elif request.method == 'POST':
-            form = AddLinkForm(request.POST)
-            if form.is_valid():
-                url = form.cleaned_data["url"]
-                print(f'[+] Adding URL: {url}')
-                depth = 0 if form.cleaned_data["depth"] == "0" else 1
-                input_kwargs = {
-                    "urls": url,
-                    "depth": depth,
-                    "update_all": False,
-                    "out_dir": CONFIG.OUTPUT_DIR,
-                }
-                add_stdout = StringIO()
-                with redirect_stdout(add_stdout):
-                   add(**input_kwargs)
-                print(add_stdout.getvalue())
+    @admin.display(description='Tags')
+    def tag_set(self, obj):
+        total_count = obj.tag_set.count()
+        return mark_safe(', '.join(
+            format_html(
+                '<code><a href="/admin/core/tag/{}/change"><b>{}</b></a></code>',
+                tag.pk,
+                tag.name,
+            )
+            for tag in obj.tag_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/core/tag/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
+
+    @admin.display(description='API Tokens')
+    def apitoken_set(self, obj):
+        total_count = obj.apitoken_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/api/apitoken/{}/change"><b>[{}]</b></a></code> {} (expires {})',
+                apitoken.pk,
+                apitoken.abid,
+                apitoken.token_redacted[:64],
+                apitoken.expires,
+            )
+            for apitoken in obj.apitoken_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/api/apitoken/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
+
+    @admin.display(description='API Outbound Webhooks')
+    def outboundwebhook_set(self, obj):
+        total_count = obj.outboundwebhook_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/api/outboundwebhook/{}/change"><b>[{}]</b></a></code> {} -> {}',
+                outboundwebhook.pk,
+                outboundwebhook.abid,
+                outboundwebhook.referenced_model,
+                outboundwebhook.endpoint,
+            )
+            for outboundwebhook in obj.outboundwebhook_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/api/outboundwebhook/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
-                context.update({
-                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
-                    "form": AddLinkForm(),
-                })
-            else:
-                context["form"] = form
 
-        return render(template_name='add.html', request=request, context=context)
 
 
 archivebox_admin = ArchiveBoxAdmin()
-archivebox_admin.register(get_user_model())
+archivebox_admin.register(get_user_model(), CustomUserAdmin)
 archivebox_admin.disable_action('delete_selected')
 
 # archivebox_admin.register(CustomPlugin)
@@ -129,6 +157,28 @@ archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_ad
 archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
 
 
+from huey_monitor.apps import HueyMonitorConfig
+HueyMonitorConfig.verbose_name = 'Background Workers'
+
+from huey_monitor.admin import TaskModel, TaskModelAdmin, SignalInfoModel, SignalInfoModelAdmin
+archivebox_admin.register(SignalInfoModel, SignalInfoModelAdmin)
+
+
+class CustomTaskModelAdmin(TaskModelAdmin):
+    actions = ["delete_selected"]
+
+    def has_delete_permission(self, request, obj=None):
+        codename = get_permission_codename("delete", self.opts)
+        return request.user.has_perm("%s.%s" % (self.opts.app_label, codename))
+
+
+archivebox_admin.register(TaskModel, CustomTaskModelAdmin)
+
+def result_url(result: TaskModel) -> str:
+    url = reverse("admin:huey_monitor_taskmodel_change", args=[str(result.id)])
+    return format_html('<a href="{url}" class="fade-in-progress-url">See progress...</a>'.format(url=url))
+
+
 class AccelleratedPaginator(Paginator):
     """
     Accellerated Pagniator ignores DISTINCT when counting total number of rows.
@@ -138,7 +188,7 @@ class AccelleratedPaginator(Paginator):
 
     @cached_property
     def count(self):
-        if self.object_list._has_filters():
+        if self.object_list._has_filters():                             # type: ignore
             # fallback to normal count method on filtered queryset
             return super().count
         else:
@@ -162,8 +212,8 @@ class ArchiveResultInline(admin.TabularInline):
     # fk_name = 'snapshot'
     extra = 0
     sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
-    readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
-    fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
+    readonly_fields = ('id', 'result_id', 'completed', 'command', 'version')
+    fields = ('start_ts', 'end_ts', *readonly_fields, 'extractor', 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
     # exclude = ('id',)
     ordering = ('end_ts',)
     show_change_link = True
@@ -172,7 +222,10 @@ class ArchiveResultInline(admin.TabularInline):
 
     def get_parent_object_from_request(self, request):
         resolved = resolve(request.path_info)
-        return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
+        try:
+            return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
+        except (self.parent_model.DoesNotExist, ValidationError):
+            return self.parent_model.objects.get(pk=self.parent_model.id_from_abid(resolved.kwargs['object_id']))
 
     @admin.display(
         description='Completed',
@@ -195,7 +248,7 @@ class ArchiveResultInline(admin.TabularInline):
         snapshot = self.get_parent_object_from_request(request)
 
         # import ipdb; ipdb.set_trace()
-        formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
+        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
         
         # default values for new entries
         formset.form.base_fields['status'].initial = 'succeeded'
@@ -251,6 +304,7 @@ from django.contrib.admin.widgets import FilteredSelectMultiple
 
 class SnapshotActionForm(ActionForm):
     tags = forms.ModelMultipleChoiceField(
+        label='Edit tags',
         queryset=Tag.objects.all(),
         required=False,
         widget=FilteredSelectMultiple(
@@ -267,48 +321,18 @@ class SnapshotActionForm(ActionForm):
     # )
 
 
-def get_abid_info(self, obj):
-    return format_html(
-        # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-        '''
-        <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-        <br/><hr/>
-        <div style="opacity: 0.8">
-        &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        <hr/>
-        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; {}: <code style="user-select: all">{}</code><br/>
-        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; "><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <span style="display:inline-block; vertical-align: -4px; width: 290px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span>
-        &nbsp; SALT: &nbsp; <code style="font-size: 10px;"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code><br/>
-        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}: <code style="user-select: all">{}</code><br/>
-        &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  {}: <code style="user-select: all">{}</code>
-        <br/><hr/>
-        &nbsp; &nbsp; <small style="opacity: 0.5">.old_id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        </div>
-        ''',
-        obj.api_url, obj.api_url, obj.api_docs_url,
-        str(obj.abid),
-        str(obj.ABID.uuid),
-        str(obj.id),
-        obj.ABID.ts, str(obj.ABID.uuid)[0:14], obj.abid_ts_src, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
-        obj.ABID.uri, str(obj.ABID.uuid)[14:26], obj.abid_uri_src, str(obj.abid_values['uri']),
-        obj.ABID.uri_salt,
-        obj.ABID.subtype, str(obj.ABID.uuid)[26:28], obj.abid_subtype_src, str(obj.abid_values['subtype']),
-        obj.ABID.rand, str(obj.ABID.uuid)[28:36], obj.abid_rand_src, str(obj.abid_values['rand'])[-7:],
-        str(getattr(obj, 'old_id', '')),
-    )
+
 
 
 @admin.register(Snapshot, site=archivebox_admin)
 class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
-    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
-    sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
-    search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
-    list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
-    fields = ('url', 'created_by', 'title', *readonly_fields)
-    ordering = ['-added']
+    list_display = ('created_at', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'created_at')
+    readonly_fields = ('admin_actions', 'status_info', 'tags_str', 'imported_timestamp', 'created_at', 'modified_at', 'downloaded_at', 'abid_info', 'link_dir')
+    search_fields = ('id', 'url', 'abid', 'timestamp', 'title', 'tags__name')
+    list_filter = ('created_at', 'downloaded_at', 'archiveresult__status', 'created_by', 'tags__name')
+    fields = ('url', 'title', 'created_by', 'bookmarked_at', *readonly_fields)
+    ordering = ['-created_at']
     actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
     inlines = [TagInline, ArchiveResultInline]
     list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
@@ -320,6 +344,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     show_full_result_count = False
 
     def changelist_view(self, request, extra_context=None):
+        self.request = request
         extra_context = extra_context or {}
         try:
             return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
@@ -327,34 +352,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
             self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
             return super().changelist_view(request, GLOBAL_CONTEXT)
 
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        snapshot = None
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-        
-        try:
-            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
-        except (Snapshot.DoesNotExist, ValidationError):
-            pass
-
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-
-        if snapshot:
-            object_id = str(snapshot.id)
-
-        return super().change_view(
-            request,
-            object_id,
-            form_url,
-            extra_context=extra_context,
-        )
 
     def get_urls(self):
         urls = super().get_urls()
@@ -370,8 +367,20 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     #     self.request = request
     #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
 
-    def tag_list(self, obj):
-        return ', '.join(tag.name for tag in obj.tags.all())
+    @admin.action(
+        description="Imported Timestamp"
+    )
+    def imported_timestamp(self, obj):
+        context = RequestContext(self.request, {
+            'bookmarked_date': obj.bookmarked,
+            'timestamp': obj.timestamp,
+        })
+
+        html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
+        return mark_safe(html.render(context))
+    
+        # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
+        # return f'{pretty_time} ({obj.timestamp})'
 
     # TODO: figure out a different way to do this, you cant nest forms so this doenst work
     # def action(self, obj):
@@ -429,12 +438,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
             obj.extension or '-',
         )
 
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            return str(e)
-
     @admin.display(
         description='Title',
         ordering='title',
@@ -464,6 +467,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         # ordering='archiveresult_count',
     )
     def files(self, obj):
+        # return '-'
         return snapshot_icons(obj)
 
 
@@ -526,46 +530,73 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     #     return super().changelist_view(request, extra_context=None)
 
     @admin.action(
-        description="Pull"
-    )
-    def update_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], out_dir=CONFIG.OUTPUT_DIR)
-
-    @admin.action(
-        description="⬇️ Title"
+        description="ℹ️ Get Title"
     )
     def update_titles(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, methods=('title','favicon'), out_dir=CONFIG.OUTPUT_DIR)
+        links = [snapshot.as_link() for snapshot in queryset]
+        if len(links) < 3:
+            # run syncronously if there are only 1 or 2 links
+            archive_links(links, overwrite=True, methods=('title','favicon'), out_dir=DATA_DIR)
+            messages.success(request, f"Title and favicon have been fetched and saved for {len(links)} URLs.")
+        else:
+            # otherwise run in a background worker
+            result = bg_archive_links((links,), kwargs={"overwrite": True, "methods": ["title", "favicon"], "out_dir": DATA_DIR})
+            messages.success(
+                request,
+                mark_safe(f"Title and favicon are updating in the background for {len(links)} URLs. {result_url(result)}"),
+            )
 
     @admin.action(
-        description="Re-Snapshot"
+        description="⬇️ Get Missing"
+    )
+    def update_snapshots(self, request, queryset):
+        links = [snapshot.as_link() for snapshot in queryset]
+
+        result = bg_archive_links((links,), kwargs={"overwrite": False, "out_dir": DATA_DIR})
+
+        messages.success(
+            request,
+            mark_safe(f"Re-trying any previously failed methods for {len(links)} URLs in the background. {result_url(result)}"),
+        )
+
+
+    @admin.action(
+        description="🆕 Archive Again"
     )
     def resnapshot_snapshot(self, request, queryset):
         for snapshot in queryset:
-            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
+            timestamp = timezone.now().isoformat('T', 'seconds')
             new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
-            add(new_url, tag=snapshot.tags_str())
+
+            result = bg_add({'urls': new_url, 'tag': snapshot.tags_str()})
+
+        messages.success(
+            request,
+            mark_safe(f"Creating new fresh snapshots for {queryset.count()} URLs in the background. {result_url(result)}"),
+        )
 
     @admin.action(
-        description="Reset"
+        description="🔄 Redo"
     )
     def overwrite_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, out_dir=CONFIG.OUTPUT_DIR)
+        links = [snapshot.as_link() for snapshot in queryset]
+
+        result = bg_archive_links((links,), kwargs={"overwrite": True, "out_dir": DATA_DIR})
+
+        messages.success(
+            request,
+            mark_safe(f"Clearing all previous results and re-downloading {len(links)} URLs in the background. {result_url(result)}"),
+        )
 
     @admin.action(
-        description="Delete"
+        description="☠️ Delete"
     )
     def delete_snapshots(self, request, queryset):
-        remove(snapshots=queryset, yes=True, delete=True, out_dir=CONFIG.OUTPUT_DIR)
+        remove(snapshots=queryset, yes=True, delete=True, out_dir=DATA_DIR)
+        messages.success(
+            request,
+            mark_safe(f"Succesfully deleted {queryset.count()} Snapshots. Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed."),
+        )
 
 
     @admin.action(
@@ -576,6 +607,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         print('[+] Adding tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.add(*tags)
+        messages.success(
+            request,
+            f"Added {len(tags)} tags to {queryset.count()} Snapshots.",
+        )
 
 
     @admin.action(
@@ -586,10 +621,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         print('[-] Removing tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.remove(*tags)
-
-
-        
-
+        messages.success(
+            request,
+            f"Removed {len(tags)} tags from {queryset.count()} Snapshots.",
+        )
 
 
 # @admin.register(SnapshotTag, site=archivebox_admin)
@@ -601,27 +636,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
 #     actions = ['delete_selected']
 #     ordering = ['-id']
 
-#     def API(self, obj):
-#         return get_abid_info(self, obj)
 
 
 @admin.register(Tag, site=archivebox_admin)
 class TagAdmin(ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
-    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
-    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
+    list_display = ('created_at', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
+    list_filter = ('created_at', 'created_by')
+    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created_at')
+    readonly_fields = ('slug', 'abid', 'created_at', 'modified_at', 'abid_info', 'snapshots')
     search_fields = ('abid', 'name', 'slug')
     fields = ('name', 'created_by', *readonly_fields)
     actions = ['delete_selected']
-    ordering = ['-created']
+    ordering = ['-created_at']
 
     paginator = AccelleratedPaginator
 
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            return str(e)
 
     def num_snapshots(self, tag):
         return format_html(
@@ -636,19 +665,19 @@ class TagAdmin(ABIDModelAdmin):
             format_html(
                 '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
                 snap.pk,
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
                 snap.url[:64],
             )
-            for snap in tag.snapshot_set.order_by('-updated')[:10]
-        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
+            for snap in tag.snapshot_set.order_by('-downloaded_at')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">{total_count} total snapshots...<a>'))
 
 
 @admin.register(ArchiveResult, site=archivebox_admin)
 class ArchiveResultAdmin(ABIDModelAdmin):
     list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
-    search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created_at', 'modified_at', 'abid_info', 'output_summary')
+    search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
     fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
     autocomplete_fields = ['snapshot']
 
@@ -657,6 +686,17 @@ class ArchiveResultAdmin(ABIDModelAdmin):
     list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
     
     paginator = AccelleratedPaginator
+    save_on_top = True
+    
+    actions = ['delete_selected']
+    
+    class Meta:
+        verbose_name = 'Archive Result'
+        verbose_name_plural = 'Archive Results'
+
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        return super().change_view(request, object_id, form_url, extra_context)
 
     @admin.display(
         description='Snapshot Info'
@@ -666,16 +706,10 @@ class ArchiveResultAdmin(ABIDModelAdmin):
             '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
             result.snapshot.timestamp,
             result.snapshot.abid,
-            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
+            result.snapshot.bookmarked_at.strftime('%Y-%m-%d %H:%M'),
             result.snapshot.url[:128],
         )
 
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            raise e
-            return str(e)
 
     @admin.display(
         description='Snapshot Tags'
@@ -698,22 +732,15 @@ class ArchiveResultAdmin(ABIDModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(CONFIG.OUTPUT_DIR) / str(result.pwd).split('data/', 1)[-1]
+        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split('data/', 1)[-1]
         output_str = format_html(
-            '<pre style="display: block; max-width: 75vw; white-space: pre-line;">{}</pre><br/>',
+            '<pre style="display: inline-block">{}</pre><br/>',
             result.output,
         )
         output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
         path_from_output_str = (snapshot_dir / result.output)
         output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
-        
-        path_exists = False
-        try:
-            path_exists = path_from_output_str.exists()
-        except OSError:
-            pass
-
-        if path_exists:
+        if path_from_output_str.exists():
             root_dir = str(path_from_output_str)
         else:
             root_dir = str(snapshot_dir)
@@ -737,18 +764,96 @@ class ArchiveResultAdmin(ABIDModelAdmin):
 
 @admin.register(APIToken, site=archivebox_admin)
 class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created', 'created_by', 'expires')
-    readonly_fields = ('abid', 'created')
+    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
     search_fields = ('id', 'abid', 'created_by__username', 'token')
     fields = ('created_by', 'token', 'expires', *readonly_fields)
 
     list_filter = ('created_by',)
-    ordering = ['-created']
+    ordering = ['-created_at']
     list_per_page = 100
 
 @admin.register(get_webhook_model(), site=archivebox_admin)
 class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('abid', 'created', *WebhookAdmin.readonly_fields)
+    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
+    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
+
+
+@admin.register(Machine, site=archivebox_admin)
+class MachineAdmin(ABIDModelAdmin):
+    list_display = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid', 'health')
+    sort_fields = ('abid', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'os_arch', 'os_family', 'os_release', 'hw_uuid')
+    # search_fields = ('id', 'abid', 'guid', 'hostname', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release')
+    
+    readonly_fields = ('guid', 'created_at', 'modified_at', 'abid_info', 'ips')
+    fields = (*readonly_fields, 'hostname', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer', 'hw_product', 'hw_uuid', 'os_arch', 'os_family', 'os_platform', 'os_kernel', 'os_release', 'stats', 'num_uses_succeeded', 'num_uses_failed')
+
+    list_filter = ('hw_in_docker', 'hw_in_vm', 'os_arch', 'os_family', 'os_platform')
+    ordering = ['-created_at']
+    list_per_page = 100
+    actions = ["delete_selected"]
+
+    @admin.display(
+        description='Public IP',
+        ordering='networkinterface__ip_public',
+    )
+    def ips(self, machine):
+        return format_html(
+            '<a href="/admin/machine/networkinterface/?q={}"><b><code>{}</code></b></a>',
+            machine.abid,
+            ', '.join(machine.networkinterface_set.values_list('ip_public', flat=True)),
+        )
+
+@admin.register(NetworkInterface, site=archivebox_admin)
+class NetworkInterfaceAdmin(ABIDModelAdmin):
+    list_display = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address', 'health')
+    sort_fields = ('abid', 'created_at', 'machine_info', 'ip_public', 'dns_server', 'isp', 'country', 'region', 'city', 'iface', 'ip_local', 'mac_address')
+    search_fields = ('abid', 'machine__abid', 'iface', 'ip_public', 'ip_local', 'mac_address', 'dns_server', 'hostname', 'isp', 'city', 'region', 'country')
+    
+    readonly_fields = ('machine', 'created_at', 'modified_at', 'abid_info', 'mac_address', 'ip_public', 'ip_local', 'dns_server')
+    fields = (*readonly_fields, 'iface', 'hostname', 'isp', 'city', 'region', 'country', 'num_uses_succeeded', 'num_uses_failed')
+
+    list_filter = ('isp', 'country', 'region')
+    ordering = ['-created_at']
+    list_per_page = 100
+    actions = ["delete_selected"]
+
+    @admin.display(
+        description='Machine',
+        ordering='machine__abid',
+    )
+    def machine_info(self, iface):
+        return format_html(
+            '<a href="/admin/machine/machine/{}/change"><b><code>[{}]</code></b> &nbsp; {}</a>',
+            iface.machine.id,
+            iface.machine.abid,
+            iface.machine.hostname,
+        )
+
+@admin.register(InstalledBinary, site=archivebox_admin)
+class InstalledBinaryAdmin(ABIDModelAdmin):
+    list_display = ('abid', 'created_at', 'machine_info', 'name', 'binprovider', 'version', 'abspath', 'sha256', 'health')
+    sort_fields = ('abid', 'created_at', 'machine_info', 'name', 'binprovider', 'version', 'abspath', 'sha256')
+    search_fields = ('abid', 'machine__abid', 'name', 'binprovider', 'version', 'abspath', 'sha256')
+    
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
+    fields = ('machine', 'name', 'binprovider', 'abspath', 'version', 'sha256', *readonly_fields, 'num_uses_succeeded', 'num_uses_failed')
+
+    list_filter = ('name', 'binprovider', 'machine_id')
+    ordering = ['-created_at']
+    list_per_page = 100
+    actions = ["delete_selected"]
+
+    @admin.display(
+        description='Machine',
+        ordering='machine__abid',
+    )
+    def machine_info(self, installed_binary):
+        return format_html(
+            '<a href="/admin/machine/machine/{}/change"><b><code>[{}]</code></b> &nbsp; {}</a>',
+            installed_binary.machine.id,
+            installed_binary.machine.abid,
+            installed_binary.machine.hostname,
+        )

archivebox/core/asgi.py

@@ -0,0 +1,26 @@
+"""
+WSGI config for archivebox project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
+"""
+
+from archivebox.config.legacy import setup_django
+
+setup_django(in_memory_db=False, check_db=True)
+
+
+from django.core.asgi import get_asgi_application
+from channels.routing import ProtocolTypeRouter
+
+
+django_asgi_app = get_asgi_application()
+
+application = ProtocolTypeRouter(
+    {
+        "http": django_asgi_app,
+        # Just HTTP for now. (We can add other protocols later.)
+    }
+)

archivebox/core/auth.py

@@ -1,13 +1,11 @@
 __package__ = 'archivebox.core'
 
 
-from ..config import (
-    LDAP
-)
+from archivebox.plugins_auth.ldap.apps import LDAP_CONFIG
 
 def register_signals():
 
-    if LDAP:
+    if LDAP_CONFIG.LDAP_ENABLED:
         import django_auth_ldap.backend
         from .auth_ldap import create_user
 

archivebox/core/auth_ldap.py

@@ -1,9 +1,7 @@
-from ..config import (
-    LDAP_CREATE_SUPERUSER
-)
+from archivebox.plugins_auth.ldap.apps import LDAP_CONFIG
 
 def create_user(sender, user=None, ldap_user=None, **kwargs):
-    if not user.id and LDAP_CREATE_SUPERUSER:
+    if not user.id and LDAP_CONFIG.LDAP_CREATE_SUPERUSER:
         user.is_superuser = True
 
     user.is_staff = True

archivebox/core/forms.py

@@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
 
 from django import forms
 
-from ..util import URL_REGEX
+from archivebox.misc.util import URL_REGEX
 from ..parsers import PARSERS
 from taggit.utils import edit_string_for_tags, parse_tags
 

archivebox/core/middleware.py

@@ -5,7 +5,7 @@ from django.utils import timezone
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.core.exceptions import ImproperlyConfigured
 
-from ..config import PUBLIC_SNAPSHOTS, REVERSE_PROXY_USER_HEADER, REVERSE_PROXY_WHITELIST
+from archivebox.config import SERVER_CONFIG
 
 
 def detect_timezone(request, activate: bool=True):
@@ -32,7 +32,7 @@ def CacheControlMiddleware(get_response):
         response = get_response(request)
 
         if '/archive/' in request.path or '/static/' in request.path:
-            policy = 'public' if PUBLIC_SNAPSHOTS else 'private'
+            policy = 'public' if SERVER_CONFIG.PUBLIC_SNAPSHOTS else 'private'
             response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
             # print('Set Cache-Control header to', response['Cache-Control'])
         return response
@@ -40,15 +40,15 @@ def CacheControlMiddleware(get_response):
     return middleware
 
 class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
-    header = 'HTTP_{normalized}'.format(normalized=REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
+    header = 'HTTP_{normalized}'.format(normalized=SERVER_CONFIG.REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
 
     def process_request(self, request):
-        if REVERSE_PROXY_WHITELIST == '':
+        if SERVER_CONFIG.REVERSE_PROXY_WHITELIST == '':
             return
 
         ip = request.META.get('REMOTE_ADDR')
 
-        for cidr in REVERSE_PROXY_WHITELIST.split(','):
+        for cidr in SERVER_CONFIG.REVERSE_PROXY_WHITELIST.split(','):
             try:
                 network = ipaddress.ip_network(cidr)
             except ValueError:

archivebox/core/migrations/0007_archiveresult.py

@@ -1,14 +1,18 @@
 # Generated by Django 3.0.8 on 2020-11-04 12:25
 
+import os
 import json
 from pathlib import Path
 
 from django.db import migrations, models
 import django.db.models.deletion
 
-from config import CONFIG
 from index.json import to_json
 
+DATA_DIR = Path(os.getcwd()).resolve()                    # archivebox user data dir
+ARCHIVE_DIR = DATA_DIR / 'archive'                      # archivebox snapshot data dir
+
+
 try:
     JSONField = models.JSONField
 except AttributeError:
@@ -22,7 +26,7 @@ def forwards_func(apps, schema_editor):
 
     snapshots = Snapshot.objects.all()
     for snapshot in snapshots:
-        out_dir = Path(CONFIG['ARCHIVE_DIR']) / snapshot.timestamp
+        out_dir = ARCHIVE_DIR / snapshot.timestamp
 
         try:
             with open(out_dir / "index.json", "r") as f:
@@ -57,7 +61,7 @@ def forwards_func(apps, schema_editor):
 
 def verify_json_index_integrity(snapshot):
     results = snapshot.archiveresult_set.all()
-    out_dir = Path(CONFIG['ARCHIVE_DIR']) / snapshot.timestamp
+    out_dir = ARCHIVE_DIR / snapshot.timestamp
     with open(out_dir / "index.json", "r") as f:
         index = json.load(f)
 

archivebox/core/migrations/0027_update_snapshot_ids.py

@@ -2,7 +2,6 @@
 
 from django.db import migrations
 
-from django.db import migrations
 from datetime import datetime
 from abid_utils.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
 
@@ -52,7 +51,7 @@ def update_snapshot_ids(apps, schema_editor):
     Snapshot = apps.get_model("core", "Snapshot")
     num_total = Snapshot.objects.all().count()
     print(f'   Updating {num_total} Snapshot.id, Snapshot.uuid values in place...')
-    for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator()):
+    for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator(chunk_size=500)):
         assert snapshot.abid
         snapshot.abid_prefix = 'snp_'
         snapshot.abid_ts_src = 'self.added'
@@ -72,7 +71,7 @@ def update_archiveresult_ids(apps, schema_editor):
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     num_total = ArchiveResult.objects.all().count()
     print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
-    for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator()):
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator(chunk_size=500)):
         assert result.abid
         result.abid_prefix = 'res_'
         result.snapshot = Snapshot.objects.get(pk=result.snapshot_id)

archivebox/core/migrations/0032_alter_archiveresult_id.py

@@ -1,9 +1,13 @@
 # Generated by Django 5.0.6 on 2024-08-18 05:20
 
 import core.models
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -14,6 +18,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
+            field=models.BigIntegerField(default=rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
         ),
     ]

archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py

@@ -2,16 +2,21 @@
 
 import core.models
 import uuid
+import random
 from django.db import migrations, models
 
 from abid_utils.abid import ABID
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
+
 def update_archiveresult_ids(apps, schema_editor):
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     num_total = ArchiveResult.objects.all().count()
     print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
-    for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()):
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator(chunk_size=500)):
         assert result.abid
         result.uuid = ABID.parse(result.abid).uuid
         result.save(update_fields=["uuid"])
@@ -30,7 +35,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='ID'),
         ),
         migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
         migrations.AlterField(

archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py

@@ -2,9 +2,13 @@
 
 import core.models
 import uuid
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -20,6 +24,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0051_snapshottag_snapshot_alter_snapshottag_snapshot_old.py

@@ -9,7 +9,7 @@ def update_snapshottag_ids(apps, schema_editor):
     SnapshotTag = apps.get_model("core", "SnapshotTag")
     num_total = SnapshotTag.objects.all().count()
     print(f'   Updating {num_total} SnapshotTag.snapshot_id values in place... (may take an hour or longer for large collections...)')
-    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator()):
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator(chunk_size=500)):
         assert snapshottag.snapshot_old_id
         snapshot = Snapshot.objects.get(old_id=snapshottag.snapshot_old_id)
         snapshottag.snapshot_id = snapshot.id

archivebox/core/migrations/0058_alter_tag_old_id.py

@@ -1,9 +1,12 @@
 # Generated by Django 5.0.6 on 2024-08-20 03:30
 
-import core.models
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -14,6 +17,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='tag',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0059_tag_id.py

@@ -1,8 +1,9 @@
 # Generated by Django 5.0.6 on 2024-08-20 03:33
 
+from datetime import datetime
 from django.db import migrations, models
-from abid_utils.models import ABID, abid_from_values
-
+from abid_utils.abid import abid_from_values
+from abid_utils.models import ABID
 
 def calculate_abid(self):
     """
@@ -49,7 +50,7 @@ def update_archiveresult_ids(apps, schema_editor):
     Tag = apps.get_model("core", "Tag")
     num_total = Tag.objects.all().count()
     print(f'   Updating {num_total} Tag.id, ArchiveResult.uuid values in place...')
-    for idx, tag in enumerate(Tag.objects.all().iterator()):
+    for idx, tag in enumerate(Tag.objects.all().iterator(chunk_size=500)):
         if not tag.slug:
             tag.slug = tag.name.lower().replace(' ', '_')
         if not tag.name:

archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py

@@ -9,7 +9,7 @@ def update_snapshottag_ids(apps, schema_editor):
     SnapshotTag = apps.get_model("core", "SnapshotTag")
     num_total = SnapshotTag.objects.all().count()
     print(f'   Updating {num_total} SnapshotTag.tag_id values in place... (may take an hour or longer for large collections...)')
-    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator()):
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator(chunk_size=500)):
         assert snapshottag.old_tag_id
         tag = Tag.objects.get(old_id=snapshottag.old_tag_id)
         snapshottag.tag_id = tag.id

archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py

@@ -3,8 +3,11 @@
 import core.models
 import django.db.models.deletion
 import uuid
+import random
 from django.db import migrations, models
 
+def rand_int_id():
+    return random.getrandbits(32)
 
 class Migration(migrations.Migration):
 
@@ -26,6 +29,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='tag',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0069_alter_archiveresult_created_alter_snapshot_added_and_more.py

@@ -0,0 +1,35 @@
+# Generated by Django 5.1 on 2024-08-28 09:40
+
+import abid_utils.models
+import django.utils.timezone
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0068_alter_archiveresult_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='added',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+    ]

archivebox/core/migrations/0070_alter_archiveresult_created_by_alter_snapshot_added_and_more.py

@@ -0,0 +1,52 @@
+# Generated by Django 5.1 on 2024-09-04 09:00
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0069_alter_archiveresult_created_alter_snapshot_added_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='added',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='id',
+            field=models.UUIDField(default=None, primary_key=True, serialize=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='old_id',
+            field=models.UUIDField(default=None, editable=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+    ]

archivebox/core/migrations/0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more.py

@@ -0,0 +1,65 @@
+# Generated by Django 5.1 on 2024-09-04 23:23
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0070_alter_archiveresult_created_by_alter_snapshot_added_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='archiveresult',
+            name='old_id',
+        ),
+        migrations.RemoveField(
+            model_name='snapshot',
+            name='old_id',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='old_id',
+        ),
+        
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='tag_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+    ]

archivebox/core/migrations/0072_rename_added_snapshot_bookmarked_at_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1 on 2024-09-05 00:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='added',
+            new_name='bookmarked_at',
+        ),
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='updated',
+            new_name='downloaded_at',
+        ),
+    ]

archivebox/core/migrations/0073_rename_created_archiveresult_created_at_and_more.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.1 on 2024-09-05 00:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0072_rename_added_snapshot_bookmarked_at_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='tag',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='tag',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+    ]

archivebox/core/migrations/0074_alter_snapshot_downloaded_at.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1 on 2024-09-05 01:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0073_rename_created_archiveresult_created_at_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='downloaded_at',
+            field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
+        ),
+    ]

archivebox/core/migrations/0075_crawl.py

@@ -0,0 +1,101 @@
+# Generated by Django 5.1.1 on 2024-10-01 02:10
+
+import abid_utils.models
+import charidfield.fields
+import django.core.validators
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0074_alter_snapshot_downloaded_at"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Crawl",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=None,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                        unique=True,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "abid",
+                    charidfield.fields.CharIDField(
+                        blank=True,
+                        db_index=True,
+                        default=None,
+                        help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+                        max_length=30,
+                        null=True,
+                        prefix="crl_",
+                        unique=True,
+                    ),
+                ),
+                (
+                    "created_at",
+                    abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+                ),
+                ("modified_at", models.DateTimeField(auto_now=True)),
+                ("urls", models.TextField()),
+                (
+                    "depth",
+                    models.PositiveSmallIntegerField(
+                        default=1,
+                        validators=[
+                            django.core.validators.MinValueValidator(0),
+                            django.core.validators.MaxValueValidator(2),
+                        ],
+                    ),
+                ),
+                (
+                    "parser",
+                    models.CharField(
+                        choices=[
+                            ("auto", "auto"),
+                            ("pocket_api", "Pocket API"),
+                            ("readwise_reader_api", "Readwise Reader API"),
+                            ("wallabag_atom", "Wallabag Atom"),
+                            ("pocket_html", "Pocket HTML"),
+                            ("pinboard_rss", "Pinboard RSS"),
+                            ("shaarli_rss", "Shaarli RSS"),
+                            ("medium_rss", "Medium RSS"),
+                            ("netscape_html", "Netscape HTML"),
+                            ("rss", "Generic RSS"),
+                            ("json", "Generic JSON"),
+                            ("jsonl", "Generic JSONL"),
+                            ("html", "Generic HTML"),
+                            ("txt", "Generic TXT"),
+                            ("url_list", "URL List"),
+                        ],
+                        default="auto",
+                        max_length=32,
+                    ),
+                ),
+                (
+                    "created_by",
+                    models.ForeignKey(
+                        default=None,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="crawl_set",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Crawl",
+                "verbose_name_plural": "Crawls",
+            },
+        ),
+    ]

archivebox/core/models.py

@@ -1,14 +1,11 @@
 __package__ = 'archivebox.core'
 
 
-from typing import Optional, List, Dict, Iterable
+from typing import Optional, Dict, Iterable
 from django_stubs_ext.db.models import TypedModelMeta
 
 import json
-import random
 
-import uuid
-from uuid import uuid4
 from pathlib import Path
 
 from django.db import models
@@ -17,24 +14,27 @@ from django.utils.text import slugify
 from django.core.cache import cache
 from django.urls import reverse, reverse_lazy
 from django.db.models import Case, When, Value, IntegerField
+from django.core.validators import MaxValueValidator, MinValueValidator 
+from django.contrib import admin
 from django.conf import settings
 
-from abid_utils.models import ABIDModel, ABIDField
+from archivebox.config import CONSTANTS
 
-from ..system import get_dir_size
-from ..util import parse_date, base_url
+from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
+from queues.tasks import bg_archive_snapshot
+from machine.models import Machine, NetworkInterface
+
+from archivebox.misc.system import get_dir_size
+from archivebox.misc.util import parse_date, base_url
 from ..index.schema import Link
 from ..index.html import snapshot_icons
 from ..extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE, EXTRACTORS
-
-
-def rand_int_id():
-    return random.getrandbits(32)
+from ..parsers import PARSERS
 
 
 # class BaseModel(models.Model):
 #     # TODO: migrate all models to a shared base class with all our standard fields and helpers:
-#     #       ulid/created/modified/owner/is_deleted/as_json/from_json/etc.
+#     #       ulid/created_at/modified_at/created_by/is_deleted/as_json/from_json/etc.
 #     #
 #     # id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
 #     # ulid = models.CharField(max_length=26, null=True, blank=True, db_index=True, unique=True)
@@ -44,28 +44,30 @@ def rand_int_id():
 
 
 
-
 class Tag(ABIDModel):
     """
     Based on django-taggit model + ABID base.
     """
     abid_prefix = 'tag_'
-    abid_ts_src = 'self.created'          # TODO: add created/modified time
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.slug'
     abid_subtype_src = '"03"'
-    abid_rand_src = 'self.old_id'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    old_id = models.BigIntegerField(unique=True, default=rand_int_id, serialize=False, verbose_name='Old ID')  # legacy PK
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='tag_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     name = models.CharField(unique=True, blank=False, max_length=100)
     slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
     # slug is autoset on save from name, never set it manually
 
     snapshot_set: models.Manager['Snapshot']
+    crawl_set: models.Manager['Crawl']
 
     class Meta(TypedModelMeta):
         verbose_name = "Tag"
@@ -74,10 +76,6 @@ class Tag(ABIDModel):
     def __str__(self):
         return self.name
 
-    # @property
-    # def old_id(self):
-    #     return self.id
-
     def slugify(self, tag, i=None):
         slug = slugify(tag)
         if i is not None:
@@ -109,11 +107,11 @@ class Tag(ABIDModel):
     @property
     def api_url(self) -> str:
         # /api/v1/core/snapshot/{uulid}
-        return reverse_lazy('api-1:get_tag', args=[self.abid])
+        return reverse_lazy('api-1:get_tag', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
 
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
 
 class SnapshotTag(models.Model):
     id = models.AutoField(primary_key=True)
@@ -126,6 +124,84 @@ class SnapshotTag(models.Model):
         unique_together = [('snapshot', 'tag')]
 
 
+# class CrawlTag(models.Model):
+#     id = models.AutoField(primary_key=True)
+
+#     crawl = models.ForeignKey('Crawl', db_column='crawl_id', on_delete=models.CASCADE, to_field='id')
+#     tag = models.ForeignKey(Tag, db_column='tag_id', on_delete=models.CASCADE, to_field='id')
+
+#     class Meta:
+#         db_table = 'core_crawl_tags'
+#         unique_together = [('crawl', 'tag')]
+
+
+class Crawl(ABIDModel):
+    abid_prefix = 'crl_'
+    abid_ts_src = 'self.created_at'
+    abid_uri_src = 'self.urls'
+    abid_subtype_src = 'self.crawler'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
+
+    # CRAWLER_CHOICES = (
+    #     ('breadth_first', 'Breadth-First'),
+    #     ('depth_first', 'Depth-First'),
+    # )
+    PARSER_CHOICES = (
+        ('auto', 'auto'),
+        *((parser_key, value[0]) for parser_key, value in PARSERS.items()),
+    )
+
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='crawl_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+
+    urls = models.TextField(blank=False, null=False)
+    depth = models.PositiveSmallIntegerField(default=1, validators=[MinValueValidator(0), MaxValueValidator(2)])
+    parser = models.CharField(choices=PARSER_CHOICES, default='auto', max_length=32)
+    
+    # crawler = models.CharField(choices=CRAWLER_CHOICES, default='breadth_first', max_length=32)
+    # tags = models.ManyToManyField(Tag, blank=True, related_name='crawl_set', through='CrawlTag')
+    # schedule = models.JSONField()
+    # config = models.JSONField()
+    
+
+    class Meta(TypedModelMeta):
+        verbose_name = 'Crawl'
+        verbose_name_plural = 'Crawls'
+
+    def __str__(self):
+        return self.parser
+
+    @cached_property
+    def crawl_dir(self):
+        return Path()
+
+    @property
+    def api_url(self) -> str:
+        # /api/v1/core/crawl/{uulid}
+        return reverse_lazy('api-1:get_crawl', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
+
+    @property
+    def api_docs_url(self) -> str:
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_crawl'
+
+    # def get_absolute_url(self):
+    #     return f'/crawls/{self.abid}'
+    
+    def crawl(self):
+        # write self.urls to sources/crawl__<user>__YYYYMMDDHHMMSS.txt
+        # run parse_links(sources/crawl__<user>__YYYYMMDDHHMMSS.txt, parser=self.parser) and for each resulting link:
+        #   create a Snapshot
+        #   enqueue task bg_archive_snapshot(snapshot)
+        pass
+
+
+
+
 class SnapshotManager(models.Manager):
     def get_queryset(self):
         return super().get_queryset().prefetch_related('tags', 'archiveresult_set')  # .annotate(archiveresult_count=models.Count('archiveresult')).distinct()
@@ -133,34 +209,43 @@ class SnapshotManager(models.Manager):
 
 class Snapshot(ABIDModel):
     abid_prefix = 'snp_'
-    abid_ts_src = 'self.added'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.url'
     abid_subtype_src = '"01"'
-    abid_rand_src = 'self.old_id'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    old_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)  # legacy pk
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)  # loaded from self._init_timestamp
+    modified_at = models.DateTimeField(auto_now=True)
+
+    # legacy ts fields
+    bookmarked_at = AutoDateTimeField(default=None, null=False, editable=True, db_index=True)
+    downloaded_at = models.DateTimeField(default=None, null=True, editable=False, db_index=True, blank=True)
+
     url = models.URLField(unique=True, db_index=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
-
-    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
-    
     tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
+    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)    
 
-    added = models.DateTimeField(auto_now_add=True, db_index=True)
-    updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
-
-    keys = ('url', 'timestamp', 'title', 'tags', 'updated')
+    keys = ('url', 'timestamp', 'title', 'tags', 'downloaded_at')
 
     archiveresult_set: models.Manager['ArchiveResult']
 
     objects = SnapshotManager()
 
-    @property
-    def uuid(self):
-        return self.id
+    def save(self, *args, **kwargs):
+        if not self.bookmarked_at:
+            self.bookmarked_at = self.created_at or self._init_timestamp
+        
+        super().save(*args, **kwargs)
+
+    def archive(self, overwrite=False, methods=None):
+        result = bg_archive_snapshot(self, overwrite=overwrite, methods=methods)
+        return result
 
     def __repr__(self) -> str:
         title = (self.title_stripped or '-')[:64]
@@ -170,13 +255,6 @@ class Snapshot(ABIDModel):
         title = (self.title_stripped or '-')[:64]
         return f'[{self.timestamp}] {self.url[:64]} ({title})'
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        try:
-            assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'Snapshot.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
-        except AssertionError as e:
-            print(e)
-
     @classmethod
     def from_json(cls, info: dict):
         info = {k: v for k, v in info.items() if k in cls.keys}
@@ -196,9 +274,10 @@ class Snapshot(ABIDModel):
         from ..index import load_link_details
         return load_link_details(self.as_link())
 
+    @admin.display(description='Tags')
     def tags_str(self, nocache=True) -> str | None:
         calc_tags_str = lambda: ','.join(sorted(tag.name for tag in self.tags.all()))
-        cache_key = f'{self.pk}-{(self.updated or self.added).timestamp()}-tags'
+        cache_key = f'{self.pk}-{(self.downloaded_at or self.bookmarked_at).timestamp()}-tags'
         
         if hasattr(self, '_prefetched_objects_cache') and 'tags' in self._prefetched_objects_cache:
             # tags are pre-fetched already, use them directly (best because db is always freshest)
@@ -217,11 +296,14 @@ class Snapshot(ABIDModel):
     @property
     def api_url(self) -> str:
         # /api/v1/core/snapshot/{uulid}
-        return reverse_lazy('api-1:get_snapshot', args=[self.abid])
+        return reverse_lazy('api-1:get_snapshot', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
     
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
+    
+    def get_absolute_url(self):
+        return f'/{self.archive_path}'
     
     @cached_property
     def title_stripped(self) -> str:
@@ -229,7 +311,7 @@ class Snapshot(ABIDModel):
 
     @cached_property
     def extension(self) -> str:
-        from ..util import extension
+        from archivebox.misc.util import extension
         return extension(self.url)
 
     @cached_property
@@ -258,15 +340,15 @@ class Snapshot(ABIDModel):
 
     @cached_property
     def link_dir(self):
-        return str(settings.CONFIG.ARCHIVE_DIR / self.timestamp)
+        return str(CONSTANTS.ARCHIVE_DIR / self.timestamp)
 
     @cached_property
     def archive_path(self):
-        return '{}/{}'.format(settings.CONFIG.ARCHIVE_DIR_NAME, self.timestamp)
+        return '{}/{}'.format(CONSTANTS.ARCHIVE_DIR_NAME, self.timestamp)
 
     @cached_property
     def archive_size(self):
-        cache_key = f'{str(self.pk)[:12]}-{(self.updated or self.added).timestamp()}-size'
+        cache_key = f'{str(self.pk)[:12]}-{(self.downloaded_at or self.bookmarked_at).timestamp()}-size'
 
         def calc_dir_size():
             try:
@@ -285,7 +367,7 @@ class Snapshot(ABIDModel):
                     for result in self.archiveresult_set.all()
                     if result.extractor == 'screenshot' and result.status =='succeeded' and result.output
                 ),
-                key=lambda result: result.created,
+                key=lambda result: result.created_at,
             ) or [None])[-1]
         else:
             result = self.archiveresult_set.filter(
@@ -318,7 +400,7 @@ class Snapshot(ABIDModel):
     def latest_title(self) -> Optional[str]:
         if self.title:
             return self.title   # whoopdedoo that was easy
-        
+
         # check if ArchiveResult set has already been prefetched, if so use it instead of fetching it from db again
         if hasattr(self, '_prefetched_objects_cache') and 'archiveresult_set' in self._prefetched_objects_cache:
             try:
@@ -332,7 +414,7 @@ class Snapshot(ABIDModel):
                 ) or [None])[-1]
             except IndexError:
                 pass
-        
+
 
         try:
             # take longest successful title from ArchiveResult db history
@@ -370,19 +452,19 @@ class Snapshot(ABIDModel):
 
 
     # def get_storage_dir(self, create=True, symlink=True) -> Path:
-    #     date_str = self.added.strftime('%Y%m%d')
+    #     date_str = self.bookmarked_at.strftime('%Y%m%d')
     #     domain_str = domain(self.url)
-    #     abs_storage_dir = Path(settings.CONFIG.ARCHIVE_DIR) / 'snapshots' / date_str / domain_str / str(self.ulid)
+    #     abs_storage_dir = Path(CONSTANTS.ARCHIVE_DIR) / 'snapshots' / date_str / domain_str / str(self.ulid)
 
     #     if create and not abs_storage_dir.is_dir():
     #         abs_storage_dir.mkdir(parents=True, exist_ok=True)
 
     #     if symlink:
     #         LINK_PATHS = [
-    #             Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'all_by_id' / str(self.ulid),
-    #             # Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'snapshots_by_id' / str(self.ulid),
-    #             Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'snapshots_by_date' / date_str / domain_str / str(self.ulid),
-    #             Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'snapshots_by_domain' / domain_str / date_str / str(self.ulid),
+    #             Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'all_by_id' / str(self.ulid),
+    #             # Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'snapshots_by_id' / str(self.ulid),
+    #             Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'snapshots_by_date' / date_str / domain_str / str(self.ulid),
+    #             Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'snapshots_by_domain' / domain_str / date_str / str(self.ulid),
     #         ]
     #         for link_path in LINK_PATHS:
     #             link_path.parent.mkdir(parents=True, exist_ok=True)
@@ -398,7 +480,7 @@ class Snapshot(ABIDModel):
 class ArchiveResultManager(models.Manager):
     def indexable(self, sorted: bool = True):
         """Return only ArchiveResults containing text suitable for full-text search (sorted in order of typical result quality)"""
-        
+
         INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
         qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS, status='succeeded')
 
@@ -418,10 +500,11 @@ class ArchiveResultManager(models.Manager):
 
 class ArchiveResult(ABIDModel):
     abid_prefix = 'res_'
-    abid_ts_src = 'self.snapshot.added'
+    abid_ts_src = 'self.snapshot.created_at'
     abid_uri_src = 'self.snapshot.url'
     abid_subtype_src = 'self.extractor'
-    abid_rand_src = 'self.old_id'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
     EXTRACTOR_CHOICES = (
         ('htmltotext', 'htmltotext'),
@@ -445,11 +528,13 @@ class ArchiveResult(ABIDModel):
         ("skipped", "skipped")
     ]
 
-    old_id = models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID')
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True, verbose_name='ID')
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='archiveresult_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')
 
     extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32)
@@ -461,26 +546,23 @@ class ArchiveResult(ABIDModel):
     end_ts = models.DateTimeField()
     status = models.CharField(max_length=16, choices=STATUS_CHOICES)
 
+    # the network interface that was used to download this result
+    # uplink = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
+
     objects = ArchiveResultManager()
 
     class Meta(TypedModelMeta):
         verbose_name = 'Archive Result'
         verbose_name_plural = 'Archive Results Log'
-        
+
 
     def __str__(self):
+        # return f'[{self.abid}] 📅 {self.start_ts.strftime("%Y-%m-%d %H:%M")} 📄 {self.extractor} {self.snapshot.url}'
         return self.extractor
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        try:
-            assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'ArchiveResult.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
-        except AssertionError as e:
-            print(e)
-
-    @property
-    def uuid(self):
-        return self.id
+    @cached_property
+    def machine(self):
+        return self.iface.machine if self.iface else None
 
     @cached_property
     def snapshot_dir(self):
@@ -489,11 +571,14 @@ class ArchiveResult(ABIDModel):
     @property
     def api_url(self) -> str:
         # /api/v1/core/archiveresult/{uulid}
-        return reverse_lazy('api-1:get_archiveresult', args=[self.abid])
-    
+        return reverse_lazy('api-1:get_archiveresult', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
+
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
+
+    def get_absolute_url(self):
+        return f'/{self.snapshot.archive_path}/{self.output_path()}'
 
     @property
     def extractor_module(self):
@@ -523,20 +608,20 @@ class ArchiveResult(ABIDModel):
 
 
     # def get_storage_dir(self, create=True, symlink=True):
-    #     date_str = self.snapshot.added.strftime('%Y%m%d')
+    #     date_str = self.snapshot.bookmarked_at.strftime('%Y%m%d')
     #     domain_str = domain(self.snapshot.url)
-    #     abs_storage_dir = Path(settings.CONFIG.ARCHIVE_DIR) / 'results' / date_str / domain_str / self.extractor / str(self.ulid)
+    #     abs_storage_dir = Path(CONSTANTS.ARCHIVE_DIR) / 'results' / date_str / domain_str / self.extractor / str(self.ulid)
 
     #     if create and not abs_storage_dir.is_dir():
     #         abs_storage_dir.mkdir(parents=True, exist_ok=True)
 
     #     if symlink:
     #         LINK_PATHS = [
-    #             Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'all_by_id' / str(self.ulid),
-    #             # Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'results_by_id' / str(self.ulid),
-    #             # Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'results_by_date' / date_str / domain_str / self.extractor / str(self.ulid),
-    #             Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'results_by_domain' / domain_str / date_str / self.extractor / str(self.ulid),
-    #             Path(settings.CONFIG.ARCHIVE_DIR).parent / 'index' / 'results_by_type' / self.extractor / date_str / domain_str / str(self.ulid),
+    #             Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'all_by_id' / str(self.ulid),
+    #             # Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'results_by_id' / str(self.ulid),
+    #             # Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'results_by_date' / date_str / domain_str / self.extractor / str(self.ulid),
+    #             Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'results_by_domain' / domain_str / date_str / self.extractor / str(self.ulid),
+    #             Path(CONSTANTS.ARCHIVE_DIR).parent / 'index' / 'results_by_type' / self.extractor / date_str / domain_str / str(self.ulid),
     #         ]
     #         for link_path in LINK_PATHS:
     #             link_path.parent.mkdir(parents=True, exist_ok=True)