fix iterator missing chunk_size

So far only for the public view, since we already have a custom search
form there, where this is easy to add.
This initial implementation supports the common set of metadata fields
that the searchbar placeholder also mentions, but adding more fields
is trivial.

.dockerignore

@@ -17,6 +17,11 @@ venv/
 .venv-old/
 .docker-venv/
 node_modules/
+chrome/
+chromeprofile/
+
+pdm.dev.lock
+pdm.lock
 
 docs/
 build/
@@ -28,4 +33,7 @@ assets/
 docker/
 
 data/
+data*/
 output/
+index.sqlite3
+index.sqlite3-wal

.gitattributes

@@ -0,0 +1,2 @@
+**/*.lock
+**/*-lock.json

.github/FUNDING.yml

@@ -1,3 +1,2 @@
-github: pirate
-patreon: theSquashSH
-custom: ["https://twitter.com/ArchiveBoxApp", "https://paypal.me/NicholasSweeting", "https://www.blockchain.com/eth/address/0x5D4c34D4a121Fe08d1dDB7969F07550f2dB9f471", "https://www.blockchain.com/btc/address/1HuxXriPE2Bbnag3jJrqa3bkNHrs297dYH"]
+github: ["ArchiveBox", "pirate"]
+custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]

.github/ISSUE_TEMPLATE/documentation_change.md

@@ -6,6 +6,7 @@ labels: ''
 assignees: ''
 
 ---
+<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
 
 ## Wiki Page URL
 <!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->

.github/dependabot.yml

@@ -0,0 +1,25 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    target-branch: "dev"
+    schedule:
+      interval: "monthly"
+    groups:
+      pip:
+        patterns:
+          - "*"
+  - package-ecosystem: "npm"
+    directory: "/"
+    target-branch: "dev"
+    schedule:
+      interval: "monthly"
+    groups:
+      npm:
+        patterns:
+          - "*"

.github/workflows/codeql-analysis.yml

@@ -1,32 +0,0 @@
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ dev ]
-  pull_request:
-    branches: [ dev ]
-  schedule:
-    - cron: '43 1 * * 2'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'python' ]
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        queries: security-extended
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+  schedule:
+    - cron: '33 17 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker.yml

@@ -11,7 +11,7 @@ on:
 
 env:
   DOCKER_IMAGE: archivebox-ci
-      
+
 jobs:
   buildx:
     runs-on: ubuntu-latest
@@ -24,21 +24,21 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
-      
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
-      
+          platforms: linux/amd64,linux/arm64
+
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
-      
+
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
-      
+
       - name: Cache Docker layers
         uses: actions/cache@v3
         with:
@@ -51,21 +51,27 @@ jobs:
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
-           username: ${{ secrets.DOCKER_USERNAME }}
-           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Collect Docker tags
+        # https://github.com/docker/metadata-action
         id: docker_meta
         uses: docker/metadata-action@v5
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
           tags: |
+              # :stable
               type=ref,event=branch
+              # :0.7.3
               type=semver,pattern={{version}}
+              # :0.7
               type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
               type=sha
-              type=raw,value=latest,enable={{is_default_branch}}
-      
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+
       - name: Build and push
         id: docker_build
         uses: docker/build-push-action@v5
@@ -77,11 +83,18 @@ jobs:
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
-       
+
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
       # This ugly bit is necessary if you don't want your cache to grow forever
       # until it hits GitHub's limit of 5GB.
       # Temp fix

.github/workflows/pip.yml

@@ -35,7 +35,7 @@ jobs:
           cache: true
 
       - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --group :all --no-self
+        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
 
       - name: Build package
         run: |

.gitignore

@@ -12,6 +12,10 @@ venv/
 .docker-venv/
 node_modules/
 
+# Ignore dev lockfiles (should always be built fresh)
+pdm.dev.lock
+requirements-dev.txt
+
 # Packaging artifacts
 .pdm-python
 .pdm-build
@@ -22,10 +26,12 @@ dist/
 
 # Data folders
 data/
-data1/
-data2/
-data3/
+data*/
 output/
+index.sqlite3
+*.sqlite*
+data.*
 
 # vim
 *.sw?
+.vscode

.gitmodules

@@ -26,3 +26,6 @@
 [submodule "archivebox/vendor/python-atomicwrites"]
 	path = archivebox/vendor/python-atomicwrites
 	url = https://github.com/untitaker/python-atomicwrites
+[submodule "archivebox/vendor/pydantic-pkgr"]
+	path = archivebox/vendor/pydantic-pkgr
+	url = https://github.com/ArchiveBox/pydantic-pkgr

.readthedocs.yaml

@@ -30,5 +30,4 @@ formats:
 # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
   install:
-    - requirements: requirements.txt
-    - requirements: docs/requirements.txt
+    - requirements: docs/requirements.txt

Dockerfile

@@ -10,7 +10,7 @@
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
 #
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
@@ -20,9 +20,23 @@ FROM python:3.11-slim-bookworm
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one personal internet archiving container" \
+    description="All-in-one self-hosted internet archiving solution" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
+    org.opencontainers.image.title="ArchiveBox" \
+    org.opencontainers.image.vendor="ArchiveBox" \
+    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
+    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'
 
 ARG TARGETPLATFORM
 ARG TARGETOS
@@ -73,7 +87,9 @@ COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
 RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
 
 # Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
     && rm -f /etc/apt/apt.conf.d/docker-clean
 
 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
@@ -106,10 +122,10 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
     && mkdir -p /etc/apt/keyrings \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    && apt-get install -qq -y -t bookworm-backports \
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
@@ -120,27 +136,13 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 
 ######### Language Environments ####################################
 
-# Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-        nodejs libatomic1 python3-minimal \
-    && rm -rf /var/lib/apt/lists/* \
-    # Update NPM to latest version
-    && npm i -g npm --cache /root/.npm \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
 # Install Python environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # && apt-get update -qq \
+    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
+    # && rm -rf /var/lib/apt/lists/* \
     # tell PDM to allow using global system python site packages
     # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
     # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
@@ -157,17 +159,37 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
+
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
+    && apt-get install -y -t bookworm-backports --no-upgrade \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
 ######### Extractor Dependencies ##################################
 
 # Install apt dependencies
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing APT extractor dependencies globally using apt..." \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    && apt-get install -qq -y -t bookworm-backports \
         curl wget git yt-dlp ffmpeg ripgrep \
         # Packages we have also needed in the past:
         # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-        # fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
     && rm -rf /var/lib/apt/lists/* \
     # Save version info
     && ( \
@@ -183,18 +205,21 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
     && apt-get update -qq \
-    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
-        # install Chromium using playwright
-        pip install playwright \
-        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-        && playwright install --with-deps chromium \
-        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
-    else \
-        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.) 
-        apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-            chromium fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        && export CHROME_BINARY="$(which chromium)"; \
-    fi \
+    && apt-get install -qq -y -t bookworm-backports \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
+        # xfonts-scalable xfonts-utils xserver-common xvfb \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
+    # install Chromium using playwright
+    && pip install playwright \
+    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+    && playwright install chromium \
+    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
     && rm -rf /var/lib/apt/lists/* \
     && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
     && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
@@ -227,7 +252,7 @@ COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    && apt-get install -qq -y -t bookworm-backports \
         build-essential \
         libssl-dev libldap2-dev libsasl2-dev \
         python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
@@ -249,8 +274,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
     # && apt-get update -qq \
     # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
-    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-    #     build-essential  \
+    # && apt-get install -qq -y -t bookworm-backports \
+        # build-essential  \
     # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
     && pip install -e "$CODE_DIR"[sonic,ldap] \
     # save docker image size and always remove compilers / build tools after building is complete
@@ -262,9 +287,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no \
+    ALLOWED_HOSTS=*
     ## No need to set explicitly, these values will be autodetected by archivebox in docker:
-    # CHROME_SANDBOX=False \
     # WGET_BINARY="wget" \
     # YOUTUBEDL_BINARY="yt-dlp" \
     # CHROME_BINARY="/usr/bin/chromium-browser" \
@@ -289,9 +319,8 @@ WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
-# Optional:
-# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-#     CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
+HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
 CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

archivebox/__init__.py

@@ -1 +1,4 @@
 __package__ = 'archivebox'
+
+
+from .monkey_patches import *

archivebox/abid_utils/__init__.py

@@ -0,0 +1 @@
+__package__ = 'abid_utils'

archivebox/abid_utils/abid.py

@@ -0,0 +1,202 @@
+from typing import NamedTuple, Any, Union, Optional
+
+import ulid
+import uuid6
+import hashlib
+from urllib.parse import urlparse
+
+from uuid import UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+
+
+
+ABID_PREFIX_LEN = 4
+ABID_SUFFIX_LEN = 26
+ABID_LEN = 30
+ABID_TS_LEN = 10
+ABID_URI_LEN = 8
+ABID_SUBTYPE_LEN = 2
+ABID_RAND_LEN = 6
+
+DEFAULT_ABID_PREFIX = 'obj_'
+
+# allows people to keep their uris secret on a per-instance basis by changing the salt.
+# the default means everyone can share the same namespace for URI hashes,
+# meaning anyone who has a URI and wants to check if you have it can guess the ABID
+DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
+
+
+class ABID(NamedTuple):
+    """
+    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
+    """
+    prefix: str            # e.g. obj_
+    ts: str                # e.g. 01HX9FPYTR
+    uri: str               # e.g. E4A5CCD9
+    subtype: str           # e.g. 01
+    rand: str              # e.g. ZYEBQE
+    
+    # salt: str = DEFAULT_ABID_URI_SALT
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self.ulid, attr)
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            return self.ulid == other.ulid
+        except AttributeError:
+            return NotImplemented
+
+    def __str__(self) -> str:
+        return self.prefix + self.suffix
+
+    def __len__(self) -> int:
+        return len(self.prefix + self.suffix)
+
+    @classmethod
+    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
+        assert buffer, f'Attempted to create ABID from null value {buffer}'
+
+        buffer = str(buffer)
+        if '_' in buffer:
+            prefix, suffix = buffer.split('_')
+        else:
+            prefix, suffix = prefix.strip('_'), buffer
+
+        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
+        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
+
+        return cls(
+            prefix=abid_part_from_prefix(prefix),
+            ts=suffix[0:10].upper(),
+            uri=suffix[10:18].upper(),
+            subtype=suffix[18:20].upper(),
+            rand=suffix[20:26].upper(),
+        )
+    
+    @property
+    def uri_salt(self) -> str:
+        return DEFAULT_ABID_URI_SALT
+
+    @property
+    def suffix(self):
+        return ''.join((self.ts, self.uri, self.subtype, self.rand))
+    
+    @property
+    def ulid(self) -> ulid.ULID:
+        return ulid.parse(self.suffix)
+
+    @property
+    def uuid(self) -> UUID:
+        return self.ulid.uuid
+
+    @property
+    def uuid6(self) -> uuid6.UUID:
+        return uuid6.UUID(hex=self.uuid.hex)
+
+    @property
+    def typeid(self) -> TypeID:
+        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
+
+    @property
+    def datetime(self) -> datetime:
+        return self.ulid.timestamp().datetime
+
+
+
+####################################################
+
+
+def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
+    """
+    if isinstance(uri, bytes):
+        uri_str: str = uri.decode()
+    else:
+        uri_str = str(uri)
+
+    # only hash the domain part of URLs
+    if '://' in uri_str:
+        try:
+            domain = urlparse(uri_str).netloc
+            if domain:
+                uri_str = domain
+        except AttributeError:
+            pass
+    
+    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
+
+    return hashlib.sha256(uri_bytes).hexdigest().upper()
+
+def abid_part_from_prefix(prefix: Optional[str]) -> str:
+    """
+    'snp_'
+    """
+    if prefix is None:
+        return 'obj_'
+
+    prefix = prefix.strip('_').lower()
+    assert len(prefix) == 3
+    return prefix + '_'
+
+def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9'     # takes first 8 characters of sha256(url)
+    """
+    uri = str(uri)
+    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
+
+def abid_part_from_ts(ts: Optional[datetime]) -> str:
+    """
+    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
+    """
+    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
+
+def abid_part_from_subtype(subtype: str) -> str:
+    """
+    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
+    Also allows us to change the ulid spec later by putting special sigil values here.
+    """
+    subtype = str(subtype)
+    if len(subtype) == ABID_SUBTYPE_LEN:
+        return subtype
+
+    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
+
+def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
+    """
+    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
+    """
+    if rand is None:
+        # if it's None we generate a new random 6 character hex string
+        return str(ulid.new())[-ABID_RAND_LEN:]
+    elif isinstance(rand, UUID):
+        # if it's a uuid we take the last 6 characters of the ULID represation of it
+        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
+    elif isinstance(rand, int):
+        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
+        rand_str = str(rand)[-ABID_RAND_LEN:]
+        padding_needed = ABID_RAND_LEN - len(rand_str)
+        rand_str = ('0'*padding_needed) + rand_str
+        return rand_str
+
+    # otherwise treat it as a string, take the last 6 characters of it verbatim
+    return str(rand)[-ABID_RAND_LEN:].upper()
+
+
+def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+
+    abid = ABID(
+        prefix=abid_part_from_prefix(prefix),
+        ts=abid_part_from_ts(ts),
+        uri=abid_part_from_uri(uri, salt=salt),
+        subtype=abid_part_from_subtype(subtype),
+        rand=abid_part_from_rand(rand),
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
+    return abid

archivebox/abid_utils/admin.py

@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+
+class ABIDModelAdmin(admin.ModelAdmin):
+    list_display = ('created', 'created_by', 'abid', '__str__')
+    sort_fields = ('created', 'created_by', 'abid', '__str__')
+    readonly_fields = ('abid', 'created', '__str__')
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if 'created_by' in form.base_fields:
+            form.base_fields['created_by'].initial = request.user
+        return form
+
+    # def save_model(self, request, obj, form, change):
+    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
+    #         obj.created_by = request.user
+    #     obj.save()

archivebox/abid_utils/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class AbidUtilsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    
+    name = 'abid_utils'

archivebox/abid_utils/models.py

@@ -0,0 +1,344 @@
+"""
+This file provides the Django ABIDField and ABIDModel base model to inherit from.
+
+It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
+"""
+
+from typing import Any, Dict, Union, List, Set, NamedTuple, cast
+
+from ulid import ULID
+from uuid import uuid4, UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+from functools import partial
+from charidfield import CharIDField  # type: ignore[import-untyped]
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+from django.db.utils import OperationalError
+from django.contrib.auth import get_user_model
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+from .abid import (
+    ABID,
+    ABID_LEN,
+    ABID_RAND_LEN,
+    ABID_SUFFIX_LEN,
+    DEFAULT_ABID_PREFIX,
+    DEFAULT_ABID_URI_SALT,
+    abid_part_from_prefix,
+    abid_from_values
+)
+
+####################################################
+
+
+# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
+ABIDField = partial(
+    CharIDField,
+    max_length=ABID_LEN,
+    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+    default=None,
+    null=True,
+    blank=True,
+    db_index=True,
+    unique=True,
+)
+
+def get_or_create_system_user_pk(username='system'):
+    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+    User = get_user_model()
+
+    # if only one user exists total, return that user
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+    # otherwise, create a dedicated "system" user
+    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    return user.pk
+
+
+class ABIDModel(models.Model):
+    """
+    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
+    """
+    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
+    abid_ts_src = 'None'                    # e.g. 'self.created'
+    abid_uri_src = 'None'                   # e.g. 'self.uri'
+    abid_subtype_src = 'None'               # e.g. 'self.extractor'
+    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
+
+    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args: Any, **kwargs: Any) -> None:
+        # when first creating a row, self.ABID is the source of truth
+        # overwrite default prefilled self.id & self.abid with generated self.ABID value
+        if self._state.adding or not self.id:
+            self.id = self.ABID.uuid
+        if self._state.adding or not self.abid:
+            self.abid = str(self.ABID)
+
+        super().save(*args, **kwargs)
+        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
+        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
+
+    @property
+    def abid_values(self) -> Dict[str, Any]:
+        return {
+            'prefix': self.abid_prefix,
+            'ts': eval(self.abid_ts_src),
+            'uri': eval(self.abid_uri_src),
+            'subtype': eval(self.abid_subtype_src),
+            'rand': eval(self.abid_rand_src),
+        }
+
+    def generate_abid(self) -> ABID:
+        """
+        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+        """
+        prefix, ts, uri, subtype, rand = self.abid_values.values()
+
+        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
+            suggested_abid = self.__class__.__name__[:3].lower()
+            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+        if not ts:
+            # default to unix epoch with 00:00:00 UTC
+            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
+            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+        if not uri:
+            uri = str(self)
+            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+        if not subtype:
+            subtype = self.__class__.__name__
+            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+        if not rand:
+            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+        abid = abid_from_values(
+            prefix=prefix,
+            ts=ts,
+            uri=uri,
+            subtype=subtype,
+            rand=rand,
+            salt=DEFAULT_ABID_URI_SALT,
+        )
+        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+        return abid
+
+    @property
+    def ABID(self) -> ABID:
+        """
+        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
+        """
+        
+        # if object is not yet saved to DB, always generate fresh ABID from values
+        if self._state.adding:
+            return self.generate_abid()
+        
+        # otherwise DB is single source of truth, load ABID from existing db pk
+        abid: ABID | None = None
+        try:
+            abid = abid or ABID.parse(self.pk)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(self.id)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(cast(str, self.abid))
+        except Exception:
+            pass
+
+        abid = abid or self.generate_abid()
+
+        return abid
+
+    @property
+    def ULID(self) -> ULID:
+        """
+        Get a ulid.ULID representation of the object's ABID.
+        """
+        return self.ABID.ulid
+
+    @property
+    def UUID(self) -> UUID:
+        """
+        Get a uuid.UUID (v4) representation of the object's ABID.
+        """
+        return self.ABID.uuid
+
+    @property
+    def TypeID(self) -> TypeID:
+        """
+        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+        """
+        return self.ABID.typeid
+
+
+
+####################################################
+
+# Django helpers
+def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
+    """
+    Return the mapping of all ABID prefixes to their models.
+    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
+    """
+    import django.apps
+    prefix_map = {}
+
+    for model in django.apps.apps.get_models():
+        abid_prefix = getattr(model, 'abid_prefix', None)
+        if abid_prefix:
+            prefix_map[abid_prefix] = model
+    return prefix_map
+
+def find_prefix_for_abid(abid: ABID) -> str:
+    """
+    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
+    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
+    """
+    # if existing abid prefix is correct, lookup is easy
+    model = find_model_from_abid(abid)
+    if model:
+        assert issubclass(model, ABIDModel)
+        return model.abid_prefix
+
+    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
+    return find_obj_from_abid_rand(abid)[0].abid_prefix
+
+def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
+    """
+    Return the Django Model that corresponds to a given ABID prefix.
+    e.g. 'tag_' -> core.models.Tag
+    """
+    prefix = abid_part_from_prefix(prefix)
+
+    import django.apps
+
+    for model in django.apps.apps.get_models():
+        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
+        if not hasattr(model, 'objects'): continue      # skip abstract models
+
+        if (model.abid_prefix == prefix):
+            return model
+
+    return None
+
+def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
+    """
+    Shortcut for find_model_from_abid_prefix(abid.prefix)
+    """
+    return find_model_from_abid_prefix(abid.prefix)
+
+def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
+    """
+    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
+    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    # convert str to ABID if necessary
+    if isinstance(rand, ABID):
+        abid: ABID = rand
+    else:
+        rand = str(rand)
+        if len(rand) < ABID_SUFFIX_LEN:
+            padding_needed = ABID_SUFFIX_LEN - len(rand)
+            rand = ('0'*padding_needed) + rand
+        abid = ABID.parse(rand)
+
+    import django.apps
+
+    partial_matches: List[ABIDModel] = []
+
+    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
+        model,
+        find_model_from_abid(abid),
+        *django.apps.apps.get_models(),
+    ))))
+    # print(abid, abid.rand, abid.uuid, models_to_try)
+
+    for model in models_to_try:
+        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
+        if not hasattr(model, 'objects'): continue      # skip abstract Models
+        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
+
+        # continue on to try fuzzy searching by randomness portion derived from uuid field
+        try:
+            qs = []
+            if hasattr(model, 'abid'):
+                qs = model.objects.filter(abid__endswith=abid.rand)
+            elif hasattr(model, 'uuid'):
+                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+            elif hasattr(model, 'id'):
+                # NOTE: this only works on SQLite where every column is a string
+                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
+                
+                # try to search for uuid=...-2354352
+                # try to search for id=...2354352
+                # try to search for id=2354352
+                qs = model.objects.filter(
+                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+                    | models.Q(id__endswith=abid.rand)
+                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
+                )
+
+            for obj in qs:
+                if obj.generate_abid() == abid:
+                    # found exact match, no need to keep iterating
+                    return [obj]
+                partial_matches.append(obj)
+        except OperationalError as err:
+            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
+
+    return partial_matches
+
+def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
+    """
+    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
+    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    model = model or find_model_from_abid(abid)
+    assert model, f'Could not find model that could match this ABID type: {abid}'
+
+    try:
+        if hasattr(model, 'abid'):
+            return model.objects.get(abid__endswith=abid.suffix)
+        if hasattr(model, 'uuid'):
+            return model.objects.get(uuid=abid.uuid)
+        return model.objects.get(id=abid.uuid)
+    except model.DoesNotExist:
+        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
+        if hasattr(model, 'abid') or (not fuzzy):
+            raise
+
+    # continue on to try fuzzy searching by randomness portion derived from uuid field
+    match_by_rand = find_obj_from_abid_rand(abid, model=model)
+    if match_by_rand:
+        if match_by_rand[0].abid_prefix != abid.prefix:
+            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
+        return match_by_rand
+
+    raise model.DoesNotExist
+

archivebox/abid_utils/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

archivebox/api/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.api'

archivebox/api/apps.py

@@ -0,0 +1,11 @@
+__package__ = 'archivebox.api'
+
+from django.apps import AppConfig
+
+
+
+class APIConfig(AppConfig):
+    name = 'api'
+
+    def ready(self):
+        pass

archivebox/api/auth.py

@@ -0,0 +1,110 @@
+__package__ = 'archivebox.api'
+
+from typing import Optional, cast
+
+from django.http import HttpRequest
+from django.contrib.auth import login
+from django.contrib.auth import authenticate
+from django.contrib.auth.models import AbstractBaseUser
+
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
+
+
+def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
+    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
+    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    
+    user = None
+
+    submitted_empty_form = token in ('string', '', None)
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
+        try:
+            token = APIToken.objects.get(token=token)
+            if token.is_valid():
+                user = token.created_by
+        except APIToken.DoesNotExist:
+            pass
+
+    if not user:
+        print('[❌] Failed to authenticate API user using API Key:', request)
+
+    return None
+
+def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
+    """Given a username and password, check if they are valid and return the corresponding user"""
+    user = None
+    
+    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
+        user = authenticate(
+            username=username,
+            password=password,
+        )
+
+    if not user:
+        print('[❌] Failed to authenticate API user using API Key:', request)
+        user = None
+
+    return cast(AbstractBaseUser | None, user)
+
+
+### Base Auth Types
+
+class APITokenAuthCheck:
+    """The base class for authentication methods that use an api.models.APIToken"""
+    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        user = auth_using_token(
+            token=key,
+            request=request,
+        )
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user
+
+class UserPassAuthCheck:
+    """The base class for authentication methods that use a username & password"""
+    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        user = auth_using_password(
+            username=username,
+            password=password,
+            request=request,
+        )
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user
+
+
+### Django-Ninja-Provided Auth Methods
+
+class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
+    """Allow authenticating by passing X-API-Key=xyz as a request header"""
+    param_name = "X-ArchiveBox-API-Key"
+
+class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
+    """Allow authenticating by passing Bearer=xyz as a request header"""
+    pass
+
+class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
+    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
+    param_name = "api_key"
+
+class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
+    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
+    pass
+
+
+### Enabled Auth Methods
+
+API_AUTH_METHODS = [
+    HeaderTokenAuth(),
+    BearerTokenAuth(),
+    QueryParamTokenAuth(), 
+    django_auth_superuser,
+    UsernameAndPasswordAuth(),
+]

archivebox/api/migrations/0001_initial.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.11 on 2024-04-25 04:19
+
+import api.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='APIToken',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expires', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

archivebox/api/migrations/0002_alter_apitoken_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.4 on 2024-04-26 05:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='apitoken',
+            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
+        ),
+    ]

archivebox/api/migrations/0003_rename_user_apitoken_created_by_apitoken_abid_and_more.py

@@ -0,0 +1,77 @@
+# Generated by Django 5.0.6 on 2024-06-03 01:52
+
+import abid_utils.models
+import charidfield.fields
+import django.db.models.deletion
+import signal_webhooks.fields
+import signal_webhooks.utils
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0002_alter_apitoken_options'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='user',
+            new_name='created_by',
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
+        ),
+        migrations.CreateModel(
+            name='OutboundWebhook',
+            fields=[
+                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
+                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
+                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
+                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
+                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
+                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
+                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
+                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
+                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
+                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
+                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
+                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('id', models.UUIDField(blank=True, null=True, unique=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
+                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'verbose_name': 'API Outbound Webhook',
+                'abstract': False,
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='outboundwebhook',
+            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
+        ),
+    ]

archivebox/api/migrations/0004_alter_apitoken_id_alter_apitoken_uuid.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.1 on 2024-08-20 10:44
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
+        ),
+    ]

archivebox/api/migrations/0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:40
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='apitoken',
+            name='uuid',
+        ),
+        migrations.RemoveField(
+            model_name='outboundwebhook',
+            name='id',
+        ),
+    ]

archivebox/api/migrations/0006_remove_outboundwebhook_uuid_apitoken_id_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.1 on 2024-08-20 22:43
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='uuid',
+            new_name='id'
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+    ]

archivebox/api/migrations/0007_alter_apitoken_created_by.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:52
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]

archivebox/api/models.py

@@ -0,0 +1,122 @@
+__package__ = 'archivebox.api'
+
+import uuid
+import secrets
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+
+from signal_webhooks.models import WebhookBase
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
+
+
+
+def generate_secret_token() -> str:
+    # returns cryptographically secure string with len() == 32
+    return secrets.token_hex(16)
+
+
+class APIToken(ABIDModel):
+    """
+    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
+    """
+    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
+    abid_prefix = 'apt_'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.token'
+    abid_subtype_src = 'self.created_by_id'
+    abid_rand_src = 'self.id'
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+
+    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
+    expires = models.DateTimeField(null=True, blank=True)
+
+    class Meta(TypedModelMeta):
+        verbose_name = "API Key"
+        verbose_name_plural = "API Keys"
+
+    def __str__(self) -> str:
+        return self.token
+
+    def __repr__(self) -> str:
+        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
+
+    def __json__(self) -> dict:
+        return {
+            "TYPE":             "APIToken",    
+            "id":               str(self.pk),
+            "abid":             str(self.ABID),
+            "created_by_id":    str(self.created_by_id),
+            "token":            self.token,
+            "created":          self.created.isoformat(),
+            "expires":          self.expires_as_iso8601,
+        }
+
+    @property
+    def expires_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
+
+        return expiry_date.isoformat()
+    
+    @property
+    def token_redacted(self):
+        return f'************{self.token[-4:]}'
+
+    def is_valid(self, for_date=None):
+        for_date = for_date or timezone.now()
+
+        if self.expires and self.expires < for_date:
+            return False
+
+        return True
+
+
+
+
+
+
+# monkey patch django-signals-webhooks to change how it shows up in Admin UI
+
+class OutboundWebhook(ABIDModel, WebhookBase):
+    """
+    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
+        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
+    """
+    abid_prefix = 'whk_'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.endpoint'
+    abid_subtype_src = 'self.ref'
+    abid_rand_src = 'self.id'
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    # More fields here: WebhookBase...
+
+    WebhookBase._meta.get_field('name').help_text = (
+        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
+    WebhookBase._meta.get_field('signal').help_text = (
+        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
+    WebhookBase._meta.get_field('ref').help_text = (
+        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
+    WebhookBase._meta.get_field('endpoint').help_text = (
+        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
+
+    class Meta(WebhookBase.Meta):
+        verbose_name = 'API Outbound Webhook'
+

archivebox/api/tests.py

@@ -0,0 +1,30 @@
+__package__ = 'archivebox.api'
+
+from django.test import TestCase
+from ninja.testing import TestClient
+
+from .routes_cli import router
+
+class ArchiveBoxCLIAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(router)
+
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])

archivebox/api/urls.py

@@ -0,0 +1,17 @@
+__package__ = 'archivebox.api'
+
+from django.urls import path
+from django.views.generic.base import RedirectView
+
+from .v1_api import urls as v1_api_urls
+
+urlpatterns = [
+    path("",                 RedirectView.as_view(url='/api/v1')),
+
+    path("v1/",              v1_api_urls),
+    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
+
+    # ... v2 can be added here ...
+    # path("v2/",              v2_api_urls),
+    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
+]

archivebox/api/v1_api.py

@@ -0,0 +1,111 @@
+__package__ = 'archivebox.api'
+
+
+from io import StringIO
+from traceback import format_exception
+from contextlib import redirect_stdout, redirect_stderr
+
+from django.http import HttpRequest, HttpResponse
+from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
+
+from ninja import NinjaAPI, Swagger
+
+# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
+
+from api.auth import API_AUTH_METHODS
+from ..config import VERSION, COMMIT_HASH
+
+
+COMMIT_HASH = COMMIT_HASH or 'unknown'
+
+html_description=f'''
+<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
+<br/>
+<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
+<br/>
+<ul>
+<li>⬅️ Manage your server: <a href="/admin/api/"><b>Setup API Keys</b></a>, <a href="/admin/">Go to your Server Admin UI</a>, <a href="/">Go to your Snapshots list</a> 
+<li>💬 Ask questions and get help here: <a href="https://zulip.archivebox.io">ArchiveBox Chat Forum</a></li>
+<li>🐞 Report API bugs here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues">Github Issues</a></li>
+<li>📚 ArchiveBox Documentation: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Github Wiki</a></li>
+<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
+</ul>
+<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
+'''
+
+
+def register_urls(api: NinjaAPI) -> NinjaAPI:
+    api.add_router('/auth/',     'api.v1_auth.router')
+    api.add_router('/core/',     'api.v1_core.router')
+    api.add_router('/cli/',      'api.v1_cli.router')
+    return api
+
+
+class NinjaAPIWithIOCapture(NinjaAPI):    
+    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
+        stdout, stderr = StringIO(), StringIO()
+
+        with redirect_stderr(stderr):
+            with redirect_stdout(stdout):
+                request.stdout = stdout
+                request.stderr = stderr
+
+                response = super().create_temporal_response(request)
+
+        print('RESPONDING NOW', response)
+
+        return response
+
+
+api = NinjaAPIWithIOCapture(
+    title='ArchiveBox API',
+    description=html_description,
+    version='1.0.0',
+    csrf=False,
+    auth=API_AUTH_METHODS,
+    urls_namespace="api-1",
+    docs=Swagger(settings={"persistAuthorization": True}),
+    # docs_decorator=login_required,
+    # renderer=ORJSONRenderer(),
+)
+api = register_urls(api)
+urls = api.urls
+
+
+@api.exception_handler(Exception)
+def generic_exception_handler(request, err):
+    status = 503
+    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
+        status = 404
+
+    print(''.join(format_exception(err)))
+
+    return api.create_response(
+        request,
+        {
+            "succeeded": False,
+            "message": f'{err.__class__.__name__}: {err}',
+            "errors": [
+                ''.join(format_exception(err)),
+                # or send simpler parent-only traceback:
+                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
+            ],
+        },
+        status=status,
+    )
+
+
+
+# import orjson
+# from ninja.renderers import BaseRenderer
+# class ORJSONRenderer(BaseRenderer):
+#     media_type = "application/json"
+#     def render(self, request, data, *, response_status):
+#         return {
+#             "success": True,
+#             "errors": [],
+#             "result": data,
+#             "stdout": ansi_to_html(stdout.getvalue().strip()),
+#             "stderr": ansi_to_html(stderr.getvalue().strip()),
+#         }
+#         return orjson.dumps(data)

archivebox/api/v1_auth.py

@@ -0,0 +1,52 @@
+__package__ = 'archivebox.api'
+
+from typing import Optional
+
+from ninja import Router, Schema
+
+from api.models import APIToken
+from api.auth import auth_using_token, auth_using_password
+
+
+router = Router(tags=['Authentication'])
+
+
+class PasswordAuthSchema(Schema):
+    """Schema for a /get_api_token request"""
+    username: Optional[str] = None
+    password: Optional[str] = None
+
+
+@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
+def get_api_token(request, auth_data: PasswordAuthSchema):
+    user = auth_using_password(
+        username=auth_data.username,
+        password=auth_data.password,
+        request=request,
+    )
+
+    if user:
+        # TODO: support multiple tokens in the future, for now we just have one per user
+        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
+
+        return api_token.__json__()
+    
+    return {"success": False, "errors": ["Invalid credentials"]}
+
+
+
+class TokenAuthSchema(Schema):
+    """Schema for a /check_api_token request"""
+    token: str
+
+
+@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
+def check_api_token(request, token_data: TokenAuthSchema):
+    user = auth_using_token(
+        token=token_data.token,
+        request=request,
+    )
+    if user:
+        return {"success": True, "user_id": str(user.pk)}
+    
+    return {"success": False, "user_id": None}

archivebox/api/v1_cli.py

@@ -0,0 +1,234 @@
+__package__ = 'archivebox.api'
+
+from typing import List, Dict, Any, Optional
+from enum import Enum
+
+from ninja import Router, Schema
+
+from ..main import (
+    add,
+    remove,
+    update,
+    list_all,
+    schedule,
+)
+from ..util import ansi_to_html
+from ..config import ONLY_NEW
+
+
+# router for API that exposes archivebox cli subcommands as REST endpoints
+router = Router(tags=['ArchiveBox CLI Sub-Commands'])
+
+
+# Schemas
+
+JSONType = List[Any] | Dict[str, Any] | bool | int | str | None
+
+class CLICommandResponseSchema(Schema):
+    success: bool
+    errors: List[str]
+    result: JSONType
+    stdout: str
+    stderr: str
+
+class FilterTypeChoices(str, Enum):
+    exact = 'exact'
+    substring = 'substring'
+    regex = 'regex'
+    domain = 'domain'
+    tag = 'tag'
+    timestamp = 'timestamp'
+
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'
+
+
+class AddCommandSchema(Schema):
+    urls: List[str]
+    tag: str = ""
+    depth: int = 0
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
+    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+class UpdateCommandSchema(Schema):
+    resume: Optional[float] = 0
+    only_new: bool = ONLY_NEW
+    index_only: bool = False
+    overwrite: bool = False
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    status: Optional[StatusChoices] = StatusChoices.unarchived
+    filter_type: Optional[str] = FilterTypeChoices.substring
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    extractors: Optional[str] = ""
+
+class ScheduleCommandSchema(Schema):
+    import_path: Optional[str] = None
+    add: bool = False
+    every: Optional[str] = None
+    tag: str = ''
+    depth: int = 0
+    overwrite: bool = False
+    update: bool = not ONLY_NEW
+    clear: bool = False
+
+class ListCommandSchema(Schema):
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_type: str = FilterTypeChoices.substring
+    status: Optional[StatusChoices] = StatusChoices.indexed
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    sort: str = 'added'
+    as_json: bool = True
+    as_html: bool = False
+    as_csv: str | bool = 'timestamp,url'
+    with_headers: bool = False
+
+class RemoveCommandSchema(Schema):
+    delete: bool = True
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    filter_type: str = FilterTypeChoices.exact
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+
+
+
+
+
+@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
+def cli_add(request, args: AddCommandSchema):
+    result = add(
+        urls=args.urls,
+        tag=args.tag,
+        depth=args.depth,
+        update=args.update,
+        update_all=args.update_all,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        init=args.init,
+        extractors=args.extractors,
+        parser=args.parser,
+    )
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+
+
+@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
+def cli_update(request, args: UpdateCommandSchema):
+    result = update(
+        resume=args.resume,
+        only_new=args.only_new,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        before=args.before,
+        after=args.after,
+        status=args.status,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
+        extractors=args.extractors,
+    )
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+
+
+@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
+def cli_schedule(request, args: ScheduleCommandSchema):
+    result = schedule(
+        import_path=args.import_path,
+        add=args.add,
+        show=args.show,
+        clear=args.clear,
+        every=args.every,
+        tag=args.tag,
+        depth=args.depth,
+        overwrite=args.overwrite,
+        update=args.update,
+    )
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+
+
+
+@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
+def cli_list(request, args: ListCommandSchema):
+    result = list_all(
+        filter_patterns=args.filter_patterns,
+        filter_type=args.filter_type,
+        status=args.status,
+        after=args.after,
+        before=args.before,
+        sort=args.sort,
+        csv=args.as_csv,
+        json=args.as_json,
+        html=args.as_html,
+        with_headers=args.with_headers,
+    )
+
+    result_format = 'txt'
+    if args.as_json:
+        result_format = "json"
+    elif args.as_html:
+        result_format = "html"
+    elif args.as_csv:
+        result_format = "csv"
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "result_format": result_format,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+    
+
+
+@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
+def cli_remove(request, args: RemoveCommandSchema):
+    result = remove(
+        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
+        delete=args.delete,
+        before=args.before,
+        after=args.after,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
+    )
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+    

archivebox/api/v1_core.py

@@ -0,0 +1,424 @@
+__package__ = 'archivebox.api'
+
+import math
+from uuid import UUID
+from typing import List, Optional, Union, Any
+from datetime import datetime
+
+from django.db.models import Q
+from django.shortcuts import get_object_or_404
+from django.core.exceptions import ValidationError
+from django.contrib.auth import get_user_model
+
+from ninja import Router, Schema, FilterSchema, Field, Query
+from ninja.pagination import paginate, PaginationBase
+
+from core.models import Snapshot, ArchiveResult, Tag
+from abid_utils.abid import ABID
+
+router = Router(tags=['Core Models'])
+
+
+
+class CustomPagination(PaginationBase):
+    class Input(Schema):
+        limit: int = 200
+        offset: int = 0
+        page: int = 0
+
+
+    class Output(Schema):
+        total_items: int
+        total_pages: int
+        page: int
+        limit: int
+        offset: int
+        num_items: int
+        items: List[Any]
+
+    def paginate_queryset(self, queryset, pagination: Input, **params):
+        limit = min(pagination.limit, 500)
+        offset = pagination.offset or (pagination.page * limit)
+        total = queryset.count()
+        total_pages = math.ceil(total / limit)
+        current_page = math.ceil(offset / (limit + 1))
+        items = queryset[offset : offset + limit]
+        return {
+            'total_items': total,
+            'total_pages': total_pages,
+            'page': current_page,
+            'limit': limit,
+            'offset': offset,
+            'num_items': len(items),
+            'items': items,
+        }
+
+
+### ArchiveResult #########################################################################
+
+class ArchiveResultSchema(Schema):
+    TYPE: str = 'core.models.ArchiveResult'
+
+    id: UUID
+    old_id: int
+    abid: str
+
+    modified: datetime
+    created: datetime
+    created_by_id: str
+    created_by_username: str
+
+    snapshot_abid: str
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: str
+
+    extractor: str
+    cmd_version: Optional[str]
+    cmd: List[str]
+    pwd: str
+    status: str
+    output: str
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
+    @staticmethod
+    def resolve_pk(obj):
+        return str(obj.pk)
+
+    @staticmethod
+    def resolve_uuid(obj):
+        return str(obj.uuid)
+
+    @staticmethod
+    def resolve_abid(obj):
+        return str(obj.ABID)
+
+    @staticmethod
+    def resolve_created(obj):
+        return obj.start_ts
+
+    @staticmethod
+    def resolve_snapshot_timestamp(obj):
+        return obj.snapshot.timestamp
+    
+    @staticmethod
+    def resolve_snapshot_url(obj):
+        return obj.snapshot.url
+
+    @staticmethod
+    def resolve_snapshot_abid(obj):
+        return str(obj.snapshot.ABID)
+
+    @staticmethod
+    def resolve_snapshot_tags(obj):
+        return obj.snapshot.tags_str()
+
+
+class ArchiveResultFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
+    snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
+    
+    status: Optional[str] = Field(None, q='status')
+    output: Optional[str] = Field(None, q='output__icontains')
+    extractor: Optional[str] = Field(None, q='extractor__icontains')
+    cmd: Optional[str] = Field(None, q='cmd__0__icontains')
+    pwd: Optional[str] = Field(None, q='pwd__icontains')
+    cmd_version: Optional[str] = Field(None, q='cmd_version')
+
+    created: Optional[datetime] = Field(None, q='updated')
+    created__gte: Optional[datetime] = Field(None, q='updated__gte')
+    created__lt: Optional[datetime] = Field(None, q='updated__lt')
+
+
+@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
+@paginate(CustomPagination)
+def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
+    """List all ArchiveResult entries matching these filters."""
+    qs = ArchiveResult.objects.all()
+    results = filters.filter(qs).distinct()
+    return results
+
+
+@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
+def get_archiveresult(request, archiveresult_id: str):
+    """Get a specific ArchiveResult by pk, abid, or old_id."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
+
+
+# @router.post("/archiveresult", response=ArchiveResultSchema)
+# def create_archiveresult(request, payload: ArchiveResultSchema):
+#     archiveresult = ArchiveResult.objects.create(**payload.dict())
+#     return archiveresult
+#
+# @router.put("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
+# def update_archiveresult(request, archiveresult_id: str, payload: ArchiveResultSchema):
+#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
+#   
+#     for attr, value in payload.dict().items():
+#         setattr(archiveresult, attr, value)
+#     archiveresult.save()
+#
+#     return archiveresult
+#
+# @router.delete("/archiveresult/{archiveresult_id}")
+# def delete_archiveresult(request, archiveresult_id: str):
+#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
+#     archiveresult.delete()
+#     return {"success": True}
+
+
+
+
+
+### Snapshot #########################################################################
+
+
+class SnapshotSchema(Schema):
+    TYPE: str = 'core.models.Snapshot'
+
+    id: UUID
+    old_id: UUID
+    abid: str
+
+    modified: datetime
+    created: datetime
+    created_by_id: str
+    created_by_username: str
+
+    url: str
+    tags: str
+    title: Optional[str]
+    timestamp: str
+    archive_path: str
+
+    bookmarked: datetime
+    added: datetime
+    updated: Optional[datetime]
+
+    num_archiveresults: int
+    archiveresults: List[ArchiveResultSchema]
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
+    @staticmethod
+    def resolve_pk(obj):
+        return str(obj.pk)
+
+    @staticmethod
+    def resolve_uuid(obj):
+        return str(obj.uuid)
+
+    @staticmethod
+    def resolve_abid(obj):
+        return str(obj.ABID)
+
+    @staticmethod
+    def resolve_tags(obj):
+        return obj.tags_str()
+
+    @staticmethod
+    def resolve_num_archiveresults(obj, context):
+        return obj.archiveresult_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_archiveresults(obj, context):
+        if context['request'].with_archiveresults:
+            return obj.archiveresult_set.all().distinct()
+        return ArchiveResult.objects.none()
+
+
+class SnapshotFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+
+    old_id: Optional[str] = Field(None, q='old_id__icontains')
+    abid: Optional[str] = Field(None, q='abid__icontains')
+
+    created_by_id: str = Field(None, q='created_by_id')
+    created_by_username: str = Field(None, q='created_by__username__icontains')
+
+    created__gte: datetime = Field(None, q='created__gte')
+    created__lt: datetime = Field(None, q='created__lt')
+    created: datetime = Field(None, q='created')
+    modified: datetime = Field(None, q='modified')
+    modified__gte: datetime = Field(None, q='modified__gte')
+    modified__lt: datetime = Field(None, q='modified__lt')
+
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+    url: Optional[str] = Field(None, q='url')
+    tag: Optional[str] = Field(None, q='tags__name')
+    title: Optional[str] = Field(None, q='title__icontains')
+    timestamp: Optional[str] = Field(None, q='timestamp__startswith')
+    
+    added__gte: Optional[datetime] = Field(None, q='added__gte')
+    added__lt: Optional[datetime] = Field(None, q='added__lt')
+
+
+
+@router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
+@paginate(CustomPagination)
+def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
+    """List all Snapshot entries matching these filters."""
+    request.with_archiveresults = with_archiveresults
+
+    qs = Snapshot.objects.all()
+    results = filters.filter(qs).distinct()
+    return results
+
+@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
+def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
+    """Get a specific Snapshot by abid, uuid, or pk."""
+    request.with_archiveresults = with_archiveresults
+    snapshot = None
+    try:
+        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+    except Snapshot.DoesNotExist:
+        pass
+
+    try:
+        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
+    except Snapshot.DoesNotExist:
+        pass
+
+    if not snapshot:
+        raise Snapshot.DoesNotExist
+
+    return snapshot
+
+
+# @router.post("/snapshot", response=SnapshotSchema)
+# def create_snapshot(request, payload: SnapshotSchema):
+#     snapshot = Snapshot.objects.create(**payload.dict())
+#     return snapshot
+#
+# @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
+# def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
+#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#
+#     for attr, value in payload.dict().items():
+#         setattr(snapshot, attr, value)
+#     snapshot.save()
+#
+#     return snapshot
+#
+# @router.delete("/snapshot/{snapshot_id}")
+# def delete_snapshot(request, snapshot_id: str):
+#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot.delete()
+#     return {"success": True}
+
+
+
+### Tag #########################################################################
+
+
+class TagSchema(Schema):
+    TYPE: str = 'core.models.Tag'
+
+    id: UUID
+    old_id: str
+    abid: str
+
+    modified: datetime
+    created: datetime
+    created_by_id: str
+    created_by_username: str
+
+    name: str
+    slug: str
+    num_snapshots: int
+    snapshots: List[SnapshotSchema]
+
+    @staticmethod
+    def resolve_old_id(obj):
+        return str(obj.old_id)
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+    
+    @staticmethod
+    def resolve_num_snapshots(obj, context):
+        return obj.snapshot_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if context['request'].with_snapshots:
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+@router.get("/tags", response=List[TagSchema], url_name="get_tags")
+@paginate(CustomPagination)
+def get_tags(request):
+    request.with_snapshots = False
+    request.with_archiveresults = False
+    return Tag.objects.all().distinct()
+
+@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
+def get_tag(request, tag_id: str, with_snapshots: bool=True):
+    request.with_snapshots = with_snapshots
+    request.with_archiveresults = False
+    tag = None
+    try:
+        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError, ValueError):
+        pass
+
+    try:
+        tag = Tag.objects.get(abid__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        pass
+
+    try:
+        tag = tag or Tag.objects.get(id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        pass
+    return tag
+
+
+
+@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
+def get_any(request, abid: str):
+    request.with_snapshots = False
+    request.with_archiveresults = False
+
+    response = None
+    try:
+        response = response or get_snapshot(request, abid)
+    except Exception:
+        pass
+
+    try:
+        response = response or get_archiveresult(request, abid)
+    except Exception:
+        pass
+
+    try:
+        response = response or get_tag(request, abid)
+    except Exception:
+        pass
+
+    return response

archivebox/builtin_plugins/base/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

archivebox/builtin_plugins/base/apps.py

@@ -0,0 +1,82 @@
+import sys
+import inspect
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+import django
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+from django.db.backends.sqlite3.base import Database as sqlite3
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+from plugantic.replayers import Replayer
+
+
+class PythonBinary(Binary):
+    name: BinName = 'python'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides: Dict[str, Any] = {
+        'env': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(Binary):
+    name: BinName = 'sqlite'
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(sqlite3),
+            'version': \
+                lambda: sqlite3.version,
+        },
+    }
+
+class DjangoBinary(Binary):
+    name: BinName = 'django'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+class BasicReplayer(Replayer):
+    name: str = 'basic'
+
+
+class BasePlugin(Plugin):
+    name: str = 'base'
+    configs: List[SerializeAsAny[ConfigSet]] = []
+    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = []
+    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
+
+
+PLUGINS = [BasePlugin()]
+
+
+class BaseConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'builtin_plugins.base'

archivebox/builtin_plugins/base/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

archivebox/builtin_plugins/base/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

archivebox/builtin_plugins/base/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

archivebox/builtin_plugins/singlefile/apps.py

@@ -0,0 +1,94 @@
+from typing import List, Optional
+from pathlib import Path
+
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+
+
+###################### Config ##########################
+
+class SinglefileToggleConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+    SAVE_SINGLEFILE: bool = True
+
+
+class SinglefileDependencyConfig(ConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileOptionsConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+    # loaded from shared config
+    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+
+DEFAULT_CONFIG = {
+    'CHECK_SSL_VALIDITY': False,
+    'SAVE_SINGLEFILE': True,
+    'TIMEOUT': 120,
+}
+
+PLUGIN_CONFIG = [
+    SinglefileToggleConfig(**DEFAULT_CONFIG),
+    SinglefileDependencyConfig(**DEFAULT_CONFIG),
+    SinglefileOptionsConfig(**DEFAULT_CONFIG),
+]
+
+###################### Binaries ############################
+
+class SinglefileBinary(Binary):
+    name: BinName = 'single-file'
+    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
+
+
+###################### Extractors ##########################
+
+class SinglefileExtractor(Extractor):
+    name: ExtractorName = 'singlefile'
+    binary: Binary = SinglefileBinary()
+
+    def get_output_path(self, snapshot) -> Path:
+        return Path(snapshot.link_dir) / 'singlefile.html'
+
+
+###################### Plugins #############################
+
+
+class SinglefilePlugin(Plugin):
+    name: str = 'singlefile'
+    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
+    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
+
+PLUGINS = [SinglefilePlugin()]
+
+###################### Django Apps #########################
+
+class SinglefileConfig(AppConfig):
+    name = 'builtin_plugins.singlefile'
+    verbose_name = 'SingleFile'
+
+    def ready(self):
+        print('Loaded singlefile plugin')

archivebox/builtin_plugins/singlefile/config.yaml

@@ -0,0 +1,66 @@
+name: singlefile
+plugin_version: '0.0.1'
+plugin_spec: '0.0.1'
+
+binaries:
+    singlefile:
+        providers:
+            - env
+            - npm
+
+commands:
+    - singlefile.exec
+    - singlefile.extract
+    - singlefile.should_extract
+    - singlefile.get_output_path
+
+extractors:
+    singlefile:
+        binary: singlefile
+        test: singlefile.should_extract
+        extract: singlefile.extract
+        output_files:
+            - singlefile.html
+
+configs:
+    ARCHIVE_METHOD_TOGGLES:
+        SAVE_SINGLEFILE:
+            type: bool
+            default: true
+
+    DEPENDENCY_CONFIG:
+        SINGLEFILE_BINARY:
+            type: str
+            default: wget
+        SINGLEFILE_ARGS:
+            type: Optional[List[str]]
+            default: null
+        SINGLEFILE_EXTRA_ARGS:
+            type: List[str]
+            default: []
+        SINGLEFILE_DEFAULT_ARGS:
+            type: List[str]
+            default: 
+            - "--timeout={TIMEOUT-10}"
+
+    ARCHIVE_METHOD_OPTIONS:
+        SINGLEFILE_USER_AGENT:
+            type: str
+            default: ""
+            alias: USER_AGENT
+        SINGLEFILE_TIMEOUT:
+            type: int
+            default: 60
+            alias: TIMEOUT
+        SINGLEFILE_CHECK_SSL_VALIDITY:
+            type: bool
+            default: true
+            alias: CHECK_SSL_VALIDITY
+        SINGLEFILE_RESTRICT_FILE_NAMES:
+            type: str
+            default: windows
+            alias: RESTRICT_FILE_NAMES
+        SINGLEFILE_COOKIES_FILE:
+            type: Optional[Path]
+            default: null
+            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

archivebox/cli/__init__.py

@@ -4,14 +4,18 @@ __command__ = 'archivebox'
 import os
 import sys
 import argparse
+import threading
+from time import sleep
 
-from typing import Optional, Dict, List, IO, Union
+from typing import Optional, Dict, List, IO, Union, Iterable
 from pathlib import Path
 
-from ..config import OUTPUT_DIR, check_data_folder, check_migrations
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations, stderr
 
 from importlib import import_module
 
+BUILTIN_LIST = list
+
 CLI_DIR = Path(__file__).resolve().parent
 
 # these common commands will appear sorted before any others for ease-of-use
@@ -33,6 +37,40 @@ is_valid_cli_module = lambda module, subcommand: (
 )
 
 
+IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting
+
+
+def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
+    """
+    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
+    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
+    """
+
+    wait_for_all: bool = thread_names == ()
+
+    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
+
+    should_wait = lambda thread: (
+        not thread_matches(thread, ignore_names)
+        and (wait_for_all or thread_matches(thread, thread_names)))
+
+    for tries in range(timeout):
+        all_threads = [*threading.enumerate()]
+        blocking_threads = [*filter(should_wait, all_threads)]
+        threads_summary = ', '.join(repr(t) for t in blocking_threads)
+        if blocking_threads:
+            sleep(1)
+            if tries == 5:                            # only show stderr message if we need to wait more than 5s
+                stderr(
+                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
+                    threads_summary,
+                )
+        else:
+            return tries
+
+    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')
+
+
 def list_subcommands() -> Dict[str, str]:
     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
 
@@ -79,6 +117,9 @@ def run_subcommand(subcommand: str,
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
 
+    # wait for webhooks, signals, and other background jobs to finish before exit
+    wait_for_bg_threads_to_exit(timeout=60)
+
 
 SUBCOMMANDS = list_subcommands()
 

archivebox/config.py

@@ -31,19 +31,23 @@ import getpass
 import platform
 import shutil
 import requests
-import django
-from sqlite3 import dbapi2 as sqlite3
 
 from hashlib import md5
 from pathlib import Path
 from datetime import datetime, timezone
-from typing import Optional, Type, Tuple, Dict, Union, List
-from subprocess import run, PIPE, DEVNULL
+from typing import Optional, Type, Tuple, Dict, Union, List, Any
+from subprocess import run, PIPE, DEVNULL, STDOUT
 from configparser import ConfigParser
 from collections import defaultdict
 import importlib.metadata
 
+from pydantic_pkgr import SemVer
+
+import django
+from django.db.backends.sqlite3.base import Database as sqlite3
+
 from .config_stubs import (
+    AttrDict,
     SimpleConfigValueDict,
     ConfigValue,
     ConfigDict,
@@ -51,6 +55,11 @@ from .config_stubs import (
     ConfigDefaultDict,
 )
 
+# load fallback libraries from vendor dir
+from .vendor import load_vendored_libs
+load_vendored_libs()
+
+
 
 ############################### Config Schema ##################################
 
@@ -72,7 +81,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'TIMEOUT':                  {'type': int,   'default': 60},
         'MEDIA_TIMEOUT':            {'type': int,   'default': 3600},
         'OUTPUT_PERMISSIONS':       {'type': str,   'default': '644'},
-        'RESTRICT_FILE_NAMES':      {'type': str,   'default': 'windows'},
+        'RESTRICT_FILE_NAMES':      {'type': str,   'default': 'windows'},  # TODO: move this to be a default WGET_ARGS
 
         'URL_DENYLIST':             {'type': str,   'default': r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', 'aliases': ('URL_BLACKLIST',)},  # to avoid downloading code assets as their own pages
         'URL_ALLOWLIST':            {'type': str,   'default': None, 'aliases': ('URL_WHITELIST',)},
@@ -87,13 +96,14 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
     'SERVER_CONFIG': {
         'SECRET_KEY':                {'type': str,   'default': None},
         'BIND_ADDR':                 {'type': str,   'default': lambda c: ['127.0.0.1:8000', '0.0.0.0:8000'][c['IN_DOCKER']]},
-        'ALLOWED_HOSTS':             {'type': str,   'default': '*'},
+        'ALLOWED_HOSTS':             {'type': str,   'default': '*'},     # e.g. archivebox.example.com,archivebox2.example.com
+        'CSRF_TRUSTED_ORIGINS':      {'type': str,   'default': lambda c: 'http://localhost:8000,http://127.0.0.1:8000,http://0.0.0.0:8000,http://{}'.format(c['BIND_ADDR'])},   # e.g. https://archivebox.example.com,https://archivebox2.example.com:8080
         'DEBUG':                     {'type': bool,  'default': False},
         'PUBLIC_INDEX':              {'type': bool,  'default': True},
         'PUBLIC_SNAPSHOTS':          {'type': bool,  'default': True},
         'PUBLIC_ADD_VIEW':           {'type': bool,  'default': False},
         'FOOTER_INFO':               {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
-        'SNAPSHOTS_PER_PAGE':        {'type': int,   'default': 40},
+        'SNAPSHOTS_PER_PAGE':        {'type': int,   'default': 100},
         'CUSTOM_TEMPLATES_DIR':      {'type': str,   'default': None},
         'TIME_ZONE':                 {'type': str,   'default': 'UTC'},
         'TIMEZONE':                  {'type': str,   'default': 'UTC'},
@@ -112,6 +122,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'LDAP_FIRSTNAME_ATTR':       {'type': str,   'default': None},
         'LDAP_LASTNAME_ATTR':        {'type': str,   'default': None},
         'LDAP_EMAIL_ATTR':           {'type': str,   'default': None},
+        'LDAP_CREATE_SUPERUSER':     {'type': bool,  'default': False},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -136,14 +147,15 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
     },
 
     'ARCHIVE_METHOD_OPTIONS': {
-        'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION',)},
-        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com'},
+        'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION','WINDOW_SIZE')},
+        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht'},
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
         'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'USER_AGENT':               {'type': str,   'default': None},
+        'CURL_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
@@ -151,7 +163,11 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CHROME_TIMEOUT':           {'type': int,   'default': 0},
         'CHROME_HEADLESS':          {'type': bool,  'default': True},
         'CHROME_SANDBOX':           {'type': bool,  'default': lambda c: not c['IN_DOCKER']},
+        'CHROME_EXTRA_ARGS':        {'type': list,  'default': None},
+
         'YOUTUBEDL_ARGS':           {'type': list,  'default': lambda c: [
+                                                                '--restrict-filenames',
+                                                                '--trim-filenames', '128',
                                                                 '--write-description',
                                                                 '--write-info-json',
                                                                 '--write-annotations',
@@ -173,6 +189,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--add-metadata',
                                                                 '--format=(bv*+ba/b)[filesize<={}][filesize_approx<=?{}]/(bv*+ba/b)'.format(c['MEDIA_MAX_SIZE'], c['MEDIA_MAX_SIZE']),
                                                                 ]},
+        'YOUTUBEDL_EXTRA_ARGS':     {'type': list,  'default': None},
 
 
         'WGET_ARGS':                {'type': list,  'default': ['--no-verbose',
@@ -184,12 +201,17 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
                                                                 '--no-parent',
                                                                 '-e', 'robots=off',
                                                                 ]},
+        'WGET_EXTRA_ARGS':          {'type': list,  'default': None},
         'CURL_ARGS':                {'type': list,  'default': ['--silent',
                                                                 '--location',
                                                                 '--compressed'
                                                                ]},
+        'CURL_EXTRA_ARGS':          {'type': list,  'default': None},
         'GIT_ARGS':                 {'type': list,  'default': ['--recursive']},
-        'SINGLEFILE_ARGS':          {'type': list,  'default' : None},
+        'SINGLEFILE_ARGS':          {'type': list,  'default': None},
+        'SINGLEFILE_EXTRA_ARGS':    {'type': list,  'default': None},
+        'MERCURY_ARGS':             {'type': list,  'default': ['--format=text']},
+        'MERCURY_EXTRA_ARGS':       {'type': list,  'default': None},
         'FAVICON_PROVIDER':         {'type': str,   'default': 'https://www.google.com/s2/favicons?domain={}'},
     },
 
@@ -253,7 +275,7 @@ CONFIG_ALIASES = {
         for key, default in section.items()
             for alias in default.get('aliases', ())
 }
-USER_CONFIG = {key for section in CONFIG_SCHEMA.values() for key in section.keys()}
+USER_CONFIG = {key: section[key] for section in CONFIG_SCHEMA.values() for key in section.keys()}
 
 def get_real_name(key: str) -> str:
     """get the current canonical name for a given deprecated config key"""
@@ -269,6 +291,9 @@ TEMPLATES_DIR_NAME = 'templates'
 ARCHIVE_DIR_NAME = 'archive'
 SOURCES_DIR_NAME = 'sources'
 LOGS_DIR_NAME = 'logs'
+CACHE_DIR_NAME = 'cache'
+PERSONAS_DIR_NAME = 'personas'
+CRONTABS_DIR_NAME = 'crontabs'
 SQL_INDEX_FILENAME = 'index.sqlite3'
 JSON_INDEX_FILENAME = 'index.json'
 HTML_INDEX_FILENAME = 'index.html'
@@ -342,9 +367,12 @@ ALLOWED_IN_OUTPUT_DIR = {
     'static',
     'sonic',
     'search.sqlite3',
+    CRONTABS_DIR_NAME,
     ARCHIVE_DIR_NAME,
     SOURCES_DIR_NAME,
     LOGS_DIR_NAME,
+    CACHE_DIR_NAME,
+    PERSONAS_DIR_NAME,
     SQL_INDEX_FILENAME,
     f'{SQL_INDEX_FILENAME}-wal',
     f'{SQL_INDEX_FILENAME}-shm',
@@ -361,26 +389,57 @@ ALLOWED_IN_OUTPUT_DIR = {
 ALLOWDENYLIST_REGEX_FLAGS: int = re.IGNORECASE | re.UNICODE | re.MULTILINE
 
 
+CONSTANTS = {
+    "PACKAGE_DIR_NAME":             {'default': lambda c: PACKAGE_DIR_NAME},
+    "TEMPLATES_DIR_NAME":           {'default': lambda c: TEMPLATES_DIR_NAME},
+    "ARCHIVE_DIR_NAME":             {'default': lambda c: ARCHIVE_DIR_NAME},
+    "SOURCES_DIR_NAME":             {'default': lambda c: SOURCES_DIR_NAME},
+    "LOGS_DIR_NAME":                {'default': lambda c: LOGS_DIR_NAME},
+    "CACHE_DIR_NAME":               {'default': lambda c: CACHE_DIR_NAME},
+    "PERSONAS_DIR_NAME":            {'default': lambda c: PERSONAS_DIR_NAME},
+    "CRONTABS_DIR_NAME":            {'default': lambda c: CRONTABS_DIR_NAME},
+    "SQL_INDEX_FILENAME":           {'default': lambda c: SQL_INDEX_FILENAME},
+    "JSON_INDEX_FILENAME":          {'default': lambda c: JSON_INDEX_FILENAME},
+    "HTML_INDEX_FILENAME":          {'default': lambda c: HTML_INDEX_FILENAME},
+    "ROBOTS_TXT_FILENAME":          {'default': lambda c: ROBOTS_TXT_FILENAME},
+    "FAVICON_FILENAME":             {'default': lambda c: FAVICON_FILENAME},
+    "CONFIG_FILENAME":              {'default': lambda c: CONFIG_FILENAME},
+    "DEFAULT_CLI_COLORS":           {'default': lambda c: DEFAULT_CLI_COLORS},
+    "ANSI":                         {'default': lambda c: ANSI},
+    "COLOR_DICT":                   {'default': lambda c: COLOR_DICT},
+    "STATICFILE_EXTENSIONS":        {'default': lambda c: STATICFILE_EXTENSIONS},
+    "ALLOWED_IN_OUTPUT_DIR":        {'default': lambda c: ALLOWED_IN_OUTPUT_DIR},
+    "ALLOWDENYLIST_REGEX_FLAGS":    {'default': lambda c: ALLOWDENYLIST_REGEX_FLAGS},
+}
+
 ############################## Version Config ##################################
 
-def get_system_user():
-    SYSTEM_USER = getpass.getuser() or os.getlogin()
+def get_system_user() -> str:
+    # some host OS's are unable to provide a username (k3s, Windows), making this complicated
+    # uid 999 is especially problematic and breaks many attempts
+    SYSTEM_USER = None
+    FALLBACK_USER_PLACHOLDER = f'user_{os.getuid()}'
+
+    # Option 1
     try:
         import pwd
-        return pwd.getpwuid(os.geteuid()).pw_name or SYSTEM_USER
-    except KeyError:
-        # Process' UID might not map to a user in cases such as running the Docker image
-        # (where `archivebox` is 999) as a different UID.
-        pass
-    except ModuleNotFoundError:
-        # pwd doesn't exist on windows
-        pass
-    except Exception:
-        # this should never happen, uncomment to debug
-        # raise
+        SYSTEM_USER = SYSTEM_USER or pwd.getpwuid(os.geteuid()).pw_name
+    except (ModuleNotFoundError, Exception):
         pass
 
-    return SYSTEM_USER
+    # Option 2
+    try:
+        SYSTEM_USER = SYSTEM_USER or getpass.getuser()
+    except Exception:
+        pass
+
+    # Option 3
+    try:
+        SYSTEM_USER = SYSTEM_USER or os.getlogin()
+    except Exception:
+        pass
+
+    return SYSTEM_USER or FALLBACK_USER_PLACHOLDER
 
 def get_version(config):
     try:
@@ -472,9 +531,13 @@ def can_upgrade(config):
 
 ############################## Derived Config ##################################
 
+
+
 # These are derived/computed values calculated *after* all user-provided config values are ingested
 # they appear in `archivebox config` output and are intended to be read-only for the user
 DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
+    **CONSTANTS,
+
     'TERM_WIDTH':               {'default': lambda c: lambda: shutil.get_terminal_size((100, 10)).columns},
     'USER':                     {'default': lambda c: get_system_user()},
     'ANSI':                     {'default': lambda c: DEFAULT_CLI_COLORS if c['USE_COLOR'] else {k: '' for k in DEFAULT_CLI_COLORS.keys()}},
@@ -487,9 +550,11 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'ARCHIVE_DIR':              {'default': lambda c: c['OUTPUT_DIR'] / ARCHIVE_DIR_NAME},
     'SOURCES_DIR':              {'default': lambda c: c['OUTPUT_DIR'] / SOURCES_DIR_NAME},
     'LOGS_DIR':                 {'default': lambda c: c['OUTPUT_DIR'] / LOGS_DIR_NAME},
+    'CACHE_DIR':                {'default': lambda c: c['OUTPUT_DIR'] / CACHE_DIR_NAME},
+    'PERSONAS_DIR':             {'default': lambda c: c['OUTPUT_DIR'] / PERSONAS_DIR_NAME},
     'CONFIG_FILE':              {'default': lambda c: Path(c['CONFIG_FILE']).resolve() if c['CONFIG_FILE'] else c['OUTPUT_DIR'] / CONFIG_FILENAME},
     'COOKIES_FILE':             {'default': lambda c: c['COOKIES_FILE'] and Path(c['COOKIES_FILE']).resolve()},
-    'CHROME_USER_DATA_DIR':     {'default': lambda c: find_chrome_data_dir() if c['CHROME_USER_DATA_DIR'] is None else (Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None)},   # None means unset, so we autodetect it with find_chrome_Data_dir(), but emptystring '' means user manually set it to '', and we should store it as None
+    'CHROME_USER_DATA_DIR':     {'default': lambda c: Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None},
     'URL_DENYLIST_PTN':         {'default': lambda c: c['URL_DENYLIST'] and re.compile(c['URL_DENYLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
     'URL_ALLOWLIST_PTN':        {'default': lambda c: c['URL_ALLOWLIST'] and re.compile(c['URL_ALLOWLIST'] or '', ALLOWDENYLIST_REGEX_FLAGS)},
     'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},  # exec is always needed to list directories
@@ -508,7 +573,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'PYTHON_VERSION':           {'default': lambda c: '{}.{}.{}'.format(*sys.version_info[:3])},
 
     'DJANGO_BINARY':            {'default': lambda c: inspect.getfile(django)},
-    'DJANGO_VERSION':           {'default': lambda c: '{}.{}.{} {} ({})'.format(*django.VERSION)},
+    'DJANGO_VERSION':           {'default': lambda c: '{}.{}.{}'.format(*django.VERSION[:3])},
     
     'SQLITE_BINARY':            {'default': lambda c: inspect.getfile(sqlite3)},
     'SQLITE_VERSION':           {'default': lambda c: sqlite3.version},
@@ -519,6 +584,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CURL_VERSION':             {'default': lambda c: bin_version(c['CURL_BINARY']) if c['USE_CURL'] else None},
     'CURL_USER_AGENT':          {'default': lambda c: c['CURL_USER_AGENT'].format(**c)},
     'CURL_ARGS':                {'default': lambda c: c['CURL_ARGS'] or []},
+    'CURL_EXTRA_ARGS':          {'default': lambda c: c['CURL_EXTRA_ARGS'] or []},
     'SAVE_FAVICON':             {'default': lambda c: c['USE_CURL'] and c['SAVE_FAVICON']},
     'SAVE_ARCHIVE_DOT_ORG':     {'default': lambda c: c['USE_CURL'] and c['SAVE_ARCHIVE_DOT_ORG']},
 
@@ -529,18 +595,22 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'SAVE_WGET':                {'default': lambda c: c['USE_WGET'] and c['SAVE_WGET']},
     'SAVE_WARC':                {'default': lambda c: c['USE_WGET'] and c['SAVE_WARC']},
     'WGET_ARGS':                {'default': lambda c: c['WGET_ARGS'] or []},
+    'WGET_EXTRA_ARGS':          {'default': lambda c: c['WGET_EXTRA_ARGS'] or []},
 
     'RIPGREP_VERSION':          {'default': lambda c: bin_version(c['RIPGREP_BINARY']) if c['USE_RIPGREP'] else None},
 
     'USE_SINGLEFILE':           {'default': lambda c: c['USE_SINGLEFILE'] and c['SAVE_SINGLEFILE']},
     'SINGLEFILE_VERSION':       {'default': lambda c: bin_version(c['SINGLEFILE_BINARY']) if c['USE_SINGLEFILE'] else None},
     'SINGLEFILE_ARGS':          {'default': lambda c: c['SINGLEFILE_ARGS'] or []},
+    'SINGLEFILE_EXTRA_ARGS':    {'default': lambda c: c['SINGLEFILE_EXTRA_ARGS'] or []},
 
     'USE_READABILITY':          {'default': lambda c: c['USE_READABILITY'] and c['SAVE_READABILITY']},
     'READABILITY_VERSION':      {'default': lambda c: bin_version(c['READABILITY_BINARY']) if c['USE_READABILITY'] else None},
 
     'USE_MERCURY':              {'default': lambda c: c['USE_MERCURY'] and c['SAVE_MERCURY']},
     'MERCURY_VERSION':          {'default': lambda c: '1.0.0' if shutil.which(str(bin_path(c['MERCURY_BINARY']))) else None},  # mercury doesnt expose version info until this is merged https://github.com/postlight/parser/pull/750
+    'MERCURY_ARGS':             {'default': lambda c: c['MERCURY_ARGS'] or []},
+    'MERCURY_EXTRA_ARGS':       {'default': lambda c: c['MERCURY_EXTRA_ARGS'] or []},
 
     'USE_GIT':                  {'default': lambda c: c['USE_GIT'] and c['SAVE_GIT']},
     'GIT_VERSION':              {'default': lambda c: bin_version(c['GIT_BINARY']) if c['USE_GIT'] else None},
@@ -550,6 +620,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'YOUTUBEDL_VERSION':        {'default': lambda c: bin_version(c['YOUTUBEDL_BINARY']) if c['USE_YOUTUBEDL'] else None},
     'SAVE_MEDIA':               {'default': lambda c: c['USE_YOUTUBEDL'] and c['SAVE_MEDIA']},
     'YOUTUBEDL_ARGS':           {'default': lambda c: c['YOUTUBEDL_ARGS'] or []},
+    'YOUTUBEDL_EXTRA_ARGS':     {'default': lambda c: c['YOUTUBEDL_EXTRA_ARGS'] or []},
 
     'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] or find_chrome_binary()},
     'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and c['CHROME_BINARY'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
@@ -568,9 +639,9 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
 
     'DEPENDENCIES':             {'default': lambda c: get_dependency_info(c)},
     'CODE_LOCATIONS':           {'default': lambda c: get_code_locations(c)},
-    'EXTERNAL_LOCATIONS':       {'default': lambda c: get_external_locations(c)},
     'DATA_LOCATIONS':           {'default': lambda c: get_data_locations(c)},
     'CHROME_OPTIONS':           {'default': lambda c: get_chrome_info(c)},
+    'CHROME_EXTRA_ARGS':        {'default': lambda c: c['CHROME_EXTRA_ARGS'] or []},
     'SAVE_ALLOWLIST_PTN':       {'default': lambda c: c['SAVE_ALLOWLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_ALLOWLIST'].items()}},
     'SAVE_DENYLIST_PTN':        {'default': lambda c: c['SAVE_DENYLIST'] and {re.compile(k, ALLOWDENYLIST_REGEX_FLAGS): v for k, v in c['SAVE_DENYLIST'].items()}},
 }
@@ -644,28 +715,29 @@ def load_config_val(key: str,
     raise Exception('Config values can only be str, bool, int, or json')
 
 
-def load_config_file(out_dir: str=None) -> Optional[Dict[str, str]]:
+def load_config_file(out_dir: str | None=None) -> Optional[ConfigDict]:
     """load the ini-formatted config file from OUTPUT_DIR/Archivebox.conf"""
 
     out_dir = out_dir or Path(os.getenv('OUTPUT_DIR', '.')).resolve()
+    assert out_dir and out_dir.is_dir()
     config_path = Path(out_dir) / CONFIG_FILENAME
     if config_path.exists():
         config_file = ConfigParser()
         config_file.optionxform = str
         config_file.read(config_path)
         # flatten into one namespace
-        config_file_vars = {
+        config_file_vars = ConfigDict({
             key.upper(): val
             for section, options in config_file.items()
                 for key, val in options.items()
-        }
+        })
         # print('[i] Loaded config file', os.path.abspath(config_path))
         # print(config_file_vars)
         return config_file_vars
     return None
 
 
-def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
+def write_config_file(config: Dict[str, str], out_dir: str | None=None) -> ConfigDict:
     """load the ini-formatted config file from OUTPUT_DIR/Archivebox.conf"""
 
     from .system import atomic_write
@@ -706,7 +778,7 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
             existing_config = dict(config_file[section])
         else:
             existing_config = {}
-        config_file[section] = {**existing_config, key: val}
+        config_file[section] = ConfigDict({**existing_config, key: val})
 
     # always make sure there's a SECRET_KEY defined for Django
     existing_secret_key = None
@@ -781,7 +853,7 @@ def load_config(defaults: ConfigDefaultDict,
             # raise
             raise SystemExit(2)
 
-    return extended_config
+    return AttrDict(extended_config)
 
 
 def parse_version_string(version: str) -> Tuple[int, int, int]:
@@ -823,7 +895,7 @@ def hint(text: Union[Tuple[str, ...], List[str], str], prefix='    ', config: Op
 
 
 # Dependency Metadata Helpers
-def bin_version(binary: Optional[str]) -> Optional[str]:
+def bin_version(binary: Optional[str], cmd: Optional[str]=None) -> Optional[str]:
     """check the presence and return valid version line of a specified binary"""
 
     abspath = bin_path(binary)
@@ -832,11 +904,15 @@ def bin_version(binary: Optional[str]) -> Optional[str]:
 
     try:
         bin_env = os.environ | {'LANG': 'C'}
-        version_str = run([abspath, "--version"], stdout=PIPE, env=bin_env).stdout.strip().decode()
+        is_cmd_str = cmd and isinstance(cmd, str)
+        version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT, env=bin_env).stdout.strip().decode()
         if not version_str:
-            version_str = run([abspath, "--version"], stdout=PIPE).stdout.strip().decode()
+            version_str = run(cmd or [abspath, "--version"], shell=is_cmd_str, stdout=PIPE, stderr=STDOUT).stdout.strip().decode()
+        
         # take first 3 columns of first line of version info
-        return ' '.join(version_str.split('\n')[0].strip().split()[:3])
+        semver = SemVer.parse(version_str)
+        if semver:
+            return str(semver)
     except OSError:
         pass
         # stderr(f'[X] Unable to find working version of dependency: {binary}', color='red')
@@ -899,27 +975,36 @@ def find_chrome_binary() -> Optional[str]:
 
 def find_chrome_data_dir() -> Optional[str]:
     """find any installed chrome user data directories in the default locations"""
-    # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
-    # make sure data dir finding precedence order always matches binary finding order
-    default_profile_paths = (
-        '~/.config/chromium',
-        '~/Library/Application Support/Chromium',
-        '~/AppData/Local/Chromium/User Data',
-        '~/.config/chrome',
-        '~/.config/google-chrome',
-        '~/Library/Application Support/Google/Chrome',
-        '~/AppData/Local/Google/Chrome/User Data',
-        '~/.config/google-chrome-stable',
-        '~/.config/google-chrome-beta',
-        '~/Library/Application Support/Google/Chrome Canary',
-        '~/AppData/Local/Google/Chrome SxS/User Data',
-        '~/.config/google-chrome-unstable',
-        '~/.config/google-chrome-dev',
-    )
-    for path in default_profile_paths:
-        full_path = Path(path).resolve()
-        if full_path.exists():
-            return full_path
+    # deprecated because this is DANGEROUS, do not re-implement/uncomment this behavior.
+
+    # Going forward we want to discourage people from using their main chrome profile for archiving.
+    # Session tokens, personal data, and cookies are often returned in server responses,
+    # when they get archived, they are essentially burned as anyone who can view the archive
+    # can use that data to masquerade as the logged-in user that did the archiving.
+    # For this reason users should always create dedicated burner profiles for archiving and not use
+    # their daily driver main accounts.
+
+    # # Precedence: Chromium, Chrome, Beta, Canary, Unstable, Dev
+    # # make sure data dir finding precedence order always matches binary finding order
+    # default_profile_paths = (
+    #     '~/.config/chromium',
+    #     '~/Library/Application Support/Chromium',
+    #     '~/AppData/Local/Chromium/User Data',
+    #     '~/.config/chrome',
+    #     '~/.config/google-chrome',
+    #     '~/Library/Application Support/Google/Chrome',
+    #     '~/AppData/Local/Google/Chrome/User Data',
+    #     '~/.config/google-chrome-stable',
+    #     '~/.config/google-chrome-beta',
+    #     '~/Library/Application Support/Google/Chrome Canary',
+    #     '~/AppData/Local/Google/Chrome SxS/User Data',
+    #     '~/.config/google-chrome-unstable',
+    #     '~/.config/google-chrome-dev',
+    # )
+    # for path in default_profile_paths:
+    #     full_path = Path(path).resolve()
+    #     if full_path.exists():
+    #         return full_path
     return None
 
 def wget_supports_compression(config):
@@ -945,11 +1030,6 @@ def get_code_locations(config: ConfigDict) -> SimpleConfigValueDict:
             'enabled': True,
             'is_valid': (config['TEMPLATES_DIR'] / 'static').exists(),
         },
-        'CUSTOM_TEMPLATES_DIR': {
-            'path': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).resolve(),
-            'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
-            'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
-        },
         # 'NODE_MODULES_DIR': {
         #     'path': ,
         #     'enabled': ,
@@ -957,45 +1037,25 @@ def get_code_locations(config: ConfigDict) -> SimpleConfigValueDict:
         # },
     }
 
-def get_external_locations(config: ConfigDict) -> ConfigValue:
-    abspath = lambda path: None if path is None else Path(path).resolve()
-    return {
-        'CHROME_USER_DATA_DIR': {
-            'path': abspath(config['CHROME_USER_DATA_DIR']),
-            'enabled': config['USE_CHROME'] and config['CHROME_USER_DATA_DIR'],
-            'is_valid': False if config['CHROME_USER_DATA_DIR'] is None else (Path(config['CHROME_USER_DATA_DIR']) / 'Default').exists(),
-        },
-        'COOKIES_FILE': {
-            'path': abspath(config['COOKIES_FILE']),
-            'enabled': config['USE_WGET'] and config['COOKIES_FILE'],
-            'is_valid': False if config['COOKIES_FILE'] is None else Path(config['COOKIES_FILE']).exists(),
-        },
-    }
-
 def get_data_locations(config: ConfigDict) -> ConfigValue:
     return {
+        # OLD: migrating to personas
+        # 'CHROME_USER_DATA_DIR': {
+        #     'path': os.path.abspath(config['CHROME_USER_DATA_DIR']),
+        #     'enabled': config['USE_CHROME'] and config['CHROME_USER_DATA_DIR'],
+        #     'is_valid': False if config['CHROME_USER_DATA_DIR'] is None else (Path(config['CHROME_USER_DATA_DIR']) / 'Default').exists(),
+        # },
+        # 'COOKIES_FILE': {
+        #     'path': os.path.abspath(config['COOKIES_FILE']),
+        #     'enabled': config['USE_WGET'] and config['COOKIES_FILE'],
+        #     'is_valid': False if config['COOKIES_FILE'] is None else Path(config['COOKIES_FILE']).exists(),
+        # },
         'OUTPUT_DIR': {
             'path': config['OUTPUT_DIR'].resolve(),
             'enabled': True,
             'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(),
             'is_mount': os.path.ismount(config['OUTPUT_DIR'].resolve()),
         },
-        'SOURCES_DIR': {
-            'path': config['SOURCES_DIR'].resolve(),
-            'enabled': True,
-            'is_valid': config['SOURCES_DIR'].exists(),
-        },
-        'LOGS_DIR': {
-            'path': config['LOGS_DIR'].resolve(),
-            'enabled': True,
-            'is_valid': config['LOGS_DIR'].exists(),
-        },
-        'ARCHIVE_DIR': {
-            'path': config['ARCHIVE_DIR'].resolve(),
-            'enabled': True,
-            'is_valid': config['ARCHIVE_DIR'].exists(),
-            'is_mount': os.path.ismount(config['ARCHIVE_DIR'].resolve()),
-        },
         'CONFIG_FILE': {
             'path': config['CONFIG_FILE'].resolve(),
             'enabled': True,
@@ -1007,6 +1067,43 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'is_valid': (config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).exists(),
             'is_mount': os.path.ismount((config['OUTPUT_DIR'] / SQL_INDEX_FILENAME).resolve()),
         },
+        'ARCHIVE_DIR': {
+            'path': config['ARCHIVE_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['ARCHIVE_DIR'].exists(),
+            'is_mount': os.path.ismount(config['ARCHIVE_DIR'].resolve()),
+        },
+        'SOURCES_DIR': {
+            'path': config['SOURCES_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['SOURCES_DIR'].exists(),
+        },
+        'PERSONAS_DIR': {
+            'path': config['PERSONAS_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['PERSONAS_DIR'].exists(),
+        },
+        'LOGS_DIR': {
+            'path': config['LOGS_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['LOGS_DIR'].exists(),
+        },
+        'CACHE_DIR': {
+            'path': config['CACHE_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['CACHE_DIR'].exists(),
+        },
+        'CUSTOM_TEMPLATES_DIR': {
+            'path': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).resolve(),
+            'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
+            'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
+        },
+        # managed by bin/docker_entrypoint.sh and python-crontab:
+        # 'CRONTABS_DIR': {
+        #     'path': config['CRONTABS_DIR'].resolve(),
+        #     'enabled': True,
+        #     'is_valid': config['CRONTABS_DIR'].exists(),
+        # },
     }
 
 def get_dependency_info(config: ConfigDict) -> ConfigValue:
@@ -1143,14 +1240,14 @@ def get_chrome_info(config: ConfigDict) -> ConfigValue:
 
 
 def load_all_config():
-    CONFIG: ConfigDict = {}
+    CONFIG: ConfigDict = ConfigDict()
     for section_name, section_config in CONFIG_SCHEMA.items():
         CONFIG = load_config(section_config, CONFIG)
 
     return load_config(DYNAMIC_CONFIG_SCHEMA, CONFIG)
 
 # add all final config values in CONFIG to globals in this file
-CONFIG = load_all_config()
+CONFIG: ConfigDict = load_all_config()
 globals().update(CONFIG)
 # this lets us do:  from .config import DEBUG, MEDIA_TIMEOUT, ...
 
@@ -1241,7 +1338,7 @@ def check_system_config(config: ConfigDict=CONFIG) -> None:
 
     # stderr('[i] Using Chrome binary: {}'.format(shutil.which(CHROME_BINARY) or CHROME_BINARY))
     # stderr('[i] Using Chrome data dir: {}'.format(os.path.abspath(CHROME_USER_DATA_DIR)))
-    if config['CHROME_USER_DATA_DIR'] is not None:
+    if config['CHROME_USER_DATA_DIR'] is not None and Path(config['CHROME_USER_DATA_DIR']).exists():
         if not (Path(config['CHROME_USER_DATA_DIR']) / 'Default').exists():
             stderr('[X] Could not find profile "Default" in CHROME_USER_DATA_DIR.', color='red')
             stderr(f'    {config["CHROME_USER_DATA_DIR"]}')
@@ -1251,8 +1348,13 @@ def check_system_config(config: ConfigDict=CONFIG) -> None:
             if '/Default' in str(config['CHROME_USER_DATA_DIR']):
                 stderr()
                 stderr('    Try removing /Default from the end e.g.:')
-                stderr('        CHROME_USER_DATA_DIR="{}"'.format(config['CHROME_USER_DATA_DIR'].split('/Default')[0]))
-            raise SystemExit(2)
+                stderr('        CHROME_USER_DATA_DIR="{}"'.format(str(config['CHROME_USER_DATA_DIR']).split('/Default')[0]))
+            
+            # hard error is too annoying here, instead just set it to nothing
+            # raise SystemExit(2)
+            config['CHROME_USER_DATA_DIR'] = None
+    else:
+        config['CHROME_USER_DATA_DIR'] = None
 
 
 def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None:
@@ -1321,6 +1423,7 @@ def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=C
         stderr('        archivebox init')
         raise SystemExit(2)
 
+
 def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CONFIG):
     output_dir = out_dir or config['OUTPUT_DIR']
     from .index.sql import list_migrations
@@ -1337,6 +1440,9 @@ def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CO
 
     (Path(output_dir) / SOURCES_DIR_NAME).mkdir(exist_ok=True)
     (Path(output_dir) / LOGS_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / CACHE_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / PERSONAS_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / PERSONAS_DIR_NAME / 'Default').mkdir(exist_ok=True)
 
 
 
@@ -1419,5 +1525,17 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
             assert sql_index_path.exists(), (
                 f'No database file {SQL_INDEX_FILENAME} found in: {config["OUTPUT_DIR"]} (Are you in an ArchiveBox collection directory?)')
 
+
+            # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
+            if settings.DEBUG_LOGFIRE:
+                from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
+                SQLite3Instrumentor().instrument()
+
+                import logfire
+
+                logfire.configure()
+                logfire.instrument_django(is_sql_commentor_enabled=True)
+                logfire.info(f'Started ArchiveBox v{CONFIG.VERSION}', argv=sys.argv)
+
     except KeyboardInterrupt:
         raise SystemExit(2)

archivebox/config_stubs.py

@@ -9,11 +9,15 @@ SimpleConfigValueDict = Dict[str, SimpleConfigValue]
 SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
 ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
 
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
 
 class BaseConfig(TypedDict):
     pass
 
-class ConfigDict(BaseConfig, total=False):
+class ConfigDict(BaseConfig, AttrDict, total=False):
     """
     # Regenerate by pasting this quine into `archivebox shell` 🥚
     from archivebox.config import ConfigDict, CONFIG_DEFAULTS
@@ -28,6 +32,7 @@ class ConfigDict(BaseConfig, total=False):
                 print(f'    {key}: {Type.__name__}')
         print()
     """
+
     IS_TTY: bool
     USE_COLOR: bool
     SHOW_PROGRESS: bool

archivebox/core/__init__.py

@@ -1 +1,2 @@
 __package__ = 'archivebox.core'
+

archivebox/core/admin.py

@@ -1,31 +1,52 @@
 __package__ = 'archivebox.core'
 
+import os
+import json
+
 from io import StringIO
 from pathlib import Path
 from contextlib import redirect_stdout
 from datetime import datetime, timezone
+from typing import Dict, Any
 
 from django.contrib import admin
-from django.urls import path
+from django.db.models import Count, Q, Prefetch
+from django.urls import path, reverse, resolve
+from django.utils import timezone
+from django.utils.functional import cached_property
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
+from django.core.paginator import Paginator
+from django.core.exceptions import ValidationError
+from django.conf import settings
 from django import forms
 
+
+from signal_webhooks.admin import WebhookAdmin
+from signal_webhooks.utils import get_webhook_model
+# from plugantic.admin import CustomPlugin
+
 from ..util import htmldecode, urldecode, ansi_to_html
 
-from core.models import Snapshot, ArchiveResult, Tag
+from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
 from core.forms import AddLinkForm
-
 from core.mixins import SearchResultsAdminMixin
+from api.models import APIToken
+from abid_utils.models import get_or_create_system_user_pk
+from abid_utils.admin import ABIDModelAdmin
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
-from config import OUTPUT_DIR, SNAPSHOTS_PER_PAGE
 from extractors import archive_links
 
+
+CONFIG = settings.CONFIG
+
+GLOBAL_CONTEXT = {'VERSION': CONFIG.VERSION, 'VERSIONS_AVAILABLE': CONFIG.VERSIONS_AVAILABLE, 'CAN_UPGRADE': CONFIG.CAN_UPGRADE}
+
 # Admin URLs
 # /admin/
 # /admin/login/
@@ -39,62 +60,301 @@ from extractors import archive_links
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
 
+class ArchiveBoxAdmin(admin.AdminSite):
+    site_header = 'ArchiveBox'
+    index_title = 'Links'
+    site_title = 'Index'
+    namespace = 'admin'
+
+    def get_urls(self):
+        return [
+            path('core/snapshot/add/', self.add_view, name='Add'),
+        ] + super().get_urls()
+
+    def add_view(self, request):
+        if not request.user.is_authenticated:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        request.current_app = self.name
+        context: Dict[str, Any] = {
+            **self.each_context(request),
+            'title': 'Add URLs',
+        }
+
+        if request.method == 'GET':
+            context['form'] = AddLinkForm()
+
+        elif request.method == 'POST':
+            form = AddLinkForm(request.POST)
+            if form.is_valid():
+                url = form.cleaned_data["url"]
+                print(f'[+] Adding URL: {url}')
+                depth = 0 if form.cleaned_data["depth"] == "0" else 1
+                input_kwargs = {
+                    "urls": url,
+                    "depth": depth,
+                    "update_all": False,
+                    "out_dir": CONFIG.OUTPUT_DIR,
+                }
+                add_stdout = StringIO()
+                with redirect_stdout(add_stdout):
+                   add(**input_kwargs)
+                print(add_stdout.getvalue())
+
+                context.update({
+                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
+                    "form": AddLinkForm(),
+                })
+            else:
+                context["form"] = form
+
+        return render(template_name='add.html', request=request, context=context)
+
+
+archivebox_admin = ArchiveBoxAdmin()
+archivebox_admin.register(get_user_model())
+archivebox_admin.disable_action('delete_selected')
+
+# archivebox_admin.register(CustomPlugin)
+
+# patch admin with methods to add data views (implemented by admin_data_views package)
+# https://github.com/MrThearMan/django-admin-data-views
+# https://mrthearman.github.io/django-admin-data-views/setup/
+############### Additional sections are defined in settings.ADMIN_DATA_VIEWS #########
+from admin_data_views.admin import get_app_list, admin_data_index_view, get_admin_data_urls, get_urls
+
+archivebox_admin.get_app_list = get_app_list.__get__(archivebox_admin, ArchiveBoxAdmin)
+archivebox_admin.admin_data_index_view = admin_data_index_view.__get__(archivebox_admin, ArchiveBoxAdmin)       # type: ignore
+archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_admin, ArchiveBoxAdmin)           # type: ignore
+archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
+
+
+class AccelleratedPaginator(Paginator):
+    """
+    Accellerated Pagniator ignores DISTINCT when counting total number of rows.
+    Speeds up SELECT Count(*) on Admin views by >20x.
+    https://hakibenita.com/optimizing-the-django-admin-paginator
+    """
+
+    @cached_property
+    def count(self):
+        if self.object_list._has_filters():
+            # fallback to normal count method on filtered queryset
+            return super().count
+        else:
+            # otherwise count total rows in a separate fast query
+            return self.object_list.model.objects.count()
+    
+        # Alternative approach for PostgreSQL: fallback count takes > 200ms
+        # from django.db import connection, transaction, OperationalError
+        # with transaction.atomic(), connection.cursor() as cursor:
+        #     cursor.execute('SET LOCAL statement_timeout TO 200;')
+        #     try:
+        #         return super().count
+        #     except OperationalError:
+        #         return 9999999999999
+
+
 class ArchiveResultInline(admin.TabularInline):
+    name = 'Archive Results Log'
     model = ArchiveResult
+    parent_model = Snapshot
+    # fk_name = 'snapshot'
+    extra = 0
+    sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
+    readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
+    fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
+    # exclude = ('id',)
+    ordering = ('end_ts',)
+    show_change_link = True
+    # # classes = ['collapse']
+    # # list_display_links = ['abid']
+
+    def get_parent_object_from_request(self, request):
+        resolved = resolve(request.path_info)
+        return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
+
+    @admin.display(
+        description='Completed',
+        ordering='end_ts',
+    )
+    def completed(self, obj):
+        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime('%Y-%m-%d %H:%M:%S'))
+
+    def result_id(self, obj):
+        return format_html('<a href="{}"><code style="font-size: 10px">[{}]</code></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
+    
+    def command(self, obj):
+        return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
+    
+    def version(self, obj):
+        return format_html('<small><code>{}</code></small>', obj.cmd_version or '-')
+    
+    def get_formset(self, request, obj=None, **kwargs):
+        formset = super().get_formset(request, obj, **kwargs)
+        snapshot = self.get_parent_object_from_request(request)
+
+        # import ipdb; ipdb.set_trace()
+        formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
+        
+        # default values for new entries
+        formset.form.base_fields['status'].initial = 'succeeded'
+        formset.form.base_fields['start_ts'].initial = timezone.now()
+        formset.form.base_fields['end_ts'].initial = timezone.now()
+        formset.form.base_fields['cmd_version'].initial = '-'
+        formset.form.base_fields['pwd'].initial = str(snapshot.link_dir)
+        formset.form.base_fields['created_by'].initial = request.user
+        formset.form.base_fields['cmd'] = forms.JSONField(initial=['-'])
+        formset.form.base_fields['output'].initial = 'Manually recorded cmd output...'
+        
+        if obj is not None:
+            # hidden values for existing entries and new entries
+            formset.form.base_fields['start_ts'].widget = formset.form.base_fields['start_ts'].hidden_widget()
+            formset.form.base_fields['end_ts'].widget = formset.form.base_fields['end_ts'].hidden_widget()
+            formset.form.base_fields['cmd'].widget = formset.form.base_fields['cmd'].hidden_widget()
+            formset.form.base_fields['pwd'].widget = formset.form.base_fields['pwd'].hidden_widget()
+            formset.form.base_fields['created_by'].widget = formset.form.base_fields['created_by'].hidden_widget()
+            formset.form.base_fields['cmd_version'].widget = formset.form.base_fields['cmd_version'].hidden_widget()
+        return formset
+    
+    def get_readonly_fields(self, request, obj=None):
+        if obj is not None:
+            return self.readonly_fields
+        else:
+            return []
+
 
 class TagInline(admin.TabularInline):
-    model = Snapshot.tags.through
+    model = Tag.snapshot_set.through       # type: ignore
+    # fk_name = 'snapshot'
+    fields = ('id', 'tag')
+    extra = 1
+    # min_num = 1
+    max_num = 1000
+    autocomplete_fields = (
+        'tag',
+    )
 
 from django.contrib.admin.helpers import ActionForm
-from django.contrib.admin.widgets import AutocompleteSelectMultiple
+from django.contrib.admin.widgets import FilteredSelectMultiple
 
-# WIP: broken by Django 3.1.2 -> 4.0 migration
-class AutocompleteTags:
-    model = Tag
-    search_fields = ['name']
-    name = 'tags'
+# class AutocompleteTags:
+#     model = Tag
+#     search_fields = ['name']
+#     name = 'name'
+#     # source_field = 'name'
+#     remote_field = Tag._meta.get_field('name')
 
-class AutocompleteTagsAdminStub:
-    name = 'admin'
+# class AutocompleteTagsAdminStub:
+#     name = 'admin'
 
 
 class SnapshotActionForm(ActionForm):
     tags = forms.ModelMultipleChoiceField(
         queryset=Tag.objects.all(),
         required=False,
-        # WIP: broken by Django 3.1.2 -> 4.0 migration
-        widget=AutocompleteSelectMultiple(
-            AutocompleteTags(),
-            AutocompleteTagsAdminStub(),
+        widget=FilteredSelectMultiple(
+            'core_tag__name',
+            False,
         ),
     )
 
     # TODO: allow selecting actions for specific extractors? is this useful?
-    # EXTRACTOR_CHOICES = [
-    #     (name, name.title())
-    #     for name, _, _ in get_default_archive_methods()
-    # ]
     # extractor = forms.ChoiceField(
-    #     choices=EXTRACTOR_CHOICES,
+    #     choices=ArchiveResult.EXTRACTOR_CHOICES,
     #     required=False,
     #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
     # )
 
 
-class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
+def get_abid_info(self, obj):
+    return format_html(
+        # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+        '''
+        <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
+        <br/><hr/>
+        <div style="opacity: 0.8">
+        &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        <hr/>
+        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; {}: <code style="user-select: all">{}</code><br/>
+        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; "><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <span style="display:inline-block; vertical-align: -4px; width: 290px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span>
+        &nbsp; SALT: &nbsp; <code style="font-size: 10px;"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code><br/>
+        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}: <code style="user-select: all">{}</code><br/>
+        &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  {}: <code style="user-select: all">{}</code>
+        <br/><hr/>
+        &nbsp; &nbsp; <small style="opacity: 0.5">.old_id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        </div>
+        ''',
+        obj.api_url, obj.api_url, obj.api_docs_url,
+        str(obj.abid),
+        str(obj.ABID.uuid),
+        str(obj.id),
+        obj.ABID.ts, str(obj.ABID.uuid)[0:14], obj.abid_ts_src, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
+        obj.ABID.uri, str(obj.ABID.uuid)[14:26], obj.abid_uri_src, str(obj.abid_values['uri']),
+        obj.ABID.uri_salt,
+        obj.ABID.subtype, str(obj.ABID.uuid)[26:28], obj.abid_subtype_src, str(obj.abid_values['subtype']),
+        obj.ABID.rand, str(obj.ABID.uuid)[28:36], obj.abid_rand_src, str(obj.abid_values['rand'])[-7:],
+        str(getattr(obj, 'old_id', '')),
+    )
+
+
+@admin.register(Snapshot, site=archivebox_admin)
+class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     list_display = ('added', 'title_str', 'files', 'size', 'url_str')
     sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('info', 'bookmarked', 'added', 'updated')
-    search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
-    fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
-    list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
+    readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
+    search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
+    list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
+    fields = ('url', 'created_by', 'title', *readonly_fields)
     ordering = ['-added']
     actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
-    autocomplete_fields = ['tags']
-    inlines = [ArchiveResultInline]
-    list_per_page = SNAPSHOTS_PER_PAGE
+    inlines = [TagInline, ArchiveResultInline]
+    list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
 
     action_form = SnapshotActionForm
+    paginator = AccelleratedPaginator
+
+    save_on_top = True
+    show_full_result_count = False
+
+    def changelist_view(self, request, extra_context=None):
+        extra_context = extra_context or {}
+        try:
+            return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
+        except Exception as e:
+            self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
+            return super().changelist_view(request, GLOBAL_CONTEXT)
+
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        snapshot = None
+
+        try:
+            snapshot = snapshot or Snapshot.objects.get(id=object_id)
+        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
+            pass
+        
+        try:
+            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
+        except (Snapshot.DoesNotExist, ValidationError):
+            pass
+
+
+        try:
+            snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
+        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
+            pass
+
+        if snapshot:
+            object_id = str(snapshot.id)
+
+        return super().change_view(
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
+        )
 
     def get_urls(self):
         urls = super().get_urls()
@@ -103,12 +363,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
         ]
         return custom_urls + urls
 
-    def get_queryset(self, request):
-        self.request = request
-        return super().get_queryset(request).prefetch_related('tags')
+    # def get_queryset(self, request):
+    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
+    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
+
+    #     self.request = request
+    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
 
     def tag_list(self, obj):
-        return ', '.join(obj.tags.values_list('name', flat=True))
+        return ', '.join(tag.name for tag in obj.tags.all())
 
     # TODO: figure out a different way to do this, you cant nest forms so this doenst work
     # def action(self, obj):
@@ -129,67 +392,84 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     #             </form>
     #         ''',
     #         csrf.get_token(self.request),
-    #         obj.id,
+    #         obj.pk,
     #     )
 
-    def info(self, obj):
+    def admin_actions(self, obj):
         return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}">Summary page ➡️</a> &nbsp; &nbsp;
+            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}/index.html#all">Result files 📑</a> &nbsp; &nbsp;
+            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/admin/core/snapshot/?id__exact={}">Admin actions ⚙️</a>
+            ''',
+            obj.timestamp,
+            obj.timestamp,
+            obj.pk,
+        )
+
+    def status_info(self, obj):
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
             '''
-            UUID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            Timestamp: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
             Archived: {} ({} files {}) &nbsp; &nbsp;
             Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
-            Status code: {} &nbsp; &nbsp;
+            Status code: {} &nbsp; &nbsp;<br/>
             Server: {} &nbsp; &nbsp;
             Content type: {} &nbsp; &nbsp;
             Extension: {} &nbsp; &nbsp;
-            <br/><br/>
-            <a href="/archive/{}">View Snapshot index ➡️</a> &nbsp; &nbsp;
-            <a href="/admin/core/snapshot/?id__exact={}">View actions ⚙️</a>
             ''',
-            obj.id,
-            obj.timestamp,
-            obj.url_hash,
             '✅' if obj.is_archived else '❌',
             obj.num_outputs,
-            self.size(obj),
+            self.size(obj) or '0kb',
             f'/archive/{obj.timestamp}/favicon.ico',
-            obj.status_code or '?',
-            obj.headers and obj.headers.get('Server') or '?',
-            obj.headers and obj.headers.get('Content-Type') or '?',
-            obj.extension or '?',
-            obj.timestamp,
-            obj.id,
+            obj.status_code or '-',
+            obj.headers and obj.headers.get('Server') or '-',
+            obj.headers and obj.headers.get('Content-Type') or '-',
+            obj.extension or '-',
         )
 
+    def API(self, obj):
+        try:
+            return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
+
+    @admin.display(
+        description='Title',
+        ordering='title',
+    )
     def title_str(self, obj):
-        canon = obj.as_link().canonical_outputs()
         tags = ''.join(
-            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.id, tag)
+            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.pk, tag.name)
             for tag in obj.tags.all()
-            if str(tag).strip()
+            if str(tag.name).strip()
         )
         return format_html(
             '<a href="/{}">'
-                '<img src="/{}/{}" class="favicon" onerror="this.remove()">'
+                '<img src="/{}/favicon.ico" class="favicon" onerror="this.remove()">'
             '</a>'
             '<a href="/{}/index.html">'
                 '<b class="status-{}">{}</b>'
             '</a>',
             obj.archive_path,
-            obj.archive_path, canon['favicon_path'],
+            obj.archive_path,
             obj.archive_path,
             'fetched' if obj.latest_title or obj.title else 'pending',
             urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
         ) + mark_safe(f' <span class="tags">{tags}</span>')
 
+    @admin.display(
+        description='Files Saved',
+        # ordering='archiveresult_count',
+    )
     def files(self, obj):
         return snapshot_icons(obj)
 
-    files.admin_order_field = 'updated'
-    files.short_description = 'Files Saved'
 
+    @admin.display(
+        # ordering='archiveresult_count'
+    )
     def size(self, obj):
         archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
         if archive_size:
@@ -204,13 +484,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             size_txt,
         )
 
-    size.admin_order_field = 'archiveresult__count'
 
+    @admin.display(
+        description='Original URL',
+        ordering='url',
+    )
     def url_str(self, obj):
         return format_html(
             '<a href="{}"><code style="user-select: all;">{}</code></a>',
             obj.url,
-            obj.url,
+            obj.url[:128],
         )
 
     def grid_view(self, request, extra_context=None):
@@ -224,7 +507,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
         # Monkey patch here plus core_tags.py
         self.change_list_template = 'private_index_grid.html'
-        self.list_per_page = SNAPSHOTS_PER_PAGE
+        self.list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
         self.list_max_show_all = self.list_per_page
 
         # Call monkey patched view
@@ -242,188 +525,230 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     #     print('[*] Got request', request.method, request.POST)
     #     return super().changelist_view(request, extra_context=None)
 
+    @admin.action(
+        description="Pull"
+    )
     def update_snapshots(self, request, queryset):
         archive_links([
             snapshot.as_link()
             for snapshot in queryset
-        ], out_dir=OUTPUT_DIR)
-    update_snapshots.short_description = "Pull"
+        ], out_dir=CONFIG.OUTPUT_DIR)
 
+    @admin.action(
+        description="⬇️ Title"
+    )
     def update_titles(self, request, queryset):
         archive_links([
             snapshot.as_link()
             for snapshot in queryset
-        ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
-    update_titles.short_description = "⬇️ Title"
+        ], overwrite=True, methods=('title','favicon'), out_dir=CONFIG.OUTPUT_DIR)
 
+    @admin.action(
+        description="Re-Snapshot"
+    )
     def resnapshot_snapshot(self, request, queryset):
         for snapshot in queryset:
             timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
             new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
             add(new_url, tag=snapshot.tags_str())
-    resnapshot_snapshot.short_description = "Re-Snapshot"
 
+    @admin.action(
+        description="Reset"
+    )
     def overwrite_snapshots(self, request, queryset):
         archive_links([
             snapshot.as_link()
             for snapshot in queryset
-        ], overwrite=True, out_dir=OUTPUT_DIR)
-    overwrite_snapshots.short_description = "Reset"
+        ], overwrite=True, out_dir=CONFIG.OUTPUT_DIR)
 
+    @admin.action(
+        description="Delete"
+    )
     def delete_snapshots(self, request, queryset):
-        remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
+        remove(snapshots=queryset, yes=True, delete=True, out_dir=CONFIG.OUTPUT_DIR)
 
-    delete_snapshots.short_description = "Delete"
 
+    @admin.action(
+        description="+"
+    )
     def add_tags(self, request, queryset):
         tags = request.POST.getlist('tags')
         print('[+] Adding tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.add(*tags)
 
-    add_tags.short_description = "+"
 
+    @admin.action(
+        description="–"
+    )
     def remove_tags(self, request, queryset):
         tags = request.POST.getlist('tags')
         print('[-] Removing tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.remove(*tags)
 
-    remove_tags.short_description = "–"
 
         
 
-    title_str.short_description = 'Title'
-    url_str.short_description = 'Original URL'
-
-    title_str.admin_order_field = 'title'
-    url_str.admin_order_field = 'url'
 
 
+# @admin.register(SnapshotTag, site=archivebox_admin)
+# class SnapshotTagAdmin(ABIDModelAdmin):
+#     list_display = ('id', 'snapshot', 'tag')
+#     sort_fields = ('id', 'snapshot', 'tag')
+#     search_fields = ('id', 'snapshot_id', 'tag_id')
+#     fields = ('snapshot', 'id')
+#     actions = ['delete_selected']
+#     ordering = ['-id']
 
-class TagAdmin(admin.ModelAdmin):
-    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
-    sort_fields = ('id', 'name', 'slug')
-    readonly_fields = ('id', 'num_snapshots', 'snapshots')
-    search_fields = ('id', 'name', 'slug')
-    fields = (*readonly_fields, 'name', 'slug')
+#     def API(self, obj):
+#         return get_abid_info(self, obj)
+
+
+@admin.register(Tag, site=archivebox_admin)
+class TagAdmin(ABIDModelAdmin):
+    list_display = ('created', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
+    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
+    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
+    search_fields = ('abid', 'name', 'slug')
+    fields = ('name', 'created_by', *readonly_fields)
     actions = ['delete_selected']
-    ordering = ['-id']
+    ordering = ['-created']
 
-    def num_snapshots(self, obj):
+    paginator = AccelleratedPaginator
+
+    def API(self, obj):
+        try:
+            return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
+
+    def num_snapshots(self, tag):
         return format_html(
             '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
-            obj.id,
-            obj.snapshot_set.count(),
+            tag.id,
+            tag.snapshot_set.count(),
         )
 
-    def snapshots(self, obj):
-        total_count = obj.snapshot_set.count()
+    def snapshots(self, tag):
+        total_count = tag.snapshot_set.count()
         return mark_safe('<br/>'.join(
             format_html(
-                '{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
+                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
+                snap.pk,
                 snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
-                snap.id,
-                snap.timestamp,
-                snap.url,
+                snap.url[:64],
             )
-            for snap in obj.snapshot_set.order_by('-updated')[:10]
-        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={obj.id}">and {total_count-10} more...<a>' if obj.snapshot_set.count() > 10 else ''))
+            for snap in tag.snapshot_set.order_by('-updated')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
 
 
-class ArchiveResultAdmin(admin.ModelAdmin):
-    list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
+@admin.register(ArchiveResult, site=archivebox_admin)
+class ArchiveResultAdmin(ABIDModelAdmin):
+    list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('id', 'uuid', 'snapshot_str', 'tags_str')
-    search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
+    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
+    search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
     autocomplete_fields = ['snapshot']
 
     list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
     ordering = ['-start_ts']
-    list_per_page = SNAPSHOTS_PER_PAGE
+    list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
+    
+    paginator = AccelleratedPaginator
 
-    def snapshot_str(self, obj):
+    @admin.display(
+        description='Snapshot Info'
+    )
+    def snapshot_info(self, result):
         return format_html(
-            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
-            '<small>{}</small>',
-            obj.snapshot.timestamp,
-            obj.snapshot.timestamp,
-            obj.snapshot.url[:128],
+            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
+            result.snapshot.timestamp,
+            result.snapshot.abid,
+            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
+            result.snapshot.url[:128],
         )
 
-    def tags_str(self, obj):
-        return obj.snapshot.tags_str()
+    def API(self, obj):
+        try:
+            return get_abid_info(self, obj)
+        except Exception as e:
+            raise e
+            return str(e)
 
-    def cmd_str(self, obj):
+    @admin.display(
+        description='Snapshot Tags'
+    )
+    def tags_str(self, result):
+        return result.snapshot.tags_str()
+
+    def cmd_str(self, result):
         return format_html(
             '<pre>{}</pre>',
-            ' '.join(obj.cmd) if isinstance(obj.cmd, list) else str(obj.cmd),
+            ' '.join(result.cmd) if isinstance(result.cmd, list) else str(result.cmd),
         )
-
-    def output_str(self, obj):
+    
+    def output_str(self, result):
         return format_html(
             '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
-            obj.snapshot.timestamp,
-            obj.output if (obj.status == 'succeeded') and obj.extractor not in ('title', 'archive_org') else 'index.html',
-            obj.output,
+            result.snapshot.timestamp,
+            result.output if (result.status == 'succeeded') and result.extractor not in ('title', 'archive_org') else 'index.html',
+            result.output,
         )
 
-    tags_str.short_description = 'tags'
-    snapshot_str.short_description = 'snapshot'
+    def output_summary(self, result):
+        snapshot_dir = Path(CONFIG.OUTPUT_DIR) / str(result.pwd).split('data/', 1)[-1]
+        output_str = format_html(
+            '<pre style="display: block; max-width: 75vw; white-space: pre-line;">{}</pre><br/>',
+            result.output,
+        )
+        output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
+        path_from_output_str = (snapshot_dir / result.output)
+        output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
+        
+        path_exists = False
+        try:
+            path_exists = path_from_output_str.exists()
+        except OSError:
+            pass
 
-class ArchiveBoxAdmin(admin.AdminSite):
-    site_header = 'ArchiveBox'
-    index_title = 'Links'
-    site_title = 'Index'
+        if path_exists:
+            root_dir = str(path_from_output_str)
+        else:
+            root_dir = str(snapshot_dir)
 
-    def get_urls(self):
-        return [
-            path('core/snapshot/add/', self.add_view, name='Add'),
-        ] + super().get_urls()
+        # print(root_dir, str(list(os.walk(root_dir))))
 
-    def add_view(self, request):
-        if not request.user.is_authenticated:
-            return redirect(f'/admin/login/?next={request.path}')
+        for root, dirs, files in os.walk(root_dir):
+            depth = root.replace(root_dir, '').count(os.sep) + 1
+            if depth > 2:
+                continue
+            indent = ' ' * 4 * (depth)
+            output_str += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
+            indentation_str = ' ' * 4 * (depth + 1)
+            for filename in sorted(files):
+                is_hidden = filename.startswith('.')
+                output_str += format_html('<span style="opacity: {}.2">{}{}</span><br/>', int(not is_hidden), indentation_str, filename.strip())
 
-        request.current_app = self.name
-        context = {
-            **self.each_context(request),
-            'title': 'Add URLs',
-        }
+        return output_str + format_html('</code></pre>')
 
-        if request.method == 'GET':
-            context['form'] = AddLinkForm()
 
-        elif request.method == 'POST':
-            form = AddLinkForm(request.POST)
-            if form.is_valid():
-                url = form.cleaned_data["url"]
-                print(f'[+] Adding URL: {url}')
-                depth = 0 if form.cleaned_data["depth"] == "0" else 1
-                input_kwargs = {
-                    "urls": url,
-                    "depth": depth,
-                    "update_all": False,
-                    "out_dir": OUTPUT_DIR,
-                }
-                add_stdout = StringIO()
-                with redirect_stdout(add_stdout):
-                   add(**input_kwargs)
-                print(add_stdout.getvalue())
 
-                context.update({
-                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
-                    "form": AddLinkForm()
-                })
-            else:
-                context["form"] = form
+@admin.register(APIToken, site=archivebox_admin)
+class APITokenAdmin(ABIDModelAdmin):
+    list_display = ('created', 'abid', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('abid', 'created', 'created_by', 'expires')
+    readonly_fields = ('abid', 'created')
+    search_fields = ('id', 'abid', 'created_by__username', 'token')
+    fields = ('created_by', 'token', 'expires', *readonly_fields)
 
-        return render(template_name='add.html', request=request, context=context)
+    list_filter = ('created_by',)
+    ordering = ['-created']
+    list_per_page = 100
 
-admin.site = ArchiveBoxAdmin()
-admin.site.register(get_user_model())
-admin.site.register(Snapshot, SnapshotAdmin)
-admin.site.register(Tag, TagAdmin)
-admin.site.register(ArchiveResult, ArchiveResultAdmin)
-admin.site.disable_action('delete_selected')
+@admin.register(get_webhook_model(), site=archivebox_admin)
+class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
+    list_display = ('created', 'created_by', 'abid', *WebhookAdmin.list_display)
+    sort_fields = ('created', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('abid', 'created', *WebhookAdmin.readonly_fields)

archivebox/core/apps.py

@@ -1,7 +1,28 @@
+__package__ = 'archivebox.core'
+
 from django.apps import AppConfig
 
 
 class CoreConfig(AppConfig):
     name = 'core'
-    # WIP: broken by Django 3.1.2 -> 4.0 migration
-    default_auto_field = 'django.db.models.UUIDField'
+
+    def ready(self):
+        # register our custom admin as the primary django admin
+        from django.contrib import admin
+        from django.contrib.admin import sites
+        from core.admin import archivebox_admin
+
+        admin.site = archivebox_admin
+        sites.site = archivebox_admin
+
+
+        # register signal handlers
+        from .auth import register_signals
+
+        register_signals()
+
+
+
+# from django.contrib.admin.apps import AdminConfig
+# class CoreAdminConfig(AdminConfig):
+#     default_site = "core.admin.get_admin_site"

archivebox/core/auth.py

@@ -0,0 +1,14 @@
+__package__ = 'archivebox.core'
+
+
+from ..config import (
+    LDAP
+)
+
+def register_signals():
+
+    if LDAP:
+        import django_auth_ldap.backend
+        from .auth_ldap import create_user
+
+        django_auth_ldap.backend.populate_user.connect(create_user)

archivebox/core/auth_ldap.py

@@ -0,0 +1,10 @@
+from ..config import (
+    LDAP_CREATE_SUPERUSER
+)
+
+def create_user(sender, user=None, ldap_user=None, **kwargs):
+    if not user.id and LDAP_CREATE_SUPERUSER:
+        user.is_superuser = True
+
+    user.is_staff = True
+    print(f'[!] WARNING: Creating new user {user} based on LDAP user {ldap_user} (is_staff={user.is_staff}, is_superuser={user.is_superuser})')

archivebox/core/forms.py

@@ -4,7 +4,7 @@ from django import forms
 
 from ..util import URL_REGEX
 from ..parsers import PARSERS
-from ..vendor.taggit_utils import edit_string_for_tags, parse_tags
+from taggit.utils import edit_string_for_tags, parse_tags
 
 PARSER_CHOICES = [
     (parser_key, parser[0])

archivebox/core/migrations/0007_archiveresult.py

@@ -17,8 +17,6 @@ except AttributeError:
 
 
 def forwards_func(apps, schema_editor):
-    from core.models import EXTRACTORS
-
     Snapshot = apps.get_model("core", "Snapshot")
     ArchiveResult = apps.get_model("core", "ArchiveResult")
 

archivebox/core/migrations/0023_alter_archiveresult_options_archiveresult_abid_and_more.py

@@ -0,0 +1,58 @@
+# Generated by Django 5.0.6 on 2024-05-13 10:56
+
+import charidfield.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0022_auto_20231023_2008'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='archiveresult',
+            options={'verbose_name': 'Result'},
+        ),
+        migrations.AddField(
+            model_name='archiveresult',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='res_', unique=True),
+        ),
+        migrations.AddField(
+            model_name='snapshot',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='snp_', unique=True),
+        ),
+        migrations.AddField(
+            model_name='snapshot',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='tag_', unique=True),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=(
+                ('htmltotext', 'htmltotext'),
+                ('git', 'git'),
+                ('singlefile', 'singlefile'),
+                ('media', 'media'),
+                ('archive_org', 'archive_org'),
+                ('readability', 'readability'),
+                ('mercury', 'mercury'),
+                ('favicon', 'favicon'),
+                ('pdf', 'pdf'),
+                ('headers', 'headers'),
+                ('screenshot', 'screenshot'),
+                ('dom', 'dom'),
+                ('title', 'title'),
+                ('wget', 'wget'),
+            ), max_length=32),
+        ),
+    ]

archivebox/core/migrations/0024_auto_20240513_1143.py

@@ -0,0 +1,100 @@
+# Generated by Django 5.0.6 on 2024-05-13 11:43
+
+from django.db import migrations
+from datetime import datetime
+from abid_utils.abid import abid_from_values, DEFAULT_ABID_URI_SALT
+
+
+def calculate_abid(self):
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+    prefix = self.abid_prefix
+    ts = eval(self.abid_ts_src)
+    uri = eval(self.abid_uri_src)
+    subtype = eval(self.abid_subtype_src)
+    rand = eval(self.abid_rand_src)
+
+    if (not prefix) or prefix == 'obj_':
+        suggested_abid = self.__class__.__name__[:3].lower()
+        raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+    if not ts:
+        ts = datetime.utcfromtimestamp(0)
+        print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+    if not uri:
+        uri = str(self)
+        print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+    if not subtype:
+        subtype = self.__class__.__name__
+        print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+    if not rand:
+        rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+        print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+    abid = abid_from_values(
+        prefix=prefix,
+        ts=ts,
+        uri=uri,
+        subtype=subtype,
+        rand=rand,
+        salt=DEFAULT_ABID_URI_SALT,
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+    return abid
+
+
+def copy_snapshot_uuids(apps, schema_editor):
+    print('   Copying snapshot.id -> snapshot.uuid...')
+    Snapshot = apps.get_model("core", "Snapshot")
+    for snapshot in Snapshot.objects.all():
+        snapshot.uuid = snapshot.id
+        snapshot.save(update_fields=["uuid"])
+
+def generate_snapshot_abids(apps, schema_editor):
+    print('   Generating snapshot.abid values...')
+    Snapshot = apps.get_model("core", "Snapshot")
+    for snapshot in Snapshot.objects.all():
+        snapshot.abid_prefix = 'snp_'
+        snapshot.abid_ts_src = 'self.added'
+        snapshot.abid_uri_src = 'self.url'
+        snapshot.abid_subtype_src = '"01"'
+        snapshot.abid_rand_src = 'self.uuid'
+
+        snapshot.abid = calculate_abid(snapshot)
+        snapshot.uuid = snapshot.abid.uuid
+        snapshot.save(update_fields=["abid", "uuid"])
+
+def generate_archiveresult_abids(apps, schema_editor):
+    print('   Generating ArchiveResult.abid values... (may take an hour or longer for large collections...)')
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    Snapshot = apps.get_model("core", "Snapshot")
+    for result in ArchiveResult.objects.all():
+        result.abid_prefix = 'res_'
+        result.snapshot = Snapshot.objects.get(pk=result.snapshot_id)
+        result.snapshot_added = result.snapshot.added
+        result.snapshot_url = result.snapshot.url
+        result.abid_ts_src = 'self.snapshot_added'
+        result.abid_uri_src = 'self.snapshot_url'
+        result.abid_subtype_src = 'self.extractor'
+        result.abid_rand_src = 'self.id'
+
+        result.abid = calculate_abid(result)
+        result.uuid = result.abid.uuid
+        result.save(update_fields=["abid", "uuid"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(copy_snapshot_uuids, reverse_code=migrations.RunPython.noop),
+        migrations.RunPython(generate_snapshot_abids, reverse_code=migrations.RunPython.noop),
+        migrations.RunPython(generate_archiveresult_abids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0025_alter_archiveresult_uuid.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-05-13 12:08
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0024_auto_20240513_1143'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0026_archiveresult_created_archiveresult_created_by_and_more.py

@@ -0,0 +1,116 @@
+# Generated by Django 5.0.6 on 2024-05-13 13:01
+
+import abid_utils.models
+import django.db.models.deletion
+import django.utils.timezone
+from django.conf import settings
+from django.db import migrations, models
+
+
+def updated_created_by_ids(apps, schema_editor):
+    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+    User = apps.get_model("auth", "User")
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    Snapshot = apps.get_model("core", "Snapshot")
+    Tag = apps.get_model("core", "Tag")
+
+    # if only one user exists total, return that user
+    if User.objects.filter(is_superuser=True).count() == 1:
+        user_id = User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+    # otherwise, create a dedicated "system" user
+    user_id = User.objects.get_or_create(username='system', is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})[0].pk
+    
+    ArchiveResult.objects.all().update(created_by_id=user_id)
+    Snapshot.objects.all().update(created_by_id=user_id)
+    Tag.objects.all().update(created_by_id=user_id)
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0025_alter_archiveresult_uuid'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='archiveresult',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='archiveresult',
+            name='created_by',
+            field=models.ForeignKey(null=True, default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='archiveresult',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='snapshot',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='snapshot',
+            name='created_by',
+            field=models.ForeignKey(null=True, default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='snapshot',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='created_by',
+            field=models.ForeignKey(null=True, default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+
+
+        migrations.RunPython(updated_created_by_ids, reverse_code=migrations.RunPython.noop),
+
+        migrations.AddField(
+            model_name='snapshot',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='tag',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]

archivebox/core/migrations/0027_update_snapshot_ids.py

@@ -0,0 +1,106 @@
+# Generated by Django 5.0.6 on 2024-08-18 02:48
+
+from django.db import migrations
+
+from django.db import migrations
+from datetime import datetime
+from abid_utils.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
+
+
+def calculate_abid(self):
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+    prefix = self.abid_prefix
+    ts = eval(self.abid_ts_src)
+    uri = eval(self.abid_uri_src)
+    subtype = eval(self.abid_subtype_src)
+    rand = eval(self.abid_rand_src)
+
+    if (not prefix) or prefix == 'obj_':
+        suggested_abid = self.__class__.__name__[:3].lower()
+        raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+    if not ts:
+        ts = datetime.utcfromtimestamp(0)
+        print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+    if not uri:
+        uri = str(self)
+        print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+    if not subtype:
+        subtype = self.__class__.__name__
+        print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+    if not rand:
+        rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+        print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+    abid = abid_from_values(
+        prefix=prefix,
+        ts=ts,
+        uri=uri,
+        subtype=subtype,
+        rand=rand,
+        salt=DEFAULT_ABID_URI_SALT,
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+    return abid
+
+def update_snapshot_ids(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    num_total = Snapshot.objects.all().count()
+    print(f'   Updating {num_total} Snapshot.id, Snapshot.uuid values in place...')
+    for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator()):
+        assert snapshot.abid
+        snapshot.abid_prefix = 'snp_'
+        snapshot.abid_ts_src = 'self.added'
+        snapshot.abid_uri_src = 'self.url'
+        snapshot.abid_subtype_src = '"01"'
+        snapshot.abid_rand_src = 'self.uuid'
+
+        snapshot.abid = calculate_abid(snapshot)
+        snapshot.uuid = snapshot.abid.uuid
+        snapshot.save(update_fields=["abid", "uuid"])
+        assert str(ABID.parse(snapshot.abid).uuid) == str(snapshot.uuid)
+        if idx % 1000 == 0:
+            print(f'Migrated {idx}/{num_total} Snapshot objects...')
+
+def update_archiveresult_ids(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    num_total = ArchiveResult.objects.all().count()
+    print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator()):
+        assert result.abid
+        result.abid_prefix = 'res_'
+        result.snapshot = Snapshot.objects.get(pk=result.snapshot_id)
+        result.snapshot_added = result.snapshot.added
+        result.snapshot_url = result.snapshot.url
+        result.abid_ts_src = 'self.snapshot_added'
+        result.abid_uri_src = 'self.snapshot_url'
+        result.abid_subtype_src = 'self.extractor'
+        result.abid_rand_src = 'self.id'
+
+        result.abid = calculate_abid(result)
+        result.uuid = result.abid.uuid
+        result.uuid = ABID.parse(result.abid).uuid
+        result.save(update_fields=["abid", "uuid"])
+        assert str(ABID.parse(result.abid).uuid) == str(result.uuid)
+        if idx % 5000 == 0:
+            print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0026_archiveresult_created_archiveresult_created_by_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_snapshot_ids, reverse_code=migrations.RunPython.noop),
+        migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
+    ]
+
+

archivebox/core/migrations/0028_alter_archiveresult_uuid.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-18 04:28
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0027_update_snapshot_ids'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4),
+        ),
+    ]

archivebox/core/migrations/0029_alter_archiveresult_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 04:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0028_alter_archiveresult_uuid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.BigIntegerField(primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

archivebox/core/migrations/0030_alter_archiveresult_uuid.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0029_alter_archiveresult_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(unique=True),
+        ),
+    ]

archivebox/core/migrations/0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:09
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0030_alter_archiveresult_uuid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.IntegerField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, null=True, unique=True),
+        ),
+    ]

archivebox/core/migrations/0032_alter_archiveresult_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:20
+
+import core.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

archivebox/core/migrations/0033_rename_id_archiveresult_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0032_alter_archiveresult_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='id',
+            new_name='old_id',
+        ),
+    ]

archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py

@@ -0,0 +1,41 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:37
+
+import core.models
+import uuid
+from django.db import migrations, models
+
+from abid_utils.abid import ABID
+
+
+def update_archiveresult_ids(apps, schema_editor):
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    num_total = ArchiveResult.objects.all().count()
+    print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()):
+        assert result.abid
+        result.uuid = ABID.parse(result.abid).uuid
+        result.save(update_fields=["uuid"])
+        assert str(ABID.parse(result.abid).uuid) == str(result.uuid)
+        if idx % 2500 == 0:
+            print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0033_rename_id_archiveresult_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'),
+        ),
+        migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0035_remove_archiveresult_uuid_archiveresult_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:49
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0034_alter_archiveresult_old_id_alter_archiveresult_uuid'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='uuid',
+            new_name='id',
+        ),
+    ]

archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:59
+
+import core.models
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0035_remove_archiveresult_uuid_archiveresult_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'),
+        ),
+    ]

archivebox/core/migrations/0037_rename_id_snapshot_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0036_alter_archiveresult_id_alter_archiveresult_old_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='id',
+            new_name='old_id',
+        ),
+    ]

archivebox/core/migrations/0038_rename_uuid_snapshot_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0037_rename_id_snapshot_old_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='uuid',
+            new_name='id',
+        ),
+    ]

archivebox/core/migrations/0039_rename_snapshot_archiveresult_snapshot_old.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0038_rename_uuid_snapshot_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='snapshot',
+            new_name='snapshot_old',
+        ),
+    ]

archivebox/core/migrations/0040_archiveresult_snapshot.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:46
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+def update_archiveresult_snapshot_ids(apps, schema_editor):
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    Snapshot = apps.get_model("core", "Snapshot")
+    num_total = ArchiveResult.objects.all().count()
+    print(f'   Updating {num_total} ArchiveResult.snapshot_id values in place... (may take an hour or longer for large collections...)')
+    for idx, result in enumerate(ArchiveResult.objects.all().only('snapshot_old_id').iterator(chunk_size=5000)):
+        assert result.snapshot_old_id
+        snapshot = Snapshot.objects.only('id').get(old_id=result.snapshot_old_id)
+        result.snapshot_id = snapshot.id
+        result.save(update_fields=["snapshot_id"])
+        assert str(result.snapshot_id) == str(snapshot.id)
+        if idx % 5000 == 0:
+            print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0039_rename_snapshot_archiveresult_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults', to='core.snapshot', to_field='id'),
+        ),
+        migrations.RunPython(update_archiveresult_snapshot_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0041_alter_archiveresult_snapshot_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:50
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0040_archiveresult_snapshot'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot_old',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults_old', to='core.snapshot'),
+        ),
+    ]

archivebox/core/migrations/0042_remove_archiveresult_snapshot_old.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:51
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0041_alter_archiveresult_snapshot_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='archiveresult',
+            name='snapshot_old',
+        ),
+    ]

archivebox/core/migrations/0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:52
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0042_remove_archiveresult_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+    ]

archivebox/core/migrations/0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more.py

@@ -0,0 +1,40 @@
+# Generated by Django 5.0.6 on 2024-08-19 23:01
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more'),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                # No-op, SnapshotTag model already exists in DB
+            ],
+            state_operations=[
+                migrations.CreateModel(
+                    name='SnapshotTag',
+                    fields=[
+                        ('id', models.AutoField(primary_key=True, serialize=False)),
+                        ('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot')),
+                        ('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.tag')),
+                    ],
+                    options={
+                        'db_table': 'core_snapshot_tags',
+                        'unique_together': {('snapshot', 'tag')},
+                    },
+                ),
+                migrations.AlterField(
+                    model_name='snapshot',
+                    name='tags',
+                    field=models.ManyToManyField(blank=True, related_name='snapshot_set', through='core.SnapshotTag', to='core.tag'),
+                ),
+            ],
+        ),
+    ]

archivebox/core/migrations/0045_alter_snapshot_old_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 01:54
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='old_id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 5.0.6 on 2024-08-20 01:55
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0045_alter_snapshot_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='old_id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0047_alter_snapshottag_unique_together_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:16
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='tag',
+            field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
+        ),
+    ]

archivebox/core/migrations/0048_alter_archiveresult_snapshot_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0047_alter_snapshottag_unique_together_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='old_id'),
+        ),
+    ]

archivebox/core/migrations/0049_rename_snapshot_snapshottag_snapshot_old_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0048_alter_archiveresult_snapshot_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshottag',
+            old_name='snapshot',
+            new_name='snapshot_old',
+        ),
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together={('snapshot_old', 'tag')},
+        ),
+    ]

archivebox/core/migrations/0050_alter_snapshottag_snapshot_old.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:30
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0049_rename_snapshot_snapshottag_snapshot_old_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot_old',
+            field=models.ForeignKey(db_column='snapshot_old_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='old_id'),
+        ),
+    ]

archivebox/core/migrations/0051_snapshottag_snapshot_alter_snapshottag_snapshot_old.py

@@ -0,0 +1,40 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:31
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def update_snapshottag_ids(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    SnapshotTag = apps.get_model("core", "SnapshotTag")
+    num_total = SnapshotTag.objects.all().count()
+    print(f'   Updating {num_total} SnapshotTag.snapshot_id values in place... (may take an hour or longer for large collections...)')
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator()):
+        assert snapshottag.snapshot_old_id
+        snapshot = Snapshot.objects.get(old_id=snapshottag.snapshot_old_id)
+        snapshottag.snapshot_id = snapshot.id
+        snapshottag.save(update_fields=["snapshot_id"])
+        assert str(snapshottag.snapshot_id) == str(snapshot.id)
+        if idx % 100 == 0:
+            print(f'Migrated {idx}/{num_total} SnapshotTag objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0050_alter_snapshottag_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='snapshottag',
+            name='snapshot',
+            field=models.ForeignKey(blank=True, db_column='snapshot_id', null=True, on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot_old',
+            field=models.ForeignKey(db_column='snapshot_old_id', on_delete=django.db.models.deletion.CASCADE, related_name='snapshottag_old_set', to='core.snapshot', to_field='old_id'),
+        ),
+        migrations.RunPython(update_snapshottag_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0052_alter_snapshottag_unique_together_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:37
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0051_snapshottag_snapshot_alter_snapshottag_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together=set(),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together={('snapshot', 'tag')},
+        ),
+    ]

archivebox/core/migrations/0053_remove_snapshottag_snapshot_old.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:38
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0052_alter_snapshottag_unique_together_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='snapshottag',
+            name='snapshot_old',
+        ),
+    ]

archivebox/core/migrations/0054_alter_snapshot_timestamp.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0053_remove_snapshottag_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='timestamp',
+            field=models.CharField(db_index=True, editable=False, max_length=32, unique=True),
+        ),
+    ]

archivebox/core/migrations/0055_alter_tag_slug.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0054_alter_snapshot_timestamp'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(editable=False, max_length=100, unique=True),
+        ),
+    ]

archivebox/core/migrations/0056_remove_tag_uuid.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0055_alter_tag_slug'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='tag',
+            name='uuid',
+        ),
+    ]

archivebox/core/migrations/0057_rename_id_tag_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:29
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0056_remove_tag_uuid'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='tag',
+            old_name='id',
+            new_name='old_id',
+        ),
+    ]

archivebox/core/migrations/0058_alter_tag_old_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:30
+
+import core.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0057_rename_id_tag_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
+        ),
+    ]

archivebox/core/migrations/0059_tag_id.py

@@ -0,0 +1,89 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:33
+
+from django.db import migrations, models
+from abid_utils.models import ABID, abid_from_values
+
+
+def calculate_abid(self):
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+    prefix = self.abid_prefix
+    ts = eval(self.abid_ts_src)
+    uri = eval(self.abid_uri_src)
+    subtype = eval(self.abid_subtype_src)
+    rand = eval(self.abid_rand_src)
+
+    if (not prefix) or prefix == 'obj_':
+        suggested_abid = self.__class__.__name__[:3].lower()
+        raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+    if not ts:
+        ts = datetime.utcfromtimestamp(0)
+        print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+    if not uri:
+        uri = str(self)
+        print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+    if not subtype:
+        subtype = self.__class__.__name__
+        print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+    if not rand:
+        rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+        print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+    abid = abid_from_values(
+        prefix=prefix,
+        ts=ts,
+        uri=uri,
+        subtype=subtype,
+        rand=rand,
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+    return abid
+
+
+def update_archiveresult_ids(apps, schema_editor):
+    Tag = apps.get_model("core", "Tag")
+    num_total = Tag.objects.all().count()
+    print(f'   Updating {num_total} Tag.id, ArchiveResult.uuid values in place...')
+    for idx, tag in enumerate(Tag.objects.all().iterator()):
+        if not tag.slug:
+            tag.slug = tag.name.lower().replace(' ', '_')
+        if not tag.name:
+            tag.name = tag.slug
+        if not (tag.name or tag.slug):
+            tag.delete()
+            continue
+
+        assert tag.slug or tag.name, f'Tag.slug must be defined! You have a Tag(id={tag.pk}) missing a slug!'
+        tag.abid_prefix = 'tag_'
+        tag.abid_ts_src = 'self.created'
+        tag.abid_uri_src = 'self.slug'
+        tag.abid_subtype_src = '"03"'
+        tag.abid_rand_src = 'self.old_id'
+        tag.abid = calculate_abid(tag)
+        tag.id = tag.abid.uuid
+        tag.save(update_fields=["abid", "id", "name", "slug"])
+        assert str(ABID.parse(tag.abid).uuid) == str(tag.id)
+        if idx % 10 == 0:
+            print(f'Migrated {idx}/{num_total} Tag objects...')
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0058_alter_tag_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+        migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0060_alter_tag_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:42
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0059_tag_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]