Merge branch 'dev' into link-removal2

.dockerignore

@@ -5,35 +5,17 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .github/
-.pdm-build/
-.pdm-python
-.eggs/
-.git/
-!.git/HEAD
-!.git/refs/heads/*
 
 venv/
 .venv/
-.venv-old/
 .docker-venv/
-node_modules/
-chrome/
-chromeprofile/
 
-pdm.dev.lock
-pdm.lock
-
-docs/
 build/
 dist/
-brew_dist/
-deb_dist/
 pip_dist/
+!pip_dist/archivebox.egg-info/requires.txt
+brew_dist/
 assets/
-docker/
 
 data/
-data*/
 output/
-index.sqlite3
-index.sqlite3-wal

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391
+select = F,E9,W
+max-line-length = 130
+max-complexity = 10
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv

.gitattributes

@@ -1,2 +0,0 @@
-**/*.lock
-**/*-lock.json

.github/FUNDING.yml

@@ -1,2 +1,3 @@
-github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
+github: pirate
+patreon: theSquashSH
+custom: ["https://twitter.com/ArchiveBoxApp", "https://paypal.me/NicholasSweeting", "https://www.blockchain.com/eth/address/0x5D4c34D4a121Fe08d1dDB7969F07550f2dB9f471", "https://www.blockchain.com/btc/address/1HuxXriPE2Bbnag3jJrqa3bkNHrs297dYH"]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,8 +1,8 @@
 ---
 name: 🐞 Bug report
 about: Create a report to help us improve
-title: 'Bug: ...'
-labels: 'bug'
+title: 'Bugfix: ...'
+labels: 'changes: bugfixes'
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/documentation_change.md

@@ -6,7 +6,6 @@ labels: ''
 assignees: ''
 
 ---
-<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
 
 ## Wiki Page URL
 <!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->

.github/dependabot.yml

@@ -1,25 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    target-branch: "dev"
-    schedule:
-      interval: "monthly"
-    groups:
-      pip:
-        patterns:
-          - "*"
-  - package-ecosystem: "npm"
-    directory: "/"
-    target-branch: "dev"
-    schedule:
-      interval: "monthly"
-    groups:
-      npm:
-        patterns:
-          - "*"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,32 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev ]
+  schedule:
+    - cron: '43 1 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        queries: security-extended
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/codeql.yml

@@ -1,92 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "dev" ]
-  pull_request:
-    branches: [ "dev" ]
-  schedule:
-    - cron: '33 17 * * 6'
-
-jobs:
-  analyze:
-    name: Analyze (${{ matrix.language }})
-    # Runner size impacts CodeQL analysis time. To learn more, please see:
-    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
-    #   - https://gh.io/supported-runners-and-hardware-resources
-    #   - https://gh.io/using-larger-runners (GitHub.com only)
-    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
-    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
-    permissions:
-      # required for all workflows
-      security-events: write
-
-      # required to fetch internal or private CodeQL packs
-      packages: read
-
-      # only required for workflows in private repositories
-      actions: read
-      contents: read
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - language: python
-          build-mode: none
-        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
-        # Use `c-cpp` to analyze code written in C, C++ or both
-        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
-        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
-        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
-        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
-        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
-        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        build-mode: ${{ matrix.build-mode }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-    # If the analyze step fails for one of the languages you are analyzing with
-    # "We were unable to automatically build your code", modify the matrix above
-    # to set the build mode to "manual" for that language. Then modify this step
-    # to build your code.
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-    - if: matrix.build-mode == 'manual'
-      run: |
-        echo 'If you are using a "manual" build mode for one or more of the' \
-          'languages you are analyzing, replace this with the commands to build' \
-          'your code, for example:'
-        echo '  make bootstrap'
-        echo '  make release'
-        exit 1
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
-      with:
-        category: "/language:${{matrix.language}}"

.github/workflows/debian.yml

@@ -9,10 +9,10 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
@@ -25,23 +25,36 @@ jobs:
             dh-python debhelper devscripts dput software-properties-common \
             python3-distutils python3-setuptools python3-wheel python3-stdeb
 
-      # - name: Build Debian/Apt sdist_dsc
-      #   run: |
-      #     ./bin/build_pip.sh
+      - name: Build Debian/Apt sdist_dsc
+        run: |
+          rm -Rf deb_dist/*
+          python3 setup.py --command-packages=stdeb.command sdist_dsc
 
-      # - name: Check ArchiveBox version
-      #   run: |
-      #     # must create dir needed for snaps to run as non-root on github actions
-      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-      #     archivebox --version
-      #     archivebox init --setup
+      - name: Build Debian/Apt bdist_deb
+        run: |
+          python3 setup.py --command-packages=stdeb.command bdist_deb
 
-      # - name: Add some links to test
-      #   run: |
-      #     cd "${{ github.workspace }}/data"
-      #     archivebox add 'https://example.com'
-      #     archivebox status
+      - name: Install archivebox from deb
+        run: |
+          cd deb_dist/
+          sudo apt-get install ./archivebox*.deb
+
+      - name: Check ArchiveBox version
+        run: |
+          # must create dir needed for snaps to run as non-root on github actions
+          sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+          mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+          archivebox init
+          archivebox config --set SAVE_READABILITY=False
+          archivebox config --set SAVE_MERCURY=False
+          archivebox config --set SAVE_SINGLEFILE=False
+          archivebox --version
+
+      - name: Add some links to test
+        run: |
+          cd "${{ github.workspace }}/data"
+          archivebox add 'https://example.com'
+          archivebox status
 
       # - name: Commit built package
       #   run: |

.github/workflows/docker.yml

@@ -1,46 +1,43 @@
 name: Build Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - '**'
-    tags:
-      - 'v*'
-  # pull_request:
+  on: workflow_dispatch
+  release:
+    types:
+      - created
 
 env:
   DOCKER_IMAGE: archivebox-ci
 
+
 jobs:
   buildx:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
+        uses: docker/setup-qemu-action@v1
+      
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64
-
+      
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
-
+      
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
-
+      
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -48,33 +45,25 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Docker Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@v1
         if: github.event_name != 'pull_request'
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
+           username: ${{ secrets.DOCKER_USERNAME }}
+           password: ${{ secrets.DOCKER_PASSWORD }}
+      
       - name: Collect Docker tags
-        # https://github.com/docker/metadata-action
         id: docker_meta
-        uses: docker/metadata-action@v5
+        uses: crazy-max/ghaction-docker-meta@v1
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :0.7.3
-              type=semver,pattern={{version}}
-              # :0.7
-              type=semver,pattern={{major}}.{{minor}}
-              # :sha-463ea54
-              type=sha
-              # :latest
-              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
-
+          tag-sha: true
+          tag-semver: |
+            {{version}}
+            {{major}}.{{minor}}
+      
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v2
         with:
           context: ./
           file: ./Dockerfile
@@ -82,25 +71,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
-
-      - name: Update README
-        uses: peter-evans/dockerhub-description@v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
-      # This ugly bit is necessary if you don't want your cache to grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/homebrew.yml

@@ -23,12 +23,11 @@ jobs:
           cd brew_dist/
           brew install --build-bottle ./archivebox.rb
           # brew bottle archivebox
-          archivebox version
 
       - name: Add some links to test
         run: |
           mkdir data && cd data
-          archivebox init --setup
+          archivebox init
           archivebox add 'https://example.com'
           archivebox version
           archivebox status

.github/workflows/jekyll-gh-pages.yml

@@ -1,58 +0,0 @@
-# Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: Build GitHub Pages website
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: true
-
-jobs:
-  # Build job
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v3
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
-
-  # Deployment job
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v2

.github/workflows/lint.yml

@@ -28,8 +28,7 @@ jobs:
 
       - name: Lint with flake8
         run: |
-          cd archivebox
           # one pass for show-stopper syntax errors or undefined names
-          flake8 . --count --show-source --statistics
+          flake8 archivebox --count --show-source --statistics
           # one pass for small stylistic things
-          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
+          flake8 archivebox --count --max-line-length="$MAX_LINE_LENGTH" --statistics

.github/workflows/pip.yml

@@ -7,42 +7,29 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
-
-    permissions:
-      id-token: write
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
         with:
-          python-version: '3.11'
+          python-version: 3.9
           architecture: x64
 
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
-
-      - uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: '3.11'
-          cache: true
-
-      - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
-
-      - name: Build package
+      - name: Build Python Package
         run: |
-          pdm build
-
-      - name: Install from build
-        run: pip install ./dist/archivebox-*.whl
+          pip3 install --upgrade pip setuptools wheel
+          rm -Rf pip_dist/*.whl
+          python3 setup.py \
+            sdist --dist-dir=./pip_dist \
+            bdist_wheel --dist-dir=./pip_dist \
+            egg_info --egg-base=./pip_dist
+          pip install pip_dist/archivebox-*.whl
 
       - name: Add some links to test
         run: |
@@ -52,15 +39,6 @@ jobs:
           archivebox version
           archivebox status
 
-      #- name: Publish package distributions to PyPI
-      #  run: pdm publish --no-build
-
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
-
       # - name: Commit built package
       #   run: |
       #     cd pip_dist/
@@ -76,3 +54,8 @@ jobs:
       #     branch: ${{ github.ref }}
       #     directory: pip_dist
 
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}

.github/workflows/test.yml

@@ -13,33 +13,26 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: [3.11]
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+        python: [3.7]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
+      - name: Set up Node JS 14.7.0
+        uses: actions/setup-node@v1
         with:
-          node-version: 20.10.0
-
-      - name: Setup PDM
-        uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: '3.11'
-          cache: true
+          node-version: 14.7.0
 
       ### Install Python & JS Dependencies
       - name: Get pip cache dir
@@ -48,7 +41,7 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         id: cache-pip
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -56,16 +49,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
-
       - name: Install pip dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel pytest bottle build
-          python -m pip install -r requirements.txt
-          python -m pip install -e .[sonic,ldap]
+          python -m pip install --upgrade pip setuptools wheel pytest bottle
+          ./bin/build_pip.sh
+          python -m pip install .
 
       - name: Get npm cache dir
         id: npm-cache
@@ -73,7 +61,7 @@ jobs:
           echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
 
       - name: Cache npm
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         id: cache-npm
         with:
           path: ${{ steps.npm-cache.outputs.dir }}
@@ -102,13 +90,13 @@ jobs:
         # TODO: remove this exception for windows once we get tests passing on that platform
         if: ${{ !contains(matrix.os, 'windows') }}
         run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+          python -m pytest -s --ignore=archivebox/vendor
 
   docker_tests:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1

.gitignore

@@ -12,13 +12,7 @@ venv/
 .docker-venv/
 node_modules/
 
-# Ignore dev lockfiles (should always be built fresh)
-pdm.dev.lock
-requirements-dev.txt
-
 # Packaging artifacts
-.pdm-python
-.pdm-build
 archivebox.egg-info
 archivebox-*.tar.gz
 build/
@@ -26,12 +20,7 @@ dist/
 
 # Data folders
 data/
-data*/
+data1/
+data2/
+data3/
 output/
-index.sqlite3
-*.sqlite*
-data.*
-
-# vim
-*.sw?
-.vscode

.gitmodules

@@ -23,9 +23,3 @@
 [submodule "archivebox/vendor/django-taggit"]
 	path = archivebox/vendor/django-taggit
 	url = https://github.com/jazzband/django-taggit
-[submodule "archivebox/vendor/python-atomicwrites"]
-	path = archivebox/vendor/python-atomicwrites
-	url = https://github.com/untitaker/python-atomicwrites
-[submodule "archivebox/vendor/pydantic-pkgr"]
-	path = archivebox/vendor/pydantic-pkgr
-	url = https://github.com/ArchiveBox/pydantic-pkgr

.readthedocs.yaml

@@ -1,33 +0,0 @@
-# Read the Docs configuration file for Sphinx projects
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    nodejs: "20"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
-  # builder: "dirhtml"
-
-# Optionally build your docs in additional formats such as PDF and ePub
-formats:
-  - pdf
-  - epub
-
-# Optional but recommended, declare the Python requirements required
-# to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt

Dockerfile

@@ -1,326 +1,117 @@
 # This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, single-file
 # Usage:
-#     git submodule update --init --recursive
-#     git pull --recurse-submodules
 #     docker build . -t archivebox --no-cache
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-# Multi-arch build:
-#     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
-#
-# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
-
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
-FROM python:3.11-slim-bookworm
+FROM python:3.9-slim-buster
 
 LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
+    maintainer="Nick Sweeting <archivebox-docker@sweeting.me>" \
+    description="All-in-one personal internet archiving container" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-
-######### Environment Variables #################################
-
-# Global system-level config
+# System-level base config
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
     PYTHONIOENCODING=UTF-8 \
     PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
 
-# Version config
-ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=20
-
-# User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911
-
-# Global paths
+# Application-level base config
 ENV CODE_DIR=/app \
+    VENV_PATH=/venv \
     DATA_DIR=/data \
-    GLOBAL_VENV=/venv \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Application-level paths
-ENV APP_VENV=/app/.venv \
-    NODE_MODULES=/app/node_modules
-
-# Build shell config
-ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading package.json
-COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
-RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && cat /etc/os-release | head -n7 \
-    && which bash && bash --version | head -n1 \
-    && which dpkg && dpkg --version | head -n1 \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    NODE_DIR=/node \
+    ARCHIVEBOX_USER="archivebox"
 
 # Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+RUN groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER
 
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        # nano iputils-ping dnsutils htop procps jq yq
+# Install system dependencies
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        apt-transport-https ca-certificates gnupg2 zlib1g-dev \
+        dumb-init gosu cron unzip curl \
     && rm -rf /var/lib/apt/lists/*
 
-######### Language Environments ####################################
-
-# Install Python environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    && ( \
-        which python3 && python3 --version | grep " $PYTHON_VERSION" \
-        && which pip && pip --version \
-        # && which pdm && pdm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+# Install apt dependencies
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        wget curl chromium git ffmpeg youtube-dl ripgrep \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+    && rm -rf /var/lib/apt/lists/*
 
+# Install apt development dependencies
+# RUN apt-get install -qq \
+#     && apt-get install -qq -y --no-install-recommends \
+#         python3 python3-dev python3-pip python3-venv python3-all \
+#         dh-python debhelper devscripts dput software-properties-common \
+#         python3-distutils python3-setuptools python3-wheel python3-stdeb
 
 # Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo 'deb https://deb.nodesource.com/node_15.x buster main' >> /etc/apt/sources.list \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
-    && apt-get install -y -t bookworm-backports --no-upgrade \
+    && apt-get install -qq -y --no-install-recommends \
         nodejs \
-    && rm -rf /var/lib/apt/lists/* \
-    # Update NPM to latest version
-    && npm i -g npm --cache /root/.npm \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### Extractor Dependencies ##################################
-
-# Install apt dependencies
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT extractor dependencies globally using apt..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        curl wget git yt-dlp ffmpeg ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
-    && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        && which rg && rg --version 2>&1 | head -n1 \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Install chromium browser using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
-        # xfonts-scalable xfonts-utils xserver-common xvfb \
-        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
-        # libxss1 dbus dbus-x11 upower \
-    # && service dbus start \
-    # install Chromium using playwright
-    && pip install playwright \
-    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-    && playwright install chromium \
-    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
-    && rm -rf /var/lib/apt/lists/* \
-    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
-    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
-    # Save version info
-    && ( \
-        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    && rm -rf /var/lib/apt/lists/*
 
 # Install Node dependencies
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
-    && npm ci --prefer-offline --no-audit --cache /root/.npm \
-    && ( \
-        which node && node --version \
-        && which npm && npm version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+WORKDIR "$NODE_DIR"
+ENV PATH="${PATH}:$NODE_DIR/node_modules/.bin" \
+    npm_config_loglevel=error
+ADD ./package.json ./package.json
+ADD ./package-lock.json ./package-lock.json
+RUN npm ci
 
-######### Build Dependencies ####################################
-
-# Install ArchiveBox Python dependencies
+# Install Python dependencies
 WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        build-essential \
-        libssl-dev libldap2-dev libsasl2-dev \
-        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
-    # && pdm use --venv in-project \
-    # && pdm run python -m ensurepip \
-    # && pdm sync --fail-fast --no-editable --group :all --no-self \
-    # && pdm export -o requirements.txt --without-hashes \
-    # && source $GLOBAL_VENV/bin/activate \
-    && pip install -r requirements.txt \
-    && apt-get purge -y \
-        build-essential \
+ENV PATH="${PATH}:$VENV_PATH/bin"
+RUN python -m venv --clear --symlinks "$VENV_PATH" \
+    && pip install --upgrade --quiet pip setuptools
+ADD ./pip_dist/archivebox.egg-info/requires.txt "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt"
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential python-dev python3-dev \
+    && grep -B 1000 -E '^$' "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin \
+    && pip install --quiet "sonic-client==0.0.5" \
+    && apt-get purge -y build-essential python-dev python3-dev \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
-# Install ArchiveBox Python package from source
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    # && apt-get update -qq \
-    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
-    # && apt-get install -qq -y -t bookworm-backports \
-        # build-essential  \
-    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
-    && pip install -e "$CODE_DIR"[sonic,ldap] \
-    # save docker image size and always remove compilers / build tools after building is complete
-    # && apt-get purge -y build-essential \
-    # && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-####################################################
+# Install ArchiveBox Python package and its dependencies
+WORKDIR "$CODE_DIR"
+ADD . "$CODE_DIR"
+RUN pip install -e .
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
 ENV IN_DOCKER=True \
-    DISPLAY=novnc:0.0 \
-    CUSTOM_TEMPLATES_DIR=/data/templates \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no \
-    ALLOWED_HOSTS=*
-    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
-    # WGET_BINARY="wget" \
-    # YOUTUBEDL_BINARY="yt-dlp" \
-    # CHROME_BINARY="/usr/bin/chromium-browser" \
-    # USE_SINGLEFILE=True \
-    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
-    # USE_READABILITY=True \
-    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
-    # USE_MERCURY=True \
-    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
+    CHROME_SANDBOX=False \
+    CHROME_BINARY="chromium" \
+    USE_SINGLEFILE=True \
+    SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \
+    USE_READABILITY=True \
+    READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor" \
+    USE_MERCURY=True \
+    MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser"
 
 # Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
-
-####################################################
+# RUN archivebox version
+RUN /app/bin/docker_entrypoint.sh archivebox version
 
 # Open up the interfaces to the outside world
-WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
-
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "0.0.0.0:8000"]

MANIFEST.in

@@ -0,0 +1,4 @@
+graft archivebox
+global-exclude .DS_Store
+global-exclude __pycache__
+global-exclude *.pyc

Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[packages]
+# see setup.py for package dependency list
+"e1839a8" = {path = ".", editable = true}
+
+[dev-packages]
+# see setup.py for dev package dependency list
+"e1839a8" = {path = ".", extras = ["dev"], editable = true}

SECURITY.md

@@ -1,34 +0,0 @@
-# Security Policy
-
----
-
-## Security Information
-
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
-
-- https://github.com/ArchiveBox/ArchiveBox#archive-layout
-- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
-
----
-
-## Reporting a Vulnerability
-
-We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
-
-1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
-
-2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
-
-3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
-
-You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-merlot

archivebox/.flake8

@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
+exclude = migrations,tests,node_modules,vendor,static,venv,.venv,.venv2,.docker-venv

archivebox/__init__.py

@@ -1,4 +1 @@
 __package__ = 'archivebox'
-
-
-from .monkey_patches import *

archivebox/abid_utils/__init__.py

@@ -1 +0,0 @@
-__package__ = 'abid_utils'

archivebox/abid_utils/abid.py

@@ -1,202 +0,0 @@
-from typing import NamedTuple, Any, Union, Optional
-
-import ulid
-import uuid6
-import hashlib
-from urllib.parse import urlparse
-
-from uuid import UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-
-
-
-ABID_PREFIX_LEN = 4
-ABID_SUFFIX_LEN = 26
-ABID_LEN = 30
-ABID_TS_LEN = 10
-ABID_URI_LEN = 8
-ABID_SUBTYPE_LEN = 2
-ABID_RAND_LEN = 6
-
-DEFAULT_ABID_PREFIX = 'obj_'
-
-# allows people to keep their uris secret on a per-instance basis by changing the salt.
-# the default means everyone can share the same namespace for URI hashes,
-# meaning anyone who has a URI and wants to check if you have it can guess the ABID
-DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
-
-
-class ABID(NamedTuple):
-    """
-    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
-    """
-    prefix: str            # e.g. obj_
-    ts: str                # e.g. 01HX9FPYTR
-    uri: str               # e.g. E4A5CCD9
-    subtype: str           # e.g. 01
-    rand: str              # e.g. ZYEBQE
-    
-    # salt: str = DEFAULT_ABID_URI_SALT
-
-    def __getattr__(self, attr: str) -> Any:
-        return getattr(self.ulid, attr)
-
-    def __eq__(self, other: Any) -> bool:
-        try:
-            return self.ulid == other.ulid
-        except AttributeError:
-            return NotImplemented
-
-    def __str__(self) -> str:
-        return self.prefix + self.suffix
-
-    def __len__(self) -> int:
-        return len(self.prefix + self.suffix)
-
-    @classmethod
-    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
-        assert buffer, f'Attempted to create ABID from null value {buffer}'
-
-        buffer = str(buffer)
-        if '_' in buffer:
-            prefix, suffix = buffer.split('_')
-        else:
-            prefix, suffix = prefix.strip('_'), buffer
-
-        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
-        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
-
-        return cls(
-            prefix=abid_part_from_prefix(prefix),
-            ts=suffix[0:10].upper(),
-            uri=suffix[10:18].upper(),
-            subtype=suffix[18:20].upper(),
-            rand=suffix[20:26].upper(),
-        )
-    
-    @property
-    def uri_salt(self) -> str:
-        return DEFAULT_ABID_URI_SALT
-
-    @property
-    def suffix(self):
-        return ''.join((self.ts, self.uri, self.subtype, self.rand))
-    
-    @property
-    def ulid(self) -> ulid.ULID:
-        return ulid.parse(self.suffix)
-
-    @property
-    def uuid(self) -> UUID:
-        return self.ulid.uuid
-
-    @property
-    def uuid6(self) -> uuid6.UUID:
-        return uuid6.UUID(hex=self.uuid.hex)
-
-    @property
-    def typeid(self) -> TypeID:
-        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
-
-    @property
-    def datetime(self) -> datetime:
-        return self.ulid.timestamp().datetime
-
-
-
-####################################################
-
-
-def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
-    """
-    if isinstance(uri, bytes):
-        uri_str: str = uri.decode()
-    else:
-        uri_str = str(uri)
-
-    # only hash the domain part of URLs
-    if '://' in uri_str:
-        try:
-            domain = urlparse(uri_str).netloc
-            if domain:
-                uri_str = domain
-        except AttributeError:
-            pass
-    
-    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
-
-    return hashlib.sha256(uri_bytes).hexdigest().upper()
-
-def abid_part_from_prefix(prefix: Optional[str]) -> str:
-    """
-    'snp_'
-    """
-    if prefix is None:
-        return 'obj_'
-
-    prefix = prefix.strip('_').lower()
-    assert len(prefix) == 3
-    return prefix + '_'
-
-def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9'     # takes first 8 characters of sha256(url)
-    """
-    uri = str(uri)
-    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
-
-def abid_part_from_ts(ts: Optional[datetime]) -> str:
-    """
-    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
-    """
-    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
-
-def abid_part_from_subtype(subtype: str) -> str:
-    """
-    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
-    Also allows us to change the ulid spec later by putting special sigil values here.
-    """
-    subtype = str(subtype)
-    if len(subtype) == ABID_SUBTYPE_LEN:
-        return subtype
-
-    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
-
-def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
-    """
-    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
-    """
-    if rand is None:
-        # if it's None we generate a new random 6 character hex string
-        return str(ulid.new())[-ABID_RAND_LEN:]
-    elif isinstance(rand, UUID):
-        # if it's a uuid we take the last 6 characters of the ULID represation of it
-        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
-    elif isinstance(rand, int):
-        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
-        rand_str = str(rand)[-ABID_RAND_LEN:]
-        padding_needed = ABID_RAND_LEN - len(rand_str)
-        rand_str = ('0'*padding_needed) + rand_str
-        return rand_str
-
-    # otherwise treat it as a string, take the last 6 characters of it verbatim
-    return str(rand)[-ABID_RAND_LEN:].upper()
-
-
-def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
-    """
-    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-    """
-
-    abid = ABID(
-        prefix=abid_part_from_prefix(prefix),
-        ts=abid_part_from_ts(ts),
-        uri=abid_part_from_uri(uri, salt=salt),
-        subtype=abid_part_from_subtype(subtype),
-        rand=abid_part_from_rand(rand),
-    )
-    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
-    return abid

archivebox/abid_utils/admin.py

@@ -1,18 +0,0 @@
-from django.contrib import admin
-
-
-class ABIDModelAdmin(admin.ModelAdmin):
-    list_display = ('created', 'created_by', 'abid', '__str__')
-    sort_fields = ('created', 'created_by', 'abid', '__str__')
-    readonly_fields = ('abid', 'created', '__str__')
-
-    def get_form(self, request, obj=None, **kwargs):
-        form = super().get_form(request, obj, **kwargs)
-        if 'created_by' in form.base_fields:
-            form.base_fields['created_by'].initial = request.user
-        return form
-
-    # def save_model(self, request, obj, form, change):
-    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
-    #         obj.created_by = request.user
-    #     obj.save()

archivebox/abid_utils/apps.py

@@ -1,7 +0,0 @@
-from django.apps import AppConfig
-
-
-class AbidUtilsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    
-    name = 'abid_utils'

archivebox/abid_utils/models.py

@@ -1,344 +0,0 @@
-"""
-This file provides the Django ABIDField and ABIDModel base model to inherit from.
-
-It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
-"""
-
-from typing import Any, Dict, Union, List, Set, NamedTuple, cast
-
-from ulid import ULID
-from uuid import uuid4, UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-from functools import partial
-from charidfield import CharIDField  # type: ignore[import-untyped]
-
-from django.conf import settings
-from django.db import models
-from django.utils import timezone
-from django.db.utils import OperationalError
-from django.contrib.auth import get_user_model
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from .abid import (
-    ABID,
-    ABID_LEN,
-    ABID_RAND_LEN,
-    ABID_SUFFIX_LEN,
-    DEFAULT_ABID_PREFIX,
-    DEFAULT_ABID_URI_SALT,
-    abid_part_from_prefix,
-    abid_from_values
-)
-
-####################################################
-
-
-# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
-ABIDField = partial(
-    CharIDField,
-    max_length=ABID_LEN,
-    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
-    default=None,
-    null=True,
-    blank=True,
-    db_index=True,
-    unique=True,
-)
-
-def get_or_create_system_user_pk(username='system'):
-    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
-
-    User = get_user_model()
-
-    # if only one user exists total, return that user
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
-
-    # otherwise, create a dedicated "system" user
-    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
-    return user.pk
-
-
-class ABIDModel(models.Model):
-    """
-    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
-    """
-    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
-    abid_ts_src = 'None'                    # e.g. 'self.created'
-    abid_uri_src = 'None'                   # e.g. 'self.uri'
-    abid_subtype_src = 'None'               # e.g. 'self.extractor'
-    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
-
-    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
-    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args: Any, **kwargs: Any) -> None:
-        # when first creating a row, self.ABID is the source of truth
-        # overwrite default prefilled self.id & self.abid with generated self.ABID value
-        if self._state.adding or not self.id:
-            self.id = self.ABID.uuid
-        if self._state.adding or not self.abid:
-            self.abid = str(self.ABID)
-
-        super().save(*args, **kwargs)
-        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
-        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
-
-    @property
-    def abid_values(self) -> Dict[str, Any]:
-        return {
-            'prefix': self.abid_prefix,
-            'ts': eval(self.abid_ts_src),
-            'uri': eval(self.abid_uri_src),
-            'subtype': eval(self.abid_subtype_src),
-            'rand': eval(self.abid_rand_src),
-        }
-
-    def generate_abid(self) -> ABID:
-        """
-        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-        """
-        prefix, ts, uri, subtype, rand = self.abid_values.values()
-
-        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
-            suggested_abid = self.__class__.__name__[:3].lower()
-            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
-
-        if not ts:
-            # default to unix epoch with 00:00:00 UTC
-            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
-            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
-
-        if not uri:
-            uri = str(self)
-            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
-
-        if not subtype:
-            subtype = self.__class__.__name__
-            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
-
-        if not rand:
-            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
-            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
-
-        abid = abid_from_values(
-            prefix=prefix,
-            ts=ts,
-            uri=uri,
-            subtype=subtype,
-            rand=rand,
-            salt=DEFAULT_ABID_URI_SALT,
-        )
-        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
-        return abid
-
-    @property
-    def ABID(self) -> ABID:
-        """
-        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
-        """
-        
-        # if object is not yet saved to DB, always generate fresh ABID from values
-        if self._state.adding:
-            return self.generate_abid()
-        
-        # otherwise DB is single source of truth, load ABID from existing db pk
-        abid: ABID | None = None
-        try:
-            abid = abid or ABID.parse(self.pk)
-        except Exception:
-            pass
-
-        try:
-            abid = abid or ABID.parse(self.id)
-        except Exception:
-            pass
-
-        try:
-            abid = abid or ABID.parse(cast(str, self.abid))
-        except Exception:
-            pass
-
-        abid = abid or self.generate_abid()
-
-        return abid
-
-    @property
-    def ULID(self) -> ULID:
-        """
-        Get a ulid.ULID representation of the object's ABID.
-        """
-        return self.ABID.ulid
-
-    @property
-    def UUID(self) -> UUID:
-        """
-        Get a uuid.UUID (v4) representation of the object's ABID.
-        """
-        return self.ABID.uuid
-
-    @property
-    def TypeID(self) -> TypeID:
-        """
-        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-        """
-        return self.ABID.typeid
-
-
-
-####################################################
-
-# Django helpers
-def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
-    """
-    Return the mapping of all ABID prefixes to their models.
-    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
-    """
-    import django.apps
-    prefix_map = {}
-
-    for model in django.apps.apps.get_models():
-        abid_prefix = getattr(model, 'abid_prefix', None)
-        if abid_prefix:
-            prefix_map[abid_prefix] = model
-    return prefix_map
-
-def find_prefix_for_abid(abid: ABID) -> str:
-    """
-    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
-    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
-    """
-    # if existing abid prefix is correct, lookup is easy
-    model = find_model_from_abid(abid)
-    if model:
-        assert issubclass(model, ABIDModel)
-        return model.abid_prefix
-
-    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
-    return find_obj_from_abid_rand(abid)[0].abid_prefix
-
-def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
-    """
-    Return the Django Model that corresponds to a given ABID prefix.
-    e.g. 'tag_' -> core.models.Tag
-    """
-    prefix = abid_part_from_prefix(prefix)
-
-    import django.apps
-
-    for model in django.apps.apps.get_models():
-        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
-        if not hasattr(model, 'objects'): continue      # skip abstract models
-
-        if (model.abid_prefix == prefix):
-            return model
-
-    return None
-
-def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
-    """
-    Shortcut for find_model_from_abid_prefix(abid.prefix)
-    """
-    return find_model_from_abid_prefix(abid.prefix)
-
-def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
-    """
-    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
-    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    # convert str to ABID if necessary
-    if isinstance(rand, ABID):
-        abid: ABID = rand
-    else:
-        rand = str(rand)
-        if len(rand) < ABID_SUFFIX_LEN:
-            padding_needed = ABID_SUFFIX_LEN - len(rand)
-            rand = ('0'*padding_needed) + rand
-        abid = ABID.parse(rand)
-
-    import django.apps
-
-    partial_matches: List[ABIDModel] = []
-
-    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
-        model,
-        find_model_from_abid(abid),
-        *django.apps.apps.get_models(),
-    ))))
-    # print(abid, abid.rand, abid.uuid, models_to_try)
-
-    for model in models_to_try:
-        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
-        if not hasattr(model, 'objects'): continue      # skip abstract Models
-        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
-
-        # continue on to try fuzzy searching by randomness portion derived from uuid field
-        try:
-            qs = []
-            if hasattr(model, 'abid'):
-                qs = model.objects.filter(abid__endswith=abid.rand)
-            elif hasattr(model, 'uuid'):
-                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-            elif hasattr(model, 'id'):
-                # NOTE: this only works on SQLite where every column is a string
-                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
-                
-                # try to search for uuid=...-2354352
-                # try to search for id=...2354352
-                # try to search for id=2354352
-                qs = model.objects.filter(
-                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-                    | models.Q(id__endswith=abid.rand)
-                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
-                )
-
-            for obj in qs:
-                if obj.generate_abid() == abid:
-                    # found exact match, no need to keep iterating
-                    return [obj]
-                partial_matches.append(obj)
-        except OperationalError as err:
-            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
-
-    return partial_matches
-
-def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
-    """
-    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
-    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    model = model or find_model_from_abid(abid)
-    assert model, f'Could not find model that could match this ABID type: {abid}'
-
-    try:
-        if hasattr(model, 'abid'):
-            return model.objects.get(abid__endswith=abid.suffix)
-        if hasattr(model, 'uuid'):
-            return model.objects.get(uuid=abid.uuid)
-        return model.objects.get(id=abid.uuid)
-    except model.DoesNotExist:
-        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
-        if hasattr(model, 'abid') or (not fuzzy):
-            raise
-
-    # continue on to try fuzzy searching by randomness portion derived from uuid field
-    match_by_rand = find_obj_from_abid_rand(abid, model=model)
-    if match_by_rand:
-        if match_by_rand[0].abid_prefix != abid.prefix:
-            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
-        return match_by_rand
-
-    raise model.DoesNotExist
-

archivebox/abid_utils/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/api/__init__.py

@@ -1 +0,0 @@
-__package__ = 'archivebox.api'

archivebox/api/apps.py

@@ -1,11 +0,0 @@
-__package__ = 'archivebox.api'
-
-from django.apps import AppConfig
-
-
-
-class APIConfig(AppConfig):
-    name = 'api'
-
-    def ready(self):
-        pass

archivebox/api/auth.py

@@ -1,110 +0,0 @@
-__package__ = 'archivebox.api'
-
-from typing import Optional, cast
-
-from django.http import HttpRequest
-from django.contrib.auth import login
-from django.contrib.auth import authenticate
-from django.contrib.auth.models import AbstractBaseUser
-
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
-
-
-def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
-    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
-    
-    user = None
-
-    submitted_empty_form = token in ('string', '', None)
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
-        try:
-            token = APIToken.objects.get(token=token)
-            if token.is_valid():
-                user = token.created_by
-        except APIToken.DoesNotExist:
-            pass
-
-    if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
-
-    return None
-
-def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
-    """Given a username and password, check if they are valid and return the corresponding user"""
-    user = None
-    
-    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
-        user = authenticate(
-            username=username,
-            password=password,
-        )
-
-    if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
-        user = None
-
-    return cast(AbstractBaseUser | None, user)
-
-
-### Base Auth Types
-
-class APITokenAuthCheck:
-    """The base class for authentication methods that use an api.models.APIToken"""
-    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_token(
-            token=key,
-            request=request,
-        )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
-
-class UserPassAuthCheck:
-    """The base class for authentication methods that use a username & password"""
-    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_password(
-            username=username,
-            password=password,
-            request=request,
-        )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
-
-
-### Django-Ninja-Provided Auth Methods
-
-class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
-    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-    param_name = "X-ArchiveBox-API-Key"
-
-class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
-    """Allow authenticating by passing Bearer=xyz as a request header"""
-    pass
-
-class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
-    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-    param_name = "api_key"
-
-class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-    pass
-
-
-### Enabled Auth Methods
-
-API_AUTH_METHODS = [
-    HeaderTokenAuth(),
-    BearerTokenAuth(),
-    QueryParamTokenAuth(), 
-    django_auth_superuser,
-    UsernameAndPasswordAuth(),
-]

archivebox/api/migrations/0001_initial.py

@@ -1,29 +0,0 @@
-# Generated by Django 4.2.11 on 2024-04-25 04:19
-
-import api.models
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='APIToken',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('expires', models.DateTimeField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-    ]

archivebox/api/migrations/0002_alter_apitoken_options.py

@@ -1,17 +0,0 @@
-# Generated by Django 5.0.4 on 2024-04-26 05:28
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='apitoken',
-            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
-        ),
-    ]

archivebox/api/migrations/0003_rename_user_apitoken_created_by_apitoken_abid_and_more.py

@@ -1,77 +0,0 @@
-# Generated by Django 5.0.6 on 2024-06-03 01:52
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import signal_webhooks.fields
-import signal_webhooks.utils
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_alter_apitoken_options'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='user',
-            new_name='created_by',
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='abid',
-            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='modified',
-            field=models.DateTimeField(auto_now=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, null=True, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
-        ),
-        migrations.CreateModel(
-            name='OutboundWebhook',
-            fields=[
-                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
-                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
-                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
-                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
-                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
-                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
-                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
-                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
-                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
-                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
-                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
-                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(blank=True, null=True, unique=True)),
-                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'verbose_name': 'API Outbound Webhook',
-                'abstract': False,
-            },
-        ),
-        migrations.AddConstraint(
-            model_name='outboundwebhook',
-            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
-        ),
-    ]

archivebox/api/migrations/0004_alter_apitoken_id_alter_apitoken_uuid.py

@@ -1,24 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 10:44
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
-        ),
-    ]

archivebox/api/migrations/0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:40
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='apitoken',
-            name='uuid',
-        ),
-        migrations.RemoveField(
-            model_name='outboundwebhook',
-            name='id',
-        ),
-    ]

archivebox/api/migrations/0006_remove_outboundwebhook_uuid_apitoken_id_and_more.py

@@ -1,29 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:43
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='uuid',
-            new_name='id'
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-    ]

archivebox/api/migrations/0007_alter_apitoken_created_by.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:52
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]

archivebox/api/models.py

@@ -1,122 +0,0 @@
-__package__ = 'archivebox.api'
-
-import uuid
-import secrets
-from datetime import timedelta
-
-from django.conf import settings
-from django.db import models
-from django.utils import timezone
-
-from signal_webhooks.models import WebhookBase
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
-
-
-
-def generate_secret_token() -> str:
-    # returns cryptographically secure string with len() == 32
-    return secrets.token_hex(16)
-
-
-class APIToken(ABIDModel):
-    """
-    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
-    """
-    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
-    abid_prefix = 'apt_'
-    abid_ts_src = 'self.created'
-    abid_uri_src = 'self.token'
-    abid_subtype_src = 'self.created_by_id'
-    abid_rand_src = 'self.id'
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-
-    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
-    expires = models.DateTimeField(null=True, blank=True)
-
-    class Meta(TypedModelMeta):
-        verbose_name = "API Key"
-        verbose_name_plural = "API Keys"
-
-    def __str__(self) -> str:
-        return self.token
-
-    def __repr__(self) -> str:
-        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
-
-    def __json__(self) -> dict:
-        return {
-            "TYPE":             "APIToken",    
-            "id":               str(self.pk),
-            "abid":             str(self.ABID),
-            "created_by_id":    str(self.created_by_id),
-            "token":            self.token,
-            "created":          self.created.isoformat(),
-            "expires":          self.expires_as_iso8601,
-        }
-
-    @property
-    def expires_as_iso8601(self):
-        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
-        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
-
-        return expiry_date.isoformat()
-    
-    @property
-    def token_redacted(self):
-        return f'************{self.token[-4:]}'
-
-    def is_valid(self, for_date=None):
-        for_date = for_date or timezone.now()
-
-        if self.expires and self.expires < for_date:
-            return False
-
-        return True
-
-
-
-
-
-
-# monkey patch django-signals-webhooks to change how it shows up in Admin UI
-
-class OutboundWebhook(ABIDModel, WebhookBase):
-    """
-    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
-        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
-    """
-    abid_prefix = 'whk_'
-    abid_ts_src = 'self.created'
-    abid_uri_src = 'self.endpoint'
-    abid_subtype_src = 'self.ref'
-    abid_rand_src = 'self.id'
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
-
-    # More fields here: WebhookBase...
-
-    WebhookBase._meta.get_field('name').help_text = (
-        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
-    WebhookBase._meta.get_field('signal').help_text = (
-        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
-    WebhookBase._meta.get_field('ref').help_text = (
-        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
-    WebhookBase._meta.get_field('endpoint').help_text = (
-        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
-
-    class Meta(WebhookBase.Meta):
-        verbose_name = 'API Outbound Webhook'
-

archivebox/api/tests.py

@@ -1,30 +0,0 @@
-__package__ = 'archivebox.api'
-
-from django.test import TestCase
-from ninja.testing import TestClient
-
-from .routes_cli import router
-
-class ArchiveBoxCLIAPITestCase(TestCase):
-    def setUp(self):
-        self.client = TestClient(router)
-
-    def test_add_endpoint(self):
-        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
-
-    def test_remove_endpoint(self):
-        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
-
-    def test_update_endpoint(self):
-        response = self.client.post("/update", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
-
-    def test_list_all_endpoint(self):
-        response = self.client.post("/list_all", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])

archivebox/api/urls.py

@@ -1,17 +0,0 @@
-__package__ = 'archivebox.api'
-
-from django.urls import path
-from django.views.generic.base import RedirectView
-
-from .v1_api import urls as v1_api_urls
-
-urlpatterns = [
-    path("",                 RedirectView.as_view(url='/api/v1')),
-
-    path("v1/",              v1_api_urls),
-    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
-
-    # ... v2 can be added here ...
-    # path("v2/",              v2_api_urls),
-    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
-]

archivebox/api/v1_api.py

@@ -1,111 +0,0 @@
-__package__ = 'archivebox.api'
-
-
-from io import StringIO
-from traceback import format_exception
-from contextlib import redirect_stdout, redirect_stderr
-
-from django.http import HttpRequest, HttpResponse
-from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-
-from ninja import NinjaAPI, Swagger
-
-# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
-
-from api.auth import API_AUTH_METHODS
-from ..config import VERSION, COMMIT_HASH
-
-
-COMMIT_HASH = COMMIT_HASH or 'unknown'
-
-html_description=f'''
-<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
-<br/>
-<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
-<br/>
-<ul>
-<li>⬅️ Manage your server: <a href="/admin/api/"><b>Setup API Keys</b></a>, <a href="/admin/">Go to your Server Admin UI</a>, <a href="/">Go to your Snapshots list</a> 
-<li>💬 Ask questions and get help here: <a href="https://zulip.archivebox.io">ArchiveBox Chat Forum</a></li>
-<li>🐞 Report API bugs here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues">Github Issues</a></li>
-<li>📚 ArchiveBox Documentation: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Github Wiki</a></li>
-<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
-</ul>
-<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-'''
-
-
-def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router('/auth/',     'api.v1_auth.router')
-    api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/cli/',      'api.v1_cli.router')
-    return api
-
-
-class NinjaAPIWithIOCapture(NinjaAPI):    
-    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
-        stdout, stderr = StringIO(), StringIO()
-
-        with redirect_stderr(stderr):
-            with redirect_stdout(stdout):
-                request.stdout = stdout
-                request.stderr = stderr
-
-                response = super().create_temporal_response(request)
-
-        print('RESPONDING NOW', response)
-
-        return response
-
-
-api = NinjaAPIWithIOCapture(
-    title='ArchiveBox API',
-    description=html_description,
-    version='1.0.0',
-    csrf=False,
-    auth=API_AUTH_METHODS,
-    urls_namespace="api-1",
-    docs=Swagger(settings={"persistAuthorization": True}),
-    # docs_decorator=login_required,
-    # renderer=ORJSONRenderer(),
-)
-api = register_urls(api)
-urls = api.urls
-
-
-@api.exception_handler(Exception)
-def generic_exception_handler(request, err):
-    status = 503
-    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
-        status = 404
-
-    print(''.join(format_exception(err)))
-
-    return api.create_response(
-        request,
-        {
-            "succeeded": False,
-            "message": f'{err.__class__.__name__}: {err}',
-            "errors": [
-                ''.join(format_exception(err)),
-                # or send simpler parent-only traceback:
-                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
-            ],
-        },
-        status=status,
-    )
-
-
-
-# import orjson
-# from ninja.renderers import BaseRenderer
-# class ORJSONRenderer(BaseRenderer):
-#     media_type = "application/json"
-#     def render(self, request, data, *, response_status):
-#         return {
-#             "success": True,
-#             "errors": [],
-#             "result": data,
-#             "stdout": ansi_to_html(stdout.getvalue().strip()),
-#             "stderr": ansi_to_html(stderr.getvalue().strip()),
-#         }
-#         return orjson.dumps(data)

archivebox/api/v1_auth.py

@@ -1,52 +0,0 @@
-__package__ = 'archivebox.api'
-
-from typing import Optional
-
-from ninja import Router, Schema
-
-from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password
-
-
-router = Router(tags=['Authentication'])
-
-
-class PasswordAuthSchema(Schema):
-    """Schema for a /get_api_token request"""
-    username: Optional[str] = None
-    password: Optional[str] = None
-
-
-@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
-def get_api_token(request, auth_data: PasswordAuthSchema):
-    user = auth_using_password(
-        username=auth_data.username,
-        password=auth_data.password,
-        request=request,
-    )
-
-    if user:
-        # TODO: support multiple tokens in the future, for now we just have one per user
-        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
-
-        return api_token.__json__()
-    
-    return {"success": False, "errors": ["Invalid credentials"]}
-
-
-
-class TokenAuthSchema(Schema):
-    """Schema for a /check_api_token request"""
-    token: str
-
-
-@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
-def check_api_token(request, token_data: TokenAuthSchema):
-    user = auth_using_token(
-        token=token_data.token,
-        request=request,
-    )
-    if user:
-        return {"success": True, "user_id": str(user.pk)}
-    
-    return {"success": False, "user_id": None}

archivebox/api/v1_cli.py

@@ -1,234 +0,0 @@
-__package__ = 'archivebox.api'
-
-from typing import List, Dict, Any, Optional
-from enum import Enum
-
-from ninja import Router, Schema
-
-from ..main import (
-    add,
-    remove,
-    update,
-    list_all,
-    schedule,
-)
-from ..util import ansi_to_html
-from ..config import ONLY_NEW
-
-
-# router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'])
-
-
-# Schemas
-
-JSONType = List[Any] | Dict[str, Any] | bool | int | str | None
-
-class CLICommandResponseSchema(Schema):
-    success: bool
-    errors: List[str]
-    result: JSONType
-    stdout: str
-    stderr: str
-
-class FilterTypeChoices(str, Enum):
-    exact = 'exact'
-    substring = 'substring'
-    regex = 'regex'
-    domain = 'domain'
-    tag = 'tag'
-    timestamp = 'timestamp'
-
-class StatusChoices(str, Enum):
-    indexed = 'indexed'
-    archived = 'archived'
-    unarchived = 'unarchived'
-    present = 'present'
-    valid = 'valid'
-    invalid = 'invalid'
-    duplicate = 'duplicate'
-    orphaned = 'orphaned'
-    corrupted = 'corrupted'
-    unrecognized = 'unrecognized'
-
-
-class AddCommandSchema(Schema):
-    urls: List[str]
-    tag: str = ""
-    depth: int = 0
-    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
-    update_all: bool = False
-    index_only: bool = False
-    overwrite: bool = False
-    init: bool = False
-    extractors: str = ""
-    parser: str = "auto"
-
-class UpdateCommandSchema(Schema):
-    resume: Optional[float] = 0
-    only_new: bool = ONLY_NEW
-    index_only: bool = False
-    overwrite: bool = False
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    status: Optional[StatusChoices] = StatusChoices.unarchived
-    filter_type: Optional[str] = FilterTypeChoices.substring
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-    extractors: Optional[str] = ""
-
-class ScheduleCommandSchema(Schema):
-    import_path: Optional[str] = None
-    add: bool = False
-    every: Optional[str] = None
-    tag: str = ''
-    depth: int = 0
-    overwrite: bool = False
-    update: bool = not ONLY_NEW
-    clear: bool = False
-
-class ListCommandSchema(Schema):
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-    filter_type: str = FilterTypeChoices.substring
-    status: Optional[StatusChoices] = StatusChoices.indexed
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    sort: str = 'added'
-    as_json: bool = True
-    as_html: bool = False
-    as_csv: str | bool = 'timestamp,url'
-    with_headers: bool = False
-
-class RemoveCommandSchema(Schema):
-    delete: bool = True
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-
-
-
-
-
-@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
-def cli_add(request, args: AddCommandSchema):
-    result = add(
-        urls=args.urls,
-        tag=args.tag,
-        depth=args.depth,
-        update=args.update,
-        update_all=args.update_all,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        init=args.init,
-        extractors=args.extractors,
-        parser=args.parser,
-    )
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
-def cli_update(request, args: UpdateCommandSchema):
-    result = update(
-        resume=args.resume,
-        only_new=args.only_new,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        before=args.before,
-        after=args.after,
-        status=args.status,
-        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
-        extractors=args.extractors,
-    )
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-
-
-@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
-def cli_schedule(request, args: ScheduleCommandSchema):
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
-    )
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-
-
-
-@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
-def cli_list(request, args: ListCommandSchema):
-    result = list_all(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
-
-    result_format = 'txt'
-    if args.as_json:
-        result_format = "json"
-    elif args.as_html:
-        result_format = "html"
-    elif args.as_csv:
-        result_format = "csv"
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": result_format,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-    
-
-
-@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
-def cli_remove(request, args: RemoveCommandSchema):
-    result = remove(
-        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
-    )
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
-    }
-    

archivebox/api/v1_core.py

@@ -1,424 +0,0 @@
-__package__ = 'archivebox.api'
-
-import math
-from uuid import UUID
-from typing import List, Optional, Union, Any
-from datetime import datetime
-
-from django.db.models import Q
-from django.shortcuts import get_object_or_404
-from django.core.exceptions import ValidationError
-from django.contrib.auth import get_user_model
-
-from ninja import Router, Schema, FilterSchema, Field, Query
-from ninja.pagination import paginate, PaginationBase
-
-from core.models import Snapshot, ArchiveResult, Tag
-from abid_utils.abid import ABID
-
-router = Router(tags=['Core Models'])
-
-
-
-class CustomPagination(PaginationBase):
-    class Input(Schema):
-        limit: int = 200
-        offset: int = 0
-        page: int = 0
-
-
-    class Output(Schema):
-        total_items: int
-        total_pages: int
-        page: int
-        limit: int
-        offset: int
-        num_items: int
-        items: List[Any]
-
-    def paginate_queryset(self, queryset, pagination: Input, **params):
-        limit = min(pagination.limit, 500)
-        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
-        total_pages = math.ceil(total / limit)
-        current_page = math.ceil(offset / (limit + 1))
-        items = queryset[offset : offset + limit]
-        return {
-            'total_items': total,
-            'total_pages': total_pages,
-            'page': current_page,
-            'limit': limit,
-            'offset': offset,
-            'num_items': len(items),
-            'items': items,
-        }
-
-
-### ArchiveResult #########################################################################
-
-class ArchiveResultSchema(Schema):
-    TYPE: str = 'core.models.ArchiveResult'
-
-    id: UUID
-    old_id: int
-    abid: str
-
-    modified: datetime
-    created: datetime
-    created_by_id: str
-    created_by_username: str
-
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: str
-
-    extractor: str
-    cmd_version: Optional[str]
-    cmd: List[str]
-    pwd: str
-    status: str
-    output: str
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
-    @staticmethod
-    def resolve_abid(obj):
-        return str(obj.ABID)
-
-    @staticmethod
-    def resolve_created(obj):
-        return obj.start_ts
-
-    @staticmethod
-    def resolve_snapshot_timestamp(obj):
-        return obj.snapshot.timestamp
-    
-    @staticmethod
-    def resolve_snapshot_url(obj):
-        return obj.snapshot.url
-
-    @staticmethod
-    def resolve_snapshot_abid(obj):
-        return str(obj.snapshot.ABID)
-
-    @staticmethod
-    def resolve_snapshot_tags(obj):
-        return obj.snapshot.tags_str()
-
-
-class ArchiveResultFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-
-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-    snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
-    snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
-    
-    status: Optional[str] = Field(None, q='status')
-    output: Optional[str] = Field(None, q='output__icontains')
-    extractor: Optional[str] = Field(None, q='extractor__icontains')
-    cmd: Optional[str] = Field(None, q='cmd__0__icontains')
-    pwd: Optional[str] = Field(None, q='pwd__icontains')
-    cmd_version: Optional[str] = Field(None, q='cmd_version')
-
-    created: Optional[datetime] = Field(None, q='updated')
-    created__gte: Optional[datetime] = Field(None, q='updated__gte')
-    created__lt: Optional[datetime] = Field(None, q='updated__lt')
-
-
-@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
-@paginate(CustomPagination)
-def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
-    """List all ArchiveResult entries matching these filters."""
-    qs = ArchiveResult.objects.all()
-    results = filters.filter(qs).distinct()
-    return results
-
-
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
-def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by pk, abid, or old_id."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
-
-
-# @router.post("/archiveresult", response=ArchiveResultSchema)
-# def create_archiveresult(request, payload: ArchiveResultSchema):
-#     archiveresult = ArchiveResult.objects.create(**payload.dict())
-#     return archiveresult
-#
-# @router.put("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
-# def update_archiveresult(request, archiveresult_id: str, payload: ArchiveResultSchema):
-#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
-#   
-#     for attr, value in payload.dict().items():
-#         setattr(archiveresult, attr, value)
-#     archiveresult.save()
-#
-#     return archiveresult
-#
-# @router.delete("/archiveresult/{archiveresult_id}")
-# def delete_archiveresult(request, archiveresult_id: str):
-#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
-#     archiveresult.delete()
-#     return {"success": True}
-
-
-
-
-
-### Snapshot #########################################################################
-
-
-class SnapshotSchema(Schema):
-    TYPE: str = 'core.models.Snapshot'
-
-    id: UUID
-    old_id: UUID
-    abid: str
-
-    modified: datetime
-    created: datetime
-    created_by_id: str
-    created_by_username: str
-
-    url: str
-    tags: str
-    title: Optional[str]
-    timestamp: str
-    archive_path: str
-
-    bookmarked: datetime
-    added: datetime
-    updated: Optional[datetime]
-
-    num_archiveresults: int
-    archiveresults: List[ArchiveResultSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
-    @staticmethod
-    def resolve_abid(obj):
-        return str(obj.ABID)
-
-    @staticmethod
-    def resolve_tags(obj):
-        return obj.tags_str()
-
-    @staticmethod
-    def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_archiveresults(obj, context):
-        if context['request'].with_archiveresults:
-            return obj.archiveresult_set.all().distinct()
-        return ArchiveResult.objects.none()
-
-
-class SnapshotFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
-
-    old_id: Optional[str] = Field(None, q='old_id__icontains')
-    abid: Optional[str] = Field(None, q='abid__icontains')
-
-    created_by_id: str = Field(None, q='created_by_id')
-    created_by_username: str = Field(None, q='created_by__username__icontains')
-
-    created__gte: datetime = Field(None, q='created__gte')
-    created__lt: datetime = Field(None, q='created__lt')
-    created: datetime = Field(None, q='created')
-    modified: datetime = Field(None, q='modified')
-    modified__gte: datetime = Field(None, q='modified__gte')
-    modified__lt: datetime = Field(None, q='modified__lt')
-
-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
-    url: Optional[str] = Field(None, q='url')
-    tag: Optional[str] = Field(None, q='tags__name')
-    title: Optional[str] = Field(None, q='title__icontains')
-    timestamp: Optional[str] = Field(None, q='timestamp__startswith')
-    
-    added__gte: Optional[datetime] = Field(None, q='added__gte')
-    added__lt: Optional[datetime] = Field(None, q='added__lt')
-
-
-
-@router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
-@paginate(CustomPagination)
-def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
-    """List all Snapshot entries matching these filters."""
-    request.with_archiveresults = with_archiveresults
-
-    qs = Snapshot.objects.all()
-    results = filters.filter(qs).distinct()
-    return results
-
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid, uuid, or pk."""
-    request.with_archiveresults = with_archiveresults
-    snapshot = None
-    try:
-        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        pass
-
-    try:
-        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
-    except Snapshot.DoesNotExist:
-        pass
-
-    if not snapshot:
-        raise Snapshot.DoesNotExist
-
-    return snapshot
-
-
-# @router.post("/snapshot", response=SnapshotSchema)
-# def create_snapshot(request, payload: SnapshotSchema):
-#     snapshot = Snapshot.objects.create(**payload.dict())
-#     return snapshot
-#
-# @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
-# def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
-#
-#     for attr, value in payload.dict().items():
-#         setattr(snapshot, attr, value)
-#     snapshot.save()
-#
-#     return snapshot
-#
-# @router.delete("/snapshot/{snapshot_id}")
-# def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
-#     snapshot.delete()
-#     return {"success": True}
-
-
-
-### Tag #########################################################################
-
-
-class TagSchema(Schema):
-    TYPE: str = 'core.models.Tag'
-
-    id: UUID
-    old_id: str
-    abid: str
-
-    modified: datetime
-    created: datetime
-    created_by_id: str
-    created_by_username: str
-
-    name: str
-    slug: str
-    num_snapshots: int
-    snapshots: List[SnapshotSchema]
-
-    @staticmethod
-    def resolve_old_id(obj):
-        return str(obj.old_id)
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-    
-    @staticmethod
-    def resolve_num_snapshots(obj, context):
-        return obj.snapshot_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if context['request'].with_snapshots:
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-@router.get("/tags", response=List[TagSchema], url_name="get_tags")
-@paginate(CustomPagination)
-def get_tags(request):
-    request.with_snapshots = False
-    request.with_archiveresults = False
-    return Tag.objects.all().distinct()
-
-@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request, tag_id: str, with_snapshots: bool=True):
-    request.with_snapshots = with_snapshots
-    request.with_archiveresults = False
-    tag = None
-    try:
-        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError, ValueError):
-        pass
-
-    try:
-        tag = Tag.objects.get(abid__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        pass
-
-    try:
-        tag = tag or Tag.objects.get(id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        pass
-    return tag
-
-
-
-@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
-def get_any(request, abid: str):
-    request.with_snapshots = False
-    request.with_archiveresults = False
-
-    response = None
-    try:
-        response = response or get_snapshot(request, abid)
-    except Exception:
-        pass
-
-    try:
-        response = response or get_archiveresult(request, abid)
-    except Exception:
-        pass
-
-    try:
-        response = response or get_tag(request, abid)
-    except Exception:
-        pass
-
-    return response

archivebox/builtin_plugins/base/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

archivebox/builtin_plugins/base/apps.py

@@ -1,82 +0,0 @@
-import sys
-import inspect
-from typing import List, Dict, Any, Optional
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-from plugantic.replayers import Replayer
-
-
-class PythonBinary(Binary):
-    name: BinName = 'python'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides: Dict[str, Any] = {
-        'env': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(Binary):
-    name: BinName = 'sqlite'
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(sqlite3),
-            'version': \
-                lambda: sqlite3.version,
-        },
-    }
-
-class DjangoBinary(Binary):
-    name: BinName = 'django'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(Replayer):
-    name: str = 'basic'
-
-
-class BasePlugin(Plugin):
-    name: str = 'base'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = []
-    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
-
-
-PLUGINS = [BasePlugin()]
-
-
-class BaseConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'builtin_plugins.base'

archivebox/builtin_plugins/base/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/builtin_plugins/base/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/base/views.py

@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,94 +0,0 @@
-from typing import List, Optional
-from pathlib import Path
-
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-
-
-###################### Config ##########################
-
-class SinglefileToggleConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
-
-    SAVE_SINGLEFILE: bool = True
-
-
-class SinglefileDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    SINGLEFILE_BINARY: str = Field(default='wget')
-    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
-    SINGLEFILE_EXTRA_ARGS: List[str] = []
-    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class SinglefileOptionsConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
-
-    # loaded from shared config
-    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
-    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
-    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
-    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
-    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
-
-
-
-DEFAULT_CONFIG = {
-    'CHECK_SSL_VALIDITY': False,
-    'SAVE_SINGLEFILE': True,
-    'TIMEOUT': 120,
-}
-
-PLUGIN_CONFIG = [
-    SinglefileToggleConfig(**DEFAULT_CONFIG),
-    SinglefileDependencyConfig(**DEFAULT_CONFIG),
-    SinglefileOptionsConfig(**DEFAULT_CONFIG),
-]
-
-###################### Binaries ############################
-
-class SinglefileBinary(Binary):
-    name: BinName = 'single-file'
-    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
-
-
-###################### Extractors ##########################
-
-class SinglefileExtractor(Extractor):
-    name: ExtractorName = 'singlefile'
-    binary: Binary = SinglefileBinary()
-
-    def get_output_path(self, snapshot) -> Path:
-        return Path(snapshot.link_dir) / 'singlefile.html'
-
-
-###################### Plugins #############################
-
-
-class SinglefilePlugin(Plugin):
-    name: str = 'singlefile'
-    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
-
-PLUGINS = [SinglefilePlugin()]
-
-###################### Django Apps #########################
-
-class SinglefileConfig(AppConfig):
-    name = 'builtin_plugins.singlefile'
-    verbose_name = 'SingleFile'
-
-    def ready(self):
-        print('Loaded singlefile plugin')

archivebox/builtin_plugins/singlefile/config.yaml

@@ -1,66 +0,0 @@
-name: singlefile
-plugin_version: '0.0.1'
-plugin_spec: '0.0.1'
-
-binaries:
-    singlefile:
-        providers:
-            - env
-            - npm
-
-commands:
-    - singlefile.exec
-    - singlefile.extract
-    - singlefile.should_extract
-    - singlefile.get_output_path
-
-extractors:
-    singlefile:
-        binary: singlefile
-        test: singlefile.should_extract
-        extract: singlefile.extract
-        output_files:
-            - singlefile.html
-
-configs:
-    ARCHIVE_METHOD_TOGGLES:
-        SAVE_SINGLEFILE:
-            type: bool
-            default: true
-
-    DEPENDENCY_CONFIG:
-        SINGLEFILE_BINARY:
-            type: str
-            default: wget
-        SINGLEFILE_ARGS:
-            type: Optional[List[str]]
-            default: null
-        SINGLEFILE_EXTRA_ARGS:
-            type: List[str]
-            default: []
-        SINGLEFILE_DEFAULT_ARGS:
-            type: List[str]
-            default: 
-            - "--timeout={TIMEOUT-10}"
-
-    ARCHIVE_METHOD_OPTIONS:
-        SINGLEFILE_USER_AGENT:
-            type: str
-            default: ""
-            alias: USER_AGENT
-        SINGLEFILE_TIMEOUT:
-            type: int
-            default: 60
-            alias: TIMEOUT
-        SINGLEFILE_CHECK_SSL_VALIDITY:
-            type: bool
-            default: true
-            alias: CHECK_SSL_VALIDITY
-        SINGLEFILE_RESTRICT_FILE_NAMES:
-            type: str
-            default: windows
-            alias: RESTRICT_FILE_NAMES
-        SINGLEFILE_COOKIES_FILE:
-            type: Optional[Path]
-            default: null
-            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/cli/__init__.py

@@ -4,25 +4,22 @@ __command__ = 'archivebox'
 import os
 import sys
 import argparse
-import threading
-from time import sleep
 
-from typing import Optional, Dict, List, IO, Union, Iterable
+from typing import Optional, Dict, List, IO, Union
 from pathlib import Path
 
-from ..config import OUTPUT_DIR, check_data_folder, check_migrations, stderr
+from ..config import OUTPUT_DIR
 
 from importlib import import_module
 
-BUILTIN_LIST = list
-
 CLI_DIR = Path(__file__).resolve().parent
 
 # these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
+meta_cmds = ('help', 'version')
+main_cmds = ('init', 'info', 'config')
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')
+
+fake_db = ("oneshot",)
 
 display_first = (*meta_cmds, *main_cmds, *archive_cmds)
 
@@ -37,40 +34,6 @@ is_valid_cli_module = lambda module, subcommand: (
 )
 
 
-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting
-
-
-def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
-    """
-    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
-    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
-    """
-
-    wait_for_all: bool = thread_names == ()
-
-    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
-
-    should_wait = lambda thread: (
-        not thread_matches(thread, ignore_names)
-        and (wait_for_all or thread_matches(thread, thread_names)))
-
-    for tries in range(timeout):
-        all_threads = [*threading.enumerate()]
-        blocking_threads = [*filter(should_wait, all_threads)]
-        threads_summary = ', '.join(repr(t) for t in blocking_threads)
-        if blocking_threads:
-            sleep(1)
-            if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                stderr(
-                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
-                    threads_summary,
-                )
-        else:
-            return tries
-
-    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')
-
-
 def list_subcommands() -> Dict[str, str]:
     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
 
@@ -98,28 +61,13 @@ def run_subcommand(subcommand: str,
                    pwd: Union[Path, str, None]=None) -> None:
     """Run a given ArchiveBox subcommand with the given list of args"""
 
-    subcommand_args = subcommand_args or []
-
     if subcommand not in meta_cmds:
         from ..config import setup_django
-
-        cmd_requires_db = subcommand in archive_cmds
-        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
-
-        if cmd_requires_db:
-            check_data_folder(pwd)
-
-        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
-
-        if cmd_requires_db:
-            check_migrations()
+        setup_django(in_memory_db=subcommand in fake_db, check_db=subcommand in archive_cmds)
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
 
-    # wait for webhooks, signals, and other background jobs to finish before exit
-    wait_for_bg_threads_to_exit(timeout=60)
-
 
 SUBCOMMANDS = list_subcommands()
 

archivebox/cli/archivebox_add.py

@@ -10,7 +10,6 @@ from typing import List, Optional, IO
 
 from ..main import add
 from ..util import docstring
-from ..parsers import PARSERS
 from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr
 
@@ -24,23 +23,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         formatter_class=SmartFormatter,
     )
     parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--update', #'-u',
+        '--update-all', #'-n',
         action='store_true',
         default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
-    parser.add_argument(
-        '--update-all', #'-n',
-        action='store_true',
-        default=False, 
-        help="Also update ALL links in index when finished adding new links",
-    )
     parser.add_argument(
         '--index-only', #'-o',
         action='store_true',
@@ -86,20 +73,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
               This does not take precedence over the configuration",
         default=""
     )
-    parser.add_argument(
-        "--parser",
-        type=str,
-        help="Parser used to read inputted URLs.",
-        default="auto",
-        choices=["auto", *PARSERS.keys()],
-    )
     command = parser.parse_args(args or ())
     urls = command.urls
-
-    stdin_urls = ''
-    if not urls:
-        stdin_urls = accept_stdin(stdin)
-
+    stdin_urls = accept_stdin(stdin)
     if (stdin_urls and urls) or (not stdin and not urls):
         stderr(
             '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
@@ -109,14 +85,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     add(
         urls=stdin_urls or urls,
         depth=command.depth,
-        tag=command.tag,
-        update=command.update,
         update_all=command.update_all,
         index_only=command.index_only,
         overwrite=command.overwrite,
         init=command.init,
         extractors=command.extract,
-        parser=command.parser,
         out_dir=pwd or OUTPUT_DIR,
     )
 

archivebox/cli/archivebox_config.py

@@ -45,10 +45,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='KEY or KEY=VALUE formatted config values to get or set',
     )
     command = parser.parse_args(args or ())
-
-    config_options_str = ''
-    if not command.config_options:
-        config_options_str = accept_stdin(stdin)
+    config_options_str = accept_stdin(stdin)
 
     config(
         config_options_str=config_options_str,

archivebox/cli/archivebox_init.py

@@ -27,23 +27,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Ignore unrecognized files in current directory and initialize anyway',
     )
-    parser.add_argument(
-        '--quick', '-q',
-        action='store_true',
-        help='Run any updates or migrations without rechecking all snapshot dirs',
-    )
-    parser.add_argument(
-        '--setup', #'-s',
-        action='store_true',
-        help='Automatically install dependencies and extras used for archiving',
-    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
     init(
         force=command.force,
-        quick=command.quick,
-        setup=command.setup,
         out_dir=pwd or OUTPUT_DIR,
     )
     

archivebox/cli/archivebox_list.py

@@ -12,7 +12,6 @@ from ..main import list_all
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
-    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -24,7 +23,7 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
-from ..logging_util import SmartFormatter, reject_stdin, stderr
+from ..logging_util import SmartFormatter, accept_stdin, stderr
 
 
 @docstring(list_all.__doc__)
@@ -45,7 +44,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     group.add_argument(
         '--json', #'-j',
         action='store_true',
-        help="Print the output in JSON format with all columns included",
+        help="Print the output in JSON format with all columns included.",
     )
     group.add_argument(
         '--html',
@@ -60,19 +59,19 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--sort', #'-s',
         type=str,
-        help="List the links sorted using the given key, e.g. timestamp or updated",
+        help="List the links sorted using the given key, e.g. timestamp or updated.",
         default=None,
     )
     parser.add_argument(
         '--before', #'-b',
         type=float,
-        help="List only links bookmarked before (less than) the given timestamp",
+        help="List only links bookmarked before the given timestamp.",
         default=None,
     )
     parser.add_argument(
         '--after', #'-a',
         type=float,
-        help="List only links bookmarked after (greater than or equal to) the given timestamp",
+        help="List only links bookmarked after the given timestamp.",
         default=None,
     )
     parser.add_argument(
@@ -97,9 +96,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type', '-t',
+        '--filter-type',
         type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
+        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -108,19 +107,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         nargs='*',
         type=str,
         default=None,
-        help='List only URLs matching these filter patterns'
+        help='List only URLs matching these filter patterns.'
     )
     command = parser.parse_args(args or ())
-    reject_stdin(stdin)
+    filter_patterns_str = accept_stdin(stdin)
 
     if command.with_headers and not (command.json or command.html or command.csv):
         stderr(
-            '[X] --with-headers can only be used with --json, --html or --csv options\n',
+            '[X] --with-headers can only be used with --json, --html or --csv options.\n',
             color='red',
         )
         raise SystemExit(2)
 
     matching_folders = list_all(
+        filter_patterns_str=filter_patterns_str,
         filter_patterns=command.filter_patterns,
         filter_type=command.filter_type,
         status=command.status,

archivebox/cli/archivebox_oneshot.py

@@ -50,11 +50,8 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help= "Path to save the single archive folder to, e.g. ./example.com_archive"
     )
     command = parser.parse_args(args or ())
-    stdin_url = None
     url = command.url
-    if not url:
-        stdin_url = accept_stdin(stdin)
-
+    stdin_url = accept_stdin(stdin)
     if (stdin_url and url) or (not stdin and not url):
         stderr(
             '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',

archivebox/cli/archivebox_remove.py

@@ -61,10 +61,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='URLs matching this filter pattern will be removed from the index.'
     )
     command = parser.parse_args(args or ())
-    
-    filter_str = None
-    if not command.filter_patterns:
-        filter_str = accept_stdin(stdin)
+    filter_str = accept_stdin(stdin)
 
     remove(
         filter_str=filter_str,

archivebox/cli/archivebox_schedule.py

@@ -39,28 +39,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=None,
         help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
     )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
     parser.add_argument(
         '--depth', # '-d',
         type=int,
         choices=[0, 1],
         default=0,
-        help='Depth to archive to [0] or 1, see "add" command help for more info',
-    )
-    parser.add_argument(
-        '--overwrite',
-        action='store_true',
-        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
-    )
-    parser.add_argument(
-        '--update',
-        action='store_true',
-        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
+        help='Depth to archive to [0] or 1, see "add" command help for more info.',
     )
     group.add_argument(
         '--clear', # '-c'
@@ -103,10 +87,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         run_all=command.run_all,
         quiet=command.quiet,
         every=command.every,
-        tag=command.tag,
         depth=command.depth,
-        overwrite=command.overwrite,
-        update=command.update,
         import_path=command.import_path,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_server.py

@@ -38,20 +38,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Enable DEBUG=True mode with more verbose errors',
     )
-    parser.add_argument(
-        '--nothreading',
-        action='store_true',
-        help='Force runserver to run in single-threaded mode',
-    )
     parser.add_argument(
         '--init',
         action='store_true',
-        help='Run a full archivebox init/upgrade before starting the server',
-    )
-    parser.add_argument(
-        '--quick-init', '-i',
-        action='store_true',
-        help='Run quick archivebox init/upgrade before starting the server',
+        help='Run archivebox init before starting the server',
     )
     parser.add_argument(
         '--createsuperuser',
@@ -62,11 +52,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     reject_stdin(__command__, stdin)
     
     server(
-        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
+        runserver_args=command.runserver_args,
         reload=command.reload,
         debug=command.debug,
         init=command.init,
-        quick_init=command.quick_init,
         createsuperuser=command.createsuperuser,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_setup.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox setup'
-
-import sys
-import argparse
-
-from typing import Optional, List, IO
-
-from ..main import setup
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-
-
-@docstring(setup.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=setup.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    # parser.add_argument(
-    #     '--force', # '-f',
-    #     action='store_true',
-    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
-    # )
-    command = parser.parse_args(args or ())   # noqa
-    reject_stdin(__command__, stdin)
-
-    setup(
-        # force=command.force,
-        out_dir=pwd or OUTPUT_DIR,
-    )
-    
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_update.py

@@ -12,7 +12,6 @@ from ..main import update
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
-    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -90,9 +89,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type', '-t',
+        '--filter-type',
         type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
+        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -111,10 +110,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=""
     )
     command = parser.parse_args(args or ())
-
-    filter_patterns_str = None
-    if not command.filter_patterns:
-        filter_patterns_str = accept_stdin(stdin)
+    filter_patterns_str = accept_stdin(stdin)
 
     update(
         resume=command.resume,

archivebox/cli/tests.py

@@ -75,23 +75,23 @@ def output_hidden(show_failing=True):
         yield
         return
 
-    sys.stdout = open('stdout.txt', 'w+', encoding='utf-8')
-    sys.stderr = open('stderr.txt', 'w+', encoding='utf-8')
+    sys.stdout = open('stdout.txt', 'w+')
+    sys.stderr = open('stderr.txt', 'w+')
     try:
         yield
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
-    except Exception:
+    except:
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
         if show_failing:
-            with open('stdout.txt', 'r', encoding='utf-8') as f:
+            with open('stdout.txt', 'r') as f:
                 print(f.read())
-            with open('stderr.txt', 'r', encoding='utf-8') as f:
+            with open('stderr.txt', 'r') as f:
                 print(f.read())
         raise
     finally:
@@ -116,7 +116,7 @@ class TestInit(unittest.TestCase):
         assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
 
     def test_conflicting_init(self):
-        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+') as f:
             f.write('test')
 
         try:
@@ -132,7 +132,7 @@ class TestInit(unittest.TestCase):
         try:
             load_main_index(out_dir=OUTPUT_DIR)
             assert False, 'load_main_index should raise an exception when no index is present'
-        except Exception:
+        except:
             pass
 
     def test_no_dirty_state(self):
@@ -161,7 +161,7 @@ class TestAdd(unittest.TestCase):
 
     def test_add_arg_file(self):
         test_file = Path(OUTPUT_DIR) / 'test.txt'
-        with open(test_file, 'w+', encoding='utf') as f:
+        with open(test_file, 'w+') as f:
             f.write(test_urls)
 
         with output_hidden():
@@ -216,7 +216,7 @@ class TestRemove(unittest.TestCase):
             with output_hidden(show_failing=False):
                 archivebox_remove.main(['--yes', '--delete', 'https://doesntexist.com'])
             assert False, 'Should raise if no URLs match'
-        except Exception:
+        except:
             pass
 
 

archivebox/config_stubs.py

@@ -9,15 +9,11 @@ SimpleConfigValueDict = Dict[str, SimpleConfigValue]
 SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
 ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
 
-class AttrDict(dict):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.__dict__ = self
 
 class BaseConfig(TypedDict):
     pass
 
-class ConfigDict(BaseConfig, AttrDict, total=False):
+class ConfigDict(BaseConfig, total=False):
     """
     # Regenerate by pasting this quine into `archivebox shell` 🥚
     from archivebox.config import ConfigDict, CONFIG_DEFAULTS
@@ -32,7 +28,6 @@ class ConfigDict(BaseConfig, AttrDict, total=False):
                 print(f'    {key}: {Type.__name__}')
         print()
     """
-
     IS_TTY: bool
     USE_COLOR: bool
     SHOW_PROGRESS: bool
@@ -46,7 +41,7 @@ class ConfigDict(BaseConfig, AttrDict, total=False):
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
     RESTRICT_FILE_NAMES: str
-    URL_DENYLIST: str
+    URL_BLACKLIST: str
 
     SECRET_KEY: Optional[str]
     BIND_ADDR: str
@@ -79,7 +74,6 @@ class ConfigDict(BaseConfig, AttrDict, total=False):
     CHROME_USER_AGENT: str
     COOKIES_FILE: Union[str, Path, None]
     CHROME_USER_DATA_DIR: Union[str, Path, None]
-    CHROME_TIMEOUT: int
     CHROME_HEADLESS: bool
     CHROME_SANDBOX: bool
 
@@ -104,7 +98,6 @@ class ConfigDict(BaseConfig, AttrDict, total=False):
     WGET_ARGS: List[str]
     CURL_ARGS: List[str]
     GIT_ARGS: List[str]
-    TAG_SEPARATOR_PATTERN: str
 
 
 ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

archivebox/core/__init__.py

@@ -1,2 +1 @@
 __package__ = 'archivebox.core'
-

archivebox/core/admin.py

@@ -1,51 +1,28 @@
 __package__ = 'archivebox.core'
 
-import os
-import json
-
 from io import StringIO
-from pathlib import Path
 from contextlib import redirect_stdout
-from datetime import datetime, timezone
-from typing import Dict, Any
 
 from django.contrib import admin
-from django.db.models import Count, Q, Prefetch
-from django.urls import path, reverse, resolve
-from django.utils import timezone
-from django.utils.functional import cached_property
+from django.urls import path
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
-from django.core.paginator import Paginator
-from django.core.exceptions import ValidationError
-from django.conf import settings
 from django import forms
 
-
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
-# from plugantic.admin import CustomPlugin
-
 from ..util import htmldecode, urldecode, ansi_to_html
 
-from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
-from core.forms import AddLinkForm
+from core.models import Snapshot, Tag
+from core.forms import AddLinkForm, TagField
+
 from core.mixins import SearchResultsAdminMixin
-from api.models import APIToken
-from abid_utils.models import get_or_create_system_user_pk
-from abid_utils.admin import ABIDModelAdmin
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
-from extractors import archive_links
-
-
-CONFIG = settings.CONFIG
-
-GLOBAL_CONTEXT = {'VERSION': CONFIG.VERSION, 'VERSIONS_AVAILABLE': CONFIG.VERSIONS_AVAILABLE, 'CAN_UPGRADE': CONFIG.CAN_UPGRADE}
+from config import OUTPUT_DIR
+from extractors import archive_snapshots
 
 # Admin URLs
 # /admin/
@@ -59,12 +36,187 @@ GLOBAL_CONTEXT = {'VERSION': CONFIG.VERSION, 'VERSIONS_AVAILABLE': CONFIG.VERSIO
 
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
+def update_snapshots(modeladmin, request, queryset):
+    archive_links([
+        snapshot.as_link()
+        for snapshot in queryset
+    ], out_dir=OUTPUT_DIR)
+update_snapshots.short_description = "Archive"
+
+def update_titles(modeladmin, request, queryset):
+    archive_links([
+        snapshot.as_link()
+        for snapshot in queryset
+    ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
+update_titles.short_description = "Pull title"
+
+def overwrite_snapshots(modeladmin, request, queryset):
+    archive_links([
+        snapshot.as_link()
+        for snapshot in queryset
+    ], overwrite=True, out_dir=OUTPUT_DIR)
+overwrite_snapshots.short_description = "Re-archive (overwrite)"
+
+def verify_snapshots(modeladmin, request, queryset):
+    for snapshot in queryset:
+        print(snapshot.timestamp, snapshot.url, snapshot.is_archived, snapshot.archive_size, len(snapshot.history))
+
+verify_snapshots.short_description = "Check"
+
+def delete_snapshots(modeladmin, request, queryset):
+    remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
+
+delete_snapshots.short_description = "Delete"
+
+
+class SnapshotAdminForm(forms.ModelForm):
+    tags = TagField(required=False)
+
+    class Meta:
+        model = Snapshot
+        fields = "__all__"
+
+    def save(self, commit=True):
+        # Based on: https://stackoverflow.com/a/49933068/3509554
+
+        # Get the unsave instance
+        instance = forms.ModelForm.save(self, False)
+        tags = self.cleaned_data.pop("tags")
+
+        #update save_m2m
+        def new_save_m2m():
+            instance.save_tags(tags)
+
+        # Do we need to save all changes now?
+        self.save_m2m = new_save_m2m
+        if commit:
+            instance.save()
+
+        return instance
+
+
+class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
+    list_display = ('added', 'title_str', 'url_str', 'files', 'size')
+    sort_fields = ('title_str', 'url_str', 'added')
+    readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
+    search_fields = ['url__icontains', 'timestamp', 'title', 'tags__name']
+    fields = (*readonly_fields, 'title', 'tags')
+    list_filter = ('added', 'updated', 'tags')
+    ordering = ['-added']
+    actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots]
+    actions_template = 'admin/actions_as_select.html'
+    form = SnapshotAdminForm
+    list_per_page = 40
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path('grid/', self.admin_site.admin_view(self.grid_view),name='grid')
+        ]
+        return custom_urls + urls
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).prefetch_related('tags')
+
+    def tag_list(self, obj):
+        return ', '.join(obj.tags.values_list('name', flat=True))
+
+    def id_str(self, obj):
+        return format_html(
+            '<code style="font-size: 10px">{}</code>',
+            obj.url_hash[:8],
+        )
+
+    def title_str(self, obj):
+        canon = obj.as_link().canonical_outputs()
+        tags = ''.join(
+            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.id, tag)
+            for tag in obj.tags.all()
+            if str(tag).strip()
+        )
+        return format_html(
+            '<a href="/{}">'
+                '<img src="/{}/{}" class="favicon" onerror="this.remove()">'
+            '</a>'
+            '<a href="/{}/index.html">'
+                '<b class="status-{}">{}</b>'
+            '</a>',
+            obj.archive_path,
+            obj.archive_path, canon['favicon_path'],
+            obj.archive_path,
+            'fetched' if obj.latest_title or obj.title else 'pending',
+            urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
+        ) + mark_safe(f' <span class="tags">{tags}</span>')
+
+    def files(self, obj):
+        return snapshot_icons(obj)
+
+    def size(self, obj):
+        archive_size = obj.archive_size
+        if archive_size:
+            size_txt = printable_filesize(archive_size)
+            if archive_size > 52428800:
+                size_txt = mark_safe(f'<b>{size_txt}</b>')
+        else:
+            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
+        return format_html(
+            '<a href="/{}" title="View all files">{}</a>',
+            obj.archive_path,
+            size_txt,
+        )
+
+    def url_str(self, obj):
+        return format_html(
+            '<a href="{}"><code>{}</code></a>',
+            obj.url,
+            obj.url.split('://www.', 1)[-1].split('://', 1)[-1][:64],
+        )
+
+    def grid_view(self, request):
+
+        # cl = self.get_changelist_instance(request)
+
+        # Save before monkey patching to restore for changelist list view
+        saved_change_list_template = self.change_list_template
+        saved_list_per_page = self.list_per_page
+        saved_list_max_show_all = self.list_max_show_all
+
+        # Monkey patch here plus core_tags.py
+        self.change_list_template = 'private_index_grid.html'
+        self.list_per_page = 20
+        self.list_max_show_all = self.list_per_page
+
+        # Call monkey patched view
+        rendered_response = self.changelist_view(request)
+
+        # Restore values
+        self.change_list_template =  saved_change_list_template
+        self.list_per_page = saved_list_per_page
+        self.list_max_show_all = saved_list_max_show_all
+
+        return rendered_response
+        
+
+    id_str.short_description = 'ID'
+    title_str.short_description = 'Title'
+    url_str.short_description = 'Original URL'
+
+    id_str.admin_order_field = 'id'
+    title_str.admin_order_field = 'title'
+    url_str.admin_order_field = 'url'
+
+class TagAdmin(admin.ModelAdmin):
+    list_display = ('slug', 'name', 'id')
+    sort_fields = ('id', 'name', 'slug')
+    readonly_fields = ('id',)
+    search_fields = ('id', 'name', 'slug')
+    fields = (*readonly_fields, 'name', 'slug')
+
 
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = 'ArchiveBox'
     index_title = 'Links'
     site_title = 'Index'
-    namespace = 'admin'
 
     def get_urls(self):
         return [
@@ -76,7 +228,7 @@ class ArchiveBoxAdmin(admin.AdminSite):
             return redirect(f'/admin/login/?next={request.path}')
 
         request.current_app = self.name
-        context: Dict[str, Any] = {
+        context = {
             **self.each_context(request),
             'title': 'Add URLs',
         }
@@ -94,7 +246,7 @@ class ArchiveBoxAdmin(admin.AdminSite):
                     "urls": url,
                     "depth": depth,
                     "update_all": False,
-                    "out_dir": CONFIG.OUTPUT_DIR,
+                    "out_dir": OUTPUT_DIR,
                 }
                 add_stdout = StringIO()
                 with redirect_stdout(add_stdout):
@@ -103,652 +255,15 @@ class ArchiveBoxAdmin(admin.AdminSite):
 
                 context.update({
                     "stdout": ansi_to_html(add_stdout.getvalue().strip()),
-                    "form": AddLinkForm(),
+                    "form": AddLinkForm()
                 })
             else:
                 context["form"] = form
 
         return render(template_name='add.html', request=request, context=context)
 
-
-archivebox_admin = ArchiveBoxAdmin()
-archivebox_admin.register(get_user_model())
-archivebox_admin.disable_action('delete_selected')
-
-# archivebox_admin.register(CustomPlugin)
-
-# patch admin with methods to add data views (implemented by admin_data_views package)
-# https://github.com/MrThearMan/django-admin-data-views
-# https://mrthearman.github.io/django-admin-data-views/setup/
-############### Additional sections are defined in settings.ADMIN_DATA_VIEWS #########
-from admin_data_views.admin import get_app_list, admin_data_index_view, get_admin_data_urls, get_urls
-
-archivebox_admin.get_app_list = get_app_list.__get__(archivebox_admin, ArchiveBoxAdmin)
-archivebox_admin.admin_data_index_view = admin_data_index_view.__get__(archivebox_admin, ArchiveBoxAdmin)       # type: ignore
-archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_admin, ArchiveBoxAdmin)           # type: ignore
-archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
-
-
-class AccelleratedPaginator(Paginator):
-    """
-    Accellerated Pagniator ignores DISTINCT when counting total number of rows.
-    Speeds up SELECT Count(*) on Admin views by >20x.
-    https://hakibenita.com/optimizing-the-django-admin-paginator
-    """
-
-    @cached_property
-    def count(self):
-        if self.object_list._has_filters():
-            # fallback to normal count method on filtered queryset
-            return super().count
-        else:
-            # otherwise count total rows in a separate fast query
-            return self.object_list.model.objects.count()
-    
-        # Alternative approach for PostgreSQL: fallback count takes > 200ms
-        # from django.db import connection, transaction, OperationalError
-        # with transaction.atomic(), connection.cursor() as cursor:
-        #     cursor.execute('SET LOCAL statement_timeout TO 200;')
-        #     try:
-        #         return super().count
-        #     except OperationalError:
-        #         return 9999999999999
-
-
-class ArchiveResultInline(admin.TabularInline):
-    name = 'Archive Results Log'
-    model = ArchiveResult
-    parent_model = Snapshot
-    # fk_name = 'snapshot'
-    extra = 0
-    sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
-    readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
-    fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
-    # exclude = ('id',)
-    ordering = ('end_ts',)
-    show_change_link = True
-    # # classes = ['collapse']
-    # # list_display_links = ['abid']
-
-    def get_parent_object_from_request(self, request):
-        resolved = resolve(request.path_info)
-        return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
-
-    @admin.display(
-        description='Completed',
-        ordering='end_ts',
-    )
-    def completed(self, obj):
-        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime('%Y-%m-%d %H:%M:%S'))
-
-    def result_id(self, obj):
-        return format_html('<a href="{}"><code style="font-size: 10px">[{}]</code></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
-    
-    def command(self, obj):
-        return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
-    
-    def version(self, obj):
-        return format_html('<small><code>{}</code></small>', obj.cmd_version or '-')
-    
-    def get_formset(self, request, obj=None, **kwargs):
-        formset = super().get_formset(request, obj, **kwargs)
-        snapshot = self.get_parent_object_from_request(request)
-
-        # import ipdb; ipdb.set_trace()
-        formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
-        
-        # default values for new entries
-        formset.form.base_fields['status'].initial = 'succeeded'
-        formset.form.base_fields['start_ts'].initial = timezone.now()
-        formset.form.base_fields['end_ts'].initial = timezone.now()
-        formset.form.base_fields['cmd_version'].initial = '-'
-        formset.form.base_fields['pwd'].initial = str(snapshot.link_dir)
-        formset.form.base_fields['created_by'].initial = request.user
-        formset.form.base_fields['cmd'] = forms.JSONField(initial=['-'])
-        formset.form.base_fields['output'].initial = 'Manually recorded cmd output...'
-        
-        if obj is not None:
-            # hidden values for existing entries and new entries
-            formset.form.base_fields['start_ts'].widget = formset.form.base_fields['start_ts'].hidden_widget()
-            formset.form.base_fields['end_ts'].widget = formset.form.base_fields['end_ts'].hidden_widget()
-            formset.form.base_fields['cmd'].widget = formset.form.base_fields['cmd'].hidden_widget()
-            formset.form.base_fields['pwd'].widget = formset.form.base_fields['pwd'].hidden_widget()
-            formset.form.base_fields['created_by'].widget = formset.form.base_fields['created_by'].hidden_widget()
-            formset.form.base_fields['cmd_version'].widget = formset.form.base_fields['cmd_version'].hidden_widget()
-        return formset
-    
-    def get_readonly_fields(self, request, obj=None):
-        if obj is not None:
-            return self.readonly_fields
-        else:
-            return []
-
-
-class TagInline(admin.TabularInline):
-    model = Tag.snapshot_set.through       # type: ignore
-    # fk_name = 'snapshot'
-    fields = ('id', 'tag')
-    extra = 1
-    # min_num = 1
-    max_num = 1000
-    autocomplete_fields = (
-        'tag',
-    )
-
-from django.contrib.admin.helpers import ActionForm
-from django.contrib.admin.widgets import FilteredSelectMultiple
-
-# class AutocompleteTags:
-#     model = Tag
-#     search_fields = ['name']
-#     name = 'name'
-#     # source_field = 'name'
-#     remote_field = Tag._meta.get_field('name')
-
-# class AutocompleteTagsAdminStub:
-#     name = 'admin'
-
-
-class SnapshotActionForm(ActionForm):
-    tags = forms.ModelMultipleChoiceField(
-        queryset=Tag.objects.all(),
-        required=False,
-        widget=FilteredSelectMultiple(
-            'core_tag__name',
-            False,
-        ),
-    )
-
-    # TODO: allow selecting actions for specific extractors? is this useful?
-    # extractor = forms.ChoiceField(
-    #     choices=ArchiveResult.EXTRACTOR_CHOICES,
-    #     required=False,
-    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
-    # )
-
-
-def get_abid_info(self, obj):
-    return format_html(
-        # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-        '''
-        <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-        <br/><hr/>
-        <div style="opacity: 0.8">
-        &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        <hr/>
-        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; {}: <code style="user-select: all">{}</code><br/>
-        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; "><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <span style="display:inline-block; vertical-align: -4px; width: 290px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span>
-        &nbsp; SALT: &nbsp; <code style="font-size: 10px;"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code><br/>
-        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}: <code style="user-select: all">{}</code><br/>
-        &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  {}: <code style="user-select: all">{}</code>
-        <br/><hr/>
-        &nbsp; &nbsp; <small style="opacity: 0.5">.old_id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        </div>
-        ''',
-        obj.api_url, obj.api_url, obj.api_docs_url,
-        str(obj.abid),
-        str(obj.ABID.uuid),
-        str(obj.id),
-        obj.ABID.ts, str(obj.ABID.uuid)[0:14], obj.abid_ts_src, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
-        obj.ABID.uri, str(obj.ABID.uuid)[14:26], obj.abid_uri_src, str(obj.abid_values['uri']),
-        obj.ABID.uri_salt,
-        obj.ABID.subtype, str(obj.ABID.uuid)[26:28], obj.abid_subtype_src, str(obj.abid_values['subtype']),
-        obj.ABID.rand, str(obj.ABID.uuid)[28:36], obj.abid_rand_src, str(obj.abid_values['rand'])[-7:],
-        str(getattr(obj, 'old_id', '')),
-    )
-
-
-@admin.register(Snapshot, site=archivebox_admin)
-class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
-    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
-    sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
-    search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
-    list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
-    fields = ('url', 'created_by', 'title', *readonly_fields)
-    ordering = ['-added']
-    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
-    inlines = [TagInline, ArchiveResultInline]
-    list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
-
-    action_form = SnapshotActionForm
-    paginator = AccelleratedPaginator
-
-    save_on_top = True
-    show_full_result_count = False
-
-    def changelist_view(self, request, extra_context=None):
-        extra_context = extra_context or {}
-        try:
-            return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
-        except Exception as e:
-            self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
-            return super().changelist_view(request, GLOBAL_CONTEXT)
-
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        snapshot = None
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-        
-        try:
-            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
-        except (Snapshot.DoesNotExist, ValidationError):
-            pass
-
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-
-        if snapshot:
-            object_id = str(snapshot.id)
-
-        return super().change_view(
-            request,
-            object_id,
-            form_url,
-            extra_context=extra_context,
-        )
-
-    def get_urls(self):
-        urls = super().get_urls()
-        custom_urls = [
-            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
-        ]
-        return custom_urls + urls
-
-    # def get_queryset(self, request):
-    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
-    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
-
-    #     self.request = request
-    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
-
-    def tag_list(self, obj):
-        return ', '.join(tag.name for tag in obj.tags.all())
-
-    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
-    # def action(self, obj):
-    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
-    #     # action: update_snapshots
-    #     # select_across: 0
-    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
-    #     return format_html(
-    #         '''
-    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
-    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
-    #                 <input type="hidden" name="_selected_action" value="{}">
-    #                 <button name="update_snapshots">Check</button>
-    #                 <button name="update_titles">Pull title + favicon</button>
-    #                 <button name="update_snapshots">Update</button>
-    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
-    #                 <button name="delete_snapshots">Permanently delete</button>
-    #             </form>
-    #         ''',
-    #         csrf.get_token(self.request),
-    #         obj.pk,
-    #     )
-
-    def admin_actions(self, obj):
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}">Summary page ➡️</a> &nbsp; &nbsp;
-            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}/index.html#all">Result files 📑</a> &nbsp; &nbsp;
-            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/admin/core/snapshot/?id__exact={}">Admin actions ⚙️</a>
-            ''',
-            obj.timestamp,
-            obj.timestamp,
-            obj.pk,
-        )
-
-    def status_info(self, obj):
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            Archived: {} ({} files {}) &nbsp; &nbsp;
-            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
-            Status code: {} &nbsp; &nbsp;<br/>
-            Server: {} &nbsp; &nbsp;
-            Content type: {} &nbsp; &nbsp;
-            Extension: {} &nbsp; &nbsp;
-            ''',
-            '✅' if obj.is_archived else '❌',
-            obj.num_outputs,
-            self.size(obj) or '0kb',
-            f'/archive/{obj.timestamp}/favicon.ico',
-            obj.status_code or '-',
-            obj.headers and obj.headers.get('Server') or '-',
-            obj.headers and obj.headers.get('Content-Type') or '-',
-            obj.extension or '-',
-        )
-
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            return str(e)
-
-    @admin.display(
-        description='Title',
-        ordering='title',
-    )
-    def title_str(self, obj):
-        tags = ''.join(
-            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.pk, tag.name)
-            for tag in obj.tags.all()
-            if str(tag.name).strip()
-        )
-        return format_html(
-            '<a href="/{}">'
-                '<img src="/{}/favicon.ico" class="favicon" onerror="this.remove()">'
-            '</a>'
-            '<a href="/{}/index.html">'
-                '<b class="status-{}">{}</b>'
-            '</a>',
-            obj.archive_path,
-            obj.archive_path,
-            obj.archive_path,
-            'fetched' if obj.latest_title or obj.title else 'pending',
-            urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
-        ) + mark_safe(f' <span class="tags">{tags}</span>')
-
-    @admin.display(
-        description='Files Saved',
-        # ordering='archiveresult_count',
-    )
-    def files(self, obj):
-        return snapshot_icons(obj)
-
-
-    @admin.display(
-        # ordering='archiveresult_count'
-    )
-    def size(self, obj):
-        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
-        if archive_size:
-            size_txt = printable_filesize(archive_size)
-            if archive_size > 52428800:
-                size_txt = mark_safe(f'<b>{size_txt}</b>')
-        else:
-            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
-        return format_html(
-            '<a href="/{}" title="View all files">{}</a>',
-            obj.archive_path,
-            size_txt,
-        )
-
-
-    @admin.display(
-        description='Original URL',
-        ordering='url',
-    )
-    def url_str(self, obj):
-        return format_html(
-            '<a href="{}"><code style="user-select: all;">{}</code></a>',
-            obj.url,
-            obj.url[:128],
-        )
-
-    def grid_view(self, request, extra_context=None):
-
-        # cl = self.get_changelist_instance(request)
-
-        # Save before monkey patching to restore for changelist list view
-        saved_change_list_template = self.change_list_template
-        saved_list_per_page = self.list_per_page
-        saved_list_max_show_all = self.list_max_show_all
-
-        # Monkey patch here plus core_tags.py
-        self.change_list_template = 'private_index_grid.html'
-        self.list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
-        self.list_max_show_all = self.list_per_page
-
-        # Call monkey patched view
-        rendered_response = self.changelist_view(request, extra_context=extra_context)
-
-        # Restore values
-        self.change_list_template = saved_change_list_template
-        self.list_per_page = saved_list_per_page
-        self.list_max_show_all = saved_list_max_show_all
-
-        return rendered_response
-
-    # for debugging, uncomment this to print all requests:
-    # def changelist_view(self, request, extra_context=None):
-    #     print('[*] Got request', request.method, request.POST)
-    #     return super().changelist_view(request, extra_context=None)
-
-    @admin.action(
-        description="Pull"
-    )
-    def update_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], out_dir=CONFIG.OUTPUT_DIR)
-
-    @admin.action(
-        description="⬇️ Title"
-    )
-    def update_titles(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, methods=('title','favicon'), out_dir=CONFIG.OUTPUT_DIR)
-
-    @admin.action(
-        description="Re-Snapshot"
-    )
-    def resnapshot_snapshot(self, request, queryset):
-        for snapshot in queryset:
-            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
-            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
-            add(new_url, tag=snapshot.tags_str())
-
-    @admin.action(
-        description="Reset"
-    )
-    def overwrite_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, out_dir=CONFIG.OUTPUT_DIR)
-
-    @admin.action(
-        description="Delete"
-    )
-    def delete_snapshots(self, request, queryset):
-        remove(snapshots=queryset, yes=True, delete=True, out_dir=CONFIG.OUTPUT_DIR)
-
-
-    @admin.action(
-        description="+"
-    )
-    def add_tags(self, request, queryset):
-        tags = request.POST.getlist('tags')
-        print('[+] Adding tags', tags, 'to Snapshots', queryset)
-        for obj in queryset:
-            obj.tags.add(*tags)
-
-
-    @admin.action(
-        description="–"
-    )
-    def remove_tags(self, request, queryset):
-        tags = request.POST.getlist('tags')
-        print('[-] Removing tags', tags, 'to Snapshots', queryset)
-        for obj in queryset:
-            obj.tags.remove(*tags)
-
-
-        
-
-
-
-# @admin.register(SnapshotTag, site=archivebox_admin)
-# class SnapshotTagAdmin(ABIDModelAdmin):
-#     list_display = ('id', 'snapshot', 'tag')
-#     sort_fields = ('id', 'snapshot', 'tag')
-#     search_fields = ('id', 'snapshot_id', 'tag_id')
-#     fields = ('snapshot', 'id')
-#     actions = ['delete_selected']
-#     ordering = ['-id']
-
-#     def API(self, obj):
-#         return get_abid_info(self, obj)
-
-
-@admin.register(Tag, site=archivebox_admin)
-class TagAdmin(ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
-    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
-    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
-    search_fields = ('abid', 'name', 'slug')
-    fields = ('name', 'created_by', *readonly_fields)
-    actions = ['delete_selected']
-    ordering = ['-created']
-
-    paginator = AccelleratedPaginator
-
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            return str(e)
-
-    def num_snapshots(self, tag):
-        return format_html(
-            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
-            tag.id,
-            tag.snapshot_set.count(),
-        )
-
-    def snapshots(self, tag):
-        total_count = tag.snapshot_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
-                snap.pk,
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
-                snap.url[:64],
-            )
-            for snap in tag.snapshot_set.order_by('-updated')[:10]
-        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
-
-
-@admin.register(ArchiveResult, site=archivebox_admin)
-class ArchiveResultAdmin(ABIDModelAdmin):
-    list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
-    sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
-    search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
-    autocomplete_fields = ['snapshot']
-
-    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
-    ordering = ['-start_ts']
-    list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
-    
-    paginator = AccelleratedPaginator
-
-    @admin.display(
-        description='Snapshot Info'
-    )
-    def snapshot_info(self, result):
-        return format_html(
-            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            result.snapshot.timestamp,
-            result.snapshot.abid,
-            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
-            result.snapshot.url[:128],
-        )
-
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            raise e
-            return str(e)
-
-    @admin.display(
-        description='Snapshot Tags'
-    )
-    def tags_str(self, result):
-        return result.snapshot.tags_str()
-
-    def cmd_str(self, result):
-        return format_html(
-            '<pre>{}</pre>',
-            ' '.join(result.cmd) if isinstance(result.cmd, list) else str(result.cmd),
-        )
-    
-    def output_str(self, result):
-        return format_html(
-            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
-            result.snapshot.timestamp,
-            result.output if (result.status == 'succeeded') and result.extractor not in ('title', 'archive_org') else 'index.html',
-            result.output,
-        )
-
-    def output_summary(self, result):
-        snapshot_dir = Path(CONFIG.OUTPUT_DIR) / str(result.pwd).split('data/', 1)[-1]
-        output_str = format_html(
-            '<pre style="display: block; max-width: 75vw; white-space: pre-line;">{}</pre><br/>',
-            result.output,
-        )
-        output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
-        path_from_output_str = (snapshot_dir / result.output)
-        output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
-        
-        path_exists = False
-        try:
-            path_exists = path_from_output_str.exists()
-        except OSError:
-            pass
-
-        if path_exists:
-            root_dir = str(path_from_output_str)
-        else:
-            root_dir = str(snapshot_dir)
-
-        # print(root_dir, str(list(os.walk(root_dir))))
-
-        for root, dirs, files in os.walk(root_dir):
-            depth = root.replace(root_dir, '').count(os.sep) + 1
-            if depth > 2:
-                continue
-            indent = ' ' * 4 * (depth)
-            output_str += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
-            indentation_str = ' ' * 4 * (depth + 1)
-            for filename in sorted(files):
-                is_hidden = filename.startswith('.')
-                output_str += format_html('<span style="opacity: {}.2">{}{}</span><br/>', int(not is_hidden), indentation_str, filename.strip())
-
-        return output_str + format_html('</code></pre>')
-
-
-
-@admin.register(APIToken, site=archivebox_admin)
-class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created', 'created_by', 'expires')
-    readonly_fields = ('abid', 'created')
-    search_fields = ('id', 'abid', 'created_by__username', 'token')
-    fields = ('created_by', 'token', 'expires', *readonly_fields)
-
-    list_filter = ('created_by',)
-    ordering = ['-created']
-    list_per_page = 100
-
-@admin.register(get_webhook_model(), site=archivebox_admin)
-class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('abid', 'created', *WebhookAdmin.readonly_fields)
+admin.site = ArchiveBoxAdmin()
+admin.site.register(get_user_model())
+admin.site.register(Snapshot, SnapshotAdmin)
+admin.site.register(Tag, TagAdmin)
+admin.site.disable_action('delete_selected')

archivebox/core/apps.py

@@ -1,28 +1,5 @@
-__package__ = 'archivebox.core'
-
 from django.apps import AppConfig
 
 
 class CoreConfig(AppConfig):
     name = 'core'
-
-    def ready(self):
-        # register our custom admin as the primary django admin
-        from django.contrib import admin
-        from django.contrib.admin import sites
-        from core.admin import archivebox_admin
-
-        admin.site = archivebox_admin
-        sites.site = archivebox_admin
-
-
-        # register signal handlers
-        from .auth import register_signals
-
-        register_signals()
-
-
-
-# from django.contrib.admin.apps import AdminConfig
-# class CoreAdminConfig(AdminConfig):
-#     default_site = "core.admin.get_admin_site"

archivebox/core/auth.py

@@ -1,14 +0,0 @@
-__package__ = 'archivebox.core'
-
-
-from ..config import (
-    LDAP
-)
-
-def register_signals():
-
-    if LDAP:
-        import django_auth_ldap.backend
-        from .auth_ldap import create_user
-
-        django_auth_ldap.backend.populate_user.connect(create_user)

archivebox/core/auth_ldap.py

@@ -1,10 +0,0 @@
-from ..config import (
-    LDAP_CREATE_SUPERUSER
-)
-
-def create_user(sender, user=None, ldap_user=None, **kwargs):
-    if not user.id and LDAP_CREATE_SUPERUSER:
-        user.is_superuser = True
-
-    user.is_staff = True
-    print(f'[!] WARNING: Creating new user {user} based on LDAP user {ldap_user} (is_staff={user.is_staff}, is_superuser={user.is_superuser})')

archivebox/core/forms.py

@@ -3,14 +3,9 @@ __package__ = 'archivebox.core'
 from django import forms
 
 from ..util import URL_REGEX
-from ..parsers import PARSERS
-from taggit.utils import edit_string_for_tags, parse_tags
+from ..vendor.taggit_utils import edit_string_for_tags, parse_tags
 
-PARSER_CHOICES = [
-    (parser_key, parser[0])
-    for parser_key, parser in PARSERS.items()
-]
-DEPTH_CHOICES = (
+CHOICES = (
     ('0', 'depth = 0 (archive just these URLs)'),
     ('1', 'depth = 1 (archive these URLs and all URLs one hop away)'),
 )
@@ -25,9 +20,7 @@ ARCHIVE_METHODS = [
 
 class AddLinkForm(forms.Form):
     url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
-    parser = forms.ChoiceField(label="URLs format", choices=[('auto', 'Auto-detect parser'), *PARSER_CHOICES], initial='auto')
-    tag = forms.CharField(label="Tags (comma separated tag1,tag2,tag3)", strip=True, required=False)
-    depth = forms.ChoiceField(label="Archive depth", choices=DEPTH_CHOICES, initial='0', widget=forms.RadioSelect(attrs={"class": "depth-selection"}))
+    depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
     archive_methods = forms.MultipleChoiceField(
         label="Archive methods (select at least 1, otherwise all will be used by default)",
         required=False,
@@ -41,7 +34,7 @@ class AddLinkForm(forms.Form):
     #     label="Exclude patterns",
     #     min_length='1',
     #     required=False,
-    #     initial=URL_DENYLIST,
+    #     initial=URL_BLACKLIST,
     # )
     # timeout = forms.IntegerField(
     #     initial=TIMEOUT,

archivebox/core/middleware.py

@@ -1,60 +0,0 @@
-__package__ = 'archivebox.core'
-
-import ipaddress
-from django.utils import timezone
-from django.contrib.auth.middleware import RemoteUserMiddleware
-from django.core.exceptions import ImproperlyConfigured
-
-from ..config import PUBLIC_SNAPSHOTS, REVERSE_PROXY_USER_HEADER, REVERSE_PROXY_WHITELIST
-
-
-def detect_timezone(request, activate: bool=True):
-    gmt_offset = (request.COOKIES.get('GMT_OFFSET') or '').strip()
-    tz = None
-    if gmt_offset.replace('-', '').isdigit():
-        tz = timezone.get_fixed_timezone(int(gmt_offset))
-        if activate:
-            timezone.activate(tz)
-    # print('GMT_OFFSET', gmt_offset, tz)
-    return tz
-
-
-def TimezoneMiddleware(get_response):
-    def middleware(request):
-        detect_timezone(request, activate=True)
-        return get_response(request)
-
-    return middleware
-
-
-def CacheControlMiddleware(get_response):
-    def middleware(request):
-        response = get_response(request)
-
-        if '/archive/' in request.path or '/static/' in request.path:
-            policy = 'public' if PUBLIC_SNAPSHOTS else 'private'
-            response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
-            # print('Set Cache-Control header to', response['Cache-Control'])
-        return response
-
-    return middleware
-
-class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
-    header = 'HTTP_{normalized}'.format(normalized=REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
-
-    def process_request(self, request):
-        if REVERSE_PROXY_WHITELIST == '':
-            return
-
-        ip = request.META.get('REMOTE_ADDR')
-
-        for cidr in REVERSE_PROXY_WHITELIST.split(','):
-            try:
-                network = ipaddress.ip_network(cidr)
-            except ValueError:
-                raise ImproperlyConfigured(
-                    "The REVERSE_PROXY_WHITELIST config paramater is in invalid format, or "
-                    "contains invalid CIDR. Correct format is a coma-separated list of IPv4/IPv6 CIDRs.")
-
-            if ipaddress.ip_address(ip) in network:
-                return super().process_request(request)

archivebox/core/migrations/0006_auto_20201012_1520.py

@@ -17,7 +17,7 @@ def forwards_func(apps, schema_editor):
         tag_set.discard("")
 
         for tag in tag_set:
-            to_add, _ = TagModel.objects.get_or_create(name=tag, defaults={'slug': slugify(tag)})
+            to_add, _ = TagModel.objects.get_or_create(name=tag, slug=slugify(tag))
             snapshot.tags.add(to_add)
 
 

archivebox/core/migrations/0007_archiveresult.py

@@ -17,6 +17,8 @@ except AttributeError:
 
 
 def forwards_func(apps, schema_editor):
+    from core.models import EXTRACTORS
+
     Snapshot = apps.get_model("core", "Snapshot")
     ArchiveResult = apps.get_model("core", "ArchiveResult")
 
@@ -34,25 +36,8 @@ def forwards_func(apps, schema_editor):
 
         for extractor in history:
             for result in history[extractor]:
-                try:
-                    ArchiveResult.objects.create(
-                        extractor=extractor,
-                        snapshot=snapshot,
-                        pwd=result["pwd"],
-                        cmd=result.get("cmd") or [],
-                        cmd_version=result.get("cmd_version") or 'unknown',
-                        start_ts=result["start_ts"],
-                        end_ts=result["end_ts"],
-                        status=result["status"],
-                        output=result.get("output") or 'null',
-                    )
-                except Exception as e:
-                    print(
-                        '    ! Skipping import due to missing/invalid index.json:',
-                        out_dir,
-                        e,
-                        '(open an issue with this index.json for help)',
-                    )
+                ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=result["cmd"], cmd_version=result["cmd_version"] or 'unknown', 
+                start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"])
 
 
 def verify_json_index_integrity(snapshot):

archivebox/core/migrations/0009_auto_20210216_1038.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 10:38
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0008_auto_20210105_1421'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='updated',
-            field=models.DateTimeField(auto_now=True, db_index=True, null=True),
-        ),
-    ]

archivebox/core/migrations/0010_auto_20210216_1055.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 10:55
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0009_auto_20210216_1038'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='start_ts',
-            field=models.DateTimeField(db_index=True),
-        ),
-    ]

archivebox/core/migrations/0011_auto_20210216_1331.py

@@ -1,24 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 13:31
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0010_auto_20210216_1055'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='archiveresult',
-            name='uuid',
-            field=models.UUIDField(default=uuid.uuid4, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='extractor',
-            field=models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
-        ),
-    ]

archivebox/core/migrations/0012_auto_20210216_1425.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 14:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0011_auto_20210216_1331'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='cmd_version',
-            field=models.CharField(blank=True, default=None, max_length=128, null=True),
-        ),
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='output',
-            field=models.CharField(max_length=1024),
-        ),
-    ]

archivebox/core/migrations/0013_auto_20210218_0729.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 07:29
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0012_auto_20210216_1425'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='title',
-            field=models.CharField(blank=True, db_index=True, max_length=256, null=True),
-        ),
-    ]

archivebox/core/migrations/0014_auto_20210218_0729.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 07:29
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0013_auto_20210218_0729'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='title',
-            field=models.CharField(blank=True, db_index=True, max_length=1024, null=True),
-        ),
-    ]

archivebox/core/migrations/0015_auto_20210218_0730.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 07:30
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0014_auto_20210218_0729'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='title',
-            field=models.CharField(blank=True, db_index=True, max_length=512, null=True),
-        ),
-    ]

archivebox/core/migrations/0016_auto_20210218_1204.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 12:04
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0015_auto_20210218_0730'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='tags',
-            field=models.ManyToManyField(blank=True, to='core.Tag'),
-        ),
-    ]

archivebox/core/migrations/0017_auto_20210219_0211.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-19 02:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0016_auto_20210218_1204'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='tag',
-            name='slug',
-            field=models.SlugField(blank=True, max_length=100, unique=True, verbose_name='slug'),
-        ),
-    ]

archivebox/core/migrations/0018_auto_20210327_0952.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.3 on 2021-03-27 09:52
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0017_auto_20210219_0211'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='tag',
-            name='name',
-            field=models.CharField(max_length=100, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='tag',
-            name='slug',
-            field=models.SlugField(blank=True, max_length=100, unique=True),
-        ),
-    ]

archivebox/core/migrations/0019_auto_20210401_0654.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-04-01 06:54
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0018_auto_20210327_0952'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='url',
-            field=models.URLField(db_index=True, unique=True),
-        ),
-    ]

archivebox/core/migrations/0020_auto_20210410_1031.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.8 on 2021-04-10 10:31
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0019_auto_20210401_0654'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='id',
-            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
-        ),
-        migrations.AlterField(
-            model_name='tag',
-            name='id',
-            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
-        ),
-    ]