Merge branch 'dev' into link-removal2

.dockerignore

@@ -5,27 +5,17 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .github/
-.pdm-build/
-.pdm-python
-.eggs/
-.git/
-!.git/HEAD
-!.git/refs/heads/*
 
 venv/
 .venv/
-.venv-old/
 .docker-venv/
-node_modules/
 
-docs/
 build/
 dist/
-brew_dist/
-deb_dist/
 pip_dist/
+!pip_dist/archivebox.egg-info/requires.txt
+brew_dist/
 assets/
-docker/
 
 data/
 output/

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391
+select = F,E9,W
+max-line-length = 130
+max-complexity = 10
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv

.github/.readthedocs.yaml

@@ -1,20 +0,0 @@
-version: 2
-build:
-  os: ubuntu-24.04
-  tools:
-    python: "3.13"
-  jobs:
-    build:
-      html:
-        - uv run --no-project --with-requirements .github/docs/requirements.txt sphinx-build -W --keep-going -b html -c .github/docs docs "$READTHEDOCS_OUTPUT/html"
-submodules:
-  include:
-    - docs
-python:
-  install:
-    - method: uv
-      command: pip
-      requirements: .github/docs/requirements.txt
-sphinx:
-  configuration: .github/docs/conf.py
-  fail_on_warning: true

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,8 +1,8 @@
 ---
 name: 🐞 Bug report
 about: Create a report to help us improve
-title: 'Bug: ...'
-labels: 'bug'
+title: 'Bugfix: ...'
+labels: 'changes: bugfixes'
 assignees: ''
 
 ---

.github/docs/README.md

@@ -1,43 +0,0 @@
-# Building the 0.7.4 documentation
-
-The `docs` submodule remains pinned to the wiki revision shipped with 0.7.4.
-This configuration renders that historical user guide without installing
-ArchiveBox, importing Django, or updating the wiki checkout.
-
-Prerequisites: [uv](https://docs.astral.sh/uv/getting-started/installation/) 0.11.3 and Python 3.13, matching CI and Read the Docs. `uv` can install Python when needed.
-
-```sh
-git submodule update --init docs
-uv run --no-project --python 3.13 --with-requirements .github/docs/requirements.txt \
-  sphinx-build -W --keep-going -b html -c .github/docs docs /tmp/archivebox-docs
-```
-
-`conf.py` adapts GitHub wiki links, heading levels, and obsolete local anchors
-in memory. Configuration options without a prose section link to their exact
-0.7.4 source definition. The original user documentation is not rewritten.
-Generated Python autodoc scaffolding is outside this standalone user guide;
-its legacy Django-dependent builder is not loaded.
-
-Read the Docs uses `.github/.readthedocs.yaml` and the same strict HTML command.
-Keep its project configuration-file setting pointed at that path.
-
-`requirements.txt` pins the complete documentation dependency set. To update it,
-edit `requirements.in` and run:
-
-```sh
-uv pip compile --universal --python-version 3.13 .github/docs/requirements.in \
-  --output-file .github/docs/requirements.txt
-```
-
-## Live version picker
-
-Read the Docs loads `https://docs.archivebox.io/docs-live-picker/_static/archivebox-versions.js`
-through Addons > Custom script. The hidden `docs-live-picker` version keeps this
-asset available independently of future main or release documentation builds. This applies the release policy and puts `dev`
-after `latest` across existing versions without rebuilding them. The script uses
-the published version inventory and preserves existing navigation URLs.
-
-The hosting checkout command checks out the original main/tag commit, then overlays
-only the historical Sphinx build files from pinned docs repair commits. Application
-source, release tags, and pinned wiki revisions remain those of the selected release.
-Keep the docs repair branches available for fetching these configuration commits.

.github/docs/conf.py

@@ -1,200 +0,0 @@
-"""Build the pinned 0.7.4 user guide without importing ArchiveBox or Django."""
-
-import json
-import os
-import re
-from pathlib import Path
-from urllib.parse import unquote
-
-from markdown_it import MarkdownIt
-from myst_parser.mdit_to_docutils.base import default_slugify
-
-ROOT = Path(__file__).resolve().parents[2]
-DOCS = ROOT / "docs"
-project = "ArchiveBox"
-author = "ArchiveBox contributors"
-copyright = "ArchiveBox contributors"
-html_show_sphinx = False
-release = json.loads((ROOT / "package.json").read_text())["version"]
-version = release
-extensions = ["myst_parser", "sphinx_rtd_theme"]
-source_suffix = {".rst": "restructuredtext", ".md": "markdown"}
-root_doc = "index"
-# These are generated autodoc scaffolding, not standalone user documentation.
-exclude_patterns = [
-    "archivebox*.rst",
-    "modules.rst",
-    "_Sidebar.md",
-    "_Footer.md",
-    ".git",
-]
-html_theme = "sphinx_rtd_theme"
-html_theme_options = {
-    "navigation_depth": 3,
-    "collapse_navigation": False,
-    "version_selector": True,
-    "language_selector": False,
-}
-html_context = {"current_version": os.environ.get("READTHEDOCS_VERSION", release)}
-html_logo = str(DOCS / "logo.png")
-html_static_path = [str(DOCS / "_static"), str(ROOT / ".github/docs/static")]
-html_title = f"ArchiveBox {release} documentation"
-myst_heading_anchors = 6
-
-
-def rewrite_prose(text, transform):
-    """Rewrite navigation outside fenced and indented code blocks."""
-    lines = text.splitlines(keepends=True)
-    output = []
-    start = 0
-    for token in MarkdownIt().parse(text):
-        if token.type not in {"fence", "code_block"}:
-            continue
-        first, last = token.map
-        output.append(transform("".join(lines[start:first])))
-        output.append("".join(lines[first:last]))
-        start = last
-    output.append(transform("".join(lines[start:])))
-    return "".join(output)
-
-
-def prepare_source(app, docname, source):
-    """Adapt GitHub wiki navigation in memory, preserving the pinned sources."""
-    text = source[0]
-    if docname == "Contents":
-        text = re.sub(
-            r"API Reference\n#############.*?(?=Meta\n####)", "", text, flags=re.DOTALL
-        )
-        text += "\n.. toctree::\n    :hidden:\n\n    Home\n"
-    if docname == "index":
-        text = text.replace("archivebox info", "archivebox status")
-        text = text.replace(
-            "pip install archivebox", f"pip install archivebox=={release}"
-        )
-        text = text.replace(
-            "ArchiveBox/ArchiveBox/tree/master",
-            f"ArchiveBox/ArchiveBox/tree/v{release}",
-        )
-        text = text.replace(
-            "`Github <https://github.com/ArchiveBox/ArchiveBox/issues>`_",
-            "`GitHub issues <https://github.com/ArchiveBox/ArchiveBox/issues>`_",
-        )
-        text = text.replace(
-            "==========\nArchiveBox\n==========",
-            f"ArchiveBox {release}\n" + "=" * (11 + len(release)),
-        )
-    if Path(app.env.doc2path(docname)).suffix == ".md":
-
-        def wiki_link(match):
-            parts = match.group(1).split("|", 1)
-            label, target = (parts[0], parts[-1])
-            page, separator, anchor = unquote(target).partition("#")
-            page = page.replace(" ", "-")
-            suffix = f"#{anchor}" if separator else ""
-            return f"[{label}]({page}.md{suffix})"
-
-        text = rewrite_prose(
-            text, lambda prose: re.sub(r"\[\[([^\]\n]+)\]\]", wiki_link, prose)
-        )
-    source[0] = text
-
-
-# GitHub wiki headings allow skipped levels; normalize their hierarchy before MyST
-# parses them, without changing fenced code blocks or the historical source files.
-def normalize_markdown(app, docname, source):
-    if Path(app.env.doc2path(docname)).suffix != ".md":
-        return
-    text = source[0]
-    if docname in {"Home", "README", "Donations", "Upgrading-or-Merging-Archives"}:
-        text = (
-            "# "
-            + (
-                "ArchiveBox overview"
-                if docname == "README"
-                else docname.replace("-", " ")
-            )
-            + "\n\n"
-            + text
-        )
-    lines = text.splitlines()
-    stack = []
-    for token in MarkdownIt().parse(text):
-        if token.type != "heading_open":
-            continue
-        original = int(token.tag[1])
-        while stack and stack[-1] >= original:
-            stack.pop()
-        stack.append(original)
-        line = token.map[0]
-        lines[line] = re.sub(r"^#{1,6}(?=\s)", "#" * len(stack), lines[line]).replace(
-            "\ufe0f", ""
-        )
-    text = "\n".join(lines) + "\n"
-
-    def link(match):
-        target = match.group(1)
-        if "://" in target or target.startswith("mailto:"):
-            return match.group(0)
-        page, sep, anchor = target.partition("#")
-        if page and (DOCS / (page + ".md")).exists():
-            page += ".md"
-        # MyST resolves Markdown GitHub slugs (e.g. background--motivation)
-        # to rendered section IDs; raw HTML links below need explicit rewrites.
-        anchor = anchor.lower()
-        if (docname == "Configuration" and not page) or page == "Configuration.md":
-            other = (DOCS / "Configuration.md").read_text()
-            for heading in re.findall(r"^#+\s+(.+)$", other, re.MULTILINE):
-                if anchor in [
-                    word.lower() for word in re.findall(r"`([^`]+)`", heading)
-                ]:
-                    anchor = default_slugify(heading.replace("`", ""))
-                    break
-            aliases = {"templates_dir": "custom_templates_dir"}
-            anchor = aliases.get(anchor, anchor)
-            if anchor in {
-                "search_backend_timeout",
-                "search_backend_engine",
-                "use_curl",
-            }:
-                code = (ROOT / "archivebox/config.py").read_text().splitlines()
-                number = next(
-                    i + 1 for i, line in enumerate(code) if anchor.upper() in line
-                )
-                return (
-                    "](https://github.com/ArchiveBox/ArchiveBox/blob/v0.7.4/archivebox/config.py#L"
-                    + str(number)
-                    + ")"
-                )
-        if (
-            docname == "README"
-            and anchor == "saves-lots-of-useful-stuff-for-each-imported-link"
-        ):
-            anchor = "output-formats"
-        if docname == "Web-Archiving-Community":
-            anchor = {
-                "blogs": "blogs-friends-of-archivebox",
-                "articles": "articles-we-like-about-internet-archiving",
-            }.get(anchor, anchor)
-        return "](" + page + ("#" + anchor if sep else "") + ")"
-
-    def rewrite_links(prose):
-        prose = re.sub(r"\]\(([^)\s]+)\)", link, prose)
-        # Raw HTML links are outside MyST's cross-reference resolver.
-        html_anchors = {
-            "background--motivation": "background-motivation",
-            "Caveats": "caveats",
-            "contents": "web-archiving-community",
-            "%EF%B8%8F-cli-usage": "cli-usage",
-        }
-        for old, new in html_anchors.items():
-            prose = prose.replace(f'href="#{old}"', f'href="#{new}"')
-        prose = prose.replace("#️-cli-usage", "#cli-usage")
-        return prose
-
-    text = rewrite_prose(text, rewrite_links)
-    source[0] = text
-
-
-def setup(app):
-    app.connect("source-read", prepare_source)
-    app.connect("source-read", normalize_markdown)

.github/docs/requirements.in

@@ -1,3 +0,0 @@
-Sphinx==8.2.3
-myst-parser==4.0.1
-sphinx-rtd-theme==3.1.0

.github/docs/requirements.txt

@@ -1,75 +0,0 @@
-# This file was autogenerated by uv via the following command:
-#    uv pip compile --universal --python-version 3.13 .github/docs/requirements.in --output-file .github/docs/requirements.txt
-alabaster==1.0.0
-    # via sphinx
-babel==2.18.0
-    # via sphinx
-certifi==2026.7.22
-    # via requests
-charset-normalizer==3.5.1
-    # via requests
-colorama==0.4.6 ; sys_platform == 'win32'
-    # via sphinx
-docutils==0.21.2
-    # via
-    #   myst-parser
-    #   sphinx
-    #   sphinx-rtd-theme
-idna==3.20
-    # via requests
-imagesize==2.0.1
-    # via sphinx
-jinja2==3.1.6
-    # via
-    #   myst-parser
-    #   sphinx
-markdown-it-py==3.0.0
-    # via
-    #   mdit-py-plugins
-    #   myst-parser
-markupsafe==3.0.3
-    # via jinja2
-mdit-py-plugins==0.6.1
-    # via myst-parser
-mdurl==0.1.2
-    # via markdown-it-py
-myst-parser==4.0.1
-    # via -r .github/docs/requirements.in
-packaging==26.3
-    # via sphinx
-pygments==2.21.0
-    # via sphinx
-pyyaml==6.0.3
-    # via myst-parser
-requests==2.34.2
-    # via sphinx
-roman-numerals==4.1.0
-    # via roman-numerals-py
-roman-numerals-py==4.1.0
-    # via sphinx
-snowballstemmer==3.1.1
-    # via sphinx
-sphinx==8.2.3
-    # via
-    #   -r .github/docs/requirements.in
-    #   myst-parser
-    #   sphinx-rtd-theme
-    #   sphinxcontrib-jquery
-sphinx-rtd-theme==3.1.0
-    # via -r .github/docs/requirements.in
-sphinxcontrib-applehelp==2.0.0
-    # via sphinx
-sphinxcontrib-devhelp==2.0.0
-    # via sphinx
-sphinxcontrib-htmlhelp==2.1.0
-    # via sphinx
-sphinxcontrib-jquery==4.1
-    # via sphinx-rtd-theme
-sphinxcontrib-jsmath==1.0.1
-    # via sphinx
-sphinxcontrib-qthelp==2.0.0
-    # via sphinx
-sphinxcontrib-serializinghtml==2.0.0
-    # via sphinx
-urllib3==2.8.0
-    # via requests

.github/docs/static/archivebox-versions.js

@@ -1,106 +0,0 @@
-/* ArchiveBox RTD picker policy. No requests, credentials, or documentation edits.
- * RTD event is read-only input: internal flyout rendering precedes the public
- * event, so changing its payload is not a supported rendering hook.
- * Targets: addons readthedocs-flyout shadow dl.versions, RTD theme 3.1 select,
- * and the legacy theme's Versions definition list. Other navigation is untouched.
- */
-(function () {
-  "use strict";
-
-  function parse(slug) {
-    const match = /^v?(\d+)\.(\d+)\.(\d+)(?:-?rc(\d*))?$/i.exec(slug);
-    if (!match) return null;
-    return { slug, major: +match[1], minor: +match[2], patch: +match[3],
-      rc: match[4] === undefined ? null : +(match[4] || 0) };
-  }
-  function compare(a, b) {
-    return a.major - b.major || a.minor - b.minor || a.patch - b.patch ||
-      (a.rc === null ? Infinity : a.rc) - (b.rc === null ? Infinity : b.rc) ||
-      a.slug.localeCompare(b.slug);
-  }
-  function select(versions) {
-    const published = versions.filter(v => v.active !== false && v.built !== false && !v.hidden);
-    const parsed = published.map(v => parse(v.slug)).filter(Boolean);
-    const minors = [...new Set(parsed.sort((a, b) => compare(b, a))
-      .map(v => `${v.major}.${v.minor}`))];
-    const winners = minors.map((minor, index) => parsed.filter(v =>
-      `${v.major}.${v.minor}` === minor && (index < 2 ? v.rc !== null : v.rc === null))
-      .sort((a, b) => compare(b, a))[0]).filter(Boolean).map(v => v.slug);
-    return ["latest", "dev"].filter(slug => published.some(v => v.slug === slug)).concat(winners);
-  }
-
-  // Export only the pure policy for Node verification; browser has no new globals.
-  if (typeof module !== "undefined" && module.exports) {
-    module.exports = { parse, select };
-    return;
-  }
-  const project = document.querySelector('meta[name="readthedocs-project-slug"]')?.content;
-  if (project && project !== "archivebox") return;
-  if (!document.querySelector('meta[name="readthedocs-addons-api-version"]')) {
-    const meta = document.createElement("meta");
-    meta.name = "readthedocs-addons-api-version";
-    meta.content = "1";
-    document.head.append(meta);
-  }
-  let wanted = null;
-  const watched = new WeakSet();
-  function reconcile(container) {
-    if (!wanted) return;
-    const isSelect = container.tagName === "SELECT";
-    const children = [...container.children].filter(n => isSelect ? n.tagName === "OPTION" : n.tagName === "DD");
-    const slug = n => isSelect ? n.value : n.querySelector("a")?.textContent.trim();
-    const selected = isSelect ? container.value : null;
-    const kept = wanted.map(s => children.find(n => slug(n) === s)).filter(Boolean);
-    const current = children.filter(n => wanted.includes(slug(n)));
-    for (const node of children) if (!wanted.includes(slug(node))) node.remove();
-    if (kept.some((node, i) => node !== current[i])) {
-      for (const node of kept) container.append(node);
-    }
-    if (isSelect) {
-      container.value = selected;
-      if (!kept.some(n => n.value === selected)) container.selectedIndex = -1;
-    }
-  }
-  function watch(container) {
-    reconcile(container);
-    if (watched.has(container)) return;
-    watched.add(container);
-    new MutationObserver(() => reconcile(container)).observe(container, { childList: true, subtree: true });
-  }
-  function scan() {
-    if (!wanted) return;
-    document.querySelectorAll("div.switch-menus > div.version-switch select").forEach(watch);
-    document.querySelectorAll(".rst-other-versions dl").forEach(dl => {
-      if (dl.querySelector("dt")?.textContent.trim() === "Versions") watch(dl);
-    });
-    document.querySelectorAll("readthedocs-flyout").forEach(host => {
-      const root = host.shadowRoot;
-      if (!root) return;
-      root.querySelectorAll("dl.versions").forEach(watch);
-      if (!watched.has(root)) {
-        watched.add(root);
-        new MutationObserver(() => root.querySelectorAll("dl.versions").forEach(watch))
-          .observe(root, { childList: true, subtree: true });
-      }
-    });
-  }
-  customElements.whenDefined("readthedocs-flyout").then(scan);
-  let discovery;
-  let stopDiscovery;
-  function ready(eventData) {
-    const config = eventData.data();
-    if (config.projects?.current?.slug !== "archivebox" || !Array.isArray(config.versions?.active)) return;
-    wanted = select(config.versions.active);
-    if (!wanted.length) return;
-    scan();
-    // Discover asynchronously inserted picker containers for at most 10 seconds.
-    // Afterwards, observers remain scoped to known picker/shadow containers only.
-    discovery?.disconnect();
-    clearTimeout(stopDiscovery);
-    discovery = new MutationObserver(scan);
-    discovery.observe(document.body, { childList: true, subtree: true });
-    stopDiscovery = setTimeout(() => discovery.disconnect(), 10000);
-  }
-  document.addEventListener("readthedocs-addons-data-ready", event => ready(event.detail));
-  if (globalThis.ReadTheDocsEventData) ready(globalThis.ReadTheDocsEventData);
-})();

.github/workflows/debian.yml

@@ -9,10 +9,10 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
@@ -25,23 +25,36 @@ jobs:
             dh-python debhelper devscripts dput software-properties-common \
             python3-distutils python3-setuptools python3-wheel python3-stdeb
 
-      # - name: Build Debian/Apt sdist_dsc
-      #   run: |
-      #     ./bin/build_pip.sh
+      - name: Build Debian/Apt sdist_dsc
+        run: |
+          rm -Rf deb_dist/*
+          python3 setup.py --command-packages=stdeb.command sdist_dsc
 
-      # - name: Check ArchiveBox version
-      #   run: |
-      #     # must create dir needed for snaps to run as non-root on github actions
-      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-      #     archivebox --version
-      #     archivebox init --setup
+      - name: Build Debian/Apt bdist_deb
+        run: |
+          python3 setup.py --command-packages=stdeb.command bdist_deb
 
-      # - name: Add some links to test
-      #   run: |
-      #     cd "${{ github.workspace }}/data"
-      #     archivebox add 'https://example.com'
-      #     archivebox status
+      - name: Install archivebox from deb
+        run: |
+          cd deb_dist/
+          sudo apt-get install ./archivebox*.deb
+
+      - name: Check ArchiveBox version
+        run: |
+          # must create dir needed for snaps to run as non-root on github actions
+          sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+          mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+          archivebox init
+          archivebox config --set SAVE_READABILITY=False
+          archivebox config --set SAVE_MERCURY=False
+          archivebox config --set SAVE_SINGLEFILE=False
+          archivebox --version
+
+      - name: Add some links to test
+        run: |
+          cd "${{ github.workspace }}/data"
+          archivebox add 'https://example.com'
+          archivebox status
 
       # - name: Commit built package
       #   run: |

.github/workflows/docker.yml

@@ -1,40 +1,34 @@
 name: Build Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches-ignore:
-      - docs
-      - 'docs/**'
-    tags:
-      - 'v*'
-  # pull_request:
+  on: workflow_dispatch
+  release:
+    types:
+      - created
 
 env:
   DOCKER_IMAGE: archivebox-ci
-      
+
+
 jobs:
   buildx:
-    # Also prevent a manual dispatch on a docs repair branch from publishing.
-    if: github.ref != 'refs/heads/docs' && !startsWith(github.ref, 'refs/heads/docs/')
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v1
       
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v1
         with:
           version: latest
           install: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
       
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
@@ -43,7 +37,7 @@ jobs:
         run: echo ${{ steps.buildx.outputs.platforms }}
       
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -51,7 +45,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Docker Login
-        uses: docker/login-action@v3
+        uses: docker/login-action@v1
         if: github.event_name != 'pull_request'
         with:
            username: ${{ secrets.DOCKER_USERNAME }}
@@ -59,19 +53,17 @@ jobs:
       
       - name: Collect Docker tags
         id: docker_meta
-        uses: docker/metadata-action@v5
+        uses: crazy-max/ghaction-docker-meta@v1
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          tags: |
-              type=ref,event=branch
-              type=semver,pattern={{version}}
-              type=semver,pattern={{major}}.{{minor}}
-              type=sha
-              type=raw,value=latest,enable={{is_default_branch}}
+          tag-sha: true
+          tag-semver: |
+            {{version}}
+            {{major}}.{{minor}}
       
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v2
         with:
           context: ./
           file: ./Dockerfile
@@ -79,18 +71,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
-       
-      # This ugly bit is necessary if you don't want your cache to grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docs.yml

@@ -1,34 +0,0 @@
-name: Historical Sphinx documentation
-
-on:
-  pull_request:
-    branches: [main]
-    paths: ['.github/docs/**', '.github/.readthedocs.yaml', '.github/workflows/docs.yml', 'docs', 'README.md', 'package.json', 'archivebox/config.py']
-  push:
-    branches: [main]
-    paths: ['.github/docs/**', '.github/.readthedocs.yaml', '.github/workflows/docs.yml', 'docs', 'README.md', 'package.json', 'archivebox/config.py']
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - run: git submodule update --init docs
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: '0.11.3'
-          python-version: '3.13'
-      - name: Build the pinned historical user guide
-        env:
-          READTHEDOCS: 'True'
-          READTHEDOCS_VERSION: latest
-        run: uv run --no-project --with-requirements .github/docs/requirements.txt sphinx-build -W --keep-going -b html -c .github/docs docs "$RUNNER_TEMP/docs-html"
-      - name: Verify the version selector
-        run: |
-          grep -q 'class="version-switch"' "$RUNNER_TEMP/docs-html/index.html"
-          grep -q 'readthedocs-addons-data-ready' "$RUNNER_TEMP/docs-html/_static/js/versions.js"

.github/workflows/homebrew.yml

@@ -23,12 +23,11 @@ jobs:
           cd brew_dist/
           brew install --build-bottle ./archivebox.rb
           # brew bottle archivebox
-          archivebox version
 
       - name: Add some links to test
         run: |
           mkdir data && cd data
-          archivebox init --setup
+          archivebox init
           archivebox add 'https://example.com'
           archivebox version
           archivebox status

.github/workflows/jekyll-gh-pages.yml

@@ -1,58 +0,0 @@
-# Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: Build GitHub Pages website
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: true
-
-jobs:
-  # Build job
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v3
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
-
-  # Deployment job
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v2

.github/workflows/lint.yml

@@ -28,8 +28,7 @@ jobs:
 
       - name: Lint with flake8
         run: |
-          cd archivebox
           # one pass for show-stopper syntax errors or undefined names
-          flake8 . --count --show-source --statistics
+          flake8 archivebox --count --show-source --statistics
           # one pass for small stylistic things
-          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
+          flake8 archivebox --count --max-line-length="$MAX_LINE_LENGTH" --statistics

.github/workflows/pip.yml

@@ -7,42 +7,29 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
-
-    permissions:
-      id-token: write
+    runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
         with:
-          python-version: '3.11'
+          python-version: 3.9
           architecture: x64
 
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
-
-      - uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: '3.11'
-          cache: true
-
-      - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --group :all --no-self
-
-      - name: Build package
+      - name: Build Python Package
         run: |
-          pdm build
-
-      - name: Install from build
-        run: pip install ./dist/archivebox-*.whl
+          pip3 install --upgrade pip setuptools wheel
+          rm -Rf pip_dist/*.whl
+          python3 setup.py \
+            sdist --dist-dir=./pip_dist \
+            bdist_wheel --dist-dir=./pip_dist \
+            egg_info --egg-base=./pip_dist
+          pip install pip_dist/archivebox-*.whl
 
       - name: Add some links to test
         run: |
@@ -52,15 +39,6 @@ jobs:
           archivebox version
           archivebox status
 
-      #- name: Publish package distributions to PyPI
-      #  run: pdm publish --no-build
-
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
-
       # - name: Commit built package
       #   run: |
       #     cd pip_dist/
@@ -76,3 +54,8 @@ jobs:
       #     branch: ${{ github.ref }}
       #     directory: pip_dist
 
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}

.github/workflows/test.yml

@@ -13,33 +13,26 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-22.04]
-        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: [3.11]
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+        python: [3.7]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
+      - name: Set up Node JS 14.7.0
+        uses: actions/setup-node@v1
         with:
-          node-version: 20.10.0
-
-      - name: Setup PDM
-        uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: '3.11'
-          cache: true
+          node-version: 14.7.0
 
       ### Install Python & JS Dependencies
       - name: Get pip cache dir
@@ -48,7 +41,7 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         id: cache-pip
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -56,16 +49,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
-
       - name: Install pip dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel pytest bottle build
-          python -m pip install -r requirements.txt
-          python -m pip install -e .[sonic,ldap]
+          python -m pip install --upgrade pip setuptools wheel pytest bottle
+          ./bin/build_pip.sh
+          python -m pip install .
 
       - name: Get npm cache dir
         id: npm-cache
@@ -73,7 +61,7 @@ jobs:
           echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
 
       - name: Cache npm
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         id: cache-npm
         with:
           path: ${{ steps.npm-cache.outputs.dir }}
@@ -102,13 +90,13 @@ jobs:
         # TODO: remove this exception for windows once we get tests passing on that platform
         if: ${{ !contains(matrix.os, 'windows') }}
         run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+          python -m pytest -s --ignore=archivebox/vendor
 
   docker_tests:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1

.gitignore

@@ -13,8 +13,6 @@ venv/
 node_modules/
 
 # Packaging artifacts
-.pdm-python
-.pdm-build
 archivebox.egg-info
 archivebox-*.tar.gz
 build/
@@ -26,6 +24,3 @@ data1/
 data2/
 data3/
 output/
-
-# vim
-*.sw?

.gitmodules

@@ -23,6 +23,3 @@
 [submodule "archivebox/vendor/django-taggit"]
 	path = archivebox/vendor/django-taggit
 	url = https://github.com/jazzband/django-taggit
-[submodule "archivebox/vendor/python-atomicwrites"]
-	path = archivebox/vendor/python-atomicwrites
-	url = https://github.com/untitaker/python-atomicwrites

.readthedocs.yaml

@@ -1,34 +0,0 @@
-# Read the Docs configuration file for Sphinx projects
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    nodejs: "20"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
-  # builder: "dirhtml"
-
-# Optionally build your docs in additional formats such as PDF and ePub
-formats:
-  - pdf
-  - epub
-
-# Optional but recommended, declare the Python requirements required
-# to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: requirements.txt
-    - requirements: docs/requirements.txt

Dockerfile

@@ -1,310 +1,117 @@
 # This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, single-file
 # Usage:
-#     git submodule update --init --recursive
-#     git pull --recurse-submodules
 #     docker build . -t archivebox --no-cache
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-# Multi-arch build:
-#     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
-#
-# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
-
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
-FROM python:3.11-slim-bookworm
+FROM python:3.9-slim-buster
 
 LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
+    maintainer="Nick Sweeting <archivebox-docker@sweeting.me>" \
     description="All-in-one personal internet archiving container" \
     homepage="https://github.com/ArchiveBox/ArchiveBox" \
     documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-
-######### Environment Variables #################################
-
-# Global system-level config
+# System-level base config
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
     PYTHONIOENCODING=UTF-8 \
     PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
 
-# Version config
-ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=24 \
-    PLAYWRIGHT_VERSION=1.59.0 \
-    YTDLP_VERSION=2026.3.17 \
-    GOSU_VERSION=1.19
-
-# User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911
-
-# Global paths
+# Application-level base config
 ENV CODE_DIR=/app \
+    VENV_PATH=/venv \
     DATA_DIR=/data \
-    GLOBAL_VENV=/venv \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Application-level paths
-ENV APP_VENV=/app/.venv \
-    NODE_MODULES=/app/node_modules
-
-# Build shell config
-ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading package.json
-COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
-RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && cat /etc/os-release | head -n7 \
-    && which bash && bash --version | head -n1 \
-    && which dpkg && dpkg --version | head -n1 \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    NODE_DIR=/node \
+    ARCHIVEBOX_USER="archivebox"
 
 # Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+RUN groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER
 
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init cron unzip grep \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/* \
-    && case "$TARGETARCH/$TARGETVARIANT" in \
-        amd64/) GOSU_ARCH=amd64; GOSU_SHA256=52c8749d0142edd234e9d6bd5237dff2d81e71f43537e2f4f66f75dd4b243dd0 ;; \
-        arm64/) GOSU_ARCH=arm64; GOSU_SHA256=3a8ef022d82c0bc4a98bcb144e77da714c25fcfa64dccc57f6aba7ae47ff1a44 ;; \
-        arm/v7) GOSU_ARCH=armhf; GOSU_SHA256=8457a0bfd28e016c2c7d8ea6e5f7eed1376033ffbd36491bb455094c8b1dc9fd ;; \
-        *) echo "Unsupported gosu platform: $TARGETARCH/$TARGETVARIANT"; exit 1 ;; \
-    esac \
-    && curl -fsSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$GOSU_ARCH" \
-    && echo "$GOSU_SHA256  /usr/local/bin/gosu" | sha256sum -c - \
-    && chmod +x /usr/local/bin/gosu \
-    && gosu --version | head -n1 | tee -a /VERSION.txt
-
-######### Language Environments ####################################
-
-# Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-        nodejs libatomic1 python3-minimal \
-    && rm -rf /var/lib/apt/lists/* \
-    # Update NPM to latest version
-    && npm i -g npm --cache /root/.npm \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Install Python environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    && ( \
-        which python3 && python3 --version | grep " $PYTHON_VERSION" \
-        && which pip && pip --version \
-        # && which pdm && pdm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-######### Extractor Dependencies ##################################
+# Install system dependencies
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        apt-transport-https ca-certificates gnupg2 zlib1g-dev \
+        dumb-init gosu cron unzip curl \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install apt dependencies
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT extractor dependencies globally using apt..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-        curl wget git ffmpeg ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-        # fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
-    && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        && which rg && rg --version 2>&1 | head -n1 \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        wget curl chromium git ffmpeg youtube-dl ripgrep \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install chromium browser using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+# Install apt development dependencies
+# RUN apt-get install -qq \
+#     && apt-get install -qq -y --no-install-recommends \
+#         python3 python3-dev python3-pip python3-venv python3-all \
+#         dh-python debhelper devscripts dput software-properties-common \
+#         python3-distutils python3-setuptools python3-wheel python3-stdeb
+
+# Install Node environment
+RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo 'deb https://deb.nodesource.com/node_15.x buster main' >> /etc/apt/sources.list \
     && apt-get update -qq \
-    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
-        # install Chromium using playwright
-        pip install "playwright==$PLAYWRIGHT_VERSION" \
-        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-        && playwright install --with-deps chromium \
-        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
-    else \
-        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.) 
-        apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-            chromium fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        && export CHROME_BINARY="$(which chromium)"; \
-    fi \
-    && rm -rf /var/lib/apt/lists/* \
-    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
-    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
-    # Save version info
-    && ( \
-        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    && apt-get install -qq -y --no-install-recommends \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install Node dependencies
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
-    && npm ci --prefer-offline --no-audit --cache /root/.npm \
-    && ( \
-        which node && node --version \
-        && which npm && npm version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+WORKDIR "$NODE_DIR"
+ENV PATH="${PATH}:$NODE_DIR/node_modules/.bin" \
+    npm_config_loglevel=error
+ADD ./package.json ./package.json
+ADD ./package-lock.json ./package-lock.json
+RUN npm ci
 
-######### Build Dependencies ####################################
-
-# Install ArchiveBox Python dependencies
+# Install Python dependencies
 WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-        build-essential \
-        libssl-dev libldap2-dev libsasl2-dev \
-        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
-    # && pdm use --venv in-project \
-    # && pdm run python -m ensurepip \
-    # && pdm sync --fail-fast --no-editable --group :all --no-self \
-    # && pdm export -o requirements.txt --without-hashes \
-    # && source $GLOBAL_VENV/bin/activate \
-    && pip install -r requirements.txt \
-    && pip install --upgrade "yt-dlp==$YTDLP_VERSION" \
-    && apt-get purge -y \
-        build-essential \
+ENV PATH="${PATH}:$VENV_PATH/bin"
+RUN python -m venv --clear --symlinks "$VENV_PATH" \
+    && pip install --upgrade --quiet pip setuptools
+ADD ./pip_dist/archivebox.egg-info/requires.txt "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt"
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential python-dev python3-dev \
+    && grep -B 1000 -E '^$' "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin \
+    && pip install --quiet "sonic-client==0.0.5" \
+    && apt-get purge -y build-essential python-dev python3-dev \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
-# Install ArchiveBox Python package from source
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    # && apt-get update -qq \
-    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
-    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
-    #     build-essential  \
-    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
-    && pip install -e "$CODE_DIR"[sonic,ldap] \
-    # save docker image size and always remove compilers / build tools after building is complete
-    # && apt-get purge -y build-essential \
-    # && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-####################################################
+# Install ArchiveBox Python package and its dependencies
+WORKDIR "$CODE_DIR"
+ADD . "$CODE_DIR"
+RUN pip install -e .
 
 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True
-    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
-    # CHROME_SANDBOX=False \
-    # WGET_BINARY="wget" \
-    # YOUTUBEDL_BINARY="yt-dlp" \
-    # CHROME_BINARY="/usr/bin/chromium-browser" \
-    # USE_SINGLEFILE=True \
-    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
-    # USE_READABILITY=True \
-    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
-    # USE_MERCURY=True \
-    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
+ENV IN_DOCKER=True \
+    CHROME_SANDBOX=False \
+    CHROME_BINARY="chromium" \
+    USE_SINGLEFILE=True \
+    SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \
+    USE_READABILITY=True \
+    READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor" \
+    USE_MERCURY=True \
+    MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser"
 
 # Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
-
-####################################################
+# RUN archivebox version
+RUN /app/bin/docker_entrypoint.sh archivebox version
 
 # Open up the interfaces to the outside world
-WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
-# Optional:
-# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-#     CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
-
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "0.0.0.0:8000"]

MANIFEST.in

@@ -0,0 +1,4 @@
+graft archivebox
+global-exclude .DS_Store
+global-exclude __pycache__
+global-exclude *.pyc

Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[packages]
+# see setup.py for package dependency list
+"e1839a8" = {path = ".", editable = true}
+
+[dev-packages]
+# see setup.py for dev package dependency list
+"e1839a8" = {path = ".", extras = ["dev"], editable = true}

SECURITY.md

@@ -1,34 +0,0 @@
-# Security Policy
-
----
-
-## Security Information
-
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
-
-- https://github.com/ArchiveBox/ArchiveBox#archive-layout
-- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
-
----
-
-## Reporting a Vulnerability
-
-We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
-
-1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
-
-2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
-
-3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
-
-You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-merlot

archivebox/.flake8

@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
+exclude = migrations,tests,node_modules,vendor,static,venv,.venv,.venv2,.docker-venv

archivebox/cli/__init__.py

@@ -8,17 +8,18 @@ import argparse
 from typing import Optional, Dict, List, IO, Union
 from pathlib import Path
 
-from ..config import OUTPUT_DIR, check_data_folder, check_migrations
+from ..config import OUTPUT_DIR
 
 from importlib import import_module
 
 CLI_DIR = Path(__file__).resolve().parent
 
 # these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
+meta_cmds = ('help', 'version')
+main_cmds = ('init', 'info', 'config')
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')
+
+fake_db = ("oneshot",)
 
 display_first = (*meta_cmds, *main_cmds, *archive_cmds)
 
@@ -60,21 +61,9 @@ def run_subcommand(subcommand: str,
                    pwd: Union[Path, str, None]=None) -> None:
     """Run a given ArchiveBox subcommand with the given list of args"""
 
-    subcommand_args = subcommand_args or []
-
     if subcommand not in meta_cmds:
         from ..config import setup_django
-
-        cmd_requires_db = subcommand in archive_cmds
-        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
-
-        if cmd_requires_db:
-            check_data_folder(pwd)
-
-        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
-
-        if cmd_requires_db:
-            check_migrations()
+        setup_django(in_memory_db=subcommand in fake_db, check_db=subcommand in archive_cmds)
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore

archivebox/cli/archivebox_add.py

@@ -10,7 +10,6 @@ from typing import List, Optional, IO
 
 from ..main import add
 from ..util import docstring
-from ..parsers import PARSERS
 from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr
 
@@ -24,23 +23,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         formatter_class=SmartFormatter,
     )
     parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--update', #'-u',
+        '--update-all', #'-n',
         action='store_true',
         default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
-    parser.add_argument(
-        '--update-all', #'-n',
-        action='store_true',
-        default=False, 
-        help="Also update ALL links in index when finished adding new links",
-    )
     parser.add_argument(
         '--index-only', #'-o',
         action='store_true',
@@ -86,20 +73,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
               This does not take precedence over the configuration",
         default=""
     )
-    parser.add_argument(
-        "--parser",
-        type=str,
-        help="Parser used to read inputted URLs.",
-        default="auto",
-        choices=["auto", *PARSERS.keys()],
-    )
     command = parser.parse_args(args or ())
     urls = command.urls
-
-    stdin_urls = ''
-    if not urls:
-        stdin_urls = accept_stdin(stdin)
-
+    stdin_urls = accept_stdin(stdin)
     if (stdin_urls and urls) or (not stdin and not urls):
         stderr(
             '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
@@ -109,14 +85,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     add(
         urls=stdin_urls or urls,
         depth=command.depth,
-        tag=command.tag,
-        update=command.update,
         update_all=command.update_all,
         index_only=command.index_only,
         overwrite=command.overwrite,
         init=command.init,
         extractors=command.extract,
-        parser=command.parser,
         out_dir=pwd or OUTPUT_DIR,
     )
 

archivebox/cli/archivebox_config.py

@@ -45,10 +45,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='KEY or KEY=VALUE formatted config values to get or set',
     )
     command = parser.parse_args(args or ())
-
-    config_options_str = ''
-    if not command.config_options:
-        config_options_str = accept_stdin(stdin)
+    config_options_str = accept_stdin(stdin)
 
     config(
         config_options_str=config_options_str,

archivebox/cli/archivebox_init.py

@@ -27,23 +27,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Ignore unrecognized files in current directory and initialize anyway',
     )
-    parser.add_argument(
-        '--quick', '-q',
-        action='store_true',
-        help='Run any updates or migrations without rechecking all snapshot dirs',
-    )
-    parser.add_argument(
-        '--setup', #'-s',
-        action='store_true',
-        help='Automatically install dependencies and extras used for archiving',
-    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
     init(
         force=command.force,
-        quick=command.quick,
-        setup=command.setup,
         out_dir=pwd or OUTPUT_DIR,
     )
     

archivebox/cli/archivebox_list.py

@@ -12,7 +12,6 @@ from ..main import list_all
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
-    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -24,7 +23,7 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
-from ..logging_util import SmartFormatter, reject_stdin, stderr
+from ..logging_util import SmartFormatter, accept_stdin, stderr
 
 
 @docstring(list_all.__doc__)
@@ -45,7 +44,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     group.add_argument(
         '--json', #'-j',
         action='store_true',
-        help="Print the output in JSON format with all columns included",
+        help="Print the output in JSON format with all columns included.",
     )
     group.add_argument(
         '--html',
@@ -60,19 +59,19 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--sort', #'-s',
         type=str,
-        help="List the links sorted using the given key, e.g. timestamp or updated",
+        help="List the links sorted using the given key, e.g. timestamp or updated.",
         default=None,
     )
     parser.add_argument(
         '--before', #'-b',
         type=float,
-        help="List only links bookmarked before (less than) the given timestamp",
+        help="List only links bookmarked before the given timestamp.",
         default=None,
     )
     parser.add_argument(
         '--after', #'-a',
         type=float,
-        help="List only links bookmarked after (greater than or equal to) the given timestamp",
+        help="List only links bookmarked after the given timestamp.",
         default=None,
     )
     parser.add_argument(
@@ -97,9 +96,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type', '-t',
+        '--filter-type',
         type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
+        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -108,19 +107,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         nargs='*',
         type=str,
         default=None,
-        help='List only URLs matching these filter patterns'
+        help='List only URLs matching these filter patterns.'
     )
     command = parser.parse_args(args or ())
-    reject_stdin(stdin)
+    filter_patterns_str = accept_stdin(stdin)
 
     if command.with_headers and not (command.json or command.html or command.csv):
         stderr(
-            '[X] --with-headers can only be used with --json, --html or --csv options\n',
+            '[X] --with-headers can only be used with --json, --html or --csv options.\n',
             color='red',
         )
         raise SystemExit(2)
 
     matching_folders = list_all(
+        filter_patterns_str=filter_patterns_str,
         filter_patterns=command.filter_patterns,
         filter_type=command.filter_type,
         status=command.status,

archivebox/cli/archivebox_oneshot.py

@@ -50,11 +50,8 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help= "Path to save the single archive folder to, e.g. ./example.com_archive"
     )
     command = parser.parse_args(args or ())
-    stdin_url = None
     url = command.url
-    if not url:
-        stdin_url = accept_stdin(stdin)
-
+    stdin_url = accept_stdin(stdin)
     if (stdin_url and url) or (not stdin and not url):
         stderr(
             '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',

archivebox/cli/archivebox_remove.py

@@ -61,10 +61,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='URLs matching this filter pattern will be removed from the index.'
     )
     command = parser.parse_args(args or ())
-    
-    filter_str = None
-    if not command.filter_patterns:
-        filter_str = accept_stdin(stdin)
+    filter_str = accept_stdin(stdin)
 
     remove(
         filter_str=filter_str,

archivebox/cli/archivebox_schedule.py

@@ -39,28 +39,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=None,
         help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
     )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
     parser.add_argument(
         '--depth', # '-d',
         type=int,
         choices=[0, 1],
         default=0,
-        help='Depth to archive to [0] or 1, see "add" command help for more info',
-    )
-    parser.add_argument(
-        '--overwrite',
-        action='store_true',
-        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
-    )
-    parser.add_argument(
-        '--update',
-        action='store_true',
-        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
+        help='Depth to archive to [0] or 1, see "add" command help for more info.',
     )
     group.add_argument(
         '--clear', # '-c'
@@ -103,10 +87,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         run_all=command.run_all,
         quiet=command.quiet,
         every=command.every,
-        tag=command.tag,
         depth=command.depth,
-        overwrite=command.overwrite,
-        update=command.update,
         import_path=command.import_path,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_server.py

@@ -38,20 +38,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Enable DEBUG=True mode with more verbose errors',
     )
-    parser.add_argument(
-        '--nothreading',
-        action='store_true',
-        help='Force runserver to run in single-threaded mode',
-    )
     parser.add_argument(
         '--init',
         action='store_true',
-        help='Run a full archivebox init/upgrade before starting the server',
-    )
-    parser.add_argument(
-        '--quick-init', '-i',
-        action='store_true',
-        help='Run quick archivebox init/upgrade before starting the server',
+        help='Run archivebox init before starting the server',
     )
     parser.add_argument(
         '--createsuperuser',
@@ -62,11 +52,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     reject_stdin(__command__, stdin)
     
     server(
-        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
+        runserver_args=command.runserver_args,
         reload=command.reload,
         debug=command.debug,
         init=command.init,
-        quick_init=command.quick_init,
         createsuperuser=command.createsuperuser,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_setup.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox setup'
-
-import sys
-import argparse
-
-from typing import Optional, List, IO
-
-from ..main import setup
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-
-
-@docstring(setup.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=setup.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    # parser.add_argument(
-    #     '--force', # '-f',
-    #     action='store_true',
-    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
-    # )
-    command = parser.parse_args(args or ())   # noqa
-    reject_stdin(__command__, stdin)
-
-    setup(
-        # force=command.force,
-        out_dir=pwd or OUTPUT_DIR,
-    )
-    
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_update.py

@@ -12,7 +12,6 @@ from ..main import update
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
-    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -90,9 +89,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type', '-t',
+        '--filter-type',
         type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
+        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -111,10 +110,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=""
     )
     command = parser.parse_args(args or ())
-
-    filter_patterns_str = None
-    if not command.filter_patterns:
-        filter_patterns_str = accept_stdin(stdin)
+    filter_patterns_str = accept_stdin(stdin)
 
     update(
         resume=command.resume,

archivebox/cli/tests.py

@@ -75,23 +75,23 @@ def output_hidden(show_failing=True):
         yield
         return
 
-    sys.stdout = open('stdout.txt', 'w+', encoding='utf-8')
-    sys.stderr = open('stderr.txt', 'w+', encoding='utf-8')
+    sys.stdout = open('stdout.txt', 'w+')
+    sys.stderr = open('stderr.txt', 'w+')
     try:
         yield
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
-    except Exception:
+    except:
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
         if show_failing:
-            with open('stdout.txt', 'r', encoding='utf-8') as f:
+            with open('stdout.txt', 'r') as f:
                 print(f.read())
-            with open('stderr.txt', 'r', encoding='utf-8') as f:
+            with open('stderr.txt', 'r') as f:
                 print(f.read())
         raise
     finally:
@@ -116,7 +116,7 @@ class TestInit(unittest.TestCase):
         assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
 
     def test_conflicting_init(self):
-        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+') as f:
             f.write('test')
 
         try:
@@ -132,7 +132,7 @@ class TestInit(unittest.TestCase):
         try:
             load_main_index(out_dir=OUTPUT_DIR)
             assert False, 'load_main_index should raise an exception when no index is present'
-        except Exception:
+        except:
             pass
 
     def test_no_dirty_state(self):
@@ -161,7 +161,7 @@ class TestAdd(unittest.TestCase):
 
     def test_add_arg_file(self):
         test_file = Path(OUTPUT_DIR) / 'test.txt'
-        with open(test_file, 'w+', encoding='utf') as f:
+        with open(test_file, 'w+') as f:
             f.write(test_urls)
 
         with output_hidden():
@@ -216,7 +216,7 @@ class TestRemove(unittest.TestCase):
             with output_hidden(show_failing=False):
                 archivebox_remove.main(['--yes', '--delete', 'https://doesntexist.com'])
             assert False, 'Should raise if no URLs match'
-        except Exception:
+        except:
             pass
 
 

archivebox/config_stubs.py

@@ -41,7 +41,7 @@ class ConfigDict(BaseConfig, total=False):
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
     RESTRICT_FILE_NAMES: str
-    URL_DENYLIST: str
+    URL_BLACKLIST: str
 
     SECRET_KEY: Optional[str]
     BIND_ADDR: str
@@ -74,7 +74,6 @@ class ConfigDict(BaseConfig, total=False):
     CHROME_USER_AGENT: str
     COOKIES_FILE: Union[str, Path, None]
     CHROME_USER_DATA_DIR: Union[str, Path, None]
-    CHROME_TIMEOUT: int
     CHROME_HEADLESS: bool
     CHROME_SANDBOX: bool
 
@@ -99,7 +98,6 @@ class ConfigDict(BaseConfig, total=False):
     WGET_ARGS: List[str]
     CURL_ARGS: List[str]
     GIT_ARGS: List[str]
-    TAG_SEPARATOR_PATTERN: str
 
 
 ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

archivebox/core/admin.py

@@ -1,9 +1,7 @@
 __package__ = 'archivebox.core'
 
 from io import StringIO
-from pathlib import Path
 from contextlib import redirect_stdout
-from datetime import datetime, timezone
 
 from django.contrib import admin
 from django.urls import path
@@ -15,16 +13,16 @@ from django import forms
 
 from ..util import htmldecode, urldecode, ansi_to_html
 
-from core.models import Snapshot, ArchiveResult, Tag
-from core.forms import AddLinkForm
+from core.models import Snapshot, Tag
+from core.forms import AddLinkForm, TagField
 
 from core.mixins import SearchResultsAdminMixin
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
-from config import OUTPUT_DIR, SNAPSHOTS_PER_PAGE
-from extractors import archive_links
+from config import OUTPUT_DIR
+from extractors import archive_snapshots
 
 # Admin URLs
 # /admin/
@@ -38,129 +36,95 @@ from extractors import archive_links
 
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
+def update_snapshots(modeladmin, request, queryset):
+    archive_links([
+        snapshot.as_link()
+        for snapshot in queryset
+    ], out_dir=OUTPUT_DIR)
+update_snapshots.short_description = "Archive"
 
-class ArchiveResultInline(admin.TabularInline):
-    model = ArchiveResult
+def update_titles(modeladmin, request, queryset):
+    archive_links([
+        snapshot.as_link()
+        for snapshot in queryset
+    ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
+update_titles.short_description = "Pull title"
 
-class TagInline(admin.TabularInline):
-    model = Snapshot.tags.through
+def overwrite_snapshots(modeladmin, request, queryset):
+    archive_links([
+        snapshot.as_link()
+        for snapshot in queryset
+    ], overwrite=True, out_dir=OUTPUT_DIR)
+overwrite_snapshots.short_description = "Re-archive (overwrite)"
 
-from django.contrib.admin.helpers import ActionForm
-from django.contrib.admin.widgets import AutocompleteSelectMultiple
+def verify_snapshots(modeladmin, request, queryset):
+    for snapshot in queryset:
+        print(snapshot.timestamp, snapshot.url, snapshot.is_archived, snapshot.archive_size, len(snapshot.history))
 
-# WIP: broken by Django 3.1.2 -> 4.0 migration
-class AutocompleteTags:
-    model = Tag
-    search_fields = ['name']
-    name = 'tags'
+verify_snapshots.short_description = "Check"
 
-class AutocompleteTagsAdminStub:
-    name = 'admin'
+def delete_snapshots(modeladmin, request, queryset):
+    remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
+
+delete_snapshots.short_description = "Delete"
 
 
-class SnapshotActionForm(ActionForm):
-    tags = forms.ModelMultipleChoiceField(
-        queryset=Tag.objects.all(),
-        required=False,
-        # WIP: broken by Django 3.1.2 -> 4.0 migration
-        widget=AutocompleteSelectMultiple(
-            AutocompleteTags(),
-            AutocompleteTagsAdminStub(),
-        ),
-    )
+class SnapshotAdminForm(forms.ModelForm):
+    tags = TagField(required=False)
 
-    # TODO: allow selecting actions for specific extractors? is this useful?
-    # EXTRACTOR_CHOICES = [
-    #     (name, name.title())
-    #     for name, _, _ in get_default_archive_methods()
-    # ]
-    # extractor = forms.ChoiceField(
-    #     choices=EXTRACTOR_CHOICES,
-    #     required=False,
-    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
-    # )
+    class Meta:
+        model = Snapshot
+        fields = "__all__"
+
+    def save(self, commit=True):
+        # Based on: https://stackoverflow.com/a/49933068/3509554
+
+        # Get the unsave instance
+        instance = forms.ModelForm.save(self, False)
+        tags = self.cleaned_data.pop("tags")
+
+        #update save_m2m
+        def new_save_m2m():
+            instance.save_tags(tags)
+
+        # Do we need to save all changes now?
+        self.save_m2m = new_save_m2m
+        if commit:
+            instance.save()
+
+        return instance
 
 
 class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
-    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
-    sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('info', 'bookmarked', 'added', 'updated')
-    search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
-    fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
-    list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
+    list_display = ('added', 'title_str', 'url_str', 'files', 'size')
+    sort_fields = ('title_str', 'url_str', 'added')
+    readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
+    search_fields = ['url__icontains', 'timestamp', 'title', 'tags__name']
+    fields = (*readonly_fields, 'title', 'tags')
+    list_filter = ('added', 'updated', 'tags')
     ordering = ['-added']
-    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
-    autocomplete_fields = ['tags']
-    inlines = [ArchiveResultInline]
-    list_per_page = SNAPSHOTS_PER_PAGE
-
-    action_form = SnapshotActionForm
+    actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots]
+    actions_template = 'admin/actions_as_select.html'
+    form = SnapshotAdminForm
+    list_per_page = 40
 
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
-            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
+            path('grid/', self.admin_site.admin_view(self.grid_view),name='grid')
         ]
         return custom_urls + urls
 
     def get_queryset(self, request):
-        self.request = request
         return super().get_queryset(request).prefetch_related('tags')
 
     def tag_list(self, obj):
         return ', '.join(obj.tags.values_list('name', flat=True))
 
-    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
-    # def action(self, obj):
-    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
-    #     # action: update_snapshots
-    #     # select_across: 0
-    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
-    #     return format_html(
-    #         '''
-    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
-    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
-    #                 <input type="hidden" name="_selected_action" value="{}">
-    #                 <button name="update_snapshots">Check</button>
-    #                 <button name="update_titles">Pull title + favicon</button>
-    #                 <button name="update_snapshots">Update</button>
-    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
-    #                 <button name="delete_snapshots">Permanently delete</button>
-    #             </form>
-    #         ''',
-    #         csrf.get_token(self.request),
-    #         obj.id,
-    #     )
-
-    def info(self, obj):
+    def id_str(self, obj):
         return format_html(
-            '''
-            UUID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            Timestamp: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            Archived: {} ({} files {}) &nbsp; &nbsp;
-            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
-            Status code: {} &nbsp; &nbsp;
-            Server: {} &nbsp; &nbsp;
-            Content type: {} &nbsp; &nbsp;
-            Extension: {} &nbsp; &nbsp;
-            <br/><br/>
-            <a href="/archive/{}">View Snapshot index ➡️</a> &nbsp; &nbsp;
-            <a href="/admin/core/snapshot/?id__exact={}">View actions ⚙️</a>
-            ''',
-            obj.id,
-            obj.timestamp,
-            obj.url_hash,
-            '✅' if obj.is_archived else '❌',
-            obj.num_outputs,
-            self.size(obj),
-            f'/archive/{obj.timestamp}/favicon.ico',
-            obj.status_code or '?',
-            obj.headers and obj.headers.get('Server') or '?',
-            obj.headers and obj.headers.get('Content-Type') or '?',
-            obj.extension or '?',
-            obj.timestamp,
-            obj.id,
+            '<code style="font-size: 10px">{}</code>',
+            obj.url_hash[:8],
         )
 
     def title_str(self, obj):
@@ -187,11 +151,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     def files(self, obj):
         return snapshot_icons(obj)
 
-    files.admin_order_field = 'updated'
-    files.short_description = 'Files Saved'
-
     def size(self, obj):
-        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
+        archive_size = obj.archive_size
         if archive_size:
             size_txt = printable_filesize(archive_size)
             if archive_size > 52428800:
@@ -204,16 +165,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             size_txt,
         )
 
-    size.admin_order_field = 'archiveresult__count'
-
     def url_str(self, obj):
         return format_html(
-            '<a href="{}"><code style="user-select: all;">{}</code></a>',
-            obj.url,
+            '<a href="{}"><code>{}</code></a>',
             obj.url,
+            obj.url.split('://www.', 1)[-1].split('://', 1)[-1][:64],
         )
 
-    def grid_view(self, request, extra_context=None):
+    def grid_view(self, request):
 
         # cl = self.get_changelist_instance(request)
 
@@ -224,153 +183,35 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
         # Monkey patch here plus core_tags.py
         self.change_list_template = 'private_index_grid.html'
-        self.list_per_page = SNAPSHOTS_PER_PAGE
+        self.list_per_page = 20
         self.list_max_show_all = self.list_per_page
 
         # Call monkey patched view
-        rendered_response = self.changelist_view(request, extra_context=extra_context)
+        rendered_response = self.changelist_view(request)
 
         # Restore values
-        self.change_list_template = saved_change_list_template
+        self.change_list_template =  saved_change_list_template
         self.list_per_page = saved_list_per_page
         self.list_max_show_all = saved_list_max_show_all
 
         return rendered_response
-
-    # for debugging, uncomment this to print all requests:
-    # def changelist_view(self, request, extra_context=None):
-    #     print('[*] Got request', request.method, request.POST)
-    #     return super().changelist_view(request, extra_context=None)
-
-    def update_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], out_dir=OUTPUT_DIR)
-    update_snapshots.short_description = "Pull"
-
-    def update_titles(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
-    update_titles.short_description = "⬇️ Title"
-
-    def resnapshot_snapshot(self, request, queryset):
-        for snapshot in queryset:
-            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
-            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
-            add(new_url, tag=snapshot.tags_str())
-    resnapshot_snapshot.short_description = "Re-Snapshot"
-
-    def overwrite_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, out_dir=OUTPUT_DIR)
-    overwrite_snapshots.short_description = "Reset"
-
-    def delete_snapshots(self, request, queryset):
-        remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
-
-    delete_snapshots.short_description = "Delete"
-
-    def add_tags(self, request, queryset):
-        tags = request.POST.getlist('tags')
-        print('[+] Adding tags', tags, 'to Snapshots', queryset)
-        for obj in queryset:
-            obj.tags.add(*tags)
-
-    add_tags.short_description = "+"
-
-    def remove_tags(self, request, queryset):
-        tags = request.POST.getlist('tags')
-        print('[-] Removing tags', tags, 'to Snapshots', queryset)
-        for obj in queryset:
-            obj.tags.remove(*tags)
-
-    remove_tags.short_description = "–"
-
         
 
+    id_str.short_description = 'ID'
     title_str.short_description = 'Title'
     url_str.short_description = 'Original URL'
 
+    id_str.admin_order_field = 'id'
     title_str.admin_order_field = 'title'
     url_str.admin_order_field = 'url'
 
-
-
 class TagAdmin(admin.ModelAdmin):
-    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
+    list_display = ('slug', 'name', 'id')
     sort_fields = ('id', 'name', 'slug')
-    readonly_fields = ('id', 'num_snapshots', 'snapshots')
+    readonly_fields = ('id',)
     search_fields = ('id', 'name', 'slug')
     fields = (*readonly_fields, 'name', 'slug')
-    actions = ['delete_selected']
-    ordering = ['-id']
 
-    def num_snapshots(self, obj):
-        return format_html(
-            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
-            obj.id,
-            obj.snapshot_set.count(),
-        )
-
-    def snapshots(self, obj):
-        total_count = obj.snapshot_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
-                snap.id,
-                snap.timestamp,
-                snap.url,
-            )
-            for snap in obj.snapshot_set.order_by('-updated')[:10]
-        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={obj.id}">and {total_count-10} more...<a>' if obj.snapshot_set.count() > 10 else ''))
-
-
-class ArchiveResultAdmin(admin.ModelAdmin):
-    list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
-    sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('id', 'uuid', 'snapshot_str', 'tags_str')
-    search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
-    autocomplete_fields = ['snapshot']
-
-    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
-    ordering = ['-start_ts']
-    list_per_page = SNAPSHOTS_PER_PAGE
-
-    def snapshot_str(self, obj):
-        return format_html(
-            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
-            '<small>{}</small>',
-            obj.snapshot.timestamp,
-            obj.snapshot.timestamp,
-            obj.snapshot.url[:128],
-        )
-
-    def tags_str(self, obj):
-        return obj.snapshot.tags_str()
-
-    def cmd_str(self, obj):
-        return format_html(
-            '<pre>{}</pre>',
-            ' '.join(obj.cmd) if isinstance(obj.cmd, list) else str(obj.cmd),
-        )
-
-    def output_str(self, obj):
-        return format_html(
-            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
-            obj.snapshot.timestamp,
-            obj.output if (obj.status == 'succeeded') and obj.extractor not in ('title', 'archive_org') else 'index.html',
-            obj.output,
-        )
-
-    tags_str.short_description = 'tags'
-    snapshot_str.short_description = 'snapshot'
 
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = 'ArchiveBox'
@@ -425,5 +266,4 @@ admin.site = ArchiveBoxAdmin()
 admin.site.register(get_user_model())
 admin.site.register(Snapshot, SnapshotAdmin)
 admin.site.register(Tag, TagAdmin)
-admin.site.register(ArchiveResult, ArchiveResultAdmin)
 admin.site.disable_action('delete_selected')

archivebox/core/apps.py

@@ -3,5 +3,3 @@ from django.apps import AppConfig
 
 class CoreConfig(AppConfig):
     name = 'core'
-    # WIP: broken by Django 3.1.2 -> 4.0 migration
-    default_auto_field = 'django.db.models.UUIDField'

archivebox/core/forms.py

@@ -3,14 +3,9 @@ __package__ = 'archivebox.core'
 from django import forms
 
 from ..util import URL_REGEX
-from ..parsers import PARSERS
 from ..vendor.taggit_utils import edit_string_for_tags, parse_tags
 
-PARSER_CHOICES = [
-    (parser_key, parser[0])
-    for parser_key, parser in PARSERS.items()
-]
-DEPTH_CHOICES = (
+CHOICES = (
     ('0', 'depth = 0 (archive just these URLs)'),
     ('1', 'depth = 1 (archive these URLs and all URLs one hop away)'),
 )
@@ -25,9 +20,7 @@ ARCHIVE_METHODS = [
 
 class AddLinkForm(forms.Form):
     url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
-    parser = forms.ChoiceField(label="URLs format", choices=[('auto', 'Auto-detect parser'), *PARSER_CHOICES], initial='auto')
-    tag = forms.CharField(label="Tags (comma separated tag1,tag2,tag3)", strip=True, required=False)
-    depth = forms.ChoiceField(label="Archive depth", choices=DEPTH_CHOICES, initial='0', widget=forms.RadioSelect(attrs={"class": "depth-selection"}))
+    depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
     archive_methods = forms.MultipleChoiceField(
         label="Archive methods (select at least 1, otherwise all will be used by default)",
         required=False,
@@ -41,7 +34,7 @@ class AddLinkForm(forms.Form):
     #     label="Exclude patterns",
     #     min_length='1',
     #     required=False,
-    #     initial=URL_DENYLIST,
+    #     initial=URL_BLACKLIST,
     # )
     # timeout = forms.IntegerField(
     #     initial=TIMEOUT,

archivebox/core/middleware.py

@@ -1,60 +0,0 @@
-__package__ = 'archivebox.core'
-
-import ipaddress
-from django.utils import timezone
-from django.contrib.auth.middleware import RemoteUserMiddleware
-from django.core.exceptions import ImproperlyConfigured
-
-from ..config import PUBLIC_SNAPSHOTS, REVERSE_PROXY_USER_HEADER, REVERSE_PROXY_WHITELIST
-
-
-def detect_timezone(request, activate: bool=True):
-    gmt_offset = (request.COOKIES.get('GMT_OFFSET') or '').strip()
-    tz = None
-    if gmt_offset.replace('-', '').isdigit():
-        tz = timezone.get_fixed_timezone(int(gmt_offset))
-        if activate:
-            timezone.activate(tz)
-    # print('GMT_OFFSET', gmt_offset, tz)
-    return tz
-
-
-def TimezoneMiddleware(get_response):
-    def middleware(request):
-        detect_timezone(request, activate=True)
-        return get_response(request)
-
-    return middleware
-
-
-def CacheControlMiddleware(get_response):
-    def middleware(request):
-        response = get_response(request)
-
-        if '/archive/' in request.path or '/static/' in request.path:
-            policy = 'public' if PUBLIC_SNAPSHOTS else 'private'
-            response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
-            # print('Set Cache-Control header to', response['Cache-Control'])
-        return response
-
-    return middleware
-
-class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
-    header = 'HTTP_{normalized}'.format(normalized=REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
-
-    def process_request(self, request):
-        if REVERSE_PROXY_WHITELIST == '':
-            return
-
-        ip = request.META.get('REMOTE_ADDR')
-
-        for cidr in REVERSE_PROXY_WHITELIST.split(','):
-            try:
-                network = ipaddress.ip_network(cidr)
-            except ValueError:
-                raise ImproperlyConfigured(
-                    "The REVERSE_PROXY_WHITELIST config paramater is in invalid format, or "
-                    "contains invalid CIDR. Correct format is a coma-separated list of IPv4/IPv6 CIDRs.")
-
-            if ipaddress.ip_address(ip) in network:
-                return super().process_request(request)

archivebox/core/migrations/0006_auto_20201012_1520.py

@@ -17,7 +17,7 @@ def forwards_func(apps, schema_editor):
         tag_set.discard("")
 
         for tag in tag_set:
-            to_add, _ = TagModel.objects.get_or_create(name=tag, defaults={'slug': slugify(tag)})
+            to_add, _ = TagModel.objects.get_or_create(name=tag, slug=slugify(tag))
             snapshot.tags.add(to_add)
 
 

archivebox/core/migrations/0007_archiveresult.py

@@ -36,25 +36,8 @@ def forwards_func(apps, schema_editor):
 
         for extractor in history:
             for result in history[extractor]:
-                try:
-                    ArchiveResult.objects.create(
-                        extractor=extractor,
-                        snapshot=snapshot,
-                        pwd=result["pwd"],
-                        cmd=result.get("cmd") or [],
-                        cmd_version=result.get("cmd_version") or 'unknown',
-                        start_ts=result["start_ts"],
-                        end_ts=result["end_ts"],
-                        status=result["status"],
-                        output=result.get("output") or 'null',
-                    )
-                except Exception as e:
-                    print(
-                        '    ! Skipping import due to missing/invalid index.json:',
-                        out_dir,
-                        e,
-                        '(open an issue with this index.json for help)',
-                    )
+                ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=result["cmd"], cmd_version=result["cmd_version"] or 'unknown', 
+                start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"])
 
 
 def verify_json_index_integrity(snapshot):

archivebox/core/migrations/0009_auto_20210216_1038.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 10:38
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0008_auto_20210105_1421'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='updated',
-            field=models.DateTimeField(auto_now=True, db_index=True, null=True),
-        ),
-    ]

archivebox/core/migrations/0010_auto_20210216_1055.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 10:55
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0009_auto_20210216_1038'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='start_ts',
-            field=models.DateTimeField(db_index=True),
-        ),
-    ]

archivebox/core/migrations/0011_auto_20210216_1331.py

@@ -1,24 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 13:31
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0010_auto_20210216_1055'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='archiveresult',
-            name='uuid',
-            field=models.UUIDField(default=uuid.uuid4, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='extractor',
-            field=models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
-        ),
-    ]

archivebox/core/migrations/0012_auto_20210216_1425.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-16 14:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0011_auto_20210216_1331'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='cmd_version',
-            field=models.CharField(blank=True, default=None, max_length=128, null=True),
-        ),
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='output',
-            field=models.CharField(max_length=1024),
-        ),
-    ]

archivebox/core/migrations/0013_auto_20210218_0729.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 07:29
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0012_auto_20210216_1425'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='title',
-            field=models.CharField(blank=True, db_index=True, max_length=256, null=True),
-        ),
-    ]

archivebox/core/migrations/0014_auto_20210218_0729.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 07:29
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0013_auto_20210218_0729'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='title',
-            field=models.CharField(blank=True, db_index=True, max_length=1024, null=True),
-        ),
-    ]

archivebox/core/migrations/0015_auto_20210218_0730.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 07:30
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0014_auto_20210218_0729'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='title',
-            field=models.CharField(blank=True, db_index=True, max_length=512, null=True),
-        ),
-    ]

archivebox/core/migrations/0016_auto_20210218_1204.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-18 12:04
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0015_auto_20210218_0730'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='tags',
-            field=models.ManyToManyField(blank=True, to='core.Tag'),
-        ),
-    ]

archivebox/core/migrations/0017_auto_20210219_0211.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-02-19 02:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0016_auto_20210218_1204'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='tag',
-            name='slug',
-            field=models.SlugField(blank=True, max_length=100, unique=True, verbose_name='slug'),
-        ),
-    ]

archivebox/core/migrations/0018_auto_20210327_0952.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.3 on 2021-03-27 09:52
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0017_auto_20210219_0211'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='tag',
-            name='name',
-            field=models.CharField(max_length=100, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='tag',
-            name='slug',
-            field=models.SlugField(blank=True, max_length=100, unique=True),
-        ),
-    ]

archivebox/core/migrations/0019_auto_20210401_0654.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.3 on 2021-04-01 06:54
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0018_auto_20210327_0952'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='snapshot',
-            name='url',
-            field=models.URLField(db_index=True, unique=True),
-        ),
-    ]

archivebox/core/migrations/0020_auto_20210410_1031.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.8 on 2021-04-10 10:31
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0019_auto_20210401_0654'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='id',
-            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
-        ),
-        migrations.AlterField(
-            model_name='tag',
-            name='id',
-            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
-        ),
-    ]

archivebox/core/migrations/0021_auto_20220914_0934.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.14 on 2022-09-14 09:34
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0020_auto_20210410_1031'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='extractor',
-            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
-        ),
-    ]

archivebox/core/migrations/0022_auto_20231023_2008.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.14 on 2023-10-23 20:08
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('core', '0021_auto_20220914_0934'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='archiveresult',
-            name='extractor',
-            field=models.CharField(choices=[('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('title', 'title'), ('readability', 'readability'), ('mercury', 'mercury'), ('htmltotext', 'htmltotext'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
-        ),
-    ]

archivebox/core/mixins.py

@@ -2,20 +2,22 @@ from django.contrib import messages
 
 from archivebox.search import query_search_index
 
-class SearchResultsAdminMixin:
-    def get_search_results(self, request, queryset, search_term: str):
-        """Enhances the search queryset with results from the search backend"""
-        
-        qs, use_distinct = super().get_search_results(request, queryset, search_term)
+class SearchResultsAdminMixin(object):
+    def get_search_results(self, request, queryset, search_term):
+        ''' Enhances the search queryset with results from the search backend.
+        '''
+        qs, use_distinct = \
+            super(SearchResultsAdminMixin, self).get_search_results(
+                request, queryset, search_term)
 
         search_term = search_term.strip()
         if not search_term:
             return qs, use_distinct
         try:
             qsearch = query_search_index(search_term)
-            qs = qs | qsearch
         except Exception as err:
-            print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
             messages.add_message(request, messages.WARNING, f'Error from the search backend, only showing results from default admin search fields - Error: {err}')
-        
-        return qs, use_distinct
+        else:
+            qs = queryset & qsearch
+        finally:
+            return qs, use_distinct

archivebox/core/models.py

@@ -1,28 +1,23 @@
 __package__ = 'archivebox.core'
 
-
 import uuid
-import json
-
 from pathlib import Path
-from typing import Optional, List
+from typing import Dict, Optional, List
+from datetime import datetime, timedelta
+from collections import defaultdict
 
-from django.db import models
+from django.db import models, transaction
 from django.utils.functional import cached_property
 from django.utils.text import slugify
-from django.core.cache import cache
-from django.urls import reverse
 from django.db.models import Case, When, Value, IntegerField
-from django.contrib.auth.models import User   # noqa
 
-from ..config import ARCHIVE_DIR, ARCHIVE_DIR_NAME
-from ..system import get_dir_size
-from ..util import parse_date, base_url, hashurl
+from ..util import parse_date
 from ..index.schema import Link
-from ..index.html import snapshot_icons
-from ..extractors import get_default_archive_methods, ARCHIVE_METHODS_INDEXING_PRECEDENCE
+from ..config import CONFIG
+from ..system import get_dir_size
 
-EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
+#EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
+EXTRACTORS = [("title", "title"), ("wget", "wget")]
 STATUS_CHOICES = [
     ("succeeded", "succeeded"),
     ("failed", "failed"),
@@ -35,18 +30,12 @@ except AttributeError:
     import jsonfield
     JSONField = jsonfield.JSONField
 
-
 class Tag(models.Model):
     """
     Based on django-taggit model
     """
-    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
-
-    name = models.CharField(unique=True, blank=False, max_length=100)
-
-    # slug is autoset on save from name, never set it manually
-    slug = models.SlugField(unique=True, blank=True, max_length=100)
-
+    name = models.CharField(verbose_name="name", unique=True, blank=False, max_length=100)
+    slug = models.SlugField(verbose_name="slug", unique=True, max_length=100)
 
     class Meta:
         verbose_name = "Tag"
@@ -65,21 +54,20 @@ class Tag(models.Model):
         if self._state.adding and not self.slug:
             self.slug = self.slugify(self.name)
 
-            # if name is different but slug conficts with another tags slug, append a counter
-            # with transaction.atomic():
-            slugs = set(
-                type(self)
-                ._default_manager.filter(slug__startswith=self.slug)
-                .values_list("slug", flat=True)
-            )
+            with transaction.atomic():
+                slugs = set(
+                    type(self)
+                    ._default_manager.filter(slug__startswith=self.slug)
+                    .values_list("slug", flat=True)
+                )
 
-            i = None
-            while True:
-                slug = self.slugify(self.name, i)
-                if slug not in slugs:
-                    self.slug = slug
-                    return super().save(*args, **kwargs)
-                i = 1 if i is None else i+1
+                i = None
+                while True:
+                    slug = self.slugify(self.name, i)
+                    if slug not in slugs:
+                        self.slug = slug
+                        return super().save(*args, **kwargs)
+                    i = 1 if i is None else i+1
         else:
             return super().save(*args, **kwargs)
 
@@ -87,16 +75,16 @@ class Tag(models.Model):
 class Snapshot(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
 
-    url = models.URLField(unique=True, db_index=True)
+    url = models.URLField(unique=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True)
 
-    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
+    title = models.CharField(max_length=128, null=True, blank=True, db_index=True)
 
     added = models.DateTimeField(auto_now_add=True, db_index=True)
-    updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
-    tags = models.ManyToManyField(Tag, blank=True)
+    updated = models.DateTimeField(null=True, blank=True, db_index=True)
+    tags = models.ManyToManyField(Tag)
 
-    keys = ('url', 'timestamp', 'title', 'tags', 'updated')
+    keys = ('id', 'url', 'timestamp', 'title', 'tags', 'updated', 'base_url')
 
     def __repr__(self) -> str:
         title = self.title or '-'
@@ -106,18 +94,63 @@ class Snapshot(models.Model):
         title = self.title or '-'
         return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
 
+    def field_names():
+        fields = self._meta.get_field_names()
+        exclude = ["tags", "archiveresult"] # Exclude relationships for now
+        return [field.name for field in fields if field.name not in exclude]
+
+
     @classmethod
     def from_json(cls, info: dict):
         info = {k: v for k, v in info.items() if k in cls.keys}
+        if "tags" in info:
+            # TODO: Handle tags
+            info.pop("tags")
+        info.pop("base_url", None) 
         return cls(**info)
 
+    def get_history(self) -> dict:
+        """
+        Generates the history dictionary out of the stored ArchiveResults
+        """
+        history_list = self.archiveresult_set.all()
+        history = defaultdict(list)
+        for history_item in history_list:
+            history[history_item.extractor].append(
+                {
+                    "cmd": history_item.cmd,
+                    "cmd_version": history_item.cmd_version,
+                    "end_ts": history_item.end_ts.isoformat(),
+                    "start_ts": history_item.start_ts.isoformat(),
+                    "pwd": history_item.pwd,
+                    "output": history_item.output,
+                    "schema": "ArchiveResult",
+                    "status": history_item.status
+                }
+            )
+        return dict(history)
+
     def as_json(self, *args) -> dict:
+        """
+        Returns the snapshot in json format.
+        id is converted to str
+        history is extracted from ArchiveResult
+        """
         args = args or self.keys
-        return {
+        output = {
             key: getattr(self, key)
             if key != 'tags' else self.tags_str()
             for key in args
         }
+        if "id" in output.keys():
+            output["id"] = str(output["id"])
+
+        output["history"] = self.get_history()
+        return output
+
+    def as_csv(self, cols: Optional[List[str]]=None, separator: str=',', ljust: int=0) -> str:
+        from ..index.csv import to_csv
+        return to_csv(self, cols=cols or self.field_names(), separator=separator, ljust=ljust)
 
     def as_link(self) -> Link:
         return Link.from_json(self.as_json())
@@ -126,35 +159,57 @@ class Snapshot(models.Model):
         from ..index import load_link_details
         return load_link_details(self.as_link())
 
-    def tags_str(self, nocache=True) -> str:
-        cache_key = f'{self.id}-{(self.updated or self.added).timestamp()}-tags'
-        calc_tags_str = lambda: ','.join(self.tags.order_by('name').values_list('name', flat=True))
-        if nocache:
-            tags_str = calc_tags_str()
-            cache.set(cache_key, tags_str)
-            return tags_str
-        return cache.get_or_set(cache_key, calc_tags_str)
-
-    def icons(self) -> str:
-        return snapshot_icons(self)
-
-    @cached_property
-    def extension(self) -> str:
-        from ..util import extension
-        return extension(self.url)
+    def tags_str(self) -> str:
+        return ','.join(self.tags.order_by('name').values_list('name', flat=True))
 
     @cached_property
     def bookmarked(self):
         return parse_date(self.timestamp)
 
     @cached_property
-    def bookmarked_date(self):
-        # TODO: remove this
-        return self.bookmarked
+    def bookmarked_date(self) -> Optional[str]:
+        from ..util import ts_to_date
+
+        max_ts = (datetime.now() + timedelta(days=30)).timestamp()
+
+        if self.timestamp and self.timestamp.replace('.', '').isdigit():
+            if 0 < float(self.timestamp) < max_ts:
+                return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
+            else:
+                return str(self.timestamp)
+        return None
 
     @cached_property
-    def is_archived(self):
-        return self.as_link().is_archived
+    def is_archived(self) -> bool:
+        from ..config import ARCHIVE_DIR
+        from ..util import domain
+
+        output_paths = (
+            domain(self.url),
+            'output.pdf',
+            'screenshot.png',
+            'output.html',
+            'media',
+            'singlefile.html'
+        )
+
+        return any(
+            (Path(ARCHIVE_DIR) / self.timestamp / path).exists()
+            for path in output_paths
+        )
+
+    @cached_property
+    def archive_dates(self) -> List[datetime]:
+        return [
+            result.start_ts
+            for result in self.archiveresult_set.all()
+        ]
+
+    @cached_property
+    def oldest_archive_date(self) -> Optional[datetime]:
+        oldest = self.archiveresult_set.all().order_by("-start_ts")[:1]
+        if len(oldest) > 0:
+            return oldest[0].start_ts
 
     @cached_property
     def num_outputs(self):
@@ -162,101 +217,113 @@ class Snapshot(models.Model):
 
     @cached_property
     def url_hash(self):
-        return hashurl(self.url)
+        return self.as_link().url_hash
 
     @cached_property
-    def base_url(self):
+    def base_url(self) -> str:
+        from ..util import base_url
         return base_url(self.url)
 
     @cached_property
-    def link_dir(self):
-        return str(ARCHIVE_DIR / self.timestamp)
+    def snapshot_dir(self):
+        from ..config import CONFIG
+        return Path(CONFIG['ARCHIVE_DIR']) / self.timestamp
 
     @cached_property
     def archive_path(self):
+        from ..config import ARCHIVE_DIR_NAME
         return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp)
 
     @cached_property
-    def archive_size(self):
-        cache_key = f'{str(self.id)[:12]}-{(self.updated or self.added).timestamp()}-size'
-
-        def calc_dir_size():
-            try:
-                return get_dir_size(self.link_dir)[0]
-            except Exception:
-                return 0
-
-        return cache.get_or_set(cache_key, calc_dir_size)
-
-    @cached_property
-    def thumbnail_url(self) -> Optional[str]:
-        result = self.archiveresult_set.filter(
-            extractor='screenshot',
-            status='succeeded'
-        ).only('output').last()
-        if result:
-            return reverse('Snapshot', args=[f'{str(self.timestamp)}/{result.output}'])
-        return None
-
-    @cached_property
-    def headers(self) -> Optional[dict]:
+    def archive_size(self) -> float:
         try:
-            return json.loads((Path(self.link_dir) / 'headers.json').read_text(encoding='utf-8').strip())
+            return get_dir_size(self.archive_path)[0]
         except Exception:
-            pass
-        return None
+            return 0
 
     @cached_property
-    def status_code(self) -> Optional[str]:
-        return self.headers and self.headers.get('Status-Code')
-
-    @cached_property
-    def history(self) -> dict:
+    def history(self):
         # TODO: use ArchiveResult for this instead of json
         return self.as_link_with_details().history
 
     @cached_property
-    def latest_title(self) -> Optional[str]:
-        if self.title:
-            return self.title   # whoopdedoo that was easy
-        
-        try:
-            # take longest successful title from ArchiveResult db history
-            return sorted(
-                self.archiveresult_set\
-                    .filter(extractor='title', status='succeeded', output__isnull=False)\
-                    .values_list('output', flat=True),
-                key=lambda r: len(r),
-            )[-1]
-        except IndexError:
-            pass
-
-        try:
-            # take longest successful title from Link json index file history
-            return sorted(
-                (
-                    result.output.strip()
-                    for result in self.history['title']
-                    if result.status == 'succeeded' and result.output.strip()
-                ),
-                key=lambda r: len(r),
-            )[-1]
-        except (KeyError, IndexError):
-            pass
-
+    def latest_title(self):
+        if ('title' in self.history
+            and self.history['title']
+            and (self.history['title'][-1].status == 'succeeded')
+            and self.history['title'][-1].output.strip()):
+            return self.history['title'][-1].output.strip()
         return None
 
-    def save_tags(self, tags: List[str]=()) -> None:
+    @cached_property
+    def domain(self) -> str:
+        from ..util import domain
+        return domain(self.url)
+
+    @cached_property
+    def is_static(self) -> bool:
+        from ..util import is_static_file
+        return is_static_file(self.url)
+
+    @cached_property
+    def details(self) -> Dict:
+        # TODO: Define what details are, and return them accordingly
+        return {"history": {}}
+
+    @property
+    def extension(self) -> str:
+        from ..util import extension
+        return extension(self.url)
+
+    def canonical_outputs(self) -> Dict[str, Optional[str]]:
+        """predict the expected output paths that should be present after archiving"""
+
+        from ..extractors.wget import wget_output_path
+        canonical = {
+            'index_path': 'index.html',
+            'favicon_path': 'favicon.ico',
+            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
+            'wget_path': wget_output_path(self),
+            'warc_path': 'warc',
+            'singlefile_path': 'singlefile.html',
+            'readability_path': 'readability/content.html',
+            'mercury_path': 'mercury/content.html',
+            'pdf_path': 'output.pdf',
+            'screenshot_path': 'screenshot.png',
+            'dom_path': 'output.html',
+            'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
+            'git_path': 'git',
+            'media_path': 'media',
+        }
+        if self.is_static:
+            # static binary files like PDF and images are handled slightly differently.
+            # they're just downloaded once and aren't archived separately multiple times, 
+            # so the wget, screenshot, & pdf urls should all point to the same file
+
+            static_path = wget_output_path(self)
+            canonical.update({
+                'title': self.basename,
+                'wget_path': static_path,
+                'pdf_path': static_path,
+                'screenshot_path': static_path,
+                'dom_path': static_path,
+                'singlefile_path': static_path,
+                'readability_path': static_path,
+                'mercury_path': static_path,
+            })
+        return canonical
+
+    def save_tags(self, tags=()):
         tags_id = []
         for tag in tags:
-            if tag.strip():
-                tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
+            tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
         self.tags.clear()
         self.tags.add(*tags_id)
 
 
 class ArchiveResultManager(models.Manager):
     def indexable(self, sorted: bool = True):
+        from ..extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
         INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
         qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS,status='succeeded')
 
@@ -267,20 +334,20 @@ class ArchiveResultManager(models.Manager):
 
 
 class ArchiveResult(models.Model):
-    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
-    uuid = models.UUIDField(default=uuid.uuid4, editable=False)
-
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
-    extractor = models.CharField(choices=EXTRACTORS, max_length=32)
     cmd = JSONField()
     pwd = models.CharField(max_length=256)
-    cmd_version = models.CharField(max_length=128, default=None, null=True, blank=True)
-    output = models.CharField(max_length=1024)
-    start_ts = models.DateTimeField(db_index=True)
+    cmd_version = models.CharField(max_length=32, default=None, null=True, blank=True)
+    output = models.CharField(max_length=512)
+    start_ts = models.DateTimeField()
     end_ts = models.DateTimeField()
     status = models.CharField(max_length=16, choices=STATUS_CHOICES)
+    extractor = models.CharField(choices=EXTRACTORS, max_length=32)
 
     objects = ArchiveResultManager()
 
     def __str__(self):
         return self.extractor
+
+    class Meta:
+        ordering = ["-start_ts"]

archivebox/core/settings.py

@@ -2,37 +2,21 @@ __package__ = 'archivebox.core'
 
 import os
 import sys
-import re
-import logging
-import tempfile
 
 from pathlib import Path
 from django.utils.crypto import get_random_string
 
-from ..config import (
+from ..config import (                                                          # noqa: F401
     DEBUG,
     SECRET_KEY,
     ALLOWED_HOSTS,
     PACKAGE_DIR,
     TEMPLATES_DIR_NAME,
-    CUSTOM_TEMPLATES_DIR,
     SQL_INDEX_FILENAME,
     OUTPUT_DIR,
-    LOGS_DIR,
-    TIMEZONE,
-
-    LDAP,
-    LDAP_SERVER_URI,
-    LDAP_BIND_DN,
-    LDAP_BIND_PASSWORD,
-    LDAP_USER_BASE,
-    LDAP_USER_FILTER,
-    LDAP_USERNAME_ATTR,
-    LDAP_FIRSTNAME_ATTR,
-    LDAP_LASTNAME_ATTR,
-    LDAP_EMAIL_ATTR,
 )
 
+
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@@ -45,8 +29,7 @@ WSGI_APPLICATION = 'core.wsgi.application'
 ROOT_URLCONF = 'core.urls'
 
 LOGIN_URL = '/accounts/login/'
-LOGOUT_REDIRECT_URL = os.environ.get('LOGOUT_REDIRECT_URL', '/')
-
+LOGOUT_REDIRECT_URL = '/'
 PASSWORD_RESET_URL = '/accounts/password_reset/'
 APPEND_SLASH = True
 
@@ -66,111 +49,19 @@ INSTALLED_APPS = [
 ]
 
 
-# For usage with https://www.jetadmin.io/integrations/django
-# INSTALLED_APPS += ['jet_django']
-# JET_PROJECT = 'archivebox'
-# JET_TOKEN = 'some-api-token-here'
-
-
 MIDDLEWARE = [
-    'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'core.middleware.ReverseProxyAuthMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
-    'core.middleware.CacheControlMiddleware',
 ]
 
-################################################################################
-### Authentication Settings
-################################################################################
-
 AUTHENTICATION_BACKENDS = [
-    'django.contrib.auth.backends.RemoteUserBackend',
     'django.contrib.auth.backends.ModelBackend',
 ]
 
-if LDAP:
-    try:
-        import ldap
-        from django_auth_ldap.config import LDAPSearch
-
-        global AUTH_LDAP_SERVER_URI
-        global AUTH_LDAP_BIND_DN
-        global AUTH_LDAP_BIND_PASSWORD
-        global AUTH_LDAP_USER_SEARCH
-        global AUTH_LDAP_USER_ATTR_MAP
-
-        AUTH_LDAP_SERVER_URI = LDAP_SERVER_URI
-        AUTH_LDAP_BIND_DN = LDAP_BIND_DN
-        AUTH_LDAP_BIND_PASSWORD = LDAP_BIND_PASSWORD
-
-        assert AUTH_LDAP_SERVER_URI and LDAP_USERNAME_ATTR and LDAP_USER_FILTER, 'LDAP_* config options must all be set if LDAP=True'
-
-        AUTH_LDAP_USER_SEARCH = LDAPSearch(
-            LDAP_USER_BASE,
-            ldap.SCOPE_SUBTREE,
-            '(&(' + LDAP_USERNAME_ATTR + '=%(user)s)' + LDAP_USER_FILTER + ')',
-        )
-
-        AUTH_LDAP_USER_ATTR_MAP = {
-            'username': LDAP_USERNAME_ATTR,
-            'first_name': LDAP_FIRSTNAME_ATTR,
-            'last_name': LDAP_LASTNAME_ATTR,
-            'email': LDAP_EMAIL_ATTR,
-        }
-
-        AUTHENTICATION_BACKENDS = [
-            'django.contrib.auth.backends.ModelBackend',
-            'django_auth_ldap.backend.LDAPBackend',
-        ]
-    except ModuleNotFoundError:
-        sys.stderr.write('[X] Error: Found LDAP=True config but LDAP packages not installed. You may need to run: pip install archivebox[ldap]\n\n')
-        # dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
-        # sys.exit(1)
-
-
-################################################################################
-### Debug Settings
-################################################################################
-
-# only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode)
-DEBUG_TOOLBAR = DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv)
-if DEBUG_TOOLBAR:
-    try:
-        import debug_toolbar   # noqa
-        DEBUG_TOOLBAR = True
-    except ImportError:
-        DEBUG_TOOLBAR = False
-
-if DEBUG_TOOLBAR:
-    INSTALLED_APPS = [*INSTALLED_APPS, 'debug_toolbar']
-    INTERNAL_IPS = ['0.0.0.0', '127.0.0.1', '*']
-    DEBUG_TOOLBAR_CONFIG = {
-        "SHOW_TOOLBAR_CALLBACK": lambda request: True,
-        "RENDER_PANELS": True,
-    }
-    DEBUG_TOOLBAR_PANELS = [
-        'debug_toolbar.panels.history.HistoryPanel',
-        'debug_toolbar.panels.versions.VersionsPanel',
-        'debug_toolbar.panels.timer.TimerPanel',
-        'debug_toolbar.panels.settings.SettingsPanel',
-        'debug_toolbar.panels.headers.HeadersPanel',
-        'debug_toolbar.panels.request.RequestPanel',
-        'debug_toolbar.panels.sql.SQLPanel',
-        'debug_toolbar.panels.staticfiles.StaticFilesPanel',
-        # 'debug_toolbar.panels.templates.TemplatesPanel',
-        'debug_toolbar.panels.cache.CachePanel',
-        'debug_toolbar.panels.signals.SignalsPanel',
-        'debug_toolbar.panels.logging.LoggingPanel',
-        'debug_toolbar.panels.redirects.RedirectsPanel',
-        'debug_toolbar.panels.profiling.ProfilingPanel',
-        'djdt_flamegraph.FlamegraphPanel',
-    ]
-    MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
 
 ################################################################################
 ### Staticfile and Template Settings
@@ -179,12 +70,10 @@ if DEBUG_TOOLBAR:
 STATIC_URL = '/static/'
 
 STATICFILES_DIRS = [
-    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
 ]
 
 TEMPLATE_DIRS = [
-    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
@@ -218,23 +107,6 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': DATABASE_NAME,
-        'OPTIONS': {
-            'timeout': 60,
-            'check_same_thread': False,
-        },
-        'TIME_ZONE': TIMEZONE,
-        # DB setup is sometimes modified at runtime by setup_django() in config.py
-    }
-}
-
-CACHE_BACKEND = 'django.core.cache.backends.locmem.LocMemCache'
-# CACHE_BACKEND = 'django.core.cache.backends.db.DatabaseCache'
-# CACHE_BACKEND = 'django.core.cache.backends.dummy.DummyCache'
-
-CACHES = {
-    'default': {
-        'BACKEND': CACHE_BACKEND,
-        'LOCATION': 'django_cache_default',
     }
 }
 
@@ -245,13 +117,12 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 ### Security Settings
 ################################################################################
 
-SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')
+SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.')
 
 ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
 
 SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
-SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
 
 CSRF_COOKIE_SECURE = False
 SESSION_COOKIE_SECURE = False
@@ -260,8 +131,6 @@ SESSION_COOKIE_AGE = 1209600  # 2 weeks
 SESSION_EXPIRE_AT_BROWSER_CLOSE = False
 SESSION_SAVE_EVERY_REQUEST = True
 
-SESSION_ENGINE = "django.contrib.sessions.backends.db"
-
 AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
     {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'},
@@ -269,8 +138,6 @@ AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
 ]
 
-# WIP: broken by Django 3.1.2 -> 4.0 migration
-DEFAULT_AUTO_FIELD = 'django.db.models.UUIDField'
 
 ################################################################################
 ### Shell Settings
@@ -289,85 +156,10 @@ if IS_SHELL:
 ################################################################################
 
 LANGUAGE_CODE = 'en-us'
-USE_I18N = True
-USE_L10N = True
-USE_TZ = True
+TIME_ZONE = 'UTC'
+USE_I18N = False
+USE_L10N = False
+USE_TZ = False
+
 DATETIME_FORMAT = 'Y-m-d g:iA'
 SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
-TIME_ZONE = TIMEZONE        # django convention is TIME_ZONE, archivebox config uses TIMEZONE, they are equivalent
-
-
-from django.conf.locale.en import formats as en_formats
-
-en_formats.DATETIME_FORMAT = DATETIME_FORMAT
-en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
-
-
-################################################################################
-### Logging Settings
-################################################################################
-
-IGNORABLE_404_URLS = [
-    re.compile(r'apple-touch-icon.*\.png$'),
-    re.compile(r'favicon\.ico$'),
-    re.compile(r'robots\.txt$'),
-    re.compile(r'.*\.(css|js)\.map$'),
-]
-
-class NoisyRequestsFilter(logging.Filter):
-    def filter(self, record):
-        logline = record.getMessage()
-
-        # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
-        for ignorable_url_pattern in IGNORABLE_404_URLS:
-            ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
-            if ignorable_log_pattern.match(logline):
-                return 0
-
-        # ignore staticfile requests that 200 or 30*
-        ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M)
-        if ignoreable_200_log_pattern.match(logline):
-            return 0
-
-        return 1
-
-if LOGS_DIR.exists():
-    ERROR_LOG = (LOGS_DIR / 'errors.log')
-else:
-    # historically too many edge cases here around creating log dir w/ correct permissions early on
-    # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr
-    ERROR_LOG = tempfile.NamedTemporaryFile().name
-
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
-        },
-        'logfile': {
-            'level': 'ERROR',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'filename': ERROR_LOG,
-            'maxBytes': 1024 * 1024 * 25,  # 25 MB
-            'backupCount': 10,
-        },
-    },
-    'filters': {
-        'noisyrequestsfilter': {
-            '()': NoisyRequestsFilter,
-        }
-    },
-    'loggers': {
-        'django': {
-            'handlers': ['console', 'logfile'],
-            'level': 'INFO',
-            'filters': ['noisyrequestsfilter'],
-        },
-        'django.server': {
-            'handlers': ['console', 'logfile'],
-            'level': 'INFO',
-            'filters': ['noisyrequestsfilter'],
-        }
-    },
-}

archivebox/core/templatetags/core_tags.py

@@ -1,15 +1,22 @@
 from django import template
+from django.urls import reverse
 from django.contrib.admin.templatetags.base import InclusionAdminNode
+from django.templatetags.static import static
 
 
 from typing import Union
 
+from core.models import ArchiveResult
 
 register = template.Library()
 
-@register.filter(name='split')
-def split(value, separator: str=','):
-    return (value or '').split(separator)
+@register.simple_tag
+def snapshot_image(snapshot):
+    result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first()
+    if result:
+        return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
+    
+    return static('archive.png')
 
 @register.filter
 def file_size(num_bytes: Union[int, float]) -> str:

archivebox/core/urls.py

@@ -2,15 +2,10 @@ from django.contrib import admin
 
 from django.urls import path, include
 from django.views import static
-from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.conf import settings
 from django.views.generic.base import RedirectView
 
-from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
-
-# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
-# from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
-# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
+from core.views import HomepageView, SnapshotView, PublicIndexView, AddView
 
 
 # print('DEBUG', settings.DEBUG)
@@ -18,8 +13,8 @@ from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, Hea
 urlpatterns = [
     path('public/', PublicIndexView.as_view(), name='public-index'),
 
-    path('robots.txt', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'robots.txt'}),
-    path('favicon.ico', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'favicon.ico'}),
+    path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
+    path('favicon.ico', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'favicon.ico'}),
 
     path('docs/', RedirectView.as_view(url='https://github.com/ArchiveBox/ArchiveBox/wiki'), name='Docs'),
 
@@ -28,7 +23,7 @@ urlpatterns = [
 
     path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
     path('add/', AddView.as_view(), name='add'),
-
+    
     path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
     path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
 
@@ -36,55 +31,39 @@ urlpatterns = [
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', admin.site.urls),
     
-    # do not add extra_context like this as not all admin views (e.g. ModelAdmin.autocomplete_view accept extra kwargs)
-    # path('admin/', admin.site.urls, {'extra_context': GLOBAL_CONTEXT}),
-
-    path('health/', HealthCheckView.as_view(), name='healthcheck'),
-    path('error/', lambda _: 1/0),
-
-    # path('jet_api/', include('jet_django.urls')),  Enable to use https://www.jetadmin.io/integrations/django
-
     path('index.html', RedirectView.as_view(url='/')),
     path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),
     path('', HomepageView.as_view(), name='Home'),
 ]
-urlpatterns += staticfiles_urlpatterns()
 
-if settings.DEBUG_TOOLBAR:
-    import debug_toolbar
-    urlpatterns += [
-        path('__debug__/', include(debug_toolbar.urls)),
-    ]
+    # # Proposed UI URLs spec
+    # path('',                 HomepageView)
+    # path('/add',             AddView)
+    # path('/public',          PublicIndexView)
+    # path('/snapshot/:slug',  SnapshotView)
+    
+    # path('/admin',           admin.site.urls)
+    # path('/accounts',        django.contrib.auth.urls)
 
+    # # Prposed REST API spec
+    # # :slugs can be uuid, short_uuid, or any of the unique index_fields
+    # path('api/v1/'),
+    # path('api/v1/core/'                      [GET])
+    # path('api/v1/core/snapshot/',            [GET, POST, PUT]),
+    # path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
+    # path('api/v1/core/archiveresult',        [GET, POST, PUT]),
+    # path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
+    # path('api/v1/core/tag/',                 [GET, POST, PUT]),
+    # path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
 
-# # Proposed FUTURE URLs spec
-# path('',                 HomepageView)
-# path('/add',             AddView)
-# path('/public',          PublicIndexView)
-# path('/snapshot/:slug',  SnapshotView)
+    # path('api/v1/cli/',                      [GET])
+    # path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
 
-# path('/admin',           admin.site.urls)
-# path('/accounts',        django.contrib.auth.urls)
+    # path('api/v1/extractors/',                    [GET])
+    # path('api/v1/extractors/:extractor/',         [GET]),
+    # path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
 
-# # Prposed REST API spec
-# # :slugs can be uuid, short_uuid, or any of the unique index_fields
-# path('api/v1/'),
-# path('api/v1/core/'                      [GET])
-# path('api/v1/core/snapshot/',            [GET, POST, PUT]),
-# path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
-# path('api/v1/core/archiveresult',        [GET, POST, PUT]),
-# path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
-# path('api/v1/core/tag/',                 [GET, POST, PUT]),
-# path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
-
-# path('api/v1/cli/',                      [GET])
-# path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
-
-# path('api/v1/extractors/',                    [GET])
-# path('api/v1/extractors/:extractor/',         [GET]),
-# path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
-
-# future, just an idea:
-# path('api/v1/scheduler/',                [GET])
-# path('api/v1/scheduler/task/',           [GET, POST, PUT]),
-# path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),
+    # future, just an idea:
+    # path('api/v1/scheduler/',                [GET])
+    # path('api/v1/scheduler/task/',           [GET, POST, PUT]),
+    # path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),

archivebox/core/views.py

@@ -4,15 +4,13 @@ from io import StringIO
 from contextlib import redirect_stdout
 
 from django.shortcuts import render, redirect
-from django.http import HttpResponse, Http404
-from django.utils.html import format_html, mark_safe
+
+from django.http import HttpResponse
 from django.views import View, static
 from django.views.generic.list import ListView
 from django.views.generic import FormView
 from django.db.models import Q
 from django.contrib.auth.mixins import UserPassesTestMixin
-from django.views.decorators.csrf import csrf_exempt
-from django.utils.decorators import method_decorator
 
 from core.models import Snapshot
 from core.forms import AddLinkForm
@@ -23,13 +21,11 @@ from ..config import (
     PUBLIC_SNAPSHOTS,
     PUBLIC_ADD_VIEW,
     VERSION,
-    COMMIT_HASH,
     FOOTER_INFO,
-    SNAPSHOTS_PER_PAGE,
 )
-from ..main import add
+from main import add
 from ..util import base_url, ansi_to_html
-from ..search import query_search_index
+from ..index.html import snapshot_icons
 
 
 class HomepageView(View):
@@ -39,7 +35,7 @@ class HomepageView(View):
 
         if PUBLIC_INDEX:
             return redirect('/public')
-
+        
         return redirect(f'/admin/login/?next={request.path}')
 
 
@@ -47,6 +43,10 @@ class SnapshotView(View):
     # render static html index from filesystem archive/<timestamp>/index.html
 
     def get(self, request, path):
+        # missing trailing slash -> redirect to index
+        if '/' not in path:
+            return redirect(f'{path}/index.html')
+
         if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
             return redirect(f'/admin/login/?next={request.path}')
 
@@ -55,182 +55,62 @@ class SnapshotView(View):
         except (IndexError, ValueError):
             slug, archivefile = path.split('/', 1)[0], 'index.html'
 
+        all_pages = list(Snapshot.objects.all())
+
         # slug is a timestamp
-        if slug.replace('.','').isdigit():
-
-            # missing trailing slash -> redirect to index
-            if '/' not in path:
-                return redirect(f'{path}/index.html')
-
-            try:
-                try:
-                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
-                    response = static.serve(request, archivefile, document_root=snapshot.link_dir, show_indexes=True)
-                    response["Link"] = f'<{snapshot.url}>; rel="canonical"'
-                    return response
-                except Snapshot.DoesNotExist:
-                    if Snapshot.objects.filter(timestamp__startswith=slug).exists():
-                        raise Snapshot.MultipleObjectsReturned
-                    else:
-                        raise
-            except Snapshot.DoesNotExist:
-                # Snapshot does not exist
-                return HttpResponse(
-                    format_html(
-                        (
-                            '<center><br/><br/><br/>'
-                            'No Snapshot directories match the given timestamp or UUID: <code>{}</code><br/><br/>'
-                            'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
-                            '</center>'
-                        ),
-                        slug,
-                        path,
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-            except Snapshot.MultipleObjectsReturned:
-                snapshot_hrefs = mark_safe('<br/>').join(
-                    format_html(
-                        '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                        snap.added.strftime('%Y-%m-%d %H:%M:%S'),
-                        snap.timestamp,
-                        snap.timestamp,
-                        snap.url,
-                        snap.title or '',
-                    )
-                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
-                )
-                return HttpResponse(
-                    format_html(
-                        (
-                            'Multiple Snapshots match the given timestamp/UUID <code>{}</code><br/><pre>'
-                        ),
-                        slug,
-                    ) + snapshot_hrefs + format_html(
-                        (
-                            '</pre><br/>'
-                            'Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'
-                        )
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-            except Http404:
-                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
-                return HttpResponse(
-                    format_html(
-                        (
-                            '<center><br/><br/><br/>'
-                            f'Snapshot <a href="/archive/{snapshot.timestamp}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a> exists in DB, but resource <b><code>{snapshot.timestamp}/'
-                            '{}'
-                            f'</code></b> does not exist in <a href="/archive/{snapshot.timestamp}/" target="_top">snapshot dir</a> yet.<br/><br/>'
-                            'Maybe this resource type is not availabe for this Snapshot,<br/>or the archiving process has not completed yet?<br/>'
-                            f'<pre><code># run this cmd to finish archiving this Snapshot<br/>archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
-                            '<div class="text-align: left; width: 100%; max-width: 400px">'
-                            '<i><b>Next steps:</i></b><br/>'
-                            f'- list all the <a href="/archive/{snapshot.timestamp}/" target="_top">Snapshot files <code>.*</code></a><br/>'
-                            f'- view the <a href="/archive/{snapshot.timestamp}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
-                            f'- go to the <a href="/admin/core/snapshot/{snapshot.id}/change/" target="_top">Snapshot admin</a> to edit<br/>'
-                            f'- go to the <a href="/admin/core/snapshot/?id__startswith={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
-                            '- or return to <a href="/" target="_top">the main index...</a></div>'
-                            '</center>'
-                        ),
-                        archivefile,
-                    ),
-                    content_type="text/html",
-                    status=404,
-                )
-        # slug is a URL
+        by_ts = {page.timestamp: page for page in all_pages}
         try:
-            try:
-                # try exact match on full url first
-                snapshot = Snapshot.objects.get(
-                    Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
-                )
-            except Snapshot.DoesNotExist:
-                # fall back to match on exact base_url
-                try:
-                    snapshot = Snapshot.objects.get(
-                        Q(url='http://' + base_url(path)) | Q(url='https://' + base_url(path))
-                    )
-                except Snapshot.DoesNotExist:
-                    # fall back to matching base_url as prefix
-                    snapshot = Snapshot.objects.get(
-                        Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
-                    )
-            return redirect(f'/archive/{snapshot.timestamp}/index.html')
-        except Snapshot.DoesNotExist:
-            return HttpResponse(
-                format_html(
-                    (
-                        '<center><br/><br/><br/>'
-                        'No Snapshots match the given url: <code>{}</code><br/><br/><br/>'
-                        'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
-                        '+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
-                        '</center>'
-                    ),
-                    base_url(path),
-                    path if '://' in path else f'https://{path}',
-                    path,
-                ),
-                content_type="text/html",
-                status=404,
-            )
-        except Snapshot.MultipleObjectsReturned:
-            snapshot_hrefs = mark_safe('<br/>').join(
-                format_html(
-                    '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                    snap.added.strftime('%Y-%m-%d %H:%M:%S'),
-                    snap.timestamp,
-                    snap.timestamp,
-                    snap.url,
-                    snap.title or '',
-                )
-                for snap in Snapshot.objects.filter(
-                    Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
-                ).only('url', 'timestamp', 'title', 'added').order_by('-added')
-            )
-            return HttpResponse(
-                format_html(
-                    (
-                        'Multiple Snapshots match the given URL <code>{}</code><br/><pre>'
-                    ),
-                    base_url(path),
-                ) + snapshot_hrefs + format_html(
-                    (
-                        '</pre><br/>'
-                        'Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'
-                    )
-                ),
-                content_type="text/html",
-                status=404,
-            )
+            # print('SERVING STATICFILE', by_ts[slug].link_dir, request.path, path)
+            response = static.serve(request, archivefile, document_root=by_ts[slug].snapshot_dir, show_indexes=True)
+            response["Link"] = f'<{by_ts[slug].url}>; rel="canonical"'
+            return response
+        except KeyError:
+            pass
 
+        # slug is a hash
+        by_hash = {page.url_hash: page for page in all_pages}
+        try:
+            timestamp = by_hash[slug].timestamp
+            return redirect(f'/archive/{timestamp}/{archivefile}')
+        except KeyError:
+            pass
+
+        # slug is a URL
+        by_url = {page.base_url: page for page in all_pages}
+        try:
+            # TODO: add multiple snapshot support by showing index of all snapshots
+            # for given url instead of redirecting to timestamp index
+            timestamp = by_url[base_url(path)].timestamp
+            return redirect(f'/archive/{timestamp}/index.html')
+        except KeyError:
+            pass
+
+        return HttpResponse(
+            'No archived link matches the given timestamp or hash.',
+            content_type="text/plain",
+            status=404,
+        )
 
 class PublicIndexView(ListView):
     template_name = 'public_index.html'
     model = Snapshot
-    paginate_by = SNAPSHOTS_PER_PAGE
-    ordering = ['-added']
+    paginate_by = 100
+    ordering = ['title']
 
     def get_context_data(self, **kwargs):
         return {
             **super().get_context_data(**kwargs),
             'VERSION': VERSION,
-            'COMMIT_HASH': COMMIT_HASH,
             'FOOTER_INFO': FOOTER_INFO,
         }
 
-    def get_queryset(self, **kwargs):
-        qs = super().get_queryset(**kwargs)
+    def get_queryset(self, **kwargs): 
+        qs = super().get_queryset(**kwargs) 
         query = self.request.GET.get('q')
-        if query and query.strip():
+        if query:
             qs = qs.filter(Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query))
-            try:
-                qs = qs | query_search_index(query)
-            except Exception as err:
-                print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
+        for snapshot in qs:
+            snapshot.icons = snapshot_icons(snapshot)
         return qs
 
     def get(self, *args, **kwargs):
@@ -240,7 +120,7 @@ class PublicIndexView(ListView):
         else:
             return redirect(f'/admin/login/?next={self.request.path}')
 
-@method_decorator(csrf_exempt, name='dispatch')
+
 class AddView(UserPassesTestMixin, FormView):
     template_name = "add.html"
     form_class = AddLinkForm
@@ -250,9 +130,9 @@ class AddView(UserPassesTestMixin, FormView):
         if self.request.method == 'GET':
             url = self.request.GET.get('url', None)
             if url:
-                return {'url': url if '://' in url else f'https://{url}'}
-
-        return super().get_initial()
+                return {'url': url}
+        else:
+            return super().get_initial()
 
     def test_func(self):
         return PUBLIC_ADD_VIEW or self.request.user.is_authenticated
@@ -265,21 +145,16 @@ class AddView(UserPassesTestMixin, FormView):
             'absolute_add_path': self.request.build_absolute_uri(self.request.path),
             'VERSION': VERSION,
             'FOOTER_INFO': FOOTER_INFO,
-            'stdout': '',
         }
 
     def form_valid(self, form):
         url = form.cleaned_data["url"]
         print(f'[+] Adding URL: {url}')
-        parser = form.cleaned_data["parser"]
-        tag = form.cleaned_data["tag"]
         depth = 0 if form.cleaned_data["depth"] == "0" else 1
         extractors = ','.join(form.cleaned_data["archive_methods"])
         input_kwargs = {
             "urls": url,
-            "tag": tag,
             "depth": depth,
-            "parser": parser,
             "update_all": False,
             "out_dir": OUTPUT_DIR,
         }
@@ -297,18 +172,3 @@ class AddView(UserPassesTestMixin, FormView):
             "form": AddLinkForm()
         })
         return render(template_name=self.template_name, request=self.request, context=context)
-
-
-class HealthCheckView(View):
-    """
-    A Django view that renders plain text "OK" for service discovery tools
-    """
-    def get(self, request):
-        """
-        Handle a GET request
-        """
-        return HttpResponse(
-            'OK',
-            content_type='text/plain',
-            status=200
-        )

archivebox/core/wsgi.py

@@ -7,10 +7,10 @@ For more information on this file, see
 https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
 """
 
-
-from archivebox.config import setup_django
-setup_django(in_memory_db=False, check_db=True)
+import os
 
 from django.core.wsgi import get_wsgi_application
 
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.settings')
+
 application = get_wsgi_application()

archivebox/extractors/__init__.py

@@ -1,31 +1,23 @@
 __package__ = 'archivebox.extractors'
 
 import os
-import sys
 from pathlib import Path
 
-from typing import Callable, Optional, List, Iterable, Union
-from datetime import datetime, timezone
-from django.db.models import QuerySet
+from typing import Optional, List, Iterable, Union
+
+from django.db.models import QuerySet, Model
 
-from ..config import (
-    SAVE_ALLOWLIST_PTN,
-    SAVE_DENYLIST_PTN,
-)
-from ..core.settings import ERROR_LOG
-from ..index.schema import ArchiveResult, Link
-from ..index.sql import write_link_to_sql_index
 from ..index import (
-    load_link_details,
-    write_link_details,
+    load_snapshot_details,
+    write_snapshot_details,
 )
 from ..util import enforce_types
 from ..logging_util import (
     log_archiving_started,
     log_archiving_paused,
     log_archiving_finished,
-    log_link_archiving_started,
-    log_link_archiving_finished,
+    log_snapshot_archiving_started,
+    log_snapshot_archiving_finished,
     log_archive_method_started,
     log_archive_method_finished,
 )
@@ -37,7 +29,6 @@ from .wget import should_save_wget, save_wget
 from .singlefile import should_save_singlefile, save_singlefile
 from .readability import should_save_readability, save_readability
 from .mercury import should_save_mercury, save_mercury
-from .htmltotext import should_save_htmltotext, save_htmltotext
 from .pdf import should_save_pdf, save_pdf
 from .screenshot import should_save_screenshot, save_screenshot
 from .dom import should_save_dom, save_dom
@@ -47,162 +38,102 @@ from .archive_org import should_save_archive_dot_org, save_archive_dot_org
 from .headers import should_save_headers, save_headers
 
 
-ShouldSaveFunction = Callable[[Link, Optional[Path], Optional[bool]], bool]
-SaveFunction = Callable[[Link, Optional[Path], int], ArchiveResult]
-ArchiveMethodEntry = tuple[str, ShouldSaveFunction, SaveFunction]
-
-def get_default_archive_methods() -> List[ArchiveMethodEntry]:
+def get_default_archive_methods():
     return [
+        ('title', should_save_title, save_title),
         ('favicon', should_save_favicon, save_favicon),
-        ('headers', should_save_headers, save_headers),
+        ('wget', should_save_wget, save_wget),
         ('singlefile', should_save_singlefile, save_singlefile),
         ('pdf', should_save_pdf, save_pdf),
         ('screenshot', should_save_screenshot, save_screenshot),
         ('dom', should_save_dom, save_dom),
-        ('wget', should_save_wget, save_wget),
-        # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
-        ('title', should_save_title, save_title),
-        ('readability', should_save_readability, save_readability),
+        ('readability', should_save_readability, save_readability), #keep readability below wget and singlefile, as it depends on them
         ('mercury', should_save_mercury, save_mercury),
-        ('htmltotext', should_save_htmltotext, save_htmltotext),
         ('git', should_save_git, save_git),
         ('media', should_save_media, save_media),
+        ('headers', should_save_headers, save_headers),
         ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
-ARCHIVE_METHODS_INDEXING_PRECEDENCE = [
-    ('readability', 1),
-    ('mercury', 2),
-    ('htmltotext', 3),
-    ('singlefile', 4),
-    ('dom', 5),
-    ('wget', 6)
-]
-
+ARCHIVE_METHODS_INDEXING_PRECEDENCE = [('readability', 1), ('singlefile', 2), ('dom', 3), ('wget', 4)]
 
 @enforce_types
-def get_archive_methods_for_link(link: Link) -> Iterable[ArchiveMethodEntry]:
-    DEFAULT_METHODS = get_default_archive_methods()
-    allowed_methods = {
-        m for pat, methods in
-        SAVE_ALLOWLIST_PTN.items()
-        if pat.search(link.url)
-        for m in methods
-    } or { m[0] for m in DEFAULT_METHODS }
-    denied_methods = {
-        m for pat, methods in
-        SAVE_DENYLIST_PTN.items()
-        if pat.search(link.url)
-        for m in methods
-    }
-    allowed_methods -= denied_methods
-
-    return (m for m in DEFAULT_METHODS if m[0] in allowed_methods)
-
-@enforce_types
-def ignore_methods(to_ignore: List[str]) -> Iterable[str]:
+def ignore_methods(to_ignore: List[str]):
     ARCHIVE_METHODS = get_default_archive_methods()
-    return [x[0] for x in ARCHIVE_METHODS if x[0] not in to_ignore]
+    methods = filter(lambda x: x[0] not in to_ignore, ARCHIVE_METHODS)
+    methods = map(lambda x: x[0], methods)
+    return list(methods)
 
 @enforce_types
-def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Link:
+def archive_snapshot(snapshot: Model, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Model:
     """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
+    from core.models import ArchiveResult
 
-    # TODO: Remove when the input is changed to be a snapshot. Suboptimal approach.
-    from core.models import Snapshot, ArchiveResult
-    try:
-        snapshot = Snapshot.objects.get(url=link.url) # TODO: This will be unnecessary once everything is a snapshot
-    except Snapshot.DoesNotExist:
-        snapshot = write_link_to_sql_index(link)
-
-    active_methods = get_archive_methods_for_link(link)
+    ARCHIVE_METHODS = get_default_archive_methods()
     
     if methods:
-        active_methods = [
-            method for method in active_methods
+        ARCHIVE_METHODS = [
+            method for method in ARCHIVE_METHODS
             if method[0] in methods
         ]
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     try:
         is_new = not Path(out_dir).exists()
         if is_new:
             os.makedirs(out_dir)
+            details = {"history": {}}
+            write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=False)
+        else:
+            details = snapshot.details #TODO: This can be retrieved from the sqlite database too.
+                                       # If that makes more sense, it can be easily changed.
 
-        link = load_link_details(link, out_dir=out_dir)
-        write_link_details(link, out_dir=out_dir, skip_sql_index=False)
-        log_link_archiving_started(link, out_dir, is_new)
-        link = link.overwrite(updated=datetime.now(timezone.utc))
+        log_snapshot_archiving_started(snapshot, out_dir, is_new)
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
-        start_ts = datetime.now(timezone.utc)
 
-        for method_name, should_run, method_function in active_methods:
+        for method_name, should_run, method_function in ARCHIVE_METHODS:
             try:
-                if method_name not in link.history:
-                    link.history[method_name] = []
+                if method_name not in details["history"]:
+                    details["history"][method_name] = []
 
-                if should_run(link, out_dir, overwrite):
+                if should_run(snapshot, out_dir, overwrite):
                     log_archive_method_started(method_name)
 
-                    result = method_function(link=link, out_dir=out_dir)
-
-                    link.history[method_name].append(result)
+                    result = method_function(snapshot=snapshot, out_dir=out_dir)
 
                     stats[result.status] += 1
                     log_archive_method_finished(result)
-                    write_search_index(link=link, texts=result.index_texts)
+                    write_search_index(snapshot=snapshot, texts=result.index_texts)
                     ArchiveResult.objects.create(snapshot=snapshot, extractor=method_name, cmd=result.cmd, cmd_version=result.cmd_version,
                                                  output=result.output, pwd=result.pwd, start_ts=result.start_ts, end_ts=result.end_ts, status=result.status)
 
-
-                    # bump the updated time on the main Snapshot here, this is critical
-                    # to be able to cache summaries of the ArchiveResults for a given
-                    # snapshot without having to load all the results from the DB each time.
-                    # (we use {Snapshot.id}-{Snapshot.updated} as the cache key and assume
-                    # ArchiveResults are unchanged as long as the updated timestamp is unchanged)
-                    snapshot.save()
                 else:
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
                     stats['skipped'] += 1
             except Exception as e:
-                # Disabled until https://github.com/ArchiveBox/ArchiveBox/issues/984
-                # and https://github.com/ArchiveBox/ArchiveBox/issues/1014
-                # are fixed.
-                """
-                raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
+                raise Exception('Exception in archive_methods.save_{}(Snapshot(url={}))'.format(
                     method_name,
-                    link.url,
+                    snapshot.url,
                 )) from e
-                """
-                # Instead, use the kludgy workaround from
-                # https://github.com/ArchiveBox/ArchiveBox/issues/984#issuecomment-1150541627
-                with open(ERROR_LOG, "a", encoding='utf-8') as f:
-                    command = ' '.join(sys.argv)
-                    ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
-                    f.write(("\n" + 'Exception in archive_methods.save_{}(Link(url={})) command={}; ts={}'.format(
-                        method_name,
-                        link.url,
-                        command,
-                        ts
-                    ) + "\n" + str(e) + "\n"))
-                    #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
 
         # print('    ', stats)
 
         try:
-            latest_title = link.history['title'][-1].output.strip()
-            if latest_title and len(latest_title) >= len(link.title or ''):
-                link = link.overwrite(title=latest_title)
+            latest_title_archive_result = snapshot.archiveresult_set.filter(extractor="title")
+            if latest_title_archive_result.count() > 0:
+                latest_title = latest_title_archive_result.output.strip()
+                if len(latest_title) >= len(snapshot.title or ''):
+                    snapshot.title = latest_title
         except Exception:
             pass
 
-        write_link_details(link, out_dir=out_dir, skip_sql_index=False)
+        write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=False)
 
-        log_link_archiving_finished(link, out_dir, is_new, stats, start_ts)
+        log_snapshot_archiving_finished(snapshot, snapshot.snapshot_dir, is_new, stats)
 
     except KeyboardInterrupt:
         try:
-            write_link_details(link, out_dir=link.link_dir)
+            write_snapshot_details(snapshot, out_dir=snapshot.snapshot_dir)
         except:
             pass
         raise
@@ -211,35 +142,29 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         print('    ! Failed to archive link: {}: {}'.format(err.__class__.__name__, err))
         raise
 
-    return link
+    return snapshot
 
 @enforce_types
-def archive_links(all_links: Union[Iterable[Link], QuerySet], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> List[Link]:
+def archive_snapshots(all_snapshots: Union[QuerySet, List[Model]], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> QuerySet:
 
-    if type(all_links) is QuerySet:
-        num_links: int = all_links.count()
-        get_link = lambda x: x.as_link()
-        all_links = all_links.iterator()
-    else:
-        num_links: int = len(all_links)
-        get_link = lambda x: x
+    all_snapshots = list(all_snapshots)
+    num_snapshots: int = len(all_snapshots)
 
-    if num_links == 0:
+    if num_snapshots == 0:
         return []
 
-    log_archiving_started(num_links)
+    log_archiving_started(num_snapshots)
     idx: int = 0
     try:
-        for link in all_links:
+        for snapshot in all_snapshots:
             idx += 1
-            to_archive = get_link(link)
-            archive_link(to_archive, overwrite=overwrite, methods=methods, out_dir=Path(link.link_dir))
+            archive_snapshot(snapshot, overwrite=overwrite, methods=methods, out_dir=Path(snapshot.snapshot_dir))
     except KeyboardInterrupt:
-        log_archiving_paused(num_links, idx, link.timestamp)
+        log_archiving_paused(num_snapshots, idx, snapshot.timestamp)
         raise SystemExit(0)
     except BaseException:
         print()
         raise
 
-    log_archiving_finished(num_links)
-    return all_links
+    log_archiving_finished(num_snapshots)
+    return all_snapshots

archivebox/extractors/archive_org.py

@@ -5,7 +5,9 @@ from pathlib import Path
 from typing import Optional, List, Dict, Tuple
 from collections import defaultdict
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -23,27 +25,30 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = '{domain}/'
+
 
 @enforce_types
-def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_archive_dot_org(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
     if not overwrite and (out_dir / 'archive.org.txt').exists():
-        # if open(path, 'r', encoding='utf-8').read().strip() != 'None':
+        # if open(path, 'r').read().strip() != 'None':
         return False
 
     return SAVE_ARCHIVE_DOT_ORG
 
 @enforce_types
-def save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_archive_dot_org(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """submit site to archive.org for archiving via their service, save returned archive url"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'archive.org.txt'
     archive_org_url = None
-    submit_url = 'https://web.archive.org/save/{}'.format(link.url)
+    submit_url = 'https://web.archive.org/save/{}'.format(snapshot.url)
     cmd = [
         CURL_BINARY,
         *CURL_ARGS,

archivebox/extractors/dom.py

@@ -3,13 +3,14 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file, atomic_write
 from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
-    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -19,30 +20,32 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'output.html'
+
 
 @enforce_types
-def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_dom(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
     if not overwrite and (out_dir / 'output.html').exists():
-        if (out_dir / 'output.html').stat().st_size > 1:
-            return False
+        return False
 
     return SAVE_DOM
 
 @enforce_types
-def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_dom(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print HTML of site to file using chrome --dump-html"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'output.html'
     output_path = out_dir / output
     cmd = [
-        *chrome_args(),
+        *chrome_args(TIMEOUT=timeout),
         '--dump-dom',
-        link.url
+        snapshot.url
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
@@ -58,7 +61,6 @@ def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     except Exception as err:
         status = 'failed'
         output = err
-        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/favicon.py

@@ -4,13 +4,14 @@ from pathlib import Path
 
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput
 from ..system import chmod_file, run
 from ..util import enforce_types, domain
 from ..config import (
     TIMEOUT,
     SAVE_FAVICON,
-    FAVICON_PROVIDER,
     CURL_BINARY,
     CURL_ARGS,
     CURL_VERSION,
@@ -20,19 +21,22 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'favicon.ico'
+
+
 @enforce_types
-def should_save_favicon(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'favicon.ico').exists():
+def should_save_favicon(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+    if not overwrite and (Path(out_dir) / 'favicon.ico').exists():
         return False
 
     return SAVE_FAVICON
 
 @enforce_types
-def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_favicon(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download site favicon from google's favicon api"""
 
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or snapshot.snapshot_dir
     output: ArchiveOutput = 'favicon.ico'
     cmd = [
         CURL_BINARY,
@@ -41,7 +45,7 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         '--output', str(output),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        FAVICON_PROVIDER.format(domain(link.url)),
+        'https://www.google.com/s2/favicons?domain={}'.format(domain(snapshot.url)),
     ]
     status = 'failed'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/git.py

@@ -4,7 +4,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -26,10 +28,16 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'git/'
+# @contents = output.glob('*.*')
+# @exists = self.contents.exists()
+# @size => get_size(self.contents)
+# @num_files => len(self.contents)
 
 @enforce_types
-def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_git(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -37,8 +45,8 @@ def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
         return False
 
     is_clonable_url = (
-        (domain(link.url) in GIT_DOMAINS)
-        or (extension(link.url) == 'git')
+        (domain(snapshot.url) in GIT_DOMAINS)
+        or (extension(snapshot.url) == 'git')
     )
     if not is_clonable_url:
         return False
@@ -47,10 +55,10 @@ def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
 
 
 @enforce_types
-def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_git(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using git"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'git'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
@@ -59,7 +67,7 @@ def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
         'clone',
         *GIT_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']),
-        without_query(without_fragment(link.url)),
+        without_query(without_fragment(snapshot.url)),
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/headers.py

@@ -4,7 +4,9 @@ from pathlib import Path
 
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput
 from ..system import atomic_write
 from ..util import (
     enforce_types,
@@ -21,20 +23,28 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
+
+# output = 'headers.json'
+
 @enforce_types
-def should_save_headers(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'headers.json').exists():
+def should_save_headers(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+
+    if not SAVE_HEADERS:
         return False
+    
+    if overwrite:
+        return True
 
-    return SAVE_HEADERS
+    output = Path(out_dir or snapshot.snapshot_dir) / 'headers.json'
+    return not output.exists()
 
 
 @enforce_types
-def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_headers(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """Download site headers"""
 
-    out_dir = Path(out_dir or link.link_dir)
+    out_dir = Path(out_dir or snapshot.snapshot_dir)
     output_folder = out_dir.absolute()
     output: ArchiveOutput = 'headers.json'
 
@@ -48,10 +58,10 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        link.url,
+        snapshot.url,
     ]
     try:
-        json_headers = get_headers(link.url, timeout=timeout)
+        json_headers = get_headers(snapshot.url, timeout=timeout)
         output_folder.mkdir(exist_ok=True)
         atomic_write(str(output_folder / "headers.json"), json_headers)
     except (Exception, OSError) as err:

archivebox/extractors/htmltotext.py

@@ -1,154 +0,0 @@
-__package__ = 'archivebox.extractors'
-
-from html.parser import HTMLParser
-import io
-from pathlib import Path
-from typing import Optional
-
-from ..config import (
-    SAVE_HTMLTOTEXT,
-    TIMEOUT,
-    VERSION,
-)
-from ..index.schema import Link, ArchiveResult, ArchiveError
-from ..logging_util import TimedProgress
-from ..system import atomic_write
-from ..util import (
-    enforce_types,
-    is_static_file,
-)
-from .title import get_html
-
-class HTMLTextExtractor(HTMLParser):
-    TEXT_ATTRS = [
-        "alt", "cite", "href", "label",
-        "list", "placeholder", "title", "value"
-    ]
-    NOTEXT_TAGS = ["script", "style", "template"]
-    NOTEXT_HREF = ["data:", "javascript:", "#"]
-
-    def __init__(self):
-        super().__init__()
-
-        self.output = io.StringIO()
-        self._tag_stack = []
-
-    def _is_text_attr(self, name, value):
-        if not isinstance(value, str):
-            return False
-        if name == "href" and any(map(lambda p: value.startswith(p), self.NOTEXT_HREF)):
-            return False
-
-        if name in self.TEXT_ATTRS:
-            return True
-
-        return False
-
-    def _parent_tag(self):
-        try:
-            return self._tag_stack[-1]
-        except IndexError:
-            return None
-
-    def _in_notext_tag(self):
-        return any([t in self._tag_stack for t in self.NOTEXT_TAGS])
-
-    def handle_starttag(self, tag, attrs):
-        self._tag_stack.append(tag)
-
-        # Don't write out attribute values if any ancestor
-        # is in NOTEXT_TAGS
-        if self._in_notext_tag():
-            return
-
-        for name, value in attrs:
-            if self._is_text_attr(name, value):
-                self.output.write(f"({value.strip()}) ")
-
-    def handle_endtag(self, tag):
-        orig_stack = self._tag_stack.copy()
-        try:
-            # Keep popping tags until we find the nearest
-            # ancestor matching this end tag
-            while tag != self._tag_stack.pop():
-                pass
-            # Write a space after every tag, to ensure that tokens
-            # in tag text aren't concatenated. This may result in
-            # excess spaces, which should be ignored by search tokenizers.
-            if not self._in_notext_tag() and tag not in self.NOTEXT_TAGS:
-                self.output.write(" ")
-        except IndexError:
-            # Got to the top of the stack, but somehow missed
-            # this end tag -- maybe malformed markup -- restore the
-            # stack
-            self._tag_stack = orig_stack
-
-    def handle_data(self, data):
-        # Don't output text data if any ancestor is in NOTEXT_TAGS
-        if self._in_notext_tag():
-            return
-
-        data = data.lstrip()
-        len_before_rstrip = len(data)
-        data = data.rstrip()
-        spaces_rstripped = len_before_rstrip - len(data)
-        if data:
-            self.output.write(data)
-            if spaces_rstripped:
-                # Add back a single space if 1 or more
-                # whitespace characters were stripped
-                self.output.write(' ')
-
-    def __str__(self):
-        return self.output.getvalue()
-
-
-@enforce_types
-def should_save_htmltotext(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
-        return False
-
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'htmltotext.txt').exists():
-        return False
-
-    return SAVE_HTMLTOTEXT
-
-
-@enforce_types
-def save_htmltotext(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
-    """extract search-indexing-friendly text from an HTML document"""
-
-    out_dir = Path(out_dir or link.link_dir)
-    output = "htmltotext.txt"
-
-    timer = TimedProgress(timeout, prefix='      ')
-    extracted_text = None
-    try:
-        extractor = HTMLTextExtractor()
-        document = get_html(link, out_dir)
-
-        if not document:
-            raise ArchiveError('htmltotext could not find HTML to parse for article text')
-
-        extractor.feed(document)
-        extractor.close()
-        extracted_text = str(extractor)
-
-        atomic_write(str(out_dir / output), extracted_text)
-    except (Exception, OSError) as err:
-        status = 'failed'
-        output = err
-        cmd = ['(internal) archivebox.extractors.htmltotext', './{singlefile,dom}.html']
-    finally:
-        timer.end()
-
-    return ArchiveResult(
-        cmd=cmd,
-        pwd=str(out_dir),
-        cmd_version=VERSION,
-        output=output,
-        status=status,
-        index_texts=[extracted_text] if extracted_text else [],
-        **timer.stats,  
-    )

archivebox/extractors/media.py

@@ -3,7 +3,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -20,9 +22,13 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'media/'
+
 @enforce_types
-def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_media(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -32,10 +38,10 @@ def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optio
     return SAVE_MEDIA
 
 @enforce_types
-def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
-    """Download playlists or individual video, audio, and subtitles using youtube-dl or yt-dlp"""
+def save_media(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
+    """Download playlists or individual video, audio, and subtitles using youtube-dl"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'media'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
@@ -43,8 +49,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
         YOUTUBEDL_BINARY,
         *YOUTUBEDL_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
-        # TODO: add --cookies-from-browser={CHROME_USER_DATA_DIR}
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
@@ -61,7 +66,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
                 pass
             else:
                 hints = (
-                    'Got youtube-dl (or yt-dlp) response code: {}.'.format(result.returncode),
+                    'Got youtube-dl response code: {}.'.format(result.returncode),
                     *result.stderr.decode().split('\n'),
                 )
                 raise ArchiveError('Failed to save media', hints)
@@ -71,34 +76,11 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
     finally:
         timer.end()
 
-    # add video description and subtitles to full-text index
-    # Let's try a few different 
-    index_texts = [
-        # errors:
-        # * 'strict' to raise a ValueError exception if there is an
-        #   encoding error. The default value of None has the same effect.
-        # * 'ignore' ignores errors. Note that ignoring encoding errors
-        #   can lead to data loss.
-        # * 'xmlcharrefreplace' is only supported when writing to a
-        #   file. Characters not supported by the encoding are replaced with
-        #   the appropriate XML character reference &#nnn;.
-        # There are a few more options described in https://docs.python.org/3/library/functions.html#open
-        text_file.read_text(encoding='utf-8', errors='xmlcharrefreplace').strip()
-        for text_file in (
-            *output_path.glob('*.description'),
-            *output_path.glob('*.srt'),
-            *output_path.glob('*.vtt'),
-            *output_path.glob('*.lrc'),
-            *output_path.glob('*.lrc'),
-        )
-    ]
-
     return ArchiveResult(
         cmd=cmd,
         pwd=str(out_dir),
         cmd_version=YOUTUBEDL_VERSION,
         output=output,
         status=status,
-        index_texts=index_texts,
         **timer.stats,
     )

archivebox/extractors/mercury.py

@@ -6,7 +6,9 @@ from subprocess import CompletedProcess
 from typing import Optional, List
 import json
 
-from ..index.schema import Link, ArchiveResult, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
@@ -37,34 +39,33 @@ def ShellError(cmd: List[str], result: CompletedProcess, lines: int=20) -> Archi
 
 
 @enforce_types
-def should_save_mercury(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_mercury(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+    if is_static_file(snapshot.url):
         return False
 
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'mercury').exists():
+    output = Path(out_dir or snapshot.snapshot_dir) / 'mercury'
+    if not overwrite and output.exists():
         return False
 
-    return SAVE_MERCURY
+    return SAVE_MERCURY and MERCURY_VERSION
 
 
 @enforce_types
-def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_mercury(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download reader friendly version using @postlight/mercury-parser"""
 
-    out_dir = Path(out_dir or link.link_dir)
+    out_dir = Path(out_dir or snapshot.snapshot_dir)
     output_folder = out_dir.absolute() / "mercury"
-    output = "mercury"
+    output = str(output_folder)
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        output_folder.mkdir(exist_ok=True)
-
         # Get plain text version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
-            link.url,
+            snapshot.url,
             "--format=text"
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
@@ -73,15 +74,10 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         except json.JSONDecodeError:
             raise ShellError(cmd, result)
         
-        if article_text.get('failed'):
-            raise ArchiveError('Mercury was not able to get article text from the URL')
-
-        atomic_write(str(output_folder / "content.txt"), article_text["content"])
-
         # Get HTML version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
-            link.url
+            snapshot.url
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:
@@ -89,10 +85,9 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         except json.JSONDecodeError:
             raise ShellError(cmd, result)
 
-        if article_text.get('failed'):
-            raise ArchiveError('Mercury was not able to get article HTML from the URL')
-
+        output_folder.mkdir(exist_ok=True)
         atomic_write(str(output_folder / "content.html"), article_json.pop("content"))
+        atomic_write(str(output_folder / "content.txt"), article_text["content"])
         atomic_write(str(output_folder / "article.json"), article_json)
 
         # Check for common failure cases

archivebox/extractors/pdf.py

@@ -3,13 +3,14 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
-    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -18,10 +19,12 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
+# output = 'output.pdf'
 
 @enforce_types
-def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_pdf(snapshot: Model, verwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -32,15 +35,15 @@ def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
 
 
 @enforce_types
-def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_pdf(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print PDF of site to file using chrome --headless"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'output.pdf'
     cmd = [
-        *chrome_args(),
+        *chrome_args(TIMEOUT=timeout),
         '--print-to-pdf',
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
@@ -55,7 +58,6 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     except Exception as err:
         status = 'failed'
         output = err
-        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/readability.py

@@ -6,11 +6,15 @@ from tempfile import NamedTemporaryFile
 from typing import Optional
 import json
 
-from ..index.schema import Link, ArchiveResult, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
+    download_url,
     is_static_file,
+    
 )
 from ..config import (
     TIMEOUT,
@@ -20,28 +24,53 @@ from ..config import (
     READABILITY_VERSION,
 )
 from ..logging_util import TimedProgress
-from .title import get_html
 
 
 @enforce_types
-def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def get_html(snapshot: Model, path: Path) -> str:
+    """
+    Try to find wget, singlefile and then dom files.
+    If none is found, download the url again.
+    """
+    canonical = snapshot.canonical_outputs()
+    abs_path = path.absolute()
+    sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
+    document = None
+    for source in sources:
+        try:
+            with open(abs_path / source, "r") as f:
+                document = f.read()
+                break
+        except (FileNotFoundError, TypeError):
+            continue
+    if document is None:
+        return download_url(snapshot.url)
+    else:
+        return document
+
+
+# output = 'readability/'
+
+@enforce_types
+def should_save_readability(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.link_dir
+    if is_static_file(snapshot.url):
         return False
 
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'readability').exists():
+    output = Path(out_dir or snapshot.snapshot_dir) / 'readability'
+    if not overwrite and output.exists():
         return False
 
-    return SAVE_READABILITY
+    return SAVE_READABILITY and READABILITY_VERSION
 
 
 @enforce_types
-def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download reader friendly version using @mozilla/readability"""
 
-    out_dir = Path(out_dir or link.link_dir)
+    out_dir = Path(out_dir or snapshot.snapshot_dir)
     output_folder = out_dir.absolute() / "readability"
-    output = "readability"
+    output = str(output_folder)
 
     # Readability Docs: https://github.com/mozilla/readability
 
@@ -49,40 +78,34 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
     # fake command to show the user so they have something to try debugging if get_html fails
     cmd = [
         CURL_BINARY,
-        link.url
+        snapshot.url
     ]
     readability_content = None
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        document = get_html(link, out_dir)
+        document = get_html(snapshot, out_dir)
         temp_doc = NamedTemporaryFile(delete=False)
         temp_doc.write(document.encode("utf-8"))
         temp_doc.close()
 
-        if not document or len(document) < 10:
-            raise ArchiveError('Readability could not find HTML to parse for article text')
-
         cmd = [
             DEPENDENCIES['READABILITY_BINARY']['path'],
-            temp_doc.name,
-            link.url,
+            temp_doc.name
         ]
-        result = run(cmd, cwd=out_dir, timeout=timeout)
-        try:
-            result_json = json.loads(result.stdout)
-            assert result_json and 'content' in result_json, 'Readability output is not valid JSON'
-        except json.JSONDecodeError:
-            raise ArchiveError('Readability was not able to archive the page (invalid JSON)', result.stdout + result.stderr)
 
+        result = run(cmd, cwd=out_dir, timeout=timeout)
+        result_json = json.loads(result.stdout)
         output_folder.mkdir(exist_ok=True)
         readability_content = result_json.pop("textContent") 
         atomic_write(str(output_folder / "content.html"), result_json.pop("content"))
         atomic_write(str(output_folder / "content.txt"), readability_content)
         atomic_write(str(output_folder / "article.json"), result_json)
 
+        # parse out number of files downloaded from last line of stderr:
+        #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
         output_tail = [
             line.strip()
-            for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
+            for line in (result.stdout + result.stderr).decode().rsplit('\n', 3)[-3:]
             if line.strip()
         ]
         hints = (
@@ -92,13 +115,10 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
 
         # Check for common failure cases
         if (result.returncode > 0):
-            raise ArchiveError(f'Readability was not able to archive the page (status={result.returncode})', hints)
+            raise ArchiveError('Readability was not able to archive the page', hints)
     except (Exception, OSError) as err:
         status = 'failed'
         output = err
-
-        # prefer Chrome dom output to singlefile because singlefile often contains huge url(data:image/...base64) strings that make the html too long to parse with readability
-        cmd = [cmd[0], './{dom,singlefile}.html']
     finally:
         timer.end()
 
@@ -108,6 +128,6 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         cmd_version=READABILITY_VERSION,
         output=output,
         status=status,
-        index_texts=[readability_content] if readability_content else [],
+        index_texts= [readability_content] if readability_content else [],
         **timer.stats,  
     )

archivebox/extractors/screenshot.py

@@ -3,13 +3,14 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
     is_static_file,
     chrome_args,
-    chrome_cleanup,
 )
 from ..config import (
     TIMEOUT,
@@ -19,10 +20,12 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'screenshot.png'
 
 @enforce_types
-def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_screenshot(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -32,15 +35,15 @@ def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite:
     return SAVE_SCREENSHOT
 
 @enforce_types
-def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_screenshot(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """take screenshot of site using chrome --headless"""
     
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'screenshot.png'
     cmd = [
-        *chrome_args(),
+        *chrome_args(TIMEOUT=timeout),
         '--screenshot',
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
@@ -55,7 +58,6 @@ def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
     except Exception as err:
         status = 'failed'
         output = err
-        chrome_cleanup()
     finally:
         timer.end()
 

archivebox/extractors/singlefile.py

@@ -5,7 +5,9 @@ from pathlib import Path
 from typing import Optional
 import json
 
-from ..index.schema import Link, ArchiveResult, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -17,15 +19,15 @@ from ..config import (
     SAVE_SINGLEFILE,
     DEPENDENCIES,
     SINGLEFILE_VERSION,
-    SINGLEFILE_ARGS,
     CHROME_BINARY,
 )
 from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_singlefile(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -36,42 +38,22 @@ def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite:
 
 
 @enforce_types
-def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_singlefile(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using single-file"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output = "singlefile.html"
 
-    browser_args = chrome_args(CHROME_TIMEOUT=0)
+    browser_args = chrome_args(TIMEOUT=0)
 
     # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
     browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:]))
-    options = [
-        *SINGLEFILE_ARGS,
-        '--browser-executable-path={}'.format(CHROME_BINARY),
-        browser_args,
-    ]
-
-    # Deduplicate options (single-file doesn't like when you use the same option two times)
-    #
-    # NOTE: Options names that come first clobber conflicting names that come later
-    # My logic is SINGLEFILE_ARGS is the option that affects the singlefile command with most 
-    # specificity, therefore the user sets it with a lot intent, therefore it should take precedence 
-    # kind of like the ergonomic principle of lexical scope in programming languages.
-    seen_option_names = []
-    def test_seen(argument):
-        option_name = argument.split("=")[0]
-        if option_name in seen_option_names:
-            return False
-        else:
-            seen_option_names.append(option_name)
-            return True
-    deduped_options = list(filter(test_seen, options))
-
     cmd = [
         DEPENDENCIES['SINGLEFILE_BINARY']['path'],
-        *deduped_options,
-        link.url,
+        '--browser-executable-path={}'.format(CHROME_BINARY),
+        browser_args,
+        snapshot.url,
+        output
         output,
     ]
 

archivebox/extractors/title.py

@@ -5,7 +5,9 @@ from html.parser import HTMLParser
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..util import (
     enforce_types,
     download_url,
@@ -26,7 +28,7 @@ from ..logging_util import TimedProgress
 
 HTML_TITLE_REGEX = re.compile(
     r'<title.*?>'                      # start matching text after <title> tag
-    r'([^<>]+)',                      # get everything up to these symbols
+    r'(.[^<>]+)',                      # get everything up to these symbols
     re.IGNORECASE | re.MULTILINE | re.DOTALL | re.UNICODE,
 )
 
@@ -58,34 +60,13 @@ class TitleParser(HTMLParser):
         if tag.lower() == "title":
             self.inside_title_tag = False
 
-@enforce_types
-def get_html(link: Link, path: Path, timeout: int=TIMEOUT) -> str:
-    """
-    Try to find wget, singlefile and then dom files.
-    If none is found, download the url again.
-    """
-    canonical = link.canonical_outputs()
-    abs_path = path.absolute()
 
-    # prefer chrome-generated DOM dump to singlefile as singlefile output often includes HUGE url(data:image/...base64) strings that crash parsers
-    sources = [canonical["dom_path"], canonical["singlefile_path"], canonical["wget_path"]]
-    document = None
-    for source in sources:
-        try:
-            with open(abs_path / source, "r", encoding="utf-8") as f:
-                document = f.read()
-                break
-        except (FileNotFoundError, TypeError):
-            continue
-    if document is None:
-        return download_url(link.url, timeout=timeout)
-    else:
-        return document
+# output = '{title}'
 
 @enforce_types
-def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+def should_save_title(snapshot: Model, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
     # if link already has valid title, skip it
-    if not overwrite and link.title and not link.title.lower().startswith('http'):
+    if not overwrite and snapshot.title and not snapshot.title.lower().startswith('http'):
         return False
 
     return SAVE_TITLE
@@ -96,7 +77,7 @@ def extract_title_with_regex(html):
     return output
 
 @enforce_types
-def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_title(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """try to guess the page's title from its content"""
 
     from core.models import Snapshot
@@ -108,12 +89,12 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        html = get_html(link, out_dir, timeout=timeout)
+        html = download_url(snapshot.url, timeout=timeout)
         try:
             # try using relatively strict html parser first
             parser = TitleParser()
@@ -127,10 +108,11 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
         
         # if title is better than the one in the db, update db with new title
         if isinstance(output, str) and output:
-            if not link.title or len(output) >= len(link.title):
-                Snapshot.objects.filter(url=link.url,
-                                        timestamp=link.timestamp)\
+            if not snapshot.title or len(output) >= len(snapshot.title):
+                Snapshot.objects.filter(url=snapshot.url,
+                                        timestamp=snapshot.timestamp)\
                                 .update(title=output)
+                snapshot.title = output
         else:
             # if no content was returned, dont save a title (because it might be a temporary error)
             if not html:

archivebox/extractors/wget.py

@@ -4,9 +4,11 @@ import re
 from pathlib import Path
 
 from typing import Optional
-from datetime import datetime, timezone
+from datetime import datetime
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -34,24 +36,24 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_wget(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    output_path = wget_output_path(link)
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and output_path and (out_dir / output_path).exists():
+def should_save_wget(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    output_path = wget_output_path(snapshot)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if not overwrite output_path and (out_dir / output_path).exists():
         return False
 
     return SAVE_WGET
 
 
 @enforce_types
-def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_wget(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using wget"""
 
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or snapshot.snapshot_dir
     if SAVE_WARC:
         warc_dir = out_dir / "warc"
         warc_dir.mkdir(exist_ok=True)
-        warc_path = warc_dir / str(int(datetime.now(timezone.utc).timestamp()))
+        warc_path = warc_dir / str(int(datetime.now().timestamp()))
 
     # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
     output: ArchiveOutput = None
@@ -68,14 +70,14 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
         *(['--compression=auto'] if WGET_AUTO_COMPRESSION else []),
         *([] if SAVE_WARC else ['--timestamping']),
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),
-        link.url,
+        snapshot.url,
     ]
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
         result = run(cmd, cwd=str(out_dir), timeout=timeout)
-        output = wget_output_path(link)
+        output = wget_output_path(snapshot)
 
         # parse out number of files downloaded from last line of stderr:
         #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
@@ -126,13 +128,13 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
 
 
 @enforce_types
-def wget_output_path(link: Link) -> Optional[str]:
+def wget_output_path(snapshot: Model) -> Optional[str]:
     """calculate the path to the wgetted .html file, since wget may
     adjust some paths to be different than the base_url path.
 
     See docs on wget --adjust-extension (-E)
     """
-    
+
     # Wget downloads can save in a number of different ways depending on the url:
     #    https://example.com
     #       > example.com/index.html
@@ -164,8 +166,8 @@ def wget_output_path(link: Link) -> Optional[str]:
     # and there's no way to get the computed output path from wget
     # in order to avoid having to reverse-engineer how they calculate it,
     # we just look in the output folder read the filename wget used from the filesystem
-    full_path = without_fragment(without_query(path(link.url))).strip('/')
-    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
+    full_path = without_fragment(without_query(path(snapshot.url))).strip('/')
+    search_dir = Path(snapshot.snapshot_dir) / domain(snapshot.url).replace(":", "+") / urldecode(full_path)
     for _ in range(4):
         if search_dir.exists():
             if search_dir.is_dir():
@@ -174,7 +176,7 @@ def wget_output_path(link: Link) -> Optional[str]:
                     if re.search(".+\\.[Ss]?[Hh][Tt][Mm][Ll]?$", str(f), re.I | re.M)
                 ]
                 if html_files:
-                    return str(html_files[0].relative_to(link.link_dir))
+                    return str(html_files[0].relative_to(snapshot.snapshot_dir))
 
                 # sometimes wget'd URLs have no ext and return non-html
                 # e.g. /some/example/rss/all -> some RSS XML content)
@@ -183,23 +185,23 @@ def wget_output_path(link: Link) -> Optional[str]:
                 last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
                 for file_present in search_dir.iterdir():
                     if file_present == last_part_of_url:
-                        return str((search_dir / file_present).relative_to(link.link_dir))
+                        return str((search_dir / file_present).relative_to(snapshot.snapshot_dir))
 
         # Move up one directory level
         search_dir = search_dir.parent
 
-        if str(search_dir) == link.link_dir:
+        if search_dir == snapshot.snapshot_dir:
             break
 
     # check for literally any file present that isnt an empty folder
-    domain_dir = Path(domain(link.url).replace(":", "+"))
-    files_within = list((Path(link.link_dir) / domain_dir).glob('**/*.*'))
+    domain_dir = Path(domain(snapshot.url).replace(":", "+"))
+    files_within = list((Path(snapshot.snapshot_dir) / domain_dir).glob('**/*.*'))
     if files_within:
-        return str((domain_dir / files_within[-1]).relative_to(link.link_dir))
+        return str((domain_dir / files_within[-1]).relative_to(snapshot.snapshot_dir))
     
-    # fallback to just the domain dir
-    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
+    # fallback to just the domain dir, dont try to introspect further inside it
+    search_dir = Path(snapshot.snapshot_dir) / domain(snapshot.url).replace(":", "+")
     if search_dir.is_dir():
-        return domain(link.url).replace(":", "+")
+        return domain(snapshot.url).replace(":", "+")
 
     return None

archivebox/index/__init__.py

@@ -9,7 +9,7 @@ from typing import List, Tuple, Dict, Optional, Iterable
 from collections import OrderedDict
 from contextlib import contextmanager
 from urllib.parse import urlparse
-from django.db.models import QuerySet, Q
+from django.db.models import QuerySet, Q, Model
 
 from ..util import (
     scheme,
@@ -22,8 +22,7 @@ from ..config import (
     JSON_INDEX_FILENAME,
     OUTPUT_DIR,
     TIMEOUT,
-    URL_DENYLIST_PTN,
-    URL_ALLOWLIST_PTN,
+    URL_BLACKLIST_PTN,
     stderr,
     OUTPUT_PERMISSIONS
 )
@@ -39,16 +38,16 @@ from ..logging_util import (
 
 from .schema import Link, ArchiveResult
 from .html import (
-    write_html_link_details,
+    write_html_snapshot_details,
 )
 from .json import (
     pyjson,
-    parse_json_link_details, 
-    write_json_link_details,
+    load_json_snapshot,
+    write_json_snapshot_details,
 )
 from .sql import (
     write_sql_main_index,
-    write_sql_link_details,
+    write_sql_snapshot_details,
 )
 
 from ..search import search_backend_enabled, query_search_index
@@ -56,10 +55,12 @@ from ..search import search_backend_enabled, query_search_index
 ### Link filtering and checking
 
 @enforce_types
-def merge_links(a: Link, b: Link) -> Link:
-    """deterministially merge two links, favoring longer field values over shorter,
+def merge_snapshots(a: Model, b: Model) -> Model:
+    """deterministially merge two snapshots, favoring longer field values over shorter,
     and "cleaner" values over worse ones.
+    TODO: Check if this makes sense with the new setup
     """
+    return a
     assert a.base_url == b.base_url, f'Cannot merge two links with different URLs ({a.base_url} != {b.base_url})'
 
     # longest url wins (because a fuzzy url will always be shorter)
@@ -110,57 +111,55 @@ def merge_links(a: Link, b: Link) -> Link:
             key=lambda result: result.start_ts,
         )))
 
-    return Link(
+    return Snapshot(
         url=url,
         timestamp=timestamp,
         title=title,
         tags=tags,
-        sources=sources,
-        history=history,
+        #sources=sources,
+        #history=history,
     )
 
 
 @enforce_types
-def validate_links(links: Iterable[Link]) -> List[Link]:
+def validate_snapshots(snapshots: List[Model]) -> List[Model]:
     timer = TimedProgress(TIMEOUT * 4)
     try:
-        links = archivable_links(links)  # remove chrome://, about:, mailto: etc.
-        links = sorted_links(links)      # deterministically sort the links based on timestamp, url
-        links = fix_duplicate_links(links)  # merge/dedupe duplicate timestamps & urls
+        snapshots = archivable_snapshots(snapshots)  # remove chrome://, about:, mailto: etc.
+        snapshots = sorted_snapshots(snapshots)      # deterministically sort the links based on timestamp, url
+        snapshots = fix_duplicate_snapshots(snapshots)  # merge/dedupe duplicate timestamps & urls
     finally:
         timer.end()
 
-    return list(links)
+    return list(snapshots)
 
 @enforce_types
-def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
+def archivable_snapshots(snapshots: Iterable[Model]) -> Iterable[Model]:
     """remove chrome://, about:// or other schemed links that cant be archived"""
-    for link in links:
+    for snapshot in snapshots:
         try:
-            urlparse(link.url)
+            urlparse(snapshot.url)
         except ValueError:
             continue
-        if scheme(link.url) not in ('http', 'https', 'ftp'):
+        if scheme(snapshot.url) not in ('http', 'https', 'ftp'):
             continue
-        if URL_DENYLIST_PTN and URL_DENYLIST_PTN.search(link.url):
-            continue
-        if URL_ALLOWLIST_PTN and (not URL_ALLOWLIST_PTN.search(link.url)):
+        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(snapshot.url):
             continue
 
-        yield link
+        yield snapshot
 
 
 @enforce_types
-def fix_duplicate_links(sorted_links: Iterable[Link]) -> Iterable[Link]:
+def fix_duplicate_snapshots(sorted_snapshots: Iterable[Model]) -> Iterable[Model]:
     """
     ensures that all non-duplicate links have monotonically increasing timestamps
+    TODO: Review how to do this with the new snapshots refactor
     """
-    # from core.models import Snapshot
-
+    return sorted_snapshots
     unique_urls: OrderedDict[str, Link] = OrderedDict()
 
-    for link in sorted_links:
-        if link.url in unique_urls:
+    for snapshot in sorted_snapshots:
+        if snapshot.url in unique_urls:
             # merge with any other links that share the same url
             link = merge_links(unique_urls[link.url], link)
         unique_urls[link.url] = link
@@ -169,9 +168,9 @@ def fix_duplicate_links(sorted_links: Iterable[Link]) -> Iterable[Link]:
 
 
 @enforce_types
-def sorted_links(links: Iterable[Link]) -> Iterable[Link]:
-    sort_func = lambda link: (link.timestamp.split('.', 1)[0], link.url)
-    return sorted(links, key=sort_func, reverse=True)
+def sorted_snapshots(snapshots: Iterable[Model]) -> Iterable[Model]:
+    sort_func = lambda snapshot: (snapshot.timestamp.split('.', 1)[0], snapshot.url)
+    return sorted(snapshots, key=sort_func, reverse=True)
 
 
 @enforce_types
@@ -225,14 +224,14 @@ def timed_index_update(out_path: Path):
 
 
 @enforce_types
-def write_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
+def write_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
     """Writes links to sqlite3 file for a given list of links"""
 
-    log_indexing_process_started(len(links))
+    log_indexing_process_started(len(snapshots))
 
     try:
         with timed_index_update(out_dir / SQL_INDEX_FILENAME):
-            write_sql_main_index(links, out_dir=out_dir)
+            write_sql_main_index(snapshots, out_dir=out_dir)
             os.chmod(out_dir / SQL_INDEX_FILENAME, int(OUTPUT_PERMISSIONS, base=8)) # set here because we don't write it with atomic writes
 
     except (KeyboardInterrupt, SystemExit):
@@ -247,7 +246,9 @@ def write_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
 
 @enforce_types
 def load_main_index(out_dir: Path=OUTPUT_DIR, warn: bool=True) -> List[Link]:
-    """parse and load existing index with any new links from import_path merged in"""
+    """
+    Returns all of the snapshots currently in index
+    """
     from core.models import Snapshot
     try:
         return Snapshot.objects.all()
@@ -268,88 +269,62 @@ def load_main_index_meta(out_dir: Path=OUTPUT_DIR) -> Optional[dict]:
 
 
 @enforce_types
-def parse_links_from_source(source_path: str, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], List[Link]]:
+def parse_snapshots_from_source(source_path: str, root_url: Optional[str]=None) -> List[Model]:
 
-    from ..parsers import parse_links
+    from ..parsers import parse_snapshots
 
-    new_links: List[Link] = []
+    new_links: List[Model] = []
 
     # parse and validate the import file
-    raw_links, parser_name = parse_links(source_path, root_url=root_url, parser=parser)
-    new_links = validate_links(raw_links)
+    raw_snapshots, parser_name = parse_snapshots(source_path, root_url=root_url)
+    new_snapshots = validate_snapshots(raw_snapshots)
 
     if parser_name:
-        num_parsed = len(raw_links)
+        num_parsed = len(raw_snapshots)
         log_parsing_finished(num_parsed, parser_name)
 
-    return new_links
+    return new_snapshots
 
 @enforce_types
-def fix_duplicate_links_in_index(snapshots: QuerySet, links: Iterable[Link]) -> Iterable[Link]:
+def filter_new_urls(snapshots: QuerySet,
+                 new_snapshots: List) -> List:
     """
-    Given a list of in-memory Links, dedupe and merge them with any conflicting Snapshots in the DB.
+    Returns a list of Snapshots corresponding to the urls that were not present in the index
     """
-    unique_urls: OrderedDict[str, Link] = OrderedDict()
+    urls = {snapshot.url: snapshot for snapshot in new_snapshots}
+    filtered_snapshots = snapshots.filter(url__in=urls.keys())
 
-    for link in links:
-        index_link = snapshots.filter(url=link.url)
-        if index_link:
-            link = merge_links(index_link[0].as_link(), link)
-
-        unique_urls[link.url] = link
-
-    return unique_urls.values()
-
-@enforce_types
-def dedupe_links(snapshots: QuerySet,
-                 new_links: List[Link]) -> List[Link]:
-    """
-    The validation of links happened at a different stage. This method will
-    focus on actual deduplication and timestamp fixing.
-    """
+    for found_snapshot in filtered_snapshots:
+        urls.pop(found_snapshot.url)
     
-    # merge existing links in out_dir and new links
-    dedup_links = fix_duplicate_links_in_index(snapshots, new_links)
+    log_deduping_finished(len(urls.keys()))
 
-    new_links = [
-        link for link in new_links
-        if not snapshots.filter(url=link.url).exists()
-    ]
-
-    dedup_links_dict = {link.url: link for link in dedup_links}
-
-    # Replace links in new_links with the dedup version
-    for i in range(len(new_links)):
-        if new_links[i].url in dedup_links_dict.keys():
-            new_links[i] = dedup_links_dict[new_links[i].url]
-    log_deduping_finished(len(new_links))
-
-    return new_links
+    return list(urls.values())
 
 ### Link Details Index
 
 @enforce_types
-def write_link_details(link: Link, out_dir: Optional[str]=None, skip_sql_index: bool=False) -> None:
-    out_dir = out_dir or link.link_dir
+def write_snapshot_details(snapshot: List[Model], out_dir: Optional[str]=None, skip_sql_index: bool=False) -> None:
+    out_dir = out_dir or snapshot.snapshot_dir
 
-    write_json_link_details(link, out_dir=out_dir)
-    write_html_link_details(link, out_dir=out_dir)
+    write_json_snapshot_details(snapshot, out_dir=out_dir)
+    write_html_snapshot_details(snapshot, out_dir=out_dir)
     if not skip_sql_index:
-        write_sql_link_details(link)
+        write_sql_snapshot_details(snapshot)
 
 
 @enforce_types
-def load_link_details(link: Link, out_dir: Optional[str]=None) -> Link:
+def load_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> Model:
     """check for an existing link archive in the given directory, 
        and load+merge it into the given link dict
     """
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
 
-    existing_link = parse_json_link_details(out_dir)
-    if existing_link:
-        return merge_links(existing_link, link)
+    existing_snapshot = load_json_snapshot(Path(out_dir))
+    if existing_snapshot:
+        return merge_snapshots(existing_snapshot, snapshot)
 
-    return link
+    return snapshot
 
 
 
@@ -359,7 +334,6 @@ LINK_FILTERS = {
     'regex': lambda pattern: Q(url__iregex=pattern),
     'domain': lambda pattern: Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}"),
     'tag': lambda pattern: Q(tags__name=pattern),
-    'timestamp': lambda pattern: Q(timestamp=pattern),
 }
 
 @enforce_types
@@ -405,56 +379,41 @@ def snapshot_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type
         return search_filter(snapshots, filter_patterns, filter_type)
 
 
-def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """indexed links without checking archive status or data directory validity"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in links
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in snapshots}
 
-def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """indexed links that are archived with a valid data directory"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in filter(is_archived, links)
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_archived, snapshots)}
 
-def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """indexed links that are unarchived with no data directory or an empty data directory"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in filter(is_unarchived, links)
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_unarchived, snapshots)}
 
-def get_present_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_present_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that actually exist in the archive/ folder"""
+    from core.models import Snapshot
 
     all_folders = {}
 
     for entry in (out_dir / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
-            link = None
+            snapshot = None
             try:
-                link = parse_json_link_details(entry.path)
+                snapshot = load_json_snapshot(Path(entry.path))
             except Exception:
                 pass
 
-            all_folders[entry.name] = link
+            all_folders[entry.name] = snapshot
 
     return all_folders
 
-def get_valid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_valid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs with a valid index matched to the main index and archived content"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in filter(is_valid, links)
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_valid, snapshots)}
 
-def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that are invalid for any reason: corrupted/duplicate/orphaned/unrecognized"""
     duplicate = get_duplicate_folders(snapshots, out_dir=OUTPUT_DIR)
     orphaned = get_orphaned_folders(snapshots, out_dir=OUTPUT_DIR)
@@ -463,7 +422,7 @@ def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Option
     return {**duplicate, **orphaned, **corrupted, **unrecognized}
 
 
-def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that conflict with other directories that have the same link URL or timestamp"""
     by_url = {}
     by_timestamp = {}
@@ -476,91 +435,92 @@ def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Opti
     )
 
     for path in chain(snapshots.iterator(), data_folders):
-        link = None
+        snapshot = None
         if type(path) is not str:
-            path = path.as_link().link_dir
+            path = path.snapshot_dir
 
         try:
-            link = parse_json_link_details(path)
+            snapshot = load_json_snapshot(Path(path))
         except Exception:
             pass
 
-        if link:
-            # link folder has same timestamp as different link folder
-            by_timestamp[link.timestamp] = by_timestamp.get(link.timestamp, 0) + 1
-            if by_timestamp[link.timestamp] > 1:
-                duplicate_folders[path] = link
+        if snapshot:
+            # snapshot folder has same timestamp as different link folder
+            by_timestamp[snapshot.timestamp] = by_timestamp.get(snapshot.timestamp, 0) + 1
+            if by_timestamp[snapshot.timestamp] > 1:
+                duplicate_folders[path] = snapshot
 
             # link folder has same url as different link folder
-            by_url[link.url] = by_url.get(link.url, 0) + 1
-            if by_url[link.url] > 1:
-                duplicate_folders[path] = link
+            by_url[snapshot.url] = by_url.get(snapshot.url, 0) + 1
+            if by_url[snapshot.url] > 1:
+                duplicate_folders[path] = snapshot
     return duplicate_folders
 
-def get_orphaned_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_orphaned_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that contain a valid index but aren't listed in the main index"""
     orphaned_folders = {}
 
     for entry in (Path(out_dir) / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
-            link = None
+            snapshot = None
             try:
-                link = parse_json_link_details(str(entry))
+                snapshot = load_json_snapshot(entry)
             except Exception:
                 pass
 
-            if link and not snapshots.filter(timestamp=entry.name).exists():
+            if snapshot and not snapshots.filter(timestamp=entry.name).exists():
                 # folder is a valid link data dir with index details, but it's not in the main index
-                orphaned_folders[str(entry)] = link
+                orphaned_folders[str(entry)] = snapshot
 
     return orphaned_folders
 
-def get_corrupted_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_corrupted_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that don't contain a valid index and aren't listed in the main index"""
     corrupted = {}
     for snapshot in snapshots.iterator():
-        link = snapshot.as_link()
-        if is_corrupt(link):
-            corrupted[link.link_dir] = link
+        if is_corrupt(snapshot):
+            corrupted[snapshot.snapshot_dir] = snapshot
     return corrupted
 
-def get_unrecognized_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_unrecognized_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that don't contain recognizable archive data and aren't listed in the main index"""
-    unrecognized_folders: Dict[str, Optional[Link]] = {}
+    unrecognized_folders: Dict[str, Optional[Model]] = {}
 
     for entry in (Path(out_dir) / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
             index_exists = (entry / "index.json").exists()
-            link = None
+            snapshot = None
             try:
-                link = parse_json_link_details(str(entry))
+                snapshot = load_json_snapshot(entry)
             except KeyError:
                 # Try to fix index
                 if index_exists:
-                    try:
+                    pass
+                    # TODO: Implement the `guess` bit for snapshots
+                    # try:
                         # Last attempt to repair the detail index
-                        link_guessed = parse_json_link_details(str(entry), guess=True)
-                        write_json_link_details(link_guessed, out_dir=str(entry))
-                        link = parse_json_link_details(str(entry))
-                    except Exception:
-                        pass
+                        # link_guessed = parse_json_snapshot_details(str(entry), guess=True)
+                        # write_json_snapshot_details(link_guessed, out_dir=str(entry))
+                        # link = parse_json_link_details(str(entry))
+                    # except Exception:
+                    #     pass
 
-            if index_exists and link is None:
+            if index_exists and snapshot is None:
                 # index exists but it's corrupted or unparseable
-                unrecognized_folders[str(entry)] = link
+                unrecognized_folders[str(entry)] = snapshot
             
             elif not index_exists:
                 # link details index doesn't exist and the folder isn't in the main index
                 timestamp = entry.name
                 if not snapshots.filter(timestamp=timestamp).exists():
-                    unrecognized_folders[str(entry)] = link
+                    unrecognized_folders[str(entry)] = snapshot
 
     return unrecognized_folders
 
 
-def is_valid(link: Link) -> bool:
-    dir_exists = Path(link.link_dir).exists()
-    index_exists = (Path(link.link_dir) / "index.json").exists()
+def is_valid(snapshot: Model) -> bool:
+    dir_exists = Path(snapshot.snapshot_dir).exists()
+    index_exists = (Path(snapshot.snapshot_dir) / "index.json").exists()
     if not dir_exists:
         # unarchived links are not included in the valid list
         return False
@@ -568,29 +528,30 @@ def is_valid(link: Link) -> bool:
         return False
     if dir_exists and index_exists:
         try:
-            parsed_link = parse_json_link_details(link.link_dir, guess=True)
-            return link.url == parsed_link.url
+            # TODO: review if the `guess` was necessary here
+            parsed_snapshot = load_json_snapshot(snapshot.snapshot_dir)
+            return snapshot.url == parsed_snapshot.url
         except Exception:
             pass
     return False
 
-def is_corrupt(link: Link) -> bool:
-    if not Path(link.link_dir).exists():
+def is_corrupt(snapshot: Model) -> bool:
+    if not Path(snapshot.snapshot_dir).exists():
         # unarchived links are not considered corrupt
         return False
 
-    if is_valid(link):
+    if is_valid(snapshot):
         return False
 
     return True
 
-def is_archived(link: Link) -> bool:
-    return is_valid(link) and link.is_archived
+def is_archived(snapshot: Model) -> bool:
+    return is_valid(snapshot) and snapshot.is_archived
     
-def is_unarchived(link: Link) -> bool:
-    if not Path(link.link_dir).exists():
+def is_unarchived(snapshot: Model) -> bool:
+    if not Path(snapshot.snapshot_dir).exists():
         return True
-    return not link.is_archived
+    return not snapshot.is_archived
 
 
 def fix_invalid_folder_locations(out_dir: Path=OUTPUT_DIR) -> Tuple[List[str], List[str]]:
@@ -600,22 +561,22 @@ def fix_invalid_folder_locations(out_dir: Path=OUTPUT_DIR) -> Tuple[List[str], L
         if entry.is_dir(follow_symlinks=True):
             if (Path(entry.path) / 'index.json').exists():
                 try:
-                    link = parse_json_link_details(entry.path)
+                    snapshot = load_json_snapshot(Path(entry.path))
                 except KeyError:
-                    link = None
-                if not link:
+                    snapshot = None
+                if not snapshot:
                     continue
 
-                if not entry.path.endswith(f'/{link.timestamp}'):
-                    dest = out_dir / ARCHIVE_DIR_NAME / link.timestamp
+                if not entry.path.endswith(f'/{snapshot.timestamp}'):
+                    dest = out_dir / ARCHIVE_DIR_NAME / snapshot.timestamp
                     if dest.exists():
                         cant_fix.append(entry.path)
                     else:
                         shutil.move(entry.path, dest)
                         fixed.append(dest)
                         timestamp = entry.path.rsplit('/', 1)[-1]
-                        assert link.link_dir == entry.path
-                        assert link.timestamp == timestamp
-                        write_json_link_details(link, out_dir=entry.path)
+                        assert snapshot.snapshot_dir == entry.path
+                        assert snapshot.timestamp == timestamp
+                        write_json_snapshot_details(snapshot, out_dir=entry.path)
 
     return fixed, cant_fix

archivebox/index/csv.py

@@ -2,12 +2,14 @@ __package__ = 'archivebox.index'
 
 from typing import List, Optional, Any
 
+from django.db.models import Model
+
 from ..util import enforce_types
 from .schema import Link
 
 
 @enforce_types
-def links_to_csv(links: List[Link],
+def snapshots_to_csv(snapshots: List[Model],
                  cols: Optional[List[str]]=None,
                  header: bool=True,
                  separator: str=',',
@@ -20,8 +22,8 @@ def links_to_csv(links: List[Link],
         header_str = separator.join(col.ljust(ljust) for col in cols)
 
     row_strs = (
-        link.to_csv(cols=cols, ljust=ljust, separator=separator)
-        for link in links
+        snapshot.as_csv(cols=cols, ljust=ljust, separator=separator)
+        for snapshot in snapshots
     )
 
     return '\n'.join((header_str, *row_strs))

archivebox/index/html.py

@@ -1,19 +1,19 @@
 __package__ = 'archivebox.index'
 
-from pathlib import Path
-from datetime import datetime, timezone
-from collections import defaultdict
+from datetime import datetime
 from typing import List, Optional, Iterator, Mapping
+from pathlib import Path
 
+from django.db.models import Model
 from django.utils.html import format_html, mark_safe
-from django.core.cache import cache
+from collections import defaultdict
 
 from .schema import Link
 from ..system import atomic_write
 from ..logging_util import printable_filesize
 from ..util import (
     enforce_types,
-    ts_to_date_str,
+    ts_to_date,
     urlencode,
     htmlencode,
     urldecode,
@@ -21,10 +21,10 @@ from ..util import (
 from ..config import (
     OUTPUT_DIR,
     VERSION,
+    GIT_SHA,
     FOOTER_INFO,
     HTML_INDEX_FILENAME,
     SAVE_ARCHIVE_DOT_ORG,
-    PREVIEW_ORIGINALS,
 )
 
 MAIN_INDEX_TEMPLATE = 'static_index.html'
@@ -48,24 +48,24 @@ def parse_html_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[str]:
     return ()
 
 @enforce_types
-def generate_index_from_links(links: List[Link], with_headers: bool):
+def generate_index_from_snapshots(snapshots: List[Model], with_headers: bool):
     if with_headers:
-        output = main_index_template(links)
+        output = main_index_template(snapshots)
     else:
-        output = main_index_template(links, template=MINIMAL_INDEX_TEMPLATE)
+        output = main_index_template(snapshots, template=MINIMAL_INDEX_TEMPLATE)
     return output
 
 @enforce_types
-def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) -> str:
+def main_index_template(snapshots: List[Model], template: str=MAIN_INDEX_TEMPLATE) -> str:
     """render the template for the entire main index"""
 
     return render_django_template(template, {
         'version': VERSION,
-        'git_sha': VERSION,  # not used anymore, but kept for backwards compatibility
-        'num_links': str(len(links)),
-        'date_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d'),
-        'time_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M'),
-        'links': [link._asdict(extended=True) for link in links],
+        'git_sha': GIT_SHA,
+        'num_snapshots': str(len(snapshots)),
+        'date_updated': datetime.now().strftime('%Y-%m-%d'),
+        'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
+        'snapshots': snapshots,
         'FOOTER_INFO': FOOTER_INFO,
     })
 
@@ -73,40 +73,37 @@ def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) ->
 ### Link Details Index
 
 @enforce_types
-def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
-    out_dir = out_dir or link.link_dir
+def write_html_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> None:
+    out_dir = out_dir or snapshot.snapshot_dir
 
-    rendered_html = link_details_template(link)
+    rendered_html = snapshot_details_template(snapshot)
     atomic_write(str(Path(out_dir) / HTML_INDEX_FILENAME), rendered_html)
 
 
 @enforce_types
-def link_details_template(link: Link) -> str:
+def snapshot_details_template(snapshot: Model) -> str:
 
     from ..extractors.wget import wget_output_path
 
-    link_info = link._asdict(extended=True)
-
     return render_django_template(LINK_DETAILS_TEMPLATE, {
-        **link_info,
-        **link_info['canonical'],
+        **snapshot.as_json(),
+        **snapshot.canonical_outputs(),
         'title': htmlencode(
-            link.title
-            or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
+            snapshot.title
+            or (snapshot.base_url if snapshot.is_archived else TITLE_LOADING_MSG)
         ),
-        'url_str': htmlencode(urldecode(link.base_url)),
+        'url_str': htmlencode(urldecode(snapshot.base_url)),
         'archive_url': urlencode(
-            wget_output_path(link)
-            or (link.domain if link.is_archived else '')
+            wget_output_path(snapshot)
+            or (snapshot.domain if snapshot.is_archived else '')
         ) or 'about:blank',
-        'extension': link.extension or 'html',
-        'tags': link.tags or 'untagged',
-        'size': printable_filesize(link.archive_size) if link.archive_size else 'pending',
-        'status': 'archived' if link.is_archived else 'not yet archived',
-        'status_color': 'success' if link.is_archived else 'danger',
-        'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
+        'extension': snapshot.extension or 'html',
+        'tags': snapshot.tags_str() or 'untagged',
+        'size': printable_filesize(snapshot.archive_size) if snapshot.archive_size else 'pending',
+        'status': 'archived' if snapshot.is_archived else 'not yet archived',
+        'status_color': 'success' if snapshot.is_archived else 'danger',
+        'oldest_archive_date': ts_to_date(snapshot.oldest_archive_date),
         'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
-        'PREVIEW_ORIGINALS': PREVIEW_ORIGINALS,
     })
 
 @enforce_types
@@ -118,78 +115,70 @@ def render_django_template(template: str, context: Mapping[str, str]) -> str:
 
 
 def snapshot_icons(snapshot) -> str:
-    cache_key = f'{snapshot.id}-{(snapshot.updated or snapshot.added).timestamp()}-snapshot-icons'
-    
-    def calc_snapshot_icons():
-        from core.models import EXTRACTORS
-        # start = datetime.now(timezone.utc)
+    from core.models import EXTRACTORS
 
-        archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
-        link = snapshot.as_link()
-        path = link.archive_path
-        canon = link.canonical_outputs()
-        output = ""
-        output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
-        icons = {
-            "singlefile": "❶",
-            "wget": "🆆",
-            "dom": "🅷",
-            "pdf": "📄",
-            "screenshot": "💻",
-            "media": "📼",
-            "git": "🅶",
-            "archive_org": "🏛",
-            "readability": "🆁",
-            "mercury": "🅼",
-            "warc": "📦"
-        }
-        exclude = ["favicon", "title", "headers", "htmltotext", "archive_org"]
-        # Missing specific entry for WARC
+    # start = datetime.now()
 
-        extractor_outputs = defaultdict(lambda: None)
-        for extractor, _ in EXTRACTORS:
-            for result in archive_results:
-                if result.extractor == extractor and result:
-                    extractor_outputs[extractor] = result
+    archive_results = snapshot.archiveresult_set.filter(status="succeeded")
+    path = snapshot.archive_path
+    canon = snapshot.canonical_outputs()
+    output = ""
+    output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
+    icons = {
+        "singlefile": "❶",
+        "wget": "🆆",
+        "dom": "🅷",
+        "pdf": "📄",
+        "screenshot": "💻",
+        "media": "📼",
+        "git": "🅶",
+        "archive_org": "🏛",
+        "readability": "🆁",
+        "mercury": "🅼",
+        "warc": "📦"
+    }
+    exclude = ["favicon", "title", "headers", "archive_org"]
+    # Missing specific entry for WARC
 
-        for extractor, _ in EXTRACTORS:
-            if extractor not in exclude:
-                existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-                # Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
-                # if existing:
-                #     existing = (Path(path) / existing)
-                #     if existing.is_file():
-                #         existing = True
-                #     elif existing.is_dir():
-                #         existing = any(existing.glob('*.*'))
-                output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
-                                             extractor, icons.get(extractor, "?"))
-            if extractor == "wget":
-                # warc isn't technically it's own extractor, so we have to add it after wget
-                
-                # get from db (faster but less thurthful)
-                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-                # get from filesystem (slower but more accurate)
-                # exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
-                output += format_html(output_template, path, canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
+    extractor_outputs = defaultdict(lambda: None)
+    for extractor, _ in EXTRACTORS:
+        for result in archive_results:
+            if result.extractor == extractor and result:
+                extractor_outputs[extractor] = result
 
-            if extractor == "archive_org":
-                # The check for archive_org is different, so it has to be handled separately
+    for extractor, _ in EXTRACTORS:
+        if extractor not in exclude:
+            existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+            # Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
+            # if existing:
+            #     existing = (Path(path) / existing)
+            #     if existing.is_file():
+            #         existing = True
+            #     elif existing.is_dir():
+            #         existing = any(existing.glob('*.*'))
+            output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
+                                         extractor, icons.get(extractor, "?"))
+        if extractor == "wget":
+            # warc isn't technically it's own extractor, so we have to add it after wget
+            
+            # get from db (faster but less thurthful)
+            exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+            # get from filesystem (slower but more accurate)
+            # exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
+            output += format_html(output_template, 'warc/', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
 
-                # get from db (faster)
-                exists = extractor in extractor_outputs and extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-                # get from filesystem (slower)
-                # target_path = Path(path) / "archive.org.txt"
-                # exists = target_path.exists()
-                output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
-                                                                                            "archive_org", icons.get("archive_org", "?"))
+        if extractor == "archive_org":
+            # The check for archive_org is different, so it has to be handled separately
 
-        result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
-        # end = datetime.now(timezone.utc)
-        # print(((end - start).total_seconds()*1000) // 1, 'ms')
-        return result
+            # get from db (faster)
+            exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+            # get from filesystem (slower)
+            # target_path = Path(path) / "archive.org.txt"
+            # exists = target_path.exists()
+            output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
+                                                                                        "archive_org", icons.get("archive_org", "?"))
 
-    return cache.get_or_set(cache_key, calc_snapshot_icons)
-    # return calc_snapshot_icons()
-
-   
+    result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
+    # end = datetime.now()
+    # print(((end - start).total_seconds()*1000) // 1, 'ms')
+    return result

archivebox/index/json.py

@@ -5,8 +5,9 @@ import sys
 import json as pyjson
 from pathlib import Path
 
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import List, Optional, Iterator, Any, Union
+from django.db.models import Model
 
 from .schema import Link
 from ..system import atomic_write
@@ -15,6 +16,7 @@ from ..config import (
     VERSION,
     OUTPUT_DIR,
     FOOTER_INFO,
+    GIT_SHA,
     DEPENDENCIES,
     JSON_INDEX_FILENAME,
     ARCHIVE_DIR_NAME,
@@ -29,7 +31,7 @@ MAIN_INDEX_HEADER = {
     'meta': {
         'project': 'ArchiveBox',
         'version': VERSION,
-        'git_sha': VERSION,  # not used anymore, but kept for backwards compatibility
+        'git_sha': GIT_SHA,
         'website': 'https://ArchiveBox.io',
         'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
         'source': 'https://github.com/ArchiveBox/ArchiveBox',
@@ -39,17 +41,18 @@ MAIN_INDEX_HEADER = {
 }
 
 @enforce_types
-def generate_json_index_from_links(links: List[Link], with_headers: bool):
+def generate_json_index_from_snapshots(snapshots: List[Model], with_headers: bool):
+    snapshots_json = [snapshot.as_json() for snapshot in snapshots]
     if with_headers:
         output = {
             **MAIN_INDEX_HEADER,
-            'num_links': len(links),
-            'updated': datetime.now(timezone.utc),
+            'num_links': len(snapshots),
+            'updated': datetime.now(),
             'last_run_cmd': sys.argv,
-            'links': links,
+            'links': snapshots_json,
         }
     else:
-        output = links
+        output = snapshots_json
     return to_json(output, indent=4, sort_keys=True)
 
 
@@ -60,18 +63,7 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     index_path = Path(out_dir) / JSON_INDEX_FILENAME
     if index_path.exists():
         with open(index_path, 'r', encoding='utf-8') as f:
-            try:
-                links = pyjson.load(f)['links']
-                if links:
-                    Link.from_json(links[0])
-            except Exception as err:
-                print("    {lightyellow}! Found an index.json in the project root but couldn't load links from it: {} {}".format(
-                    err.__class__.__name__,
-                    err,
-                    **ANSI,
-                ))
-                return ()
-
+            links = pyjson.load(f)['links']
             for link_json in links:
                 try:
                     yield Link.from_json(link_json)
@@ -91,41 +83,46 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
 ### Link Details Index
 
 @enforce_types
-def write_json_link_details(link: Link, out_dir: Optional[str]=None) -> None:
-    """write a json file with some info about the link"""
+def write_json_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> None:
+    """write a json file with some info about the snapshot"""
     
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or snapshot.snapshot_dir
     path = Path(out_dir) / JSON_INDEX_FILENAME
-    atomic_write(str(path), link._asdict(extended=True))
+    atomic_write(str(path), snapshot.as_json())
 
 
 @enforce_types
-def parse_json_link_details(out_dir: Union[Path, str], guess: Optional[bool]=False) -> Optional[Link]:
-    """load the json link index from a given directory"""
+def load_json_snapshot(out_dir: Path) -> Optional[Model]:
+    """
+    Loads the detail from the local json index
+    """
+    from core.models import Snapshot
+
     existing_index = Path(out_dir) / JSON_INDEX_FILENAME
     if existing_index.exists():
         with open(existing_index, 'r', encoding='utf-8') as f:
             try:
-                link_json = pyjson.load(f)
-                return Link.from_json(link_json, guess)
+                output = pyjson.load(f)
+                output = Snapshot.from_json(output)
+                return output
             except pyjson.JSONDecodeError:
                 pass
     return None
 
 
 @enforce_types
-def parse_json_links_details(out_dir: Union[Path, str]) -> Iterator[Link]:
-    """read through all the archive data folders and return the parsed links"""
+def parse_json_snapshot_details(out_dir: Union[Path, str]) -> Iterator[dict]:
+    """read through all the archive data folders and return the parsed snapshots"""
 
     for entry in os.scandir(Path(out_dir) / ARCHIVE_DIR_NAME):
         if entry.is_dir(follow_symlinks=True):
             if (Path(entry.path) / 'index.json').exists():
                 try:
-                    link = parse_json_link_details(entry.path)
+                    snapshot_details = load_json_snapshot(Path(entry.path))
                 except KeyError:
-                    link = None
-                if link:
-                    yield link
+                    snapshot_details = None
+                if snapshot_details:
+                    yield snapshot_details
 
 
 

archivebox/index/schema.py

@@ -10,17 +10,16 @@ __package__ = 'archivebox.index'
 
 from pathlib import Path
 
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta
 
 from typing import List, Dict, Any, Optional, Union
 
 from dataclasses import dataclass, asdict, field, fields
 
-from django.utils.functional import cached_property
 
 from ..system import get_dir_size
-from ..util import ts_to_date_str, parse_date
-from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME, FAVICON_PROVIDER
+
+from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
 
 class ArchiveError(Exception):
     def __init__(self, message, hints=None):
@@ -56,11 +55,11 @@ class ArchiveResult:
         assert isinstance(self.end_ts, datetime)
         assert isinstance(self.cmd, list)
         assert all(isinstance(arg, str) and arg for arg in self.cmd)
-
-        # TODO: replace emptystrings in these three with None / remove them from the DB
-        assert self.pwd is None or isinstance(self.pwd, str)
-        assert self.cmd_version is None or isinstance(self.cmd_version, str)
+        assert self.pwd is None or isinstance(self.pwd, str) and self.pwd
+        assert self.cmd_version is None or isinstance(self.cmd_version, str) and self.cmd_version
         assert self.output is None or isinstance(self.output, (str, Exception))
+        if isinstance(self.output, str):
+            assert self.output
 
     @classmethod
     def guess_ts(_cls, dict_info):
@@ -134,6 +133,7 @@ class Link:
     updated: Optional[datetime] = None
     schema: str = 'Link'
 
+
     def __str__(self) -> str:
         return f'[{self.timestamp}] {self.url} "{self.title}"'
 
@@ -190,7 +190,6 @@ class Link:
         }
         if extended:
             info.update({
-                'snapshot_id': self.snapshot_id,
                 'link_dir': self.link_dir,
                 'archive_path': self.archive_path,
                 
@@ -202,9 +201,6 @@ class Link:
                 'basename': self.basename,
                 'extension': self.extension,
                 'is_static': self.is_static,
-                
-                'tags_str': (self.tags or '').strip(','),   # only used to render static index in index/html.py, remove if no longer needed there
-                'icons': None,           # only used to render static index in index/html.py, remove if no longer needed there
 
                 'bookmarked_date': self.bookmarked_date,
                 'updated_date': self.updated_date,
@@ -259,11 +255,6 @@ class Link:
 
         return to_csv(self, cols=cols or self.field_names(), separator=separator, ljust=ljust)
 
-    @cached_property
-    def snapshot_id(self):
-        from core.models import Snapshot
-        return str(Snapshot.objects.only('id').get(url=self.url).id)
-
     @classmethod
     def field_names(cls):
         return [f.name for f in fields(cls)]
@@ -325,11 +316,13 @@ class Link:
     ### Pretty Printing Helpers
     @property
     def bookmarked_date(self) -> Optional[str]:
-        max_ts = (datetime.now(timezone.utc) + timedelta(days=30)).timestamp()
+        from ..util import ts_to_date
+
+        max_ts = (datetime.now() + timedelta(days=30)).timestamp()
 
         if self.timestamp and self.timestamp.replace('.', '').isdigit():
             if 0 < float(self.timestamp) < max_ts:
-                return ts_to_date_str(datetime.fromtimestamp(float(self.timestamp)))
+                return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
             else:
                 return str(self.timestamp)
         return None
@@ -337,12 +330,13 @@ class Link:
 
     @property
     def updated_date(self) -> Optional[str]:
-        return ts_to_date_str(self.updated) if self.updated else None
+        from ..util import ts_to_date
+        return ts_to_date(self.updated) if self.updated else None
 
     @property
     def archive_dates(self) -> List[datetime]:
         return [
-            parse_date(result.start_ts)
+            result.start_ts
             for method in self.history.keys()
                 for result in self.history[method]
         ]
@@ -414,6 +408,7 @@ class Link:
         return latest
 
 
+
     def canonical_outputs(self) -> Dict[str, Optional[str]]:
         """predict the expected output paths that should be present after archiving"""
 
@@ -423,13 +418,12 @@ class Link:
         canonical = {
             'index_path': 'index.html',
             'favicon_path': 'favicon.ico',
-            'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
+            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
             'wget_path': wget_output_path(self),
             'warc_path': 'warc/',
             'singlefile_path': 'singlefile.html',
             'readability_path': 'readability/content.html',
             'mercury_path': 'mercury/content.html',
-            'htmltotext_path': 'htmltotext.txt',
             'pdf_path': 'output.pdf',
             'screenshot_path': 'screenshot.png',
             'dom_path': 'output.html',
@@ -453,7 +447,6 @@ class Link:
                 'singlefile_path': static_path,
                 'readability_path': static_path,
                 'mercury_path': static_path,
-                'htmltotext_path': static_path,
             })
         return canonical
 

archivebox/index/sql.py

@@ -1,19 +1,15 @@
 __package__ = 'archivebox.index'
 
-import re
-
 from io import StringIO
 from pathlib import Path
 from typing import List, Tuple, Iterator
-from django.db.models import QuerySet
+from django.db.models import QuerySet, Model
 from django.db import transaction
+from datetime import datetime
 
 from .schema import Link
-from ..util import enforce_types, parse_date
-from ..config import (
-    OUTPUT_DIR,
-    TAG_SEPARATOR_PATTERN,
-)
+from ..util import enforce_types
+from ..config import OUTPUT_DIR
 
 
 ### Main Links Index
@@ -28,97 +24,55 @@ def parse_sql_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     )
 
 @enforce_types
-def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
-    if atomic:
-        with transaction.atomic():
-            return snapshots.delete()
-    return snapshots.delete()
+def remove_from_sql_main_index(snapshots: QuerySet, out_dir: Path=OUTPUT_DIR) -> None:
+    with transaction.atomic():
+        snapshots.delete()
 
 @enforce_types
-def write_link_to_sql_index(link: Link):
-    from core.models import Snapshot, ArchiveResult
-    info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}
-
-    tag_list = list(dict.fromkeys(
-        tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')
-    ))
-    info.pop('tags')
-
+def write_snapshot_to_index(snapshot: Model):
+    from core.models import Snapshot
     try:
-        info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp
+        timestamp = Snapshot.objects.get(url=snapshot.url).timestamp
     except Snapshot.DoesNotExist:
-        while Snapshot.objects.filter(timestamp=info["timestamp"]).exists():
-            info["timestamp"] = str(float(info["timestamp"]) + 1.0)
-
-        snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info)
-    snapshot.save_tags(tag_list)
-
-    for extractor, entries in link.history.items():
-        for entry in entries:
-            if isinstance(entry, dict):
-                result, _ = ArchiveResult.objects.get_or_create(
-                    snapshot_id=snapshot.id,
-                    extractor=extractor,
-                    start_ts=parse_date(entry['start_ts']),
-                    defaults={
-                        'end_ts': parse_date(entry['end_ts']),
-                        'cmd': entry['cmd'],
-                        'output': entry['output'],
-                        'cmd_version': entry.get('cmd_version') or 'unknown',
-                        'pwd': entry['pwd'],
-                        'status': entry['status'],
-                    }
-                )
-            else:
-                result, _ = ArchiveResult.objects.update_or_create(
-                    snapshot_id=snapshot.id,
-                    extractor=extractor,
-                    start_ts=parse_date(entry.start_ts),
-                    defaults={
-                        'end_ts': parse_date(entry.end_ts),
-                        'cmd': entry.cmd,
-                        'output': entry.output,
-                        'cmd_version': entry.cmd_version or 'unknown',
-                        'pwd': entry.pwd,
-                        'status': entry.status,
-                    }
-                )
+        timestamp = snapshot.timestamp
+        if not timestamp:
+            timestamp = str(datetime.now().timestamp())
+        while Snapshot.objects.filter(timestamp=timestamp).exists():
+            print("the timestamp is: ", timestamp)
+            timestamp = str(float(timestamp) + 1.0)
 
+    snapshot.timestamp = timestamp
+    snapshot.save()
     return snapshot
 
 
 @enforce_types
-def write_sql_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
-    for link in links:
-        # with transaction.atomic():
-            # write_link_to_sql_index(link)
-        write_link_to_sql_index(link)
+def write_sql_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
+    with transaction.atomic():
+        for snapshot in snapshots:
+            write_snapshot_to_index(snapshot)
             
 
 @enforce_types
-def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
+def write_sql_snapshot_details(snapshot: Model, out_dir: Path=OUTPUT_DIR) -> None:
     from core.models import Snapshot
 
-    # with transaction.atomic():
-    #     try:
-    #         snap = Snapshot.objects.get(url=link.url)
-    #     except Snapshot.DoesNotExist:
-    #         snap = write_link_to_sql_index(link)
-    #     snap.title = link.title
-    try:
-        snap = Snapshot.objects.get(url=link.url)
-    except Snapshot.DoesNotExist:
-        snap = write_link_to_sql_index(link)
+    with transaction.atomic():
+        try:
+            snap = Snapshot.objects.get(url=snapshot.url)
+        except Snapshot.DoesNotExist:
+            snap = write_snapshot_to_index(snapshot)
+        snap.title = snapshot.title
 
-    snap.title = link.title
+        # TODO: If there are actual tags, this will break
+        #tag_set = (
+        #    set(tag.strip() for tag in (snapshot.tags.all() or '').split(','))
+        #)
+        #tag_list = list(tag_set) or []
 
-    tag_list = list(
-        {tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')}
-        | set(snap.tags.values_list('name', flat=True))
-    )
-
-    snap.save()
-    snap.save_tags(tag_list)
+        snap.save()
+        #snap.save_tags(tag_list)
+        return snap
 
 
 

archivebox/logging_util.py

@@ -3,30 +3,26 @@ __package__ = 'archivebox'
 import re
 import os
 import sys
-import stat
 import time
 import argparse
 from math import log
 from multiprocessing import Process
 from pathlib import Path
 
-from datetime import datetime, timezone
+from datetime import datetime
 from dataclasses import dataclass
-from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
+from typing import Optional, List, Dict, Union, IO, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .index.schema import Link, ArchiveResult
 
-from .system import get_dir_size
 from .util import enforce_types
 from .config import (
     ConfigDict,
     OUTPUT_DIR,
     PYTHON_ENCODING,
-    VERSION,
     ANSI,
     IS_TTY,
-    IN_DOCKER,
     TERM_WIDTH,
     SHOW_PROGRESS,
     SOURCES_DIR_NAME,
@@ -54,37 +50,6 @@ class RuntimeStats:
 _LAST_RUN_STATS = RuntimeStats()
 
 
-def debug_dict_summary(obj: Dict[Any, Any]) -> None:
-    stderr(' '.join(f'{key}={str(val).ljust(6)}' for key, val in obj.items()))
-
-
-def get_fd_info(fd) -> Dict[str, Any]:
-    NAME = fd.name[1:-1]
-    FILENO = fd.fileno()
-    MODE = os.fstat(FILENO).st_mode
-    IS_TTY = hasattr(fd, 'isatty') and fd.isatty()
-    IS_PIPE = stat.S_ISFIFO(MODE)
-    IS_FILE = stat.S_ISREG(MODE)
-    IS_TERMINAL =  not (IS_PIPE or IS_FILE)
-    IS_LINE_BUFFERED = fd.line_buffering
-    IS_READABLE = fd.readable()
-    return {
-        'NAME': NAME, 'FILENO': FILENO, 'MODE': MODE,
-        'IS_TTY': IS_TTY, 'IS_PIPE': IS_PIPE, 'IS_FILE': IS_FILE,
-        'IS_TERMINAL': IS_TERMINAL, 'IS_LINE_BUFFERED': IS_LINE_BUFFERED,
-        'IS_READABLE': IS_READABLE,
-    }
-    
-
-# # Log debug information about stdin, stdout, and stderr
-# sys.stdout.write('[>&1] this is python stdout\n')
-# sys.stderr.write('[>&2] this is python stderr\n')
-
-# debug_dict_summary(get_fd_info(sys.stdin))
-# debug_dict_summary(get_fd_info(sys.stdout))
-# debug_dict_summary(get_fd_info(sys.stderr))
-
-
 
 class SmartFormatter(argparse.HelpFormatter):
     """Patched formatter that prints newlines in argparse help strings"""
@@ -97,40 +62,22 @@ class SmartFormatter(argparse.HelpFormatter):
 def reject_stdin(caller: str, stdin: Optional[IO]=sys.stdin) -> None:
     """Tell the user they passed stdin to a command that doesn't accept it"""
 
-    if not stdin:
-        return None
-
-    if IN_DOCKER:
-        # when TTY is disabled in docker we cant tell if stdin is being piped in or not
-        # if we try to read stdin when its not piped we will hang indefinitely waiting for it
-        return None
-
-    if not stdin.isatty():
-        # stderr('READING STDIN TO REJECT...')
-        stdin_raw_text = stdin.read()
-        if stdin_raw_text.strip():
-            # stderr('GOT STDIN!', len(stdin_str))
-            stderr(f'[!] The "{caller}" command does not accept stdin (ignoring).', color='red')
+    if stdin and not stdin.isatty():
+        stdin_raw_text = stdin.read().strip()
+        if stdin_raw_text:
+            stderr(f'[X] The "{caller}" command does not accept stdin.', color='red')
             stderr(f'    Run archivebox "{caller} --help" to see usage and examples.')
             stderr()
-            # raise SystemExit(1)
-    return None
+            raise SystemExit(1)
 
 
 def accept_stdin(stdin: Optional[IO]=sys.stdin) -> Optional[str]:
     """accept any standard input and return it as a string or None"""
-    
     if not stdin:
         return None
-
-    if not stdin.isatty():
-        # stderr('READING STDIN TO ACCEPT...')
-        stdin_str = stdin.read()
-
-        if stdin_str:
-            # stderr('GOT STDIN...', len(stdin_str))
-            return stdin_str
-
+    elif stdin and not stdin.isatty():
+        stdin_str = stdin.read().strip()
+        return stdin_str or None
     return None
 
 
@@ -138,19 +85,17 @@ class TimedProgress:
     """Show a progress bar and measure elapsed time until .end() is called"""
 
     def __init__(self, seconds, prefix=''):
-
         self.SHOW_PROGRESS = SHOW_PROGRESS
         if self.SHOW_PROGRESS:
             self.p = Process(target=progress_bar, args=(seconds, prefix))
             self.p.start()
 
-        self.stats = {'start_ts': datetime.now(timezone.utc), 'end_ts': None}
+        self.stats = {'start_ts': datetime.now(), 'end_ts': None}
 
     def end(self):
         """immediately end progress, clear the progressbar line, and save end_ts"""
 
-
-        end_ts = datetime.now(timezone.utc)
+        end_ts = datetime.now()
         self.stats['end_ts'] = end_ts
         
         if self.SHOW_PROGRESS:
@@ -159,10 +104,7 @@ class TimedProgress:
                 # kill the progress bar subprocess
                 try:
                     self.p.close()   # must be closed *before* its terminnated
-                except (KeyboardInterrupt, SystemExit):
-                    print()
-                    raise
-                except BaseException:                                           # lgtm [py/catch-base-exception]
+                except:
                     pass
                 self.p.terminate()
                 self.p.join()
@@ -228,12 +170,14 @@ def progress_bar(seconds: int, prefix: str='') -> None:
         # sys.stdout.flush()
     except (KeyboardInterrupt, BrokenPipeError):
         print()
+        pass
 
 
 def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str):
+    from .config import VERSION, ANSI
     cmd = ' '.join(('archivebox', subcommand, *subcommand_args))
     stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{reset}'.format(
-        now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
+        now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
         VERSION=VERSION,
         cmd=cmd,
         **ANSI,
@@ -245,7 +189,7 @@ def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional
 
 
 def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool):
-    _LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc)
+    _LAST_RUN_STATS.parse_start_ts = datetime.now()
     print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format(
         _LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'),
         len(urls) if isinstance(urls, list) else len(urls.split('\n')),
@@ -258,7 +202,7 @@ def log_source_saved(source_file: str):
     print('    > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1]))
 
 def log_parsing_finished(num_parsed: int, parser_name: str):
-    _LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc)
+    _LAST_RUN_STATS.parse_end_ts = datetime.now()
     print('    > Parsed {} URLs from input ({})'.format(num_parsed, parser_name))
 
 def log_deduping_finished(num_new_links: int):
@@ -272,7 +216,7 @@ def log_crawl_started(new_links):
 ### Indexing Stage
 
 def log_indexing_process_started(num_links: int):
-    start_ts = datetime.now(timezone.utc)
+    start_ts = datetime.now()
     _LAST_RUN_STATS.index_start_ts = start_ts
     print()
     print('{black}[*] [{}] Writing {} links to main index...{reset}'.format(
@@ -283,24 +227,23 @@ def log_indexing_process_started(num_links: int):
 
 
 def log_indexing_process_finished():
-    end_ts = datetime.now(timezone.utc)
+    end_ts = datetime.now()
     _LAST_RUN_STATS.index_end_ts = end_ts
 
 
 def log_indexing_started(out_path: str):
     if IS_TTY:
-        sys.stdout.write(f'    > ./{Path(out_path).relative_to(OUTPUT_DIR)}')
+        sys.stdout.write(f'    > {out_path}')
 
 
 def log_indexing_finished(out_path: str):
-    print(f'\r    √ ./{Path(out_path).relative_to(OUTPUT_DIR)}')
+    print(f'\r    √ {out_path}')
 
 
 ### Archiving Stage
 
 def log_archiving_started(num_links: int, resume: Optional[float]=None):
-
-    start_ts = datetime.now(timezone.utc)
+    start_ts = datetime.now()
     _LAST_RUN_STATS.archiving_start_ts = start_ts
     print()
     if resume:
@@ -318,8 +261,7 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
         ))
 
 def log_archiving_paused(num_links: int, idx: int, timestamp: str):
-
-    end_ts = datetime.now(timezone.utc)
+    end_ts = datetime.now()
     _LAST_RUN_STATS.archiving_end_ts = end_ts
     print()
     print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
@@ -330,14 +272,13 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str):
         total=num_links,
     ))
     print()
+    print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI))
+    print('        archivebox server  # then visit http://127.0.0.1:8000')
     print('    Continue archiving where you left off by running:')
     print('        archivebox update --resume={}'.format(timestamp))
 
 def log_archiving_finished(num_links: int):
-
-    from core.models import Snapshot
-
-    end_ts = datetime.now(timezone.utc)
+    end_ts = datetime.now()
     _LAST_RUN_STATS.archiving_end_ts = end_ts
     assert _LAST_RUN_STATS.archiving_start_ts is not None
     seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
@@ -357,15 +298,12 @@ def log_archiving_finished(num_links: int):
     print('    - {} links skipped'.format(_LAST_RUN_STATS.skipped))
     print('    - {} links updated'.format(_LAST_RUN_STATS.succeeded + _LAST_RUN_STATS.failed))
     print('    - {} links had errors'.format(_LAST_RUN_STATS.failed))
-    
-    if Snapshot.objects.count() < 50:
-        print()
-        print('    {lightred}Hint:{reset} To manage your archive in a Web UI, run:'.format(**ANSI))
-        print('        archivebox server 0.0.0.0:8000')
+    print()
+    print('    {lightred}Hint:{reset} To manage your archive in a Web UI, run:'.format(**ANSI))
+    print('        archivebox server 0.0.0.0:8000')
 
 
-def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
-
+def log_snapshot_archiving_started(snapshot: Model, snapshot_dir: str, is_new: bool):
     # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
     #     http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
     #     > output/archive/1478739709
@@ -373,17 +311,17 @@ def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
     print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format(
         symbol_color=ANSI['green' if is_new else 'black'],
         symbol='+' if is_new else '√',
-        now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
-        title=link.title or link.base_url,
+        now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+        title=snapshot.title or snapshot.base_url,
         **ANSI,
     ))
-    print('    {blue}{url}{reset}'.format(url=link.url, **ANSI))
+    print('    {blue}{url}{reset}'.format(url=snapshot.url, **ANSI))
     print('    {} {}'.format(
         '>' if is_new else '√',
-        pretty_path(link_dir),
+        pretty_path(snapshot_dir),
     ))
 
-def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats: dict, start_ts: datetime):
+def log_snapshot_archiving_finished(snapshot: Model, snapshot_dir: str, is_new: bool, stats: dict):
     total = sum(stats.values())
 
     if stats['failed'] > 0 :
@@ -393,15 +331,6 @@ def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats
     else:
         _LAST_RUN_STATS.succeeded += 1
 
-    try:
-        size = get_dir_size(link_dir)
-    except FileNotFoundError:
-        size = (0, None, '0')
-
-    end_ts = datetime.now(timezone.utc)
-    duration = str(end_ts - start_ts).split('.')[0]
-    print('        {black}{} files ({}) in {}s {reset}'.format(size[2], printable_filesize(size[0]), duration, **ANSI))
-
 
 def log_archive_method_started(method: str):
     print('      > {}'.format(method))
@@ -413,7 +342,7 @@ def log_archive_method_finished(result: "ArchiveResult"):
     """
     # Prettify CMD string and make it safe to copy-paste by quoting arguments
     quoted_cmd = ' '.join(
-        '"{}"'.format(arg) if (' ' in arg) or (':' in arg) else arg
+        '"{}"'.format(arg) if ' ' in arg else arg
         for arg in result.cmd
     )
 
@@ -436,30 +365,18 @@ def log_archive_method_finished(result: "ArchiveResult"):
         # Prettify error output hints string and limit to five lines
         hints = getattr(result.output, 'hints', None) or ()
         if hints:
-            if isinstance(hints, (list, tuple, type(_ for _ in ()))):
-                hints = [hint.decode() for hint in hints if isinstance(hint, bytes)]
-            else:
-                if isinstance(hints, bytes):
-                    hints = hints.decode()
-                hints = hints.split('\n')
-
+            hints = hints if isinstance(hints, (list, tuple)) else hints.split('\n')
             hints = (
                 '    {}{}{}'.format(ANSI['lightyellow'], line.strip(), ANSI['reset'])
-                for line in list(hints)[:5] if line.strip()
+                for line in hints[:5] if line.strip()
             )
 
-        docker_hints = ()
-        if IN_DOCKER:
-            docker_hints = (
-                '  docker run -it -v $PWD/data:/data archivebox/archivebox /bin/bash',
-            )
 
         # Collect and prefix output lines with indentation
         output_lines = [
             *hint_header,
             *hints,
             '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']),
-            *docker_hints,
             *(['    cd {};'.format(result.pwd)] if result.pwd else []),
             '    {}'.format(quoted_cmd),
         ]
@@ -478,57 +395,57 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str):
     ))
     print('    {}'.format(' '.join(filter_patterns or ())))
 
-def log_list_finished(links):
-    from .index.csv import links_to_csv
+def log_list_finished(snapshots):
+    from .index.csv import snapshots_to_csv
     print()
     print('---------------------------------------------------------------------------------------------------')
-    print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
+    print(snapshots_to_csv(snapshots, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
     print('---------------------------------------------------------------------------------------------------')
     print()
 
 
-def log_removal_started(links: List["Link"], yes: bool, delete: bool):
-    print('{lightyellow}[i] Found {} matching URLs to remove.{reset}'.format(len(links), **ANSI))
+def log_removal_started(snapshots: List["Snapshot"], yes: bool, delete: bool):
+    print('{lightyellow}[i] Found {} matching URLs to remove.{reset}'.format(len(snapshots), **ANSI))
     if delete:
-        file_counts = [link.num_outputs for link in links if Path(link.link_dir).exists()]
+        file_counts = [snapshot.num_outputs for snapshot in snapshots if Path(snapshot.snapshot_dir).exists()]
         print(
-            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
+            f'    {len(snapshots)} Snapshots will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
             f'    ({len(file_counts)} data folders with {sum(file_counts)} archived files will be deleted!)'
         )
     else:
         print(
-            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
+            '    Matching snapshots will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
             '    (Pass --delete if you also want to permanently delete the data folders)'
         )
 
     if not yes:
         print()
-        print('{lightyellow}[?] Do you want to proceed with removing these {} links?{reset}'.format(len(links), **ANSI))
+        print('{lightyellow}[?] Do you want to proceed with removing these {} snapshots?{reset}'.format(len(snapshots), **ANSI))
         try:
             assert input('    y/[n]: ').lower() == 'y'
         except (KeyboardInterrupt, EOFError, AssertionError):
             raise SystemExit(0)
 
-def log_removal_finished(all_links: int, to_remove: int):
-    if all_links == 0:
+def log_removal_finished(all_snapshots: int, to_remove: int):
+    if to_remove == 0:
         print()
-        print('{red}[X] No matching links found.{reset}'.format(**ANSI))
+        print('{red}[X] No matching snapshots found.{reset}'.format(**ANSI))
     else:
         print()
-        print('{red}[√] Removed {} out of {} links from the archive index.{reset}'.format(
+        print('{red}[√] Removed {} out of {} snapshots from the archive index.{reset}'.format(
             to_remove,
-            all_links,
+            all_snapshots,
             **ANSI,
         ))
-        print('    Index now contains {} links.'.format(all_links - to_remove))
+        print('    Index now contains {} snapshots.'.format(all_snapshots - to_remove))
 
 
 def log_shell_welcome_msg():
     from .cli import list_subcommands
 
     print('{green}# ArchiveBox Imports{reset}'.format(**ANSI))
-    print('{green}from archivebox.core.models import Snapshot, ArchiveResult, Tag, User{reset}'.format(**ANSI))
-    print('{green}from archivebox.cli import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
+    print('{green}from core.models import Snapshot, User{reset}'.format(**ANSI))
+    print('{green}from archivebox import *\n    {}{reset}'.format("\n    ".join(list_subcommands().keys()), **ANSI))
     print()
     print('[i] Welcome to the ArchiveBox Shell!')
     print('    https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage')
@@ -543,27 +460,11 @@ def log_shell_welcome_msg():
 ### Helpers
 
 @enforce_types
-def pretty_path(path: Union[Path, str], pwd: Union[Path, str]=OUTPUT_DIR) -> str:
+def pretty_path(path: Union[Path, str]) -> str:
     """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
-    pwd = str(Path(pwd))  # .resolve()
-    path = str(path)
-
-    if not path:
-        return path
-
-    # replace long absolute paths with ./ relative ones to save on terminal output width
-    if path.startswith(pwd) and (pwd != '/'):
-        path = path.replace(pwd, '.', 1)
-    
-    # quote paths containing spaces
-    if ' ' in path:
-        path = f'"{path}"'
-
-    # if path is just a plain dot, replace it back with the absolute path for clarity
-    if path == '.':
-        path = pwd
-
-    return path
+    pwd = Path('.').resolve()
+    # parent = os.path.abspath(os.path.join(pwd, os.path.pardir))
+    return str(path).replace(str(pwd) + '/', './')
 
 
 @enforce_types
@@ -598,13 +499,12 @@ def printable_config(config: ConfigDict, prefix: str='') -> str:
 def printable_folder_status(name: str, folder: Dict) -> str:
     if folder['enabled']:
         if folder['is_valid']:
-            color, symbol, note, num_files = 'green', '√', 'valid', ''
+            color, symbol, note = 'green', '√', 'valid'
         else:
             color, symbol, note, num_files = 'red', 'X', 'invalid', '?'
     else:
         color, symbol, note, num_files = 'lightyellow', '-', 'disabled', '-'
 
-
     if folder['path']:
         if Path(folder['path']).exists():
             num_files = (
@@ -614,12 +514,14 @@ def printable_folder_status(name: str, folder: Dict) -> str:
             )
         else:
             num_files = 'missing'
-        
-    if folder.get('is_mount'):
-        # add symbol @ next to filecount if path is a remote filesystem mount
-        num_files = f'{num_files} @' if num_files else '@'
 
-    path = pretty_path(folder['path'])
+    path = str(folder['path']).replace(str(OUTPUT_DIR), '.') if folder['path'] else ''
+    if path and ' ' in path:
+        path = f'"{path}"'
+
+    # if path is just a plain dot, replace it back with the full path for clarity
+    if path == '.':
+        path = str(OUTPUT_DIR)
 
     return ' '.join((
         ANSI[color],
@@ -650,7 +552,9 @@ def printable_dependency_version(name: str, dependency: Dict) -> str:
     else:
         color, symbol, note, version = 'lightyellow', '-', 'disabled', '-'
 
-    path = pretty_path(dependency['path'])
+    path = str(dependency["path"]).replace(str(OUTPUT_DIR), '.') if dependency["path"] else ''
+    if path and ' ' in path:
+        path = f'"{path}"'
 
     return ' '.join((
         ANSI[color],

archivebox/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "archivebox",
-  "version": "0.7.4",
-  "description": "ArchiveBox: The self-hosted internet archive",
-  "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
-  "repository": "github:ArchiveBox/ArchiveBox",
-  "license": "MIT",
-  "dependencies": {
-    "@postlight/parser": "^2.2.3",
-    "readability-extractor": "github:ArchiveBox/readability-extractor",
-    "single-file-cli": "^2.0.83"
-  }
-}

archivebox/package.json

@@ -0,0 +1 @@
+../package.json

archivebox/parsers/__init__.py

@@ -11,17 +11,17 @@ import re
 from io import StringIO
 
 from typing import IO, Tuple, List, Optional
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path 
 
+from django.db.models import Model
+
 from ..system import atomic_write
 from ..config import (
     ANSI,
     OUTPUT_DIR,
     SOURCES_DIR_NAME,
     TIMEOUT,
-    stderr,
-    hint,
 )
 from ..util import (
     basename,
@@ -33,138 +33,108 @@ from ..util import (
 from ..index.schema import Link
 from ..logging_util import TimedProgress, log_source_saved
 
-from . import pocket_api
-from . import readwise_reader_api
-from . import wallabag_atom
-from . import pocket_html
-from . import pinboard_rss
-from . import shaarli_rss
-from . import medium_rss
+from .pocket_html import parse_pocket_html_export
+from .pocket_api import parse_pocket_api_export
+from .pinboard_rss import parse_pinboard_rss_export
+from .wallabag_atom import parse_wallabag_atom_export
+from .shaarli_rss import parse_shaarli_rss_export
+from .medium_rss import parse_medium_rss_export
+from .netscape_html import parse_netscape_html_export
+from .generic_rss import parse_generic_rss_export
+from .generic_json import parse_generic_json_export
+from .generic_html import parse_generic_html_export
+from .generic_txt import parse_generic_txt_export
 
-from . import netscape_html
-from . import generic_rss
-from . import generic_json
-from . import generic_html
-from . import generic_txt
-from . import url_list
-
-
-PARSERS = {
+PARSERS = (
     # Specialized parsers
-    pocket_api.KEY:     (pocket_api.NAME,       pocket_api.PARSER),
-    readwise_reader_api.KEY: (readwise_reader_api.NAME, readwise_reader_api.PARSER),
-    wallabag_atom.KEY:  (wallabag_atom.NAME,    wallabag_atom.PARSER),
-    pocket_html.KEY:    (pocket_html.NAME,      pocket_html.PARSER),
-    pinboard_rss.KEY:   (pinboard_rss.NAME,     pinboard_rss.PARSER),
-    shaarli_rss.KEY:    (shaarli_rss.NAME,      shaarli_rss.PARSER),
-    medium_rss.KEY:     (medium_rss.NAME,       medium_rss.PARSER),
-
+    ('Pocket API', parse_pocket_api_export),
+    ('Wallabag ATOM', parse_wallabag_atom_export),
+    ('Pocket HTML', parse_pocket_html_export),
+    ('Pinboard RSS', parse_pinboard_rss_export),
+    ('Shaarli RSS', parse_shaarli_rss_export),
+    ('Medium RSS', parse_medium_rss_export),
+    
     # General parsers
-    netscape_html.KEY:  (netscape_html.NAME,    netscape_html.PARSER),
-    generic_rss.KEY:    (generic_rss.NAME,      generic_rss.PARSER),
-    generic_json.KEY:   (generic_json.NAME,     generic_json.PARSER),
-    generic_html.KEY:   (generic_html.NAME,     generic_html.PARSER),
+    ('Netscape HTML', parse_netscape_html_export),
+    ('Generic RSS', parse_generic_rss_export),
+    ('Generic JSON', parse_generic_json_export),
+    ('Generic HTML', parse_generic_html_export),
 
-    # Catchall fallback parser
-    generic_txt.KEY:    (generic_txt.NAME,      generic_txt.PARSER),
-
-    # Explicitly specified parsers
-    url_list.KEY:       (url_list.NAME,         url_list.PARSER),
-}
+    # Fallback parser
+    ('Plain Text', parse_generic_txt_export),
+)
 
 
 @enforce_types
-def parse_links_memory(urls: List[str], root_url: Optional[str]=None):
+def parse_snapshots_memory(urls: List[str], root_url: Optional[str]=None):
     """
     parse a list of URLS without touching the filesystem
     """
+    check_url_parsing_invariants()
 
     timer = TimedProgress(TIMEOUT * 4)
     #urls = list(map(lambda x: x + "\n", urls))
     file = StringIO()
     file.writelines(urls)
     file.name = "io_string"
-    links, parser = run_parser_functions(file, timer, root_url=root_url)
+    snapshots, parser = run_parser_functions(file, timer, root_url=root_url)
     timer.end()
 
     if parser is None:
         return [], 'Failed to parse'
-    return links, parser
+    return snapshots, parser
     
 
 @enforce_types
-def parse_links(source_file: str, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], str]:
+def parse_snapshots(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Model], str]:
     """parse a list of URLs with their metadata from an 
        RSS feed, bookmarks export, or text file
     """
 
+    check_url_parsing_invariants()
+
     timer = TimedProgress(TIMEOUT * 4)
     with open(source_file, 'r', encoding='utf-8') as file:
-        links, parser = run_parser_functions(file, timer, root_url=root_url, parser=parser)
+        snapshots, parser = run_parser_functions(file, timer, root_url=root_url)
 
     timer.end()
     if parser is None:
         return [], 'Failed to parse'
-    return links, parser
+    return snapshots, parser
 
 
-def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], Optional[str]]:
-    most_links: List[Link] = []
+def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None) -> Tuple[List[Model], Optional[str]]:
+    most_snapshots: List[Model] = []
     best_parser_name = None
 
-    if parser != "auto":
-        parser_name, parser_func = PARSERS[parser]
-        parsed_links = list(parser_func(to_parse, root_url=root_url))
-        if not parsed_links:
-            stderr()
-            stderr(f'[X] No links found using {parser_name} parser', color='red')
-            hint('Try a different parser or double check the input?')
-            stderr()
-        timer.end()
-        return parsed_links, parser_name
-
-    for parser_id in PARSERS:
-        parser_name, parser_func = PARSERS[parser_id]
+    for parser_name, parser_func in PARSERS:
         try:
-            parsed_links = list(parser_func(to_parse, root_url=root_url))
-            if not parsed_links:
-                raise Exception(f'No links found using {parser_name} parser')
+            parsed_snapshots = list(parser_func(to_parse, root_url=root_url))
+            if not parsed_snapshots:
+                raise Exception('no links found')
 
             # print(f'[√] Parser {parser_name} succeeded: {len(parsed_links)} links parsed')
-            if len(parsed_links) > len(most_links):
-                most_links = parsed_links
+            if len(parsed_snapshots) > len(most_snapshots):
+                most_snapshots = parsed_snapshots
                 best_parser_name = parser_name
                 
         except Exception as err:                                                # noqa
             # Parsers are tried one by one down the list, and the first one
-            # that succeeds is used. To debug why a certain parser was not used
-            # due to python error or format incompatibility, uncomment this line:
+            # that succeeds is used. To see why a certain parser was not used
+            # due to error or format incompatibility, uncomment this line:
             
             # print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err))
             # raise
             pass
     timer.end()
-    return most_links, best_parser_name
+    return most_snapshots, best_parser_name
 
 
 @enforce_types
 def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str:
-    ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
+    ts = str(datetime.now().timestamp()).split('.', 1)[0]
     source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts))
-
-    referenced_texts = ''
-
-    for entry in raw_text.split():
-        # Skip URLs - Path() can't handle them and they aren't local files
-        if any(entry.startswith(s) for s in ('http://', 'https://', 'ftp://')):
-            continue
-        try:
-            if Path(entry).exists():
-                referenced_texts += Path(entry).read_text()
-        except Exception as err:
-            print(err)
-
-    atomic_write(source_path, raw_text + '\n' + referenced_texts)
+    atomic_write(source_path, raw_text)
     log_source_saved(source_file=source_path)
     return source_path
 
@@ -172,7 +142,7 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir:
 @enforce_types
 def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: Path=OUTPUT_DIR) -> str:
     """download a given url's content into output/sources/domain-<timestamp>.txt"""
-    ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
+    ts = str(datetime.now().timestamp()).split('.', 1)[0]
     source_path = str(OUTPUT_DIR / SOURCES_DIR_NAME / filename.format(basename=basename(path), ts=ts))
 
     if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')):
@@ -191,7 +161,7 @@ def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{ba
                 ANSI['reset'],
             ))
             print('    ', e)
-            raise e
+            raise SystemExit(1)
 
     else:
         # Source is a path to a local file on the filesystem
@@ -205,52 +175,31 @@ def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{ba
     return source_path
 
 
-# Check that plain text regex URL parsing works as expected
-#   this is last-line-of-defense to make sure the URL_REGEX isn't
-#   misbehaving due to some OS-level or environment level quirks (e.g. bad regex lib)
-#   the consequences of bad URL parsing could be disastrous and lead to many
-#   incorrect/badly parsed links being added to the archive, so this is worth the cost of checking
-_test_url_strs = {
-    'example.com': 0,
-    '/example.com': 0,
-    '//example.com': 0,
-    ':/example.com': 0,
-    '://example.com': 0,
-    'htt://example8.com': 0,
-    '/htt://example.com': 0,
-    'https://example': 1,
-    'https://localhost/2345': 1,
-    'https://localhost:1234/123': 1,
-    '://': 0,
-    'https://': 0,
-    'http://': 0,
-    'ftp://': 0,
-    'ftp://example.com': 0,
-    'https://example.com': 1,
-    'https://example.com/': 1,
-    'https://a.example.com': 1,
-    'https://a.example.com/': 1,
-    'https://a.example.com/what/is/happening.html': 1,
-    'https://a.example.com/what/ís/happening.html': 1,
-    'https://a.example.com/what/is/happening.html?what=1&2%20b#höw-about-this=1a': 1,
-    'https://a.example.com/what/is/happéning/?what=1&2%20b#how-aboüt-this=1a': 1,
-    'HTtpS://a.example.com/what/is/happening/?what=1&2%20b#how-about-this=1af&2f%20b': 1,
-    'https://example.com/?what=1#how-about-this=1&2%20baf': 1,
-    'https://example.com?what=1#how-about-this=1&2%20baf': 1,
-    '<test>http://example7.com</test>': 1,
-    'https://<test>': 0,
-    'https://[test]': 0,
-    'http://"test"': 0,
-    'http://\'test\'': 0,
-    '[https://example8.com/what/is/this.php?what=1]': 1,
-    '[and http://example9.com?what=1&other=3#and-thing=2]': 1,
-    '<what>https://example10.com#and-thing=2 "</about>': 1,
-    'abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def': 1,
-    'sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi': 1,
-    '<or>http://examplehttp://15.badc</that>': 2,
-    'https://a.example.com/one.html?url=http://example.com/inside/of/another?=http://': 2,
-    '[https://a.example.com/one.html?url=http://example.com/inside/of/another?=](http://a.example.com)': 3,
-}
-for url_str, num_urls in _test_url_strs.items():
-    assert len(re.findall(URL_REGEX, url_str)) == num_urls, (
-        f'{url_str} does not contain {num_urls} urls')
+def check_url_parsing_invariants() -> None:
+    """Check that plain text regex URL parsing works as expected"""
+
+    # this is last-line-of-defense to make sure the URL_REGEX isn't
+    # misbehaving, as the consequences could be disastrous and lead to many
+    # incorrect/badly parsed links being added to the archive
+
+    test_urls = '''
+    https://example1.com/what/is/happening.html?what=1#how-about-this=1
+    https://example2.com/what/is/happening/?what=1#how-about-this=1
+    HTtpS://example3.com/what/is/happening/?what=1#how-about-this=1f
+    https://example4.com/what/is/happening.html
+    https://example5.com/
+    https://example6.com
+
+    <test>http://example7.com</test>
+    [https://example8.com/what/is/this.php?what=1]
+    [and http://example9.com?what=1&other=3#and-thing=2]
+    <what>https://example10.com#and-thing=2 "</about>
+    abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def
+    sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi
+    example13.bada
+    and example14.badb
+    <or>htt://example15.badc</that>
+    '''
+    # print('\n'.join(re.findall(URL_REGEX, test_urls)))
+    assert len(re.findall(URL_REGEX, test_urls)) == 12
+

archivebox/parsers/generic_html.py

@@ -4,9 +4,10 @@ __package__ = 'archivebox.parsers'
 import re
 
 from typing import IO, Iterable, Optional
-from datetime import datetime, timezone
+from datetime import datetime
+
+from django.db.models import Model
 
-from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -29,8 +30,9 @@ class HrefParser(HTMLParser):
 
 
 @enforce_types
-def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None, **_kwargs) -> Iterable[Link]:
+def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None, **_kwargs) -> Iterable[Model]:
     """Parse Generic HTML for href tags and use only the url (support for title coming later)"""
+    from core.models import Snapshot
 
     html_file.seek(0)
     for line in html_file:
@@ -44,15 +46,10 @@ def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None,
                 url = urljoin(root_url, url)
             
             for archivable_url in re.findall(URL_REGEX, url):
-                yield Link(
+                yield Snapshot(
                     url=htmldecode(archivable_url),
-                    timestamp=str(datetime.now(timezone.utc).timestamp()),
+                    timestamp=str(datetime.now().timestamp()),
                     title=None,
-                    tags=None,
-                    sources=[html_file.name],
+                    #tags=None,
+                    #sources=[html_file.name],
                 )
-
-
-KEY = 'html'
-NAME = 'Generic HTML'
-PARSER = parse_generic_html_export

archivebox/parsers/generic_json.py

@@ -3,9 +3,10 @@ __package__ = 'archivebox.parsers'
 import json
 
 from typing import IO, Iterable
-from datetime import datetime, timezone
+from datetime import datetime
+
+from django.db.models import Model
 
-from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -13,14 +14,12 @@ from ..util import (
 
 
 @enforce_types
-def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)"""
+    from core.models import Snapshot
 
     json_file.seek(0)
-
-    # sometimes the first line is a comment or filepath, so we get everything after the first {
-    json_file_json_str = '{' + json_file.read().split('{', 1)[-1]
-    links = json.loads(json_file_json_str)
+    links = json.load(json_file)
     json_date = lambda s: datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z')
 
     for link in links:
@@ -33,7 +32,7 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
                 raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
 
             # Parse the timestamp
-            ts_str = str(datetime.now(timezone.utc).timestamp())
+            ts_str = str(datetime.now().timestamp())
             if link.get('timestamp'):
                 # chrome/ff histories use a very precise timestamp
                 ts_str = str(link['timestamp'] / 10000000)  
@@ -59,15 +58,10 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
             elif link.get('name'):
                 title = link['name'].strip()
 
-            yield Link(
+            yield Snapshot(
                 url=htmldecode(url),
                 timestamp=ts_str,
                 title=htmldecode(title) or None,
-                tags=htmldecode(link.get('tags')) or '',
-                sources=[json_file.name],
+                #tags=htmldecode(link.get('tags')) or '',
+                #sources=[json_file.name],
             )
-
-
-KEY = 'json'
-NAME = 'Generic JSON'
-PARSER = parse_generic_json_export

archivebox/parsers/generic_rss.py

@@ -4,7 +4,8 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -12,8 +13,9 @@ from ..util import (
 )
 
 @enforce_types
-def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse RSS XML-format files into links"""
+    from core.models import Snapshot
 
     rss_file.seek(0)
     items = rss_file.read().split('<item>')
@@ -40,15 +42,10 @@ def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z")
         title = str_between(get_row('title'), '<![CDATA[', ']]').strip()
 
-        yield Link(
+        yield Snapshot(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            tags=None,
-            sources=[rss_file.name],
+            #tags=None,
+            #sources=[rss_file.name],
         )
-
-
-KEY = 'rss'
-NAME = 'Generic RSS'
-PARSER = parse_generic_rss_export

archivebox/parsers/generic_txt.py

@@ -4,10 +4,11 @@ __description__ = 'Plain Text'
 import re
 
 from typing import IO, Iterable
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -16,8 +17,10 @@ from ..util import (
 
 
 @enforce_types
-def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
-    """Parse links from a text file, ignoring other text"""
+def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Model]:
+    """Parse raw links from each line in a text file"""
+    # TODO: Check if we should add sources list to the database
+    from core.models import Snapshot
 
     text_file.seek(0)
     for line in text_file.readlines():
@@ -27,12 +30,12 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
         # if the line is a local file path that resolves, then we can archive it
         try:
             if Path(line).exists():
-                yield Link(
+                yield Snapshot(
                     url=line,
-                    timestamp=str(datetime.now(timezone.utc).timestamp()),
+                    timestamp=str(datetime.now().timestamp()),
                     title=None,
-                    tags=None,
-                    sources=[text_file.name],
+                    #tags=None,
+                    #sources=[text_file.name],
                 )
         except (OSError, PermissionError):
             # nvm, not a valid path...
@@ -40,26 +43,22 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
 
         # otherwise look for anything that looks like a URL in the line
         for url in re.findall(URL_REGEX, line):
-            yield Link(
+            yield Snapshot(
                 url=htmldecode(url),
-                timestamp=str(datetime.now(timezone.utc).timestamp()),
+                timestamp=str(datetime.now().timestamp()),
                 title=None,
-                tags=None,
-                sources=[text_file.name],
+                #tags=None,
+                #sources=[text_file.name],
             )
 
             # look inside the URL for any sub-urls, e.g. for archive.org links
             # https://web.archive.org/web/20200531203453/https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             # -> https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             for sub_url in re.findall(URL_REGEX, line[1:]):
-                yield Link(
+                yield Snapshot(
                     url=htmldecode(sub_url),
-                    timestamp=str(datetime.now(timezone.utc).timestamp()),
+                    timestamp=str(datetime.now().timestamp()),
                     title=None,
-                    tags=None,
-                    sources=[text_file.name],
+                    #tags=None,
+                    #sources=[text_file.name],
                 )
-
-KEY = 'txt'
-NAME = 'Generic TXT'
-PARSER = parse_generic_txt_export