Merge branch 'dev'

add command: --parser option (fixes #235)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,8 +1,8 @@
 ---
 name: 🐞 Bug report
 about: Create a report to help us improve
-title: 'Bugfix: ...'
-labels: 'changes: bugfixes'
+title: 'Bug: ...'
+labels: 'bug'
 assignees: ''
 
 ---

.github/workflows/docker.yml

@@ -1,10 +1,13 @@
 name: Build Docker image
 
 on:
-  on: workflow_dispatch
-  release:
-    types:
-      - created
+  workflow_dispatch:
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+  # pull_request:
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -53,13 +56,16 @@ jobs:
       
       - name: Collect Docker tags
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: crazy-max/ghaction-docker-meta@v2
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          tag-sha: true
-          tag-semver: |
-            {{version}}
-            {{major}}.{{minor}}
+          flavor: |
+              latest=auto
+          tags: |
+              type=ref,event=branch
+              type=semver,pattern={{version}}
+              type=semver,pattern={{major}}.{{minor}}
+              type=sha
       
       - name: Build and push
         id: docker_build
@@ -72,7 +78,7 @@ jobs:
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
-          platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/pip.yml

@@ -39,6 +39,12 @@ jobs:
           archivebox version
           archivebox status
 
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
+
       # - name: Commit built package
       #   run: |
       #     cd pip_dist/
@@ -54,8 +60,3 @@ jobs:
       #     branch: ${{ github.ref }}
       #     directory: pip_dist
 
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}

.gitmodules

@@ -23,3 +23,6 @@
 [submodule "archivebox/vendor/django-taggit"]
 	path = archivebox/vendor/django-taggit
 	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites

Dockerfile

@@ -50,19 +50,13 @@ RUN apt-get update -qq \
         fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
     && rm -rf /var/lib/apt/lists/*
 
-# Install apt development dependencies
-# RUN apt-get install -qq \
-#     && apt-get install -qq -y --no-install-recommends \
-#         python3 python3-dev python3-pip python3-venv python3-all \
-#         dh-python debhelper devscripts dput software-properties-common \
-#         python3-distutils python3-setuptools python3-wheel python3-stdeb
-
 # Install Node environment
 RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
     && echo 'deb https://deb.nodesource.com/node_15.x buster main' >> /etc/apt/sources.list \
     && apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
         nodejs \
+    # && npm install -g npm \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Node dependencies
@@ -78,16 +72,27 @@ WORKDIR "$CODE_DIR"
 ENV PATH="${PATH}:$VENV_PATH/bin"
 RUN python -m venv --clear --symlinks "$VENV_PATH" \
     && pip install --upgrade --quiet pip setuptools
-ADD ./pip_dist/archivebox.egg-info/requires.txt "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt"
+ADD "./setup.py" "$CODE_DIR/"
+ADD "./package.json" "$CODE_DIR/archivebox/"
 RUN apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
         build-essential python-dev python3-dev \
-    && grep -B 1000 -E '^$' "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin \
-    && pip install --quiet "sonic-client==0.0.5" \
+    && echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \
+    && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
+    && pip install --quiet -r /tmp/requirements.txt \
     && apt-get purge -y build-essential python-dev python3-dev \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
+# Install apt development dependencies
+# RUN apt-get install -qq \
+#     && apt-get install -qq -y --no-install-recommends \
+#         python3 python3-dev python3-pip python3-venv python3-all \
+#         dh-python debhelper devscripts dput software-properties-common \
+#         python3-distutils python3-setuptools python3-wheel python3-stdeb
+# RUN python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.extras_require["dev"]))' > /tmp/dev_requirements.txt \
+    # && pip install --quiet -r /tmp/dev_requirements.txt
+
 # Install ArchiveBox Python package and its dependencies
 WORKDIR "$CODE_DIR"
 ADD . "$CODE_DIR"
@@ -113,5 +118,8 @@ RUN /app/bin/docker_entrypoint.sh archivebox version
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
+HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+    CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
+
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

MANIFEST.in

@@ -2,3 +2,5 @@ graft archivebox
 global-exclude .DS_Store
 global-exclude __pycache__
 global-exclude *.pyc
+
+prune tests/

archivebox/cli/__init__.py

@@ -8,18 +8,17 @@ import argparse
 from typing import Optional, Dict, List, IO, Union
 from pathlib import Path
 
-from ..config import OUTPUT_DIR
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations
 
 from importlib import import_module
 
 CLI_DIR = Path(__file__).resolve().parent
 
 # these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')
-main_cmds = ('init', 'info', 'config')
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')
-
-fake_db = ("oneshot",)
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
 
 display_first = (*meta_cmds, *main_cmds, *archive_cmds)
 
@@ -61,9 +60,21 @@ def run_subcommand(subcommand: str,
                    pwd: Union[Path, str, None]=None) -> None:
     """Run a given ArchiveBox subcommand with the given list of args"""
 
+    subcommand_args = subcommand_args or []
+
     if subcommand not in meta_cmds:
         from ..config import setup_django
-        setup_django(in_memory_db=subcommand in fake_db, check_db=subcommand in archive_cmds)
+
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+
+        if cmd_requires_db:
+            check_data_folder(pwd)
+
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
+
+        if cmd_requires_db:
+            check_migrations()
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore

archivebox/cli/archivebox_add.py

@@ -10,6 +10,7 @@ from typing import List, Optional, IO
 
 from ..main import add
 from ..util import docstring
+from ..parsers import PARSERS
 from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr
 
@@ -22,6 +23,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         add_help=True,
         formatter_class=SmartFormatter,
     )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
     parser.add_argument(
         '--update-all', #'-n',
         action='store_true',
@@ -73,9 +80,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
               This does not take precedence over the configuration",
         default=""
     )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        help="Parser used to read inputted URLs.",
+        default="auto",
+        choices=["auto", *PARSERS.keys()],
+    )
     command = parser.parse_args(args or ())
     urls = command.urls
-    stdin_urls = accept_stdin(stdin)
+
+    stdin_urls = ''
+    if not urls:
+        stdin_urls = accept_stdin(stdin)
+
     if (stdin_urls and urls) or (not stdin and not urls):
         stderr(
             '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
@@ -85,11 +103,13 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     add(
         urls=stdin_urls or urls,
         depth=command.depth,
+        tag=command.tag,
         update_all=command.update_all,
         index_only=command.index_only,
         overwrite=command.overwrite,
         init=command.init,
         extractors=command.extract,
+        parser=command.parser,
         out_dir=pwd or OUTPUT_DIR,
     )
 

archivebox/cli/archivebox_config.py

@@ -45,7 +45,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='KEY or KEY=VALUE formatted config values to get or set',
     )
     command = parser.parse_args(args or ())
-    config_options_str = accept_stdin(stdin)
+
+    config_options_str = ''
+    if not command.config_options:
+        config_options_str = accept_stdin(stdin)
 
     config(
         config_options_str=config_options_str,

archivebox/cli/archivebox_init.py

@@ -27,11 +27,23 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Ignore unrecognized files in current directory and initialize anyway',
     )
+    parser.add_argument(
+        '--quick', '-q',
+        action='store_true',
+        help='Run any updates or migrations without rechecking all snapshot dirs',
+    )
+    parser.add_argument(
+        '--setup', #'-s',
+        action='store_true',
+        help='Automatically install dependencies and extras used for archiving',
+    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
     init(
         force=command.force,
+        quick=command.quick,
+        setup=command.setup,
         out_dir=pwd or OUTPUT_DIR,
     )
     

archivebox/cli/archivebox_list.py

@@ -12,6 +12,7 @@ from ..main import list_all
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
+    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -23,7 +24,7 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
-from ..logging_util import SmartFormatter, accept_stdin, stderr
+from ..logging_util import SmartFormatter, reject_stdin, stderr
 
 
 @docstring(list_all.__doc__)
@@ -44,7 +45,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     group.add_argument(
         '--json', #'-j',
         action='store_true',
-        help="Print the output in JSON format with all columns included.",
+        help="Print the output in JSON format with all columns included",
     )
     group.add_argument(
         '--html',
@@ -59,19 +60,19 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--sort', #'-s',
         type=str,
-        help="List the links sorted using the given key, e.g. timestamp or updated.",
+        help="List the links sorted using the given key, e.g. timestamp or updated",
         default=None,
     )
     parser.add_argument(
         '--before', #'-b',
         type=float,
-        help="List only links bookmarked before the given timestamp.",
+        help="List only links bookmarked before (less than) the given timestamp",
         default=None,
     )
     parser.add_argument(
         '--after', #'-a',
         type=float,
-        help="List only links bookmarked after the given timestamp.",
+        help="List only links bookmarked after (greater than or equal to) the given timestamp",
         default=None,
     )
     parser.add_argument(
@@ -96,9 +97,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type',
+        '--filter-type', '-t',
         type=str,
-        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
+        choices=(*LINK_FILTERS.keys(), 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -107,20 +108,19 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         nargs='*',
         type=str,
         default=None,
-        help='List only URLs matching these filter patterns.'
+        help='List only URLs matching these filter patterns'
     )
     command = parser.parse_args(args or ())
-    filter_patterns_str = accept_stdin(stdin)
+    reject_stdin(stdin)
 
     if command.with_headers and not (command.json or command.html or command.csv):
         stderr(
-            '[X] --with-headers can only be used with --json, --html or --csv options.\n',
+            '[X] --with-headers can only be used with --json, --html or --csv options\n',
             color='red',
         )
         raise SystemExit(2)
 
     matching_folders = list_all(
-        filter_patterns_str=filter_patterns_str,
         filter_patterns=command.filter_patterns,
         filter_type=command.filter_type,
         status=command.status,

archivebox/cli/archivebox_oneshot.py

@@ -50,8 +50,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help= "Path to save the single archive folder to, e.g. ./example.com_archive"
     )
     command = parser.parse_args(args or ())
+    stdin_url = None
     url = command.url
-    stdin_url = accept_stdin(stdin)
+    if not url:
+        stdin_url = accept_stdin(stdin)
+
     if (stdin_url and url) or (not stdin and not url):
         stderr(
             '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',

archivebox/cli/archivebox_remove.py

@@ -61,7 +61,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='URLs matching this filter pattern will be removed from the index.'
     )
     command = parser.parse_args(args or ())
-    filter_str = accept_stdin(stdin)
+    
+    filter_str = None
+    if not command.filter_patterns:
+        filter_str = accept_stdin(stdin)
 
     remove(
         filter_str=filter_str,

archivebox/cli/archivebox_schedule.py

@@ -44,7 +44,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         type=int,
         choices=[0, 1],
         default=0,
-        help='Depth to archive to [0] or 1, see "add" command help for more info.',
+        help='Depth to archive to [0] or 1, see "add" command help for more info',
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
     )
     group.add_argument(
         '--clear', # '-c'
@@ -88,6 +93,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         quiet=command.quiet,
         every=command.every,
         depth=command.depth,
+        overwrite=command.overwrite,
         import_path=command.import_path,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_server.py

@@ -38,10 +38,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Enable DEBUG=True mode with more verbose errors',
     )
+    parser.add_argument(
+        '--nothreading',
+        action='store_true',
+        help='Force runserver to run in single-threaded mode',
+    )
     parser.add_argument(
         '--init',
         action='store_true',
-        help='Run archivebox init before starting the server',
+        help='Run a full archivebox init/upgrade before starting the server',
+    )
+    parser.add_argument(
+        '--quick-init', '-i',
+        action='store_true',
+        help='Run quick archivebox init/upgrade before starting the server',
     )
     parser.add_argument(
         '--createsuperuser',
@@ -52,10 +62,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     reject_stdin(__command__, stdin)
     
     server(
-        runserver_args=command.runserver_args,
+        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
         reload=command.reload,
         debug=command.debug,
         init=command.init,
+        quick_init=command.quick_init,
         createsuperuser=command.createsuperuser,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_setup.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox setup'
+
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import setup
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+
+
+@docstring(setup.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=setup.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    # parser.add_argument(
+    #     '--force', # '-f',
+    #     action='store_true',
+    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
+    # )
+    command = parser.parse_args(args or ())   # noqa
+    reject_stdin(__command__, stdin)
+
+    setup(
+        # force=command.force,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_update.py

@@ -12,6 +12,7 @@ from ..main import update
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
+    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -89,9 +90,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type',
+        '--filter-type', '-t',
         type=str,
-        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
+        choices=(*LINK_FILTERS.keys(), 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -110,7 +111,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=""
     )
     command = parser.parse_args(args or ())
-    filter_patterns_str = accept_stdin(stdin)
+
+    filter_patterns_str = None
+    if not command.filter_patterns:
+        filter_patterns_str = accept_stdin(stdin)
 
     update(
         resume=command.resume,

archivebox/cli/tests.py

@@ -75,23 +75,23 @@ def output_hidden(show_failing=True):
         yield
         return
 
-    sys.stdout = open('stdout.txt', 'w+')
-    sys.stderr = open('stderr.txt', 'w+')
+    sys.stdout = open('stdout.txt', 'w+', encoding='utf-8')
+    sys.stderr = open('stderr.txt', 'w+', encoding='utf-8')
     try:
         yield
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
-    except:
+    except Exception:
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
         if show_failing:
-            with open('stdout.txt', 'r') as f:
+            with open('stdout.txt', 'r', encoding='utf-8') as f:
                 print(f.read())
-            with open('stderr.txt', 'r') as f:
+            with open('stderr.txt', 'r', encoding='utf-8') as f:
                 print(f.read())
         raise
     finally:
@@ -116,7 +116,7 @@ class TestInit(unittest.TestCase):
         assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
 
     def test_conflicting_init(self):
-        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+') as f:
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
             f.write('test')
 
         try:
@@ -132,7 +132,7 @@ class TestInit(unittest.TestCase):
         try:
             load_main_index(out_dir=OUTPUT_DIR)
             assert False, 'load_main_index should raise an exception when no index is present'
-        except:
+        except Exception:
             pass
 
     def test_no_dirty_state(self):
@@ -161,7 +161,7 @@ class TestAdd(unittest.TestCase):
 
     def test_add_arg_file(self):
         test_file = Path(OUTPUT_DIR) / 'test.txt'
-        with open(test_file, 'w+') as f:
+        with open(test_file, 'w+', encoding='utf') as f:
             f.write(test_urls)
 
         with output_hidden():
@@ -216,7 +216,7 @@ class TestRemove(unittest.TestCase):
             with output_hidden(show_failing=False):
                 archivebox_remove.main(['--yes', '--delete', 'https://doesntexist.com'])
             assert False, 'Should raise if no URLs match'
-        except:
+        except Exception:
             pass
 
 

archivebox/config.py

@@ -29,10 +29,12 @@ import json
 import getpass
 import platform
 import shutil
+import sqlite3
 import django
 
 from hashlib import md5
 from pathlib import Path
+from datetime import datetime, timezone
 from typing import Optional, Type, Tuple, Dict, Union, List
 from subprocess import run, PIPE, DEVNULL
 from configparser import ConfigParser
@@ -77,6 +79,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'PUBLIC_SNAPSHOTS':         {'type': bool,  'default': True},
         'PUBLIC_ADD_VIEW':          {'type': bool,  'default': False},
         'FOOTER_INFO':              {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
+        'SNAPSHOTS_PER_PAGE':       {'type': int,   'default': 40},
+        'CUSTOM_TEMPLATES_DIR':     {'type': str,   'default': None},
+        'TIME_ZONE':                {'type': str,   'default': 'UTC'},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -99,30 +104,35 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
 
     'ARCHIVE_METHOD_OPTIONS': {
         'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION',)},
-        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com'},
+        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com'},
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
+        'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'CURL_USER_AGENT':          {'type': str,   'default': 'ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': 'ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36'},
+        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
 
         'CHROME_HEADLESS':          {'type': bool,  'default': True},
         'CHROME_SANDBOX':           {'type': bool,  'default': lambda c: not c['IN_DOCKER']},
-        'YOUTUBEDL_ARGS':           {'type': list,  'default': ['--write-description',
+        'YOUTUBEDL_ARGS':           {'type': list,  'default': lambda c: [
+                                                                '--write-description',
                                                                 '--write-info-json',
                                                                 '--write-annotations',
                                                                 '--write-thumbnail',
                                                                 '--no-call-home',
+                                                                '--write-sub',
                                                                 '--all-subs',
+                                                                '--write-auto-sub',
+                                                                '--convert-subs=srt',
                                                                 '--yes-playlist',
                                                                 '--continue',
                                                                 '--ignore-errors',
                                                                 '--geo-bypass',
                                                                 '--add-metadata',
-                                                                '--max-filesize=750m',
+                                                                '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']),
                                                                 ]},
                                                                     
 
@@ -152,6 +162,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         # SONIC
         'SONIC_COLLECTION':         {'type': str,   'default': 'archivebox'},
         'SONIC_BUCKET':             {'type': str,   'default': 'snapshots'},
+        'SEARCH_BACKEND_TIMEOUT':   {'type': int,   'default': 90},
     },
 
     'DEPENDENCY_CONFIG': {
@@ -169,9 +180,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CURL_BINARY':              {'type': str,   'default': 'curl'},
         'GIT_BINARY':               {'type': str,   'default': 'git'},
         'WGET_BINARY':              {'type': str,   'default': 'wget'},
-        'SINGLEFILE_BINARY':        {'type': str,   'default': 'single-file'},
-        'READABILITY_BINARY':       {'type': str,   'default': 'readability-extractor'},
-        'MERCURY_BINARY':           {'type': str,   'default': 'mercury-parser'},
+        'SINGLEFILE_BINARY':        {'type': str,   'default': lambda c: bin_path('single-file')},
+        'READABILITY_BINARY':       {'type': str,   'default': lambda c: bin_path('readability-extractor')},
+        'MERCURY_BINARY':           {'type': str,   'default': lambda c: bin_path('mercury-parser')},
         'YOUTUBEDL_BINARY':         {'type': str,   'default': 'youtube-dl'},
         'NODE_BINARY':              {'type': str,   'default': 'node'},
         'RIPGREP_BINARY':           {'type': str,   'default': 'rg'},
@@ -263,7 +274,38 @@ STATICFILE_EXTENSIONS = {
     # html, htm, shtml, xhtml, xml, aspx, php, cgi
 }
 
-
+# When initializing archivebox in a new directory, we check to make sure the dir is
+# actually empty so that we dont clobber someone's home directory or desktop by accident.
+# These files are exceptions to the is_empty check when we're trying to init a new dir,
+# as they could be from a previous archivebox version, system artifacts, dependencies, etc.
+ALLOWED_IN_OUTPUT_DIR = {
+    '.gitignore',
+    'lost+found',
+    '.DS_Store',
+    '.venv',
+    'venv',
+    'virtualenv',
+    '.virtualenv',
+    'node_modules',
+    'package.json',
+    'package-lock.json',
+    'yarn.lock',
+    'static',
+    'sonic',
+    ARCHIVE_DIR_NAME,
+    SOURCES_DIR_NAME,
+    LOGS_DIR_NAME,
+    SQL_INDEX_FILENAME,
+    f'{SQL_INDEX_FILENAME}-wal',
+    f'{SQL_INDEX_FILENAME}-shm',
+    JSON_INDEX_FILENAME,
+    HTML_INDEX_FILENAME,
+    ROBOTS_TXT_FILENAME,
+    FAVICON_FILENAME,
+    CONFIG_FILENAME,
+    f'{CONFIG_FILENAME}.bak',
+    'static_index.json',
+}
 
 ############################## Derived Config ##################################
 
@@ -275,6 +317,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
 
     'PACKAGE_DIR':              {'default': lambda c: Path(__file__).resolve().parent},
     'TEMPLATES_DIR':            {'default': lambda c: c['PACKAGE_DIR'] / TEMPLATES_DIR_NAME},
+    'CUSTOM_TEMPLATES_DIR':     {'default': lambda c: c['CUSTOM_TEMPLATES_DIR'] and Path(c['CUSTOM_TEMPLATES_DIR'])},
 
     'OUTPUT_DIR':               {'default': lambda c: Path(c['OUTPUT_DIR']).resolve() if c['OUTPUT_DIR'] else Path(os.curdir).resolve()},
     'ARCHIVE_DIR':              {'default': lambda c: c['OUTPUT_DIR'] / ARCHIVE_DIR_NAME},
@@ -285,9 +328,8 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CHROME_USER_DATA_DIR':     {'default': lambda c: find_chrome_data_dir() if c['CHROME_USER_DATA_DIR'] is None else (Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None)},   # None means unset, so we autodetect it with find_chrome_Data_dir(), but emptystring '' means user manually set it to '', and we should store it as None
     'URL_BLACKLIST_PTN':        {'default': lambda c: c['URL_BLACKLIST'] and re.compile(c['URL_BLACKLIST'] or '', re.IGNORECASE | re.UNICODE | re.MULTILINE)},
 
-    'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0]},
-    'VERSION':                  {'default': lambda c: json.loads((Path(c['PACKAGE_DIR']) / 'package.json').read_text().strip())['version']},
-    'GIT_SHA':                  {'default': lambda c: c['VERSION'].split('+')[-1] or 'unknown'},
+    'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0] or bin_path('archivebox')},
+    'VERSION':                  {'default': lambda c: json.loads((Path(c['PACKAGE_DIR']) / 'package.json').read_text(encoding='utf-8').strip())['version']},
 
     'PYTHON_BINARY':            {'default': lambda c: sys.executable},
     'PYTHON_ENCODING':          {'default': lambda c: sys.stdout.encoding.upper()},
@@ -331,8 +373,8 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'SAVE_MEDIA':               {'default': lambda c: c['USE_YOUTUBEDL'] and c['SAVE_MEDIA']},
     'YOUTUBEDL_ARGS':           {'default': lambda c: c['YOUTUBEDL_ARGS'] or []},
 
-    'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
-    'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] if c['CHROME_BINARY'] else find_chrome_binary()},
+    'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] or find_chrome_binary()},
+    'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and c['CHROME_BINARY'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
     'CHROME_VERSION':           {'default': lambda c: bin_version(c['CHROME_BINARY']) if c['USE_CHROME'] else None},
     
     'SAVE_PDF':                 {'default': lambda c: c['USE_CHROME'] and c['SAVE_PDF']},
@@ -459,7 +501,7 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
     config_file.optionxform = str
     config_file.read(config_path)
 
-    with open(config_path, 'r') as old:
+    with open(config_path, 'r', encoding='utf-8') as old:
         atomic_write(f'{config_path}.bak', old.read())
 
     find_section = lambda key: [name for name, opts in CONFIG_SCHEMA.items() if key in opts][0]
@@ -480,32 +522,33 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
 
     if (not existing_secret_key) or ('not a valid secret' in existing_secret_key):
         from django.utils.crypto import get_random_string
-        chars = 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.'
+        chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
         random_secret_key = get_random_string(50, chars)
         if 'SERVER_CONFIG' in config_file:
             config_file['SERVER_CONFIG']['SECRET_KEY'] = random_secret_key
         else:
             config_file['SERVER_CONFIG'] = {'SECRET_KEY': random_secret_key}
 
-    with open(config_path, 'w+') as new:
+    with open(config_path, 'w+', encoding='utf-8') as new:
         config_file.write(new)
     
     try:
         # validate the config by attempting to re-parse it
         CONFIG = load_all_config()
-        return {
-            key.upper(): CONFIG.get(key.upper())
-            for key in config.keys()
-        }
-    except:
+    except BaseException:                                                       # lgtm [py/catch-base-exception]
         # something went horribly wrong, rever to the previous version
-        with open(f'{config_path}.bak', 'r') as old:
+        with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
             atomic_write(config_path, old.read())
 
+        raise
+
     if Path(f'{config_path}.bak').exists():
         os.remove(f'{config_path}.bak')
-
-    return {}
+    
+    return {
+        key.upper(): CONFIG.get(key.upper())
+        for key in config.keys()
+    }
 
    
 
@@ -542,7 +585,7 @@ def load_config(defaults: ConfigDefaultDict,
             stderr('    For config documentation and examples see:')
             stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration')
             stderr()
-            raise
+            # raise
             raise SystemExit(2)
     
     return extended_config
@@ -703,6 +746,11 @@ def get_code_locations(config: ConfigDict) -> SimpleConfigValueDict:
             'enabled': True,
             'is_valid': (config['TEMPLATES_DIR'] / 'static').exists(),
         },
+        'CUSTOM_TEMPLATES_DIR': {
+            'path': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).resolve(),
+            'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
+            'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
+        },
         # 'NODE_MODULES_DIR': {
         #     'path': ,
         #     'enabled': ,
@@ -980,9 +1028,9 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None:
                     info['version'] or 'unable to detect version',
                 )
             )
-            if dependency in ('SINGLEFILE_BINARY', 'READABILITY_BINARY', 'MERCURY_BINARY'):
-                hint(('npm install --prefix . "git+https://github.com/ArchiveBox/ArchiveBox.git"',
-                    f'or archivebox config --set SAVE_{dependency.rsplit("_", 1)[0]}=False to silence this warning',
+            if dependency in ('YOUTUBEDL_BINARY', 'CHROME_BINARY', 'SINGLEFILE_BINARY', 'READABILITY_BINARY', 'MERCURY_BINARY'):
+                hint(('To install all packages automatically run: archivebox setup',
+                    f'or to disable it and silence this warning: archivebox config --set SAVE_{dependency.rsplit("_", 1)[0]}=False',
                     ''), prefix='      ')
         stderr('')
 
@@ -1017,8 +1065,8 @@ def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=C
     output_dir = out_dir or config['OUTPUT_DIR']
     assert isinstance(output_dir, (str, Path))
 
-    sql_index_exists = (Path(output_dir) / SQL_INDEX_FILENAME).exists()
-    if not sql_index_exists:
+    archive_dir_exists = (Path(output_dir) / ARCHIVE_DIR_NAME).exists()
+    if not archive_dir_exists:
         stderr('[X] No archivebox index found in the current directory.', color='red')
         stderr(f'    {output_dir}', color='lightyellow')
         stderr()
@@ -1030,26 +1078,22 @@ def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=C
         stderr('        archivebox init')
         raise SystemExit(2)
 
+def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CONFIG):
+    output_dir = out_dir or config['OUTPUT_DIR']
     from .index.sql import list_migrations
 
     pending_migrations = [name for status, name in list_migrations() if not status]
 
-    if (not sql_index_exists) or pending_migrations:
-        if sql_index_exists:
-            pending_operation = f'apply the {len(pending_migrations)} pending migrations'
-        else:
-            pending_operation = 'generate the new SQL main index'
-
+    if pending_migrations:
         stderr('[X] This collection was created with an older version of ArchiveBox and must be upgraded first.', color='lightyellow')
         stderr(f'    {output_dir}')
         stderr()
-        stderr(f'    To upgrade it to the latest version and {pending_operation} run:')
+        stderr(f'    To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:')
         stderr('        archivebox init')
         raise SystemExit(3)
 
-    sources_dir = Path(output_dir) / SOURCES_DIR_NAME
-    if not sources_dir.exists():
-        sources_dir.mkdir()
+    (Path(output_dir) / SOURCES_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / LOGS_DIR_NAME).mkdir(exist_ok=True)
 
 
 
@@ -1061,24 +1105,72 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
     assert isinstance(output_dir, Path) and isinstance(config['PACKAGE_DIR'], Path)
 
     try:
-        import django
+        from django.core.management import call_command
+
         sys.path.append(str(config['PACKAGE_DIR']))
         os.environ.setdefault('OUTPUT_DIR', str(output_dir))
         assert (config['PACKAGE_DIR'] / 'core' / 'settings.py').exists(), 'settings.py was not found at archivebox/core/settings.py'
         os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
 
+        # Check to make sure JSON extension is available in our Sqlite3 instance
+        try:
+            cursor = sqlite3.connect(':memory:').cursor()
+            cursor.execute('SELECT JSON(\'{"a": "b"}\')')
+        except sqlite3.OperationalError as exc:
+            stderr(f'[X] Your SQLite3 version is missing the required JSON1 extension: {exc}', color='red')
+            hint([
+                'Upgrade your Python version or install the extension manually:',
+                'https://code.djangoproject.com/wiki/JSON1Extension'
+            ])
+
         if in_memory_db:
-            # Put the db in memory and run migrations in case any command requires it
-            from django.core.management import call_command
+            # some commands (e.g. oneshot) dont store a long-lived sqlite3 db file on disk.
+            # in those cases we create a temporary in-memory db and run the migrations
+            # immediately to get a usable in-memory-database at startup
             os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
             django.setup()
             call_command("migrate", interactive=False, verbosity=0)
         else:
+            # Otherwise use default sqlite3 file-based database and initialize django
+            # without running migrations automatically (user runs them manually by calling init)
             django.setup()
+            
+
+        from django.conf import settings
+
+        # log startup message to the error log
+        with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f:
+            command = ' '.join(sys.argv)
+            ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
+            f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
+
 
         if check_db:
+            # Enable WAL mode in sqlite3
+            from django.db import connection
+            with connection.cursor() as cursor:
+                current_mode = cursor.execute("PRAGMA journal_mode")
+                if current_mode != 'wal':
+                    cursor.execute("PRAGMA journal_mode=wal;")
+
+            # Create cache table in DB if needed
+            try:
+                from django.core.cache import cache
+                cache.get('test', None)
+            except django.db.utils.OperationalError:
+                call_command("createcachetable", verbosity=0)
+
+
+            # if archivebox gets imported multiple times, we have to close
+            # the sqlite3 whenever we init from scratch to avoid multiple threads
+            # sharing the same connection by accident
+            from django.db import connections
+            for conn in connections.all():
+                conn.close_if_unusable_or_obsolete()
+
             sql_index_path = Path(output_dir) / SQL_INDEX_FILENAME
             assert sql_index_path.exists(), (
-                f'No database file {SQL_INDEX_FILENAME} found in OUTPUT_DIR: {config["OUTPUT_DIR"]}')
+                f'No database file {SQL_INDEX_FILENAME} found in: {config["OUTPUT_DIR"]} (Are you in an ArchiveBox collection directory?)')
+
     except KeyboardInterrupt:
         raise SystemExit(2)

archivebox/core/admin.py

@@ -1,7 +1,9 @@
 __package__ = 'archivebox.core'
 
 from io import StringIO
+from pathlib import Path
 from contextlib import redirect_stdout
+from datetime import datetime, timezone
 
 from django.contrib import admin
 from django.urls import path
@@ -13,16 +15,16 @@ from django import forms
 
 from ..util import htmldecode, urldecode, ansi_to_html
 
-from core.models import Snapshot, Tag
-from core.forms import AddLinkForm, TagField
+from core.models import Snapshot, ArchiveResult, Tag
+from core.forms import AddLinkForm
 
 from core.mixins import SearchResultsAdminMixin
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
-from config import OUTPUT_DIR
-from extractors import archive_snapshots
+from config import OUTPUT_DIR, SNAPSHOTS_PER_PAGE
+from extractors import archive_links
 
 # Admin URLs
 # /admin/
@@ -36,95 +38,126 @@ from extractors import archive_snapshots
 
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
-def update_snapshots(modeladmin, request, queryset):
-    archive_links([
-        snapshot.as_link()
-        for snapshot in queryset
-    ], out_dir=OUTPUT_DIR)
-update_snapshots.short_description = "Archive"
 
-def update_titles(modeladmin, request, queryset):
-    archive_links([
-        snapshot.as_link()
-        for snapshot in queryset
-    ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
-update_titles.short_description = "Pull title"
+class ArchiveResultInline(admin.TabularInline):
+    model = ArchiveResult
 
-def overwrite_snapshots(modeladmin, request, queryset):
-    archive_links([
-        snapshot.as_link()
-        for snapshot in queryset
-    ], overwrite=True, out_dir=OUTPUT_DIR)
-overwrite_snapshots.short_description = "Re-archive (overwrite)"
+class TagInline(admin.TabularInline):
+    model = Snapshot.tags.through
 
-def verify_snapshots(modeladmin, request, queryset):
-    for snapshot in queryset:
-        print(snapshot.timestamp, snapshot.url, snapshot.is_archived, snapshot.archive_size, len(snapshot.history))
+from django.contrib.admin.helpers import ActionForm
+from django.contrib.admin.widgets import AutocompleteSelectMultiple
 
-verify_snapshots.short_description = "Check"
+class AutocompleteTags:
+    model = Tag
+    search_fields = ['name']
 
-def delete_snapshots(modeladmin, request, queryset):
-    remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
-
-delete_snapshots.short_description = "Delete"
+class AutocompleteTagsAdminStub:
+    name = 'admin'
 
 
-class SnapshotAdminForm(forms.ModelForm):
-    tags = TagField(required=False)
+class SnapshotActionForm(ActionForm):
+    tags = forms.ModelMultipleChoiceField(
+        queryset=Tag.objects.all(),
+        required=False,
+        widget=AutocompleteSelectMultiple(
+            AutocompleteTags(),
+            AutocompleteTagsAdminStub(),
+        ),
+    )
 
-    class Meta:
-        model = Snapshot
-        fields = "__all__"
-
-    def save(self, commit=True):
-        # Based on: https://stackoverflow.com/a/49933068/3509554
-
-        # Get the unsave instance
-        instance = forms.ModelForm.save(self, False)
-        tags = self.cleaned_data.pop("tags")
-
-        #update save_m2m
-        def new_save_m2m():
-            instance.save_tags(tags)
-
-        # Do we need to save all changes now?
-        self.save_m2m = new_save_m2m
-        if commit:
-            instance.save()
-
-        return instance
+    # TODO: allow selecting actions for specific extractors? is this useful?
+    # EXTRACTOR_CHOICES = [
+    #     (name, name.title())
+    #     for name, _, _ in get_default_archive_methods()
+    # ]
+    # extractor = forms.ChoiceField(
+    #     choices=EXTRACTOR_CHOICES,
+    #     required=False,
+    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
+    # )
 
 
 class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
-    list_display = ('added', 'title_str', 'url_str', 'files', 'size')
-    sort_fields = ('title_str', 'url_str', 'added')
-    readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
-    search_fields = ['url__icontains', 'timestamp', 'title', 'tags__name']
-    fields = (*readonly_fields, 'title', 'tags')
-    list_filter = ('added', 'updated', 'tags')
+    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'added', 'files')
+    readonly_fields = ('info', 'bookmarked', 'added', 'updated')
+    search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
+    fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
+    list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
     ordering = ['-added']
-    actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots]
-    actions_template = 'admin/actions_as_select.html'
-    form = SnapshotAdminForm
-    list_per_page = 40
+    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
+    autocomplete_fields = ['tags']
+    inlines = [ArchiveResultInline]
+    list_per_page = SNAPSHOTS_PER_PAGE
+
+    action_form = SnapshotActionForm
 
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
-            path('grid/', self.admin_site.admin_view(self.grid_view),name='grid')
+            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
         ]
         return custom_urls + urls
 
     def get_queryset(self, request):
+        self.request = request
         return super().get_queryset(request).prefetch_related('tags')
 
     def tag_list(self, obj):
         return ', '.join(obj.tags.values_list('name', flat=True))
 
-    def id_str(self, obj):
+    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
+    # def action(self, obj):
+    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
+    #     # action: update_snapshots
+    #     # select_across: 0
+    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
+    #     return format_html(
+    #         '''
+    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
+    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
+    #                 <input type="hidden" name="_selected_action" value="{}">
+    #                 <button name="update_snapshots">Check</button>
+    #                 <button name="update_titles">Pull title + favicon</button>
+    #                 <button name="update_snapshots">Update</button>
+    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
+    #                 <button name="delete_snapshots">Permanently delete</button>
+    #             </form>
+    #         ''',
+    #         csrf.get_token(self.request),
+    #         obj.id,
+    #     )
+
+    def info(self, obj):
         return format_html(
-            '<code style="font-size: 10px">{}</code>',
-            obj.url_hash[:8],
+            '''
+            UUID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
+            Timestamp: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
+            URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            Archived: {} ({} files {}) &nbsp; &nbsp;
+            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
+            Status code: {} &nbsp; &nbsp;
+            Server: {} &nbsp; &nbsp;
+            Content type: {} &nbsp; &nbsp;
+            Extension: {} &nbsp; &nbsp;
+            <br/><br/>
+            <a href="/archive/{}">View Snapshot index ➡️</a> &nbsp; &nbsp;
+            <a href="/admin/core/snapshot/?id__exact={}">View actions ⚙️</a>
+            ''',
+            obj.id,
+            obj.timestamp,
+            obj.url_hash,
+            '✅' if obj.is_archived else '❌',
+            obj.num_outputs,
+            self.size(obj),
+            f'/archive/{obj.timestamp}/favicon.ico',
+            obj.status_code or '?',
+            obj.headers and obj.headers.get('Server') or '?',
+            obj.headers and obj.headers.get('Content-Type') or '?',
+            obj.extension or '?',
+            obj.timestamp,
+            obj.id,
         )
 
     def title_str(self, obj):
@@ -151,8 +184,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     def files(self, obj):
         return snapshot_icons(obj)
 
+    files.admin_order_field = 'updated'
+    files.short_description = 'Files Saved'
+
     def size(self, obj):
-        archive_size = obj.archive_size
+        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
         if archive_size:
             size_txt = printable_filesize(archive_size)
             if archive_size > 52428800:
@@ -165,14 +201,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             size_txt,
         )
 
+    size.admin_order_field = 'archiveresult__count'
+
     def url_str(self, obj):
         return format_html(
-            '<a href="{}"><code>{}</code></a>',
+            '<a href="{}"><code style="user-select: all;">{}</code></a>',
+            obj.url,
             obj.url,
-            obj.url.split('://www.', 1)[-1].split('://', 1)[-1][:64],
         )
 
-    def grid_view(self, request):
+    def grid_view(self, request, extra_context=None):
 
         # cl = self.get_changelist_instance(request)
 
@@ -183,35 +221,153 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
         # Monkey patch here plus core_tags.py
         self.change_list_template = 'private_index_grid.html'
-        self.list_per_page = 20
+        self.list_per_page = SNAPSHOTS_PER_PAGE
         self.list_max_show_all = self.list_per_page
 
         # Call monkey patched view
-        rendered_response = self.changelist_view(request)
+        rendered_response = self.changelist_view(request, extra_context=extra_context)
 
         # Restore values
-        self.change_list_template =  saved_change_list_template
+        self.change_list_template = saved_change_list_template
         self.list_per_page = saved_list_per_page
         self.list_max_show_all = saved_list_max_show_all
 
         return rendered_response
+
+    # for debugging, uncomment this to print all requests:
+    # def changelist_view(self, request, extra_context=None):
+    #     print('[*] Got request', request.method, request.POST)
+    #     return super().changelist_view(request, extra_context=None)
+
+    def update_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], out_dir=OUTPUT_DIR)
+    update_snapshots.short_description = "Pull"
+
+    def update_titles(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
+    update_titles.short_description = "⬇️ Title"
+
+    def resnapshot_snapshot(self, request, queryset):
+        for snapshot in queryset:
+            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
+            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
+            add(new_url, tag=snapshot.tags_str())
+    resnapshot_snapshot.short_description = "Re-Snapshot"
+
+    def overwrite_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, out_dir=OUTPUT_DIR)
+    overwrite_snapshots.short_description = "Reset"
+
+    def delete_snapshots(self, request, queryset):
+        remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
+
+    delete_snapshots.short_description = "Delete"
+
+    def add_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[+] Adding tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.add(*tags)
+
+    add_tags.short_description = "+"
+
+    def remove_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[-] Removing tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.remove(*tags)
+
+    remove_tags.short_description = "–"
+
         
 
-    id_str.short_description = 'ID'
     title_str.short_description = 'Title'
     url_str.short_description = 'Original URL'
 
-    id_str.admin_order_field = 'id'
     title_str.admin_order_field = 'title'
     url_str.admin_order_field = 'url'
 
+
+
 class TagAdmin(admin.ModelAdmin):
-    list_display = ('slug', 'name', 'id')
+    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
     sort_fields = ('id', 'name', 'slug')
-    readonly_fields = ('id',)
+    readonly_fields = ('id', 'num_snapshots', 'snapshots')
     search_fields = ('id', 'name', 'slug')
     fields = (*readonly_fields, 'name', 'slug')
+    actions = ['delete_selected']
+    ordering = ['-id']
 
+    def num_snapshots(self, obj):
+        return format_html(
+            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
+            obj.id,
+            obj.snapshot_set.count(),
+        )
+
+    def snapshots(self, obj):
+        total_count = obj.snapshot_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
+                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.id,
+                snap.timestamp,
+                snap.url,
+            )
+            for snap in obj.snapshot_set.order_by('-updated')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={obj.id}">and {total_count-10} more...<a>' if obj.snapshot_set.count() > 10 else ''))
+
+
+class ArchiveResultAdmin(admin.ModelAdmin):
+    list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
+    sort_fields = ('start_ts', 'extractor', 'status')
+    readonly_fields = ('id', 'uuid', 'snapshot_str', 'tags_str')
+    search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
+    autocomplete_fields = ['snapshot']
+
+    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
+    ordering = ['-start_ts']
+    list_per_page = SNAPSHOTS_PER_PAGE
+
+    def snapshot_str(self, obj):
+        return format_html(
+            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
+            '<small>{}</small>',
+            obj.snapshot.timestamp,
+            obj.snapshot.timestamp,
+            obj.snapshot.url[:128],
+        )
+
+    def tags_str(self, obj):
+        return obj.snapshot.tags_str()
+
+    def cmd_str(self, obj):
+        return format_html(
+            '<pre>{}</pre>',
+            ' '.join(obj.cmd) if isinstance(obj.cmd, list) else str(obj.cmd),
+        )
+
+    def output_str(self, obj):
+        return format_html(
+            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
+            obj.snapshot.timestamp,
+            obj.output if (obj.status == 'succeeded') and obj.extractor not in ('title', 'archive_org') else 'index.html',
+            obj.output,
+        )
+
+    tags_str.short_description = 'tags'
+    snapshot_str.short_description = 'snapshot'
 
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = 'ArchiveBox'
@@ -266,4 +422,5 @@ admin.site = ArchiveBoxAdmin()
 admin.site.register(get_user_model())
 admin.site.register(Snapshot, SnapshotAdmin)
 admin.site.register(Tag, TagAdmin)
+admin.site.register(ArchiveResult, ArchiveResultAdmin)
 admin.site.disable_action('delete_selected')

archivebox/core/apps.py

@@ -3,3 +3,4 @@ from django.apps import AppConfig
 
 class CoreConfig(AppConfig):
     name = 'core'
+    default_auto_field = 'django.db.models.UUIDField'

archivebox/core/forms.py

@@ -3,9 +3,14 @@ __package__ = 'archivebox.core'
 from django import forms
 
 from ..util import URL_REGEX
+from ..parsers import PARSERS
 from ..vendor.taggit_utils import edit_string_for_tags, parse_tags
 
-CHOICES = (
+PARSER_CHOICES = [
+    (parser_key, parser[0])
+    for parser_key, parser in PARSERS.items()
+]
+DEPTH_CHOICES = (
     ('0', 'depth = 0 (archive just these URLs)'),
     ('1', 'depth = 1 (archive these URLs and all URLs one hop away)'),
 )
@@ -20,7 +25,9 @@ ARCHIVE_METHODS = [
 
 class AddLinkForm(forms.Form):
     url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
-    depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
+    parser = forms.ChoiceField(label="URLs format", choices=[('auto', 'Auto-detect parser'), *PARSER_CHOICES], initial='auto')
+    tag = forms.CharField(label="Tags (comma separated tag1,tag2,tag3)", strip=True, required=False)
+    depth = forms.ChoiceField(label="Archive depth", choices=DEPTH_CHOICES, initial='0', widget=forms.RadioSelect(attrs={"class": "depth-selection"}))
     archive_methods = forms.MultipleChoiceField(
         label="Archive methods (select at least 1, otherwise all will be used by default)",
         required=False,

archivebox/core/middleware.py

@@ -0,0 +1,37 @@
+__package__ = 'archivebox.core'
+
+from django.utils import timezone
+
+from ..config import PUBLIC_SNAPSHOTS
+
+
+def detect_timezone(request, activate: bool=True):
+    gmt_offset = (request.COOKIES.get('GMT_OFFSET') or '').strip()
+    tz = None
+    if gmt_offset.replace('-', '').isdigit():
+        tz = timezone.get_fixed_timezone(int(gmt_offset))
+        if activate:
+            timezone.activate(tz)
+    # print('GMT_OFFSET', gmt_offset, tz)
+    return tz
+
+
+def TimezoneMiddleware(get_response):
+    def middleware(request):
+        detect_timezone(request, activate=True)
+        return get_response(request)
+
+    return middleware
+
+
+def CacheControlMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        if '/archive/' in request.path or '/static/' in request.path:
+            policy = 'public' if PUBLIC_SNAPSHOTS else 'private'
+            response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
+            # print('Set Cache-Control header to', response['Cache-Control'])
+        return response
+
+    return middleware

archivebox/core/migrations/0006_auto_20201012_1520.py

@@ -17,7 +17,7 @@ def forwards_func(apps, schema_editor):
         tag_set.discard("")
 
         for tag in tag_set:
-            to_add, _ = TagModel.objects.get_or_create(name=tag, slug=slugify(tag))
+            to_add, _ = TagModel.objects.get_or_create(name=tag, defaults={'slug': slugify(tag)})
             snapshot.tags.add(to_add)
 
 

archivebox/core/migrations/0007_archiveresult.py

@@ -36,8 +36,25 @@ def forwards_func(apps, schema_editor):
 
         for extractor in history:
             for result in history[extractor]:
-                ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=result["cmd"], cmd_version=result["cmd_version"] or 'unknown', 
-                start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"])
+                try:
+                    ArchiveResult.objects.create(
+                        extractor=extractor,
+                        snapshot=snapshot,
+                        pwd=result["pwd"],
+                        cmd=result.get("cmd") or [],
+                        cmd_version=result.get("cmd_version") or 'unknown',
+                        start_ts=result["start_ts"],
+                        end_ts=result["end_ts"],
+                        status=result["status"],
+                        output=result.get("output") or 'null',
+                    )
+                except Exception as e:
+                    print(
+                        '    ! Skipping import due to missing/invalid index.json:',
+                        out_dir,
+                        e,
+                        '(open an issue with this index.json for help)',
+                    )
 
 
 def verify_json_index_integrity(snapshot):

archivebox/core/migrations/0009_auto_20210216_1038.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-16 10:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0008_auto_20210105_1421'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='updated',
+            field=models.DateTimeField(auto_now=True, db_index=True, null=True),
+        ),
+    ]

archivebox/core/migrations/0010_auto_20210216_1055.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-16 10:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_auto_20210216_1038'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='start_ts',
+            field=models.DateTimeField(db_index=True),
+        ),
+    ]

archivebox/core/migrations/0011_auto_20210216_1331.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.3 on 2021-02-16 13:31
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0010_auto_20210216_1055'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+        ),
+    ]

archivebox/core/migrations/0012_auto_20210216_1425.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2021-02-16 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0011_auto_20210216_1331'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='cmd_version',
+            field=models.CharField(blank=True, default=None, max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='output',
+            field=models.CharField(max_length=1024),
+        ),
+    ]

archivebox/core/migrations/0013_auto_20210218_0729.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 07:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0012_auto_20210216_1425'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=256, null=True),
+        ),
+    ]

archivebox/core/migrations/0014_auto_20210218_0729.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 07:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_auto_20210218_0729'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=1024, null=True),
+        ),
+    ]

archivebox/core/migrations/0015_auto_20210218_0730.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 07:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0014_auto_20210218_0729'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=512, null=True),
+        ),
+    ]

archivebox/core/migrations/0016_auto_20210218_1204.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0015_auto_20210218_0730'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='tags',
+            field=models.ManyToManyField(blank=True, to='core.Tag'),
+        ),
+    ]

archivebox/core/migrations/0017_auto_20210219_0211.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-19 02:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0016_auto_20210218_1204'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(blank=True, max_length=100, unique=True, verbose_name='slug'),
+        ),
+    ]

archivebox/core/migrations/0018_auto_20210327_0952.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2021-03-27 09:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0017_auto_20210219_0211'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(blank=True, max_length=100, unique=True),
+        ),
+    ]

archivebox/core/migrations/0019_auto_20210401_0654.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-04-01 06:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0018_auto_20210327_0952'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='url',
+            field=models.URLField(db_index=True, unique=True),
+        ),
+    ]

archivebox/core/migrations/0020_auto_20210410_1031.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.8 on 2021-04-10 10:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0019_auto_20210401_0654'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

archivebox/core/mixins.py

@@ -2,22 +2,20 @@ from django.contrib import messages
 
 from archivebox.search import query_search_index
 
-class SearchResultsAdminMixin(object):
-    def get_search_results(self, request, queryset, search_term):
-        ''' Enhances the search queryset with results from the search backend.
-        '''
-        qs, use_distinct = \
-            super(SearchResultsAdminMixin, self).get_search_results(
-                request, queryset, search_term)
+class SearchResultsAdminMixin:
+    def get_search_results(self, request, queryset, search_term: str):
+        """Enhances the search queryset with results from the search backend"""
+        
+        qs, use_distinct = super().get_search_results(request, queryset, search_term)
 
         search_term = search_term.strip()
         if not search_term:
             return qs, use_distinct
         try:
             qsearch = query_search_index(search_term)
+            qs = qs | qsearch
         except Exception as err:
+            print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
             messages.add_message(request, messages.WARNING, f'Error from the search backend, only showing results from default admin search fields - Error: {err}')
-        else:
-            qs = queryset & qsearch
-        finally:
-            return qs, use_distinct
+        
+        return qs, use_distinct

archivebox/core/models.py

@@ -1,23 +1,28 @@
 __package__ = 'archivebox.core'
 
-import uuid
-from pathlib import Path
-from typing import Dict, Optional, List
-from datetime import datetime, timedelta
-from collections import defaultdict
 
-from django.db import models, transaction
+import uuid
+import json
+
+from pathlib import Path
+from typing import Optional, List
+
+from django.db import models
 from django.utils.functional import cached_property
 from django.utils.text import slugify
+from django.core.cache import cache
+from django.urls import reverse
 from django.db.models import Case, When, Value, IntegerField
+from django.contrib.auth.models import User   # noqa
 
-from ..util import parse_date
-from ..index.schema import Link
-from ..config import CONFIG
+from ..config import ARCHIVE_DIR, ARCHIVE_DIR_NAME
 from ..system import get_dir_size
+from ..util import parse_date, base_url, hashurl
+from ..index.schema import Link
+from ..index.html import snapshot_icons
+from ..extractors import get_default_archive_methods, ARCHIVE_METHODS_INDEXING_PRECEDENCE
 
-#EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
-EXTRACTORS = [("title", "title"), ("wget", "wget")]
+EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
 STATUS_CHOICES = [
     ("succeeded", "succeeded"),
     ("failed", "failed"),
@@ -30,12 +35,18 @@ except AttributeError:
     import jsonfield
     JSONField = jsonfield.JSONField
 
+
 class Tag(models.Model):
     """
     Based on django-taggit model
     """
-    name = models.CharField(verbose_name="name", unique=True, blank=False, max_length=100)
-    slug = models.SlugField(verbose_name="slug", unique=True, max_length=100)
+    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
+
+    name = models.CharField(unique=True, blank=False, max_length=100)
+
+    # slug is autoset on save from name, never set it manually
+    slug = models.SlugField(unique=True, blank=True, max_length=100)
+
 
     class Meta:
         verbose_name = "Tag"
@@ -54,20 +65,21 @@ class Tag(models.Model):
         if self._state.adding and not self.slug:
             self.slug = self.slugify(self.name)
 
-            with transaction.atomic():
-                slugs = set(
-                    type(self)
-                    ._default_manager.filter(slug__startswith=self.slug)
-                    .values_list("slug", flat=True)
-                )
+            # if name is different but slug conficts with another tags slug, append a counter
+            # with transaction.atomic():
+            slugs = set(
+                type(self)
+                ._default_manager.filter(slug__startswith=self.slug)
+                .values_list("slug", flat=True)
+            )
 
-                i = None
-                while True:
-                    slug = self.slugify(self.name, i)
-                    if slug not in slugs:
-                        self.slug = slug
-                        return super().save(*args, **kwargs)
-                    i = 1 if i is None else i+1
+            i = None
+            while True:
+                slug = self.slugify(self.name, i)
+                if slug not in slugs:
+                    self.slug = slug
+                    return super().save(*args, **kwargs)
+                i = 1 if i is None else i+1
         else:
             return super().save(*args, **kwargs)
 
@@ -75,16 +87,16 @@ class Tag(models.Model):
 class Snapshot(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
 
-    url = models.URLField(unique=True)
+    url = models.URLField(unique=True, db_index=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True)
 
-    title = models.CharField(max_length=128, null=True, blank=True, db_index=True)
+    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
 
     added = models.DateTimeField(auto_now_add=True, db_index=True)
-    updated = models.DateTimeField(null=True, blank=True, db_index=True)
-    tags = models.ManyToManyField(Tag)
+    updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
+    tags = models.ManyToManyField(Tag, blank=True)
 
-    keys = ('id', 'url', 'timestamp', 'title', 'tags', 'updated', 'base_url')
+    keys = ('url', 'timestamp', 'title', 'tags', 'updated')
 
     def __repr__(self) -> str:
         title = self.title or '-'
@@ -94,63 +106,18 @@ class Snapshot(models.Model):
         title = self.title or '-'
         return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
 
-    def field_names():
-        fields = self._meta.get_field_names()
-        exclude = ["tags", "archiveresult"] # Exclude relationships for now
-        return [field.name for field in fields if field.name not in exclude]
-
-
     @classmethod
     def from_json(cls, info: dict):
         info = {k: v for k, v in info.items() if k in cls.keys}
-        if "tags" in info:
-            # TODO: Handle tags
-            info.pop("tags")
-        info.pop("base_url", None) 
         return cls(**info)
 
-    def get_history(self) -> dict:
-        """
-        Generates the history dictionary out of the stored ArchiveResults
-        """
-        history_list = self.archiveresult_set.all()
-        history = defaultdict(list)
-        for history_item in history_list:
-            history[history_item.extractor].append(
-                {
-                    "cmd": history_item.cmd,
-                    "cmd_version": history_item.cmd_version,
-                    "end_ts": history_item.end_ts.isoformat(),
-                    "start_ts": history_item.start_ts.isoformat(),
-                    "pwd": history_item.pwd,
-                    "output": history_item.output,
-                    "schema": "ArchiveResult",
-                    "status": history_item.status
-                }
-            )
-        return dict(history)
-
     def as_json(self, *args) -> dict:
-        """
-        Returns the snapshot in json format.
-        id is converted to str
-        history is extracted from ArchiveResult
-        """
         args = args or self.keys
-        output = {
+        return {
             key: getattr(self, key)
             if key != 'tags' else self.tags_str()
             for key in args
         }
-        if "id" in output.keys():
-            output["id"] = str(output["id"])
-
-        output["history"] = self.get_history()
-        return output
-
-    def as_csv(self, cols: Optional[List[str]]=None, separator: str=',', ljust: int=0) -> str:
-        from ..index.csv import to_csv
-        return to_csv(self, cols=cols or self.field_names(), separator=separator, ljust=ljust)
 
     def as_link(self) -> Link:
         return Link.from_json(self.as_json())
@@ -159,57 +126,35 @@ class Snapshot(models.Model):
         from ..index import load_link_details
         return load_link_details(self.as_link())
 
-    def tags_str(self) -> str:
-        return ','.join(self.tags.order_by('name').values_list('name', flat=True))
+    def tags_str(self, nocache=True) -> str:
+        cache_key = f'{self.id}-{(self.updated or self.added).timestamp()}-tags'
+        calc_tags_str = lambda: ','.join(self.tags.order_by('name').values_list('name', flat=True))
+        if nocache:
+            tags_str = calc_tags_str()
+            cache.set(cache_key, tags_str)
+            return tags_str
+        return cache.get_or_set(cache_key, calc_tags_str)
+
+    def icons(self) -> str:
+        return snapshot_icons(self)
+
+    @cached_property
+    def extension(self) -> str:
+        from ..util import extension
+        return extension(self.url)
 
     @cached_property
     def bookmarked(self):
         return parse_date(self.timestamp)
 
     @cached_property
-    def bookmarked_date(self) -> Optional[str]:
-        from ..util import ts_to_date
-
-        max_ts = (datetime.now() + timedelta(days=30)).timestamp()
-
-        if self.timestamp and self.timestamp.replace('.', '').isdigit():
-            if 0 < float(self.timestamp) < max_ts:
-                return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
-            else:
-                return str(self.timestamp)
-        return None
+    def bookmarked_date(self):
+        # TODO: remove this
+        return self.bookmarked
 
     @cached_property
-    def is_archived(self) -> bool:
-        from ..config import ARCHIVE_DIR
-        from ..util import domain
-
-        output_paths = (
-            domain(self.url),
-            'output.pdf',
-            'screenshot.png',
-            'output.html',
-            'media',
-            'singlefile.html'
-        )
-
-        return any(
-            (Path(ARCHIVE_DIR) / self.timestamp / path).exists()
-            for path in output_paths
-        )
-
-    @cached_property
-    def archive_dates(self) -> List[datetime]:
-        return [
-            result.start_ts
-            for result in self.archiveresult_set.all()
-        ]
-
-    @cached_property
-    def oldest_archive_date(self) -> Optional[datetime]:
-        oldest = self.archiveresult_set.all().order_by("-start_ts")[:1]
-        if len(oldest) > 0:
-            return oldest[0].start_ts
+    def is_archived(self):
+        return self.as_link().is_archived
 
     @cached_property
     def num_outputs(self):
@@ -217,113 +162,101 @@ class Snapshot(models.Model):
 
     @cached_property
     def url_hash(self):
-        return self.as_link().url_hash
+        return hashurl(self.url)
 
     @cached_property
-    def base_url(self) -> str:
-        from ..util import base_url
+    def base_url(self):
         return base_url(self.url)
 
     @cached_property
-    def snapshot_dir(self):
-        from ..config import CONFIG
-        return Path(CONFIG['ARCHIVE_DIR']) / self.timestamp
+    def link_dir(self):
+        return str(ARCHIVE_DIR / self.timestamp)
 
     @cached_property
     def archive_path(self):
-        from ..config import ARCHIVE_DIR_NAME
         return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp)
 
     @cached_property
-    def archive_size(self) -> float:
-        try:
-            return get_dir_size(self.archive_path)[0]
-        except Exception:
-            return 0
+    def archive_size(self):
+        cache_key = f'{str(self.id)[:12]}-{(self.updated or self.added).timestamp()}-size'
+
+        def calc_dir_size():
+            try:
+                return get_dir_size(self.link_dir)[0]
+            except Exception:
+                return 0
+
+        return cache.get_or_set(cache_key, calc_dir_size)
 
     @cached_property
-    def history(self):
+    def thumbnail_url(self) -> Optional[str]:
+        result = self.archiveresult_set.filter(
+            extractor='screenshot',
+            status='succeeded'
+        ).only('output').last()
+        if result:
+            return reverse('Snapshot', args=[f'{str(self.timestamp)}/{result.output}'])
+        return None
+
+    @cached_property
+    def headers(self) -> Optional[dict]:
+        try:
+            return json.loads((Path(self.link_dir) / 'headers.json').read_text(encoding='utf-8').strip())
+        except Exception:
+            pass
+        return None
+
+    @cached_property
+    def status_code(self) -> Optional[str]:
+        return self.headers and self.headers.get('Status-Code')
+
+    @cached_property
+    def history(self) -> dict:
         # TODO: use ArchiveResult for this instead of json
         return self.as_link_with_details().history
 
     @cached_property
-    def latest_title(self):
-        if ('title' in self.history
-            and self.history['title']
-            and (self.history['title'][-1].status == 'succeeded')
-            and self.history['title'][-1].output.strip()):
-            return self.history['title'][-1].output.strip()
+    def latest_title(self) -> Optional[str]:
+        if self.title:
+            return self.title   # whoopdedoo that was easy
+        
+        try:
+            # take longest successful title from ArchiveResult db history
+            return sorted(
+                self.archiveresult_set\
+                    .filter(extractor='title', status='succeeded', output__isnull=False)\
+                    .values_list('output', flat=True),
+                key=lambda r: len(r),
+            )[-1]
+        except IndexError:
+            pass
+
+        try:
+            # take longest successful title from Link json index file history
+            return sorted(
+                (
+                    result.output.strip()
+                    for result in self.history['title']
+                    if result.status == 'succeeded' and result.output.strip()
+                ),
+                key=lambda r: len(r),
+            )[-1]
+        except (KeyError, IndexError):
+            pass
+
         return None
 
-    @cached_property
-    def domain(self) -> str:
-        from ..util import domain
-        return domain(self.url)
-
-    @cached_property
-    def is_static(self) -> bool:
-        from ..util import is_static_file
-        return is_static_file(self.url)
-
-    @cached_property
-    def details(self) -> Dict:
-        # TODO: Define what details are, and return them accordingly
-        return {"history": {}}
-
-    @property
-    def extension(self) -> str:
-        from ..util import extension
-        return extension(self.url)
-
-    def canonical_outputs(self) -> Dict[str, Optional[str]]:
-        """predict the expected output paths that should be present after archiving"""
-
-        from ..extractors.wget import wget_output_path
-        canonical = {
-            'index_path': 'index.html',
-            'favicon_path': 'favicon.ico',
-            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
-            'wget_path': wget_output_path(self),
-            'warc_path': 'warc',
-            'singlefile_path': 'singlefile.html',
-            'readability_path': 'readability/content.html',
-            'mercury_path': 'mercury/content.html',
-            'pdf_path': 'output.pdf',
-            'screenshot_path': 'screenshot.png',
-            'dom_path': 'output.html',
-            'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
-            'git_path': 'git',
-            'media_path': 'media',
-        }
-        if self.is_static:
-            # static binary files like PDF and images are handled slightly differently.
-            # they're just downloaded once and aren't archived separately multiple times, 
-            # so the wget, screenshot, & pdf urls should all point to the same file
-
-            static_path = wget_output_path(self)
-            canonical.update({
-                'title': self.basename,
-                'wget_path': static_path,
-                'pdf_path': static_path,
-                'screenshot_path': static_path,
-                'dom_path': static_path,
-                'singlefile_path': static_path,
-                'readability_path': static_path,
-                'mercury_path': static_path,
-            })
-        return canonical
-
-    def save_tags(self, tags=()):
+    def save_tags(self, tags: List[str]=()) -> None:
         tags_id = []
         for tag in tags:
-            tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
+            if tag.strip():
+                tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
         self.tags.clear()
         self.tags.add(*tags_id)
 
 
 class ArchiveResultManager(models.Manager):
     def indexable(self, sorted: bool = True):
-        from ..extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
         INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
         qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS,status='succeeded')
 
@@ -334,20 +267,20 @@ class ArchiveResultManager(models.Manager):
 
 
 class ArchiveResult(models.Model):
+    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
+    uuid = models.UUIDField(default=uuid.uuid4, editable=False)
+
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
+    extractor = models.CharField(choices=EXTRACTORS, max_length=32)
     cmd = JSONField()
     pwd = models.CharField(max_length=256)
-    cmd_version = models.CharField(max_length=32, default=None, null=True, blank=True)
-    output = models.CharField(max_length=512)
-    start_ts = models.DateTimeField()
+    cmd_version = models.CharField(max_length=128, default=None, null=True, blank=True)
+    output = models.CharField(max_length=1024)
+    start_ts = models.DateTimeField(db_index=True)
     end_ts = models.DateTimeField()
     status = models.CharField(max_length=16, choices=STATUS_CHOICES)
-    extractor = models.CharField(choices=EXTRACTORS, max_length=32)
 
     objects = ArchiveResultManager()
 
     def __str__(self):
         return self.extractor
-
-    class Meta:
-        ordering = ["-start_ts"]

archivebox/core/settings.py

@@ -2,21 +2,26 @@ __package__ = 'archivebox.core'
 
 import os
 import sys
+import re
+import logging
+import tempfile
 
 from pathlib import Path
 from django.utils.crypto import get_random_string
 
-from ..config import (                                                          # noqa: F401
+from ..config import (
     DEBUG,
     SECRET_KEY,
     ALLOWED_HOSTS,
     PACKAGE_DIR,
     TEMPLATES_DIR_NAME,
+    CUSTOM_TEMPLATES_DIR,
     SQL_INDEX_FILENAME,
     OUTPUT_DIR,
+    LOGS_DIR,
+    TIME_ZONE,
 )
 
-
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@@ -50,18 +55,54 @@ INSTALLED_APPS = [
 
 
 MIDDLEWARE = [
+    'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
+    'core.middleware.CacheControlMiddleware',
 ]
 
 AUTHENTICATION_BACKENDS = [
     'django.contrib.auth.backends.ModelBackend',
 ]
 
+# only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode)
+DEBUG_TOOLBAR = DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv)
+if DEBUG_TOOLBAR:
+    try:
+        import debug_toolbar   # noqa
+        DEBUG_TOOLBAR = True
+    except ImportError:
+        DEBUG_TOOLBAR = False
+
+if DEBUG_TOOLBAR:
+    INSTALLED_APPS = [*INSTALLED_APPS, 'debug_toolbar']
+    INTERNAL_IPS = ['0.0.0.0', '127.0.0.1', '*']
+    DEBUG_TOOLBAR_CONFIG = {
+        "SHOW_TOOLBAR_CALLBACK": lambda request: True,
+        "RENDER_PANELS": True,
+    }
+    DEBUG_TOOLBAR_PANELS = [
+        'debug_toolbar.panels.history.HistoryPanel',
+        'debug_toolbar.panels.versions.VersionsPanel',
+        'debug_toolbar.panels.timer.TimerPanel',
+        'debug_toolbar.panels.settings.SettingsPanel',
+        'debug_toolbar.panels.headers.HeadersPanel',
+        'debug_toolbar.panels.request.RequestPanel',
+        'debug_toolbar.panels.sql.SQLPanel',
+        'debug_toolbar.panels.staticfiles.StaticFilesPanel',
+        # 'debug_toolbar.panels.templates.TemplatesPanel',
+        'debug_toolbar.panels.cache.CachePanel',
+        'debug_toolbar.panels.signals.SignalsPanel',
+        'debug_toolbar.panels.logging.LoggingPanel',
+        'debug_toolbar.panels.redirects.RedirectsPanel',
+        'debug_toolbar.panels.profiling.ProfilingPanel',
+        'djdt_flamegraph.FlamegraphPanel',
+    ]
+    MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
 
 ################################################################################
 ### Staticfile and Template Settings
@@ -70,10 +111,12 @@ AUTHENTICATION_BACKENDS = [
 STATIC_URL = '/static/'
 
 STATICFILES_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
 ]
 
 TEMPLATE_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
@@ -107,6 +150,23 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': DATABASE_NAME,
+        'OPTIONS': {
+            'timeout': 60,
+            'check_same_thread': False,
+        },
+        'TIME_ZONE': 'UTC',
+        # DB setup is sometimes modified at runtime by setup_django() in config.py
+    }
+}
+
+CACHE_BACKEND = 'django.core.cache.backends.locmem.LocMemCache'
+# CACHE_BACKEND = 'django.core.cache.backends.db.DatabaseCache'
+# CACHE_BACKEND = 'django.core.cache.backends.dummy.DummyCache'
+
+CACHES = {
+    'default': {
+        'BACKEND': CACHE_BACKEND,
+        'LOCATION': 'django_cache_default',
     }
 }
 
@@ -117,12 +177,13 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 ### Security Settings
 ################################################################################
 
-SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.')
+SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')
 
 ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
 
 SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
 
 CSRF_COOKIE_SECURE = False
 SESSION_COOKIE_SECURE = False
@@ -131,6 +192,8 @@ SESSION_COOKIE_AGE = 1209600  # 2 weeks
 SESSION_EXPIRE_AT_BROWSER_CLOSE = False
 SESSION_SAVE_EVERY_REQUEST = True
 
+SESSION_ENGINE = "django.contrib.sessions.backends.db"
+
 AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
     {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'},
@@ -156,10 +219,84 @@ if IS_SHELL:
 ################################################################################
 
 LANGUAGE_CODE = 'en-us'
-TIME_ZONE = 'UTC'
-USE_I18N = False
-USE_L10N = False
-USE_TZ = False
-
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
 DATETIME_FORMAT = 'Y-m-d g:iA'
 SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
+TIME_ZONE = TIME_ZONE                            # noqa
+
+from django.conf.locale.en import formats as en_formats
+
+en_formats.DATETIME_FORMAT = DATETIME_FORMAT
+en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
+
+
+################################################################################
+### Logging Settings
+################################################################################
+
+IGNORABLE_404_URLS = [
+    re.compile(r'apple-touch-icon.*\.png$'),
+    re.compile(r'favicon\.ico$'),
+    re.compile(r'robots\.txt$'),
+    re.compile(r'.*\.(css|js)\.map$'),
+]
+
+class NoisyRequestsFilter(logging.Filter):
+    def filter(self, record):
+        logline = record.getMessage()
+
+        # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
+        for ignorable_url_pattern in IGNORABLE_404_URLS:
+            ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
+            if ignorable_log_pattern.match(logline):
+                return 0
+
+        # ignore staticfile requests that 200 or 30*
+        ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M)
+        if ignoreable_200_log_pattern.match(logline):
+            return 0
+
+        return 1
+
+if LOGS_DIR.exists():
+    ERROR_LOG = (LOGS_DIR / 'errors.log')
+else:
+    # meh too many edge cases here around creating log dir w/ correct permissions
+    # cant be bothered, just trash the log and let them figure it out via stdout/stderr
+    ERROR_LOG = tempfile.NamedTemporaryFile().name
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+        'logfile': {
+            'level': 'ERROR',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': ERROR_LOG,
+            'maxBytes': 1024 * 1024 * 25,  # 25 MB
+            'backupCount': 10,
+        },
+    },
+    'filters': {
+        'noisyrequestsfilter': {
+            '()': NoisyRequestsFilter,
+        }
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['console', 'logfile'],
+            'level': 'INFO',
+            'filters': ['noisyrequestsfilter'],
+        },
+        'django.server': {
+            'handlers': ['console', 'logfile'],
+            'level': 'INFO',
+            'filters': ['noisyrequestsfilter'],
+        }
+    },
+}

archivebox/core/templatetags/core_tags.py

@@ -1,22 +1,15 @@
 from django import template
-from django.urls import reverse
 from django.contrib.admin.templatetags.base import InclusionAdminNode
-from django.templatetags.static import static
 
 
 from typing import Union
 
-from core.models import ArchiveResult
 
 register = template.Library()
 
-@register.simple_tag
-def snapshot_image(snapshot):
-    result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first()
-    if result:
-        return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
-    
-    return static('archive.png')
+@register.filter(name='split')
+def split(value, separator: str=','):
+    return (value or '').split(separator)
 
 @register.filter
 def file_size(num_bytes: Union[int, float]) -> str:

archivebox/core/urls.py

@@ -2,6 +2,7 @@ from django.contrib import admin
 
 from django.urls import path, include
 from django.views import static
+from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.conf import settings
 from django.views.generic.base import RedirectView
 
@@ -13,8 +14,8 @@ from core.views import HomepageView, SnapshotView, PublicIndexView, AddView
 urlpatterns = [
     path('public/', PublicIndexView.as_view(), name='public-index'),
 
-    path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
-    path('favicon.ico', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'favicon.ico'}),
+    path('robots.txt', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'robots.txt'}),
+    path('favicon.ico', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'favicon.ico'}),
 
     path('docs/', RedirectView.as_view(url='https://github.com/ArchiveBox/ArchiveBox/wiki'), name='Docs'),
 
@@ -35,35 +36,43 @@ urlpatterns = [
     path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),
     path('', HomepageView.as_view(), name='Home'),
 ]
+urlpatterns += staticfiles_urlpatterns()
 
-    # # Proposed UI URLs spec
-    # path('',                 HomepageView)
-    # path('/add',             AddView)
-    # path('/public',          PublicIndexView)
-    # path('/snapshot/:slug',  SnapshotView)
-    
-    # path('/admin',           admin.site.urls)
-    # path('/accounts',        django.contrib.auth.urls)
+if settings.DEBUG_TOOLBAR:
+    import debug_toolbar
+    urlpatterns += [
+        path('__debug__/', include(debug_toolbar.urls)),
+    ]
 
-    # # Prposed REST API spec
-    # # :slugs can be uuid, short_uuid, or any of the unique index_fields
-    # path('api/v1/'),
-    # path('api/v1/core/'                      [GET])
-    # path('api/v1/core/snapshot/',            [GET, POST, PUT]),
-    # path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
-    # path('api/v1/core/archiveresult',        [GET, POST, PUT]),
-    # path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
-    # path('api/v1/core/tag/',                 [GET, POST, PUT]),
-    # path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
 
-    # path('api/v1/cli/',                      [GET])
-    # path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
+# # Proposed FUTURE URLs spec
+# path('',                 HomepageView)
+# path('/add',             AddView)
+# path('/public',          PublicIndexView)
+# path('/snapshot/:slug',  SnapshotView)
 
-    # path('api/v1/extractors/',                    [GET])
-    # path('api/v1/extractors/:extractor/',         [GET]),
-    # path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
+# path('/admin',           admin.site.urls)
+# path('/accounts',        django.contrib.auth.urls)
 
-    # future, just an idea:
-    # path('api/v1/scheduler/',                [GET])
-    # path('api/v1/scheduler/task/',           [GET, POST, PUT]),
-    # path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),
+# # Prposed REST API spec
+# # :slugs can be uuid, short_uuid, or any of the unique index_fields
+# path('api/v1/'),
+# path('api/v1/core/'                      [GET])
+# path('api/v1/core/snapshot/',            [GET, POST, PUT]),
+# path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
+# path('api/v1/core/archiveresult',        [GET, POST, PUT]),
+# path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
+# path('api/v1/core/tag/',                 [GET, POST, PUT]),
+# path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
+
+# path('api/v1/cli/',                      [GET])
+# path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
+
+# path('api/v1/extractors/',                    [GET])
+# path('api/v1/extractors/:extractor/',         [GET]),
+# path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
+
+# future, just an idea:
+# path('api/v1/scheduler/',                [GET])
+# path('api/v1/scheduler/task/',           [GET, POST, PUT]),
+# path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),

archivebox/core/views.py

@@ -4,8 +4,8 @@ from io import StringIO
 from contextlib import redirect_stdout
 
 from django.shortcuts import render, redirect
-
-from django.http import HttpResponse
+from django.http import HttpResponse, Http404
+from django.utils.html import format_html, mark_safe
 from django.views import View, static
 from django.views.generic.list import ListView
 from django.views.generic import FormView
@@ -22,10 +22,11 @@ from ..config import (
     PUBLIC_ADD_VIEW,
     VERSION,
     FOOTER_INFO,
+    SNAPSHOTS_PER_PAGE,
 )
-from main import add
+from ..main import add
 from ..util import base_url, ansi_to_html
-from ..index.html import snapshot_icons
+from ..search import query_search_index
 
 
 class HomepageView(View):
@@ -43,10 +44,6 @@ class SnapshotView(View):
     # render static html index from filesystem archive/<timestamp>/index.html
 
     def get(self, request, path):
-        # missing trailing slash -> redirect to index
-        if '/' not in path:
-            return redirect(f'{path}/index.html')
-
         if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
             return redirect(f'/admin/login/?next={request.path}')
 
@@ -55,47 +52,164 @@ class SnapshotView(View):
         except (IndexError, ValueError):
             slug, archivefile = path.split('/', 1)[0], 'index.html'
 
-        all_pages = list(Snapshot.objects.all())
-
         # slug is a timestamp
-        by_ts = {page.timestamp: page for page in all_pages}
-        try:
-            # print('SERVING STATICFILE', by_ts[slug].link_dir, request.path, path)
-            response = static.serve(request, archivefile, document_root=by_ts[slug].snapshot_dir, show_indexes=True)
-            response["Link"] = f'<{by_ts[slug].url}>; rel="canonical"'
-            return response
-        except KeyError:
-            pass
+        if slug.replace('.','').isdigit():
 
-        # slug is a hash
-        by_hash = {page.url_hash: page for page in all_pages}
-        try:
-            timestamp = by_hash[slug].timestamp
-            return redirect(f'/archive/{timestamp}/{archivefile}')
-        except KeyError:
-            pass
+            # missing trailing slash -> redirect to index
+            if '/' not in path:
+                return redirect(f'{path}/index.html')
 
+            try:
+                try:
+                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
+                    response = static.serve(request, archivefile, document_root=snapshot.link_dir, show_indexes=True)
+                    response["Link"] = f'<{snapshot.url}>; rel="canonical"'
+                    return response
+                except Snapshot.DoesNotExist:
+                    if Snapshot.objects.filter(timestamp__startswith=slug).exists():
+                        raise Snapshot.MultipleObjectsReturned
+                    else:
+                        raise
+            except Snapshot.DoesNotExist:
+                # Snapshot does not exist
+                return HttpResponse(
+                    format_html(
+                        (
+                            '<center><br/><br/><br/>'
+                            'No Snapshot directories match the given timestamp or UUID: <code>{}</code><br/><br/>'
+                            'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
+                            '</center>'
+                        ),
+                        slug,
+                        path,
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
+            except Snapshot.MultipleObjectsReturned:
+                snapshot_hrefs = mark_safe('<br/>').join(
+                    format_html(
+                        '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                        snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                        snap.timestamp,
+                        snap.timestamp,
+                        snap.url,
+                        snap.title or '',
+                    )
+                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
+                )
+                return HttpResponse(
+                    format_html(
+                        (
+                            'Multiple Snapshots match the given timestamp/UUID <code>{}</code><br/><pre>'
+                        ),
+                        slug,
+                    ) + snapshot_hrefs + format_html(
+                        (
+                            '</pre><br/>'
+                            'Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'
+                        )
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
+            except Http404:
+                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
+                return HttpResponse(
+                    format_html(
+                        (
+                            '<center><br/><br/><br/>'
+                            f'Snapshot <a href="/archive/{snapshot.timestamp}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a> exists in DB, but resource <b><code>{snapshot.timestamp}/'
+                            '{}'
+                            f'</code></b> does not exist in <a href="/archive/{snapshot.timestamp}/" target="_top">snapshot dir</a> yet.<br/><br/>'
+                            'Maybe this resource type is not availabe for this Snapshot,<br/>or the archiving process has not completed yet?<br/>'
+                            f'<pre><code># run this cmd to finish archiving this Snapshot<br/>archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
+                            '<div class="text-align: left; width: 100%; max-width: 400px">'
+                            '<i><b>Next steps:</i></b><br/>'
+                            f'- list all the <a href="/archive/{snapshot.timestamp}/" target="_top">Snapshot files <code>.*</code></a><br/>'
+                            f'- view the <a href="/archive/{snapshot.timestamp}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
+                            f'- go to the <a href="/admin/core/snapshot/{snapshot.id}/change/" target="_top">Snapshot admin</a> to edit<br/>'
+                            f'- go to the <a href="/admin/core/snapshot/?id__startswith={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
+                            '- or return to <a href="/" target="_top">the main index...</a></div>'
+                            '</center>'
+                        ),
+                        archivefile,
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
         # slug is a URL
-        by_url = {page.base_url: page for page in all_pages}
         try:
-            # TODO: add multiple snapshot support by showing index of all snapshots
-            # for given url instead of redirecting to timestamp index
-            timestamp = by_url[base_url(path)].timestamp
-            return redirect(f'/archive/{timestamp}/index.html')
-        except KeyError:
-            pass
-
-        return HttpResponse(
-            'No archived link matches the given timestamp or hash.',
-            content_type="text/plain",
-            status=404,
-        )
+            try:
+                # try exact match on full url first
+                snapshot = Snapshot.objects.get(
+                    Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
+                )
+            except Snapshot.DoesNotExist:
+                # fall back to match on exact base_url
+                try:
+                    snapshot = Snapshot.objects.get(
+                        Q(url='http://' + base_url(path)) | Q(url='https://' + base_url(path))
+                    )
+                except Snapshot.DoesNotExist:
+                    # fall back to matching base_url as prefix
+                    snapshot = Snapshot.objects.get(
+                        Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
+                    )
+            return redirect(f'/archive/{snapshot.timestamp}/index.html')
+        except Snapshot.DoesNotExist:
+            return HttpResponse(
+                format_html(
+                    (
+                        '<center><br/><br/><br/>'
+                        'No Snapshots match the given url: <code>{}</code><br/><br/><br/>'
+                        'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
+                        '+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
+                        '</center>'
+                    ),
+                    base_url(path),
+                    path if '://' in path else f'https://{path}',
+                    path,
+                ),
+                content_type="text/html",
+                status=404,
+            )
+        except Snapshot.MultipleObjectsReturned:
+            snapshot_hrefs = mark_safe('<br/>').join(
+                format_html(
+                    '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                    snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                    snap.timestamp,
+                    snap.timestamp,
+                    snap.url,
+                    snap.title or '',
+                )
+                for snap in Snapshot.objects.filter(
+                    Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
+                ).only('url', 'timestamp', 'title', 'added').order_by('-added')
+            )
+            return HttpResponse(
+                format_html(
+                    (
+                        'Multiple Snapshots match the given URL <code>{}</code><br/><pre>'
+                    ),
+                    base_url(path),
+                ) + snapshot_hrefs + format_html(
+                    (
+                        '</pre><br/>'
+                        'Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'
+                    )
+                ),
+                content_type="text/html",
+                status=404,
+            )
+        
 
 class PublicIndexView(ListView):
     template_name = 'public_index.html'
     model = Snapshot
-    paginate_by = 100
-    ordering = ['title']
+    paginate_by = SNAPSHOTS_PER_PAGE
+    ordering = ['-added']
 
     def get_context_data(self, **kwargs):
         return {
@@ -105,12 +219,14 @@ class PublicIndexView(ListView):
         }
 
     def get_queryset(self, **kwargs): 
-        qs = super().get_queryset(**kwargs) 
+        qs = super().get_queryset(**kwargs)
         query = self.request.GET.get('q')
-        if query:
+        if query and query.strip():
             qs = qs.filter(Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query))
-        for snapshot in qs:
-            snapshot.icons = snapshot_icons(snapshot)
+            try:
+                qs = qs | query_search_index(query)
+            except Exception as err:
+                print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
         return qs
 
     def get(self, *args, **kwargs):
@@ -130,9 +246,9 @@ class AddView(UserPassesTestMixin, FormView):
         if self.request.method == 'GET':
             url = self.request.GET.get('url', None)
             if url:
-                return {'url': url}
-        else:
-            return super().get_initial()
+                return {'url': url if '://' in url else f'https://{url}'}
+        
+        return super().get_initial()
 
     def test_func(self):
         return PUBLIC_ADD_VIEW or self.request.user.is_authenticated
@@ -145,16 +261,21 @@ class AddView(UserPassesTestMixin, FormView):
             'absolute_add_path': self.request.build_absolute_uri(self.request.path),
             'VERSION': VERSION,
             'FOOTER_INFO': FOOTER_INFO,
+            'stdout': '',
         }
 
     def form_valid(self, form):
         url = form.cleaned_data["url"]
         print(f'[+] Adding URL: {url}')
+        parser = form.cleaned_data["parser"]
+        tag = form.cleaned_data["tag"]
         depth = 0 if form.cleaned_data["depth"] == "0" else 1
         extractors = ','.join(form.cleaned_data["archive_methods"])
         input_kwargs = {
             "urls": url,
+            "tag": tag,
             "depth": depth,
+            "parser": parser,
             "update_all": False,
             "out_dir": OUTPUT_DIR,
         }

archivebox/core/wsgi.py

@@ -7,10 +7,10 @@ For more information on this file, see
 https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
 """
 
-import os
+
+from archivebox.config import setup_django
+setup_django(in_memory_db=False, check_db=True)
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.settings')
-
 application = get_wsgi_application()

archivebox/extractors/__init__.py

@@ -4,20 +4,22 @@ import os
 from pathlib import Path
 
 from typing import Optional, List, Iterable, Union
+from datetime import datetime, timezone
+from django.db.models import QuerySet
 
-from django.db.models import QuerySet, Model
-
+from ..index.schema import Link
+from ..index.sql import write_link_to_sql_index
 from ..index import (
-    load_snapshot_details,
-    write_snapshot_details,
+    load_link_details,
+    write_link_details,
 )
 from ..util import enforce_types
 from ..logging_util import (
     log_archiving_started,
     log_archiving_paused,
     log_archiving_finished,
-    log_snapshot_archiving_started,
-    log_snapshot_archiving_finished,
+    log_link_archiving_started,
+    log_link_archiving_finished,
     log_archive_method_started,
     log_archive_method_finished,
 )
@@ -42,16 +44,16 @@ def get_default_archive_methods():
     return [
         ('title', should_save_title, save_title),
         ('favicon', should_save_favicon, save_favicon),
-        ('wget', should_save_wget, save_wget),
+        ('headers', should_save_headers, save_headers),
         ('singlefile', should_save_singlefile, save_singlefile),
         ('pdf', should_save_pdf, save_pdf),
         ('screenshot', should_save_screenshot, save_screenshot),
         ('dom', should_save_dom, save_dom),
-        ('readability', should_save_readability, save_readability), #keep readability below wget and singlefile, as it depends on them
+        ('wget', should_save_wget, save_wget),
+        ('readability', should_save_readability, save_readability),  # keep readability below wget and singlefile, as it depends on them
         ('mercury', should_save_mercury, save_mercury),
         ('git', should_save_git, save_git),
         ('media', should_save_media, save_media),
-        ('headers', should_save_headers, save_headers),
         ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
@@ -65,9 +67,15 @@ def ignore_methods(to_ignore: List[str]):
     return list(methods)
 
 @enforce_types
-def archive_snapshot(snapshot: Model, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Model:
+def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Link:
     """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
-    from core.models import ArchiveResult
+
+    # TODO: Remove when the input is changed to be a snapshot. Suboptimal approach.
+    from core.models import Snapshot, ArchiveResult
+    try:
+        snapshot = Snapshot.objects.get(url=link.url) # TODO: This will be unnecessary once everything is a snapshot
+    except Snapshot.DoesNotExist:
+        snapshot = write_link_to_sql_index(link)
 
     ARCHIVE_METHODS = get_default_archive_methods()
     
@@ -77,63 +85,69 @@ def archive_snapshot(snapshot: Model, overwrite: bool=False, methods: Optional[I
             if method[0] in methods
         ]
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     try:
         is_new = not Path(out_dir).exists()
         if is_new:
             os.makedirs(out_dir)
-            details = {"history": {}}
-            write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=False)
-        else:
-            details = snapshot.details #TODO: This can be retrieved from the sqlite database too.
-                                       # If that makes more sense, it can be easily changed.
 
-        log_snapshot_archiving_started(snapshot, out_dir, is_new)
+        link = load_link_details(link, out_dir=out_dir)
+        write_link_details(link, out_dir=out_dir, skip_sql_index=False)
+        log_link_archiving_started(link, out_dir, is_new)
+        link = link.overwrite(updated=datetime.now(timezone.utc))
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
+        start_ts = datetime.now(timezone.utc)
 
         for method_name, should_run, method_function in ARCHIVE_METHODS:
             try:
-                if method_name not in details["history"]:
-                    details["history"][method_name] = []
+                if method_name not in link.history:
+                    link.history[method_name] = []
 
-                if should_run(snapshot, out_dir, overwrite):
+                if should_run(link, out_dir, overwrite):
                     log_archive_method_started(method_name)
 
-                    result = method_function(snapshot=snapshot, out_dir=out_dir)
+                    result = method_function(link=link, out_dir=out_dir)
+
+                    link.history[method_name].append(result)
 
                     stats[result.status] += 1
                     log_archive_method_finished(result)
-                    write_search_index(snapshot=snapshot, texts=result.index_texts)
+                    write_search_index(link=link, texts=result.index_texts)
                     ArchiveResult.objects.create(snapshot=snapshot, extractor=method_name, cmd=result.cmd, cmd_version=result.cmd_version,
                                                  output=result.output, pwd=result.pwd, start_ts=result.start_ts, end_ts=result.end_ts, status=result.status)
 
+
+                    # bump the updated time on the main Snapshot here, this is critical
+                    # to be able to cache summaries of the ArchiveResults for a given
+                    # snapshot without having to load all the results from the DB each time.
+                    # (we use {Snapshot.id}-{Snapshot.updated} as the cache key and assume
+                    # ArchiveResults are unchanged as long as the updated timestamp is unchanged)
+                    snapshot.save()
                 else:
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
                     stats['skipped'] += 1
             except Exception as e:
-                raise Exception('Exception in archive_methods.save_{}(Snapshot(url={}))'.format(
+                raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
                     method_name,
-                    snapshot.url,
+                    link.url,
                 )) from e
 
         # print('    ', stats)
 
         try:
-            latest_title_archive_result = snapshot.archiveresult_set.filter(extractor="title")
-            if latest_title_archive_result.count() > 0:
-                latest_title = latest_title_archive_result.output.strip()
-                if len(latest_title) >= len(snapshot.title or ''):
-                    snapshot.title = latest_title
+            latest_title = link.history['title'][-1].output.strip()
+            if latest_title and len(latest_title) >= len(link.title or ''):
+                link = link.overwrite(title=latest_title)
         except Exception:
             pass
 
-        write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=False)
+        write_link_details(link, out_dir=out_dir, skip_sql_index=False)
 
-        log_snapshot_archiving_finished(snapshot, snapshot.snapshot_dir, is_new, stats)
+        log_link_archiving_finished(link, link.link_dir, is_new, stats, start_ts)
 
     except KeyboardInterrupt:
         try:
-            write_snapshot_details(snapshot, out_dir=snapshot.snapshot_dir)
+            write_link_details(link, out_dir=link.link_dir)
         except:
             pass
         raise
@@ -142,29 +156,35 @@ def archive_snapshot(snapshot: Model, overwrite: bool=False, methods: Optional[I
         print('    ! Failed to archive link: {}: {}'.format(err.__class__.__name__, err))
         raise
 
-    return snapshot
+    return link
 
 @enforce_types
-def archive_snapshots(all_snapshots: Union[QuerySet, List[Model]], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> QuerySet:
+def archive_links(all_links: Union[Iterable[Link], QuerySet], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> List[Link]:
 
-    all_snapshots = list(all_snapshots)
-    num_snapshots: int = len(all_snapshots)
+    if type(all_links) is QuerySet:
+        num_links: int = all_links.count()
+        get_link = lambda x: x.as_link()
+        all_links = all_links.iterator()
+    else:
+        num_links: int = len(all_links)
+        get_link = lambda x: x
 
-    if num_snapshots == 0:
+    if num_links == 0:
         return []
 
-    log_archiving_started(num_snapshots)
+    log_archiving_started(num_links)
     idx: int = 0
     try:
-        for snapshot in all_snapshots:
+        for link in all_links:
             idx += 1
-            archive_snapshot(snapshot, overwrite=overwrite, methods=methods, out_dir=Path(snapshot.snapshot_dir))
+            to_archive = get_link(link)
+            archive_link(to_archive, overwrite=overwrite, methods=methods, out_dir=Path(link.link_dir))
     except KeyboardInterrupt:
-        log_archiving_paused(num_snapshots, idx, snapshot.timestamp)
+        log_archiving_paused(num_links, idx, link.timestamp)
         raise SystemExit(0)
     except BaseException:
         print()
         raise
 
-    log_archiving_finished(num_snapshots)
-    return all_snapshots
+    log_archiving_finished(num_links)
+    return all_links

archivebox/extractors/archive_org.py

@@ -5,9 +5,7 @@ from pathlib import Path
 from typing import Optional, List, Dict, Tuple
 from collections import defaultdict
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -25,30 +23,27 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-# output = '{domain}/'
-
 
 @enforce_types
-def should_save_archive_dot_org(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
-    if is_static_file(snapshot.url):
+def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
     if not overwrite and (out_dir / 'archive.org.txt').exists():
-        # if open(path, 'r').read().strip() != 'None':
+        # if open(path, 'r', encoding='utf-8').read().strip() != 'None':
         return False
 
     return SAVE_ARCHIVE_DOT_ORG
 
 @enforce_types
-def save_archive_dot_org(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """submit site to archive.org for archiving via their service, save returned archive url"""
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'archive.org.txt'
     archive_org_url = None
-    submit_url = 'https://web.archive.org/save/{}'.format(snapshot.url)
+    submit_url = 'https://web.archive.org/save/{}'.format(link.url)
     cmd = [
         CURL_BINARY,
         *CURL_ARGS,

archivebox/extractors/dom.py

@@ -3,9 +3,7 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file, atomic_write
 from ..util import (
     enforce_types,
@@ -20,13 +18,10 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-# output = 'output.html'
-
 
 @enforce_types
-def should_save_dom(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
-    if is_static_file(snapshot.url):
+def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -36,16 +31,16 @@ def should_save_dom(snapshot: Model, overwrite: Optional[bool]=False, out_dir: O
     return SAVE_DOM
 
 @enforce_types
-def save_dom(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print HTML of site to file using chrome --dump-html"""
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'output.html'
     output_path = out_dir / output
     cmd = [
         *chrome_args(TIMEOUT=timeout),
         '--dump-dom',
-        snapshot.url
+        link.url
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/favicon.py

@@ -4,9 +4,7 @@ from pathlib import Path
 
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput
+from ..index.schema import Link, ArchiveResult, ArchiveOutput
 from ..system import chmod_file, run
 from ..util import enforce_types, domain
 from ..config import (
@@ -21,22 +19,19 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-# output = 'favicon.ico'
-
-
 @enforce_types
-def should_save_favicon(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or snapshot.snapshot_dir
-    if not overwrite and (Path(out_dir) / 'favicon.ico').exists():
+def should_save_favicon(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'favicon.ico').exists():
         return False
 
     return SAVE_FAVICON
 
 @enforce_types
-def save_favicon(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download site favicon from google's favicon api"""
 
-    out_dir = out_dir or snapshot.snapshot_dir
+    out_dir = out_dir or link.link_dir
     output: ArchiveOutput = 'favicon.ico'
     cmd = [
         CURL_BINARY,
@@ -45,7 +40,7 @@ def save_favicon(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIM
         '--output', str(output),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        'https://www.google.com/s2/favicons?domain={}'.format(domain(snapshot.url)),
+        'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
     ]
     status = 'failed'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/git.py

@@ -4,9 +4,7 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -28,16 +26,10 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-# output = 'git/'
-# @contents = output.glob('*.*')
-# @exists = self.contents.exists()
-# @size => get_size(self.contents)
-# @num_files => len(self.contents)
 
 @enforce_types
-def should_save_git(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or snapshot.snapshot_dir
-    if is_static_file(snapshot.url):
+def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -45,8 +37,8 @@ def should_save_git(snapshot: Model, overwrite: Optional[bool]=False, out_dir: O
         return False
 
     is_clonable_url = (
-        (domain(snapshot.url) in GIT_DOMAINS)
-        or (extension(snapshot.url) == 'git')
+        (domain(link.url) in GIT_DOMAINS)
+        or (extension(link.url) == 'git')
     )
     if not is_clonable_url:
         return False
@@ -55,10 +47,10 @@ def should_save_git(snapshot: Model, overwrite: Optional[bool]=False, out_dir: O
 
 
 @enforce_types
-def save_git(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using git"""
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'git'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
@@ -67,7 +59,7 @@ def save_git(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT
         'clone',
         *GIT_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']),
-        without_query(without_fragment(snapshot.url)),
+        without_query(without_fragment(link.url)),
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/headers.py

@@ -4,9 +4,7 @@ from pathlib import Path
 
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput
+from ..index.schema import Link, ArchiveResult, ArchiveOutput
 from ..system import atomic_write
 from ..util import (
     enforce_types,
@@ -23,28 +21,20 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
-
-# output = 'headers.json'
-
 @enforce_types
-def should_save_headers(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or snapshot.snapshot_dir
-
-    if not SAVE_HEADERS:
+def should_save_headers(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'headers.json').exists():
         return False
-    
-    if overwrite:
-        return True
 
-    output = Path(out_dir or snapshot.snapshot_dir) / 'headers.json'
-    return not output.exists()
+    return SAVE_HEADERS
 
 
 @enforce_types
-def save_headers(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """Download site headers"""
 
-    out_dir = Path(out_dir or snapshot.snapshot_dir)
+    out_dir = Path(out_dir or link.link_dir)
     output_folder = out_dir.absolute()
     output: ArchiveOutput = 'headers.json'
 
@@ -58,10 +48,10 @@ def save_headers(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIME
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        snapshot.url,
+        link.url,
     ]
     try:
-        json_headers = get_headers(snapshot.url, timeout=timeout)
+        json_headers = get_headers(link.url, timeout=timeout)
         output_folder.mkdir(exist_ok=True)
         atomic_write(str(output_folder / "headers.json"), json_headers)
     except (Exception, OSError) as err:

archivebox/extractors/media.py

@@ -3,9 +3,7 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -22,13 +20,9 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-# output = 'media/'
-
 @enforce_types
-def should_save_media(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or snapshot.snapshot_dir
-
-    if is_static_file(snapshot.url):
+def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -38,10 +32,10 @@ def should_save_media(snapshot: Model, overwrite: Optional[bool]=False, out_dir:
     return SAVE_MEDIA
 
 @enforce_types
-def save_media(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
+def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
     """Download playlists or individual video, audio, and subtitles using youtube-dl"""
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'media'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
@@ -49,7 +43,7 @@ def save_media(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=MEDIA
         YOUTUBEDL_BINARY,
         *YOUTUBEDL_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
-        snapshot.url,
+        link.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
@@ -76,11 +70,24 @@ def save_media(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=MEDIA
     finally:
         timer.end()
 
+    # add video description and subtitles to full-text index
+    index_texts = [
+        text_file.read_text(encoding='utf-8').strip()
+        for text_file in (
+            *output_path.glob('*.description'),
+            *output_path.glob('*.srt'),
+            *output_path.glob('*.vtt'),
+            *output_path.glob('*.lrc'),
+            *output_path.glob('*.lrc'),
+        )
+    ]
+
     return ArchiveResult(
         cmd=cmd,
         pwd=str(out_dir),
         cmd_version=YOUTUBEDL_VERSION,
         output=output,
         status=status,
+        index_texts=index_texts,
         **timer.stats,
     )

archivebox/extractors/mercury.py

@@ -6,9 +6,7 @@ from subprocess import CompletedProcess
 from typing import Optional, List
 import json
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
@@ -39,33 +37,34 @@ def ShellError(cmd: List[str], result: CompletedProcess, lines: int=20) -> Archi
 
 
 @enforce_types
-def should_save_mercury(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or snapshot.snapshot_dir
-    if is_static_file(snapshot.url):
+def should_save_mercury(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
-    output = Path(out_dir or snapshot.snapshot_dir) / 'mercury'
-    if not overwrite and output.exists():
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'mercury').exists():
         return False
 
-    return SAVE_MERCURY and MERCURY_VERSION
+    return SAVE_MERCURY
 
 
 @enforce_types
-def save_mercury(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download reader friendly version using @postlight/mercury-parser"""
 
-    out_dir = Path(out_dir or snapshot.snapshot_dir)
+    out_dir = Path(out_dir or link.link_dir)
     output_folder = out_dir.absolute() / "mercury"
-    output = str(output_folder)
+    output = "mercury"
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
+        output_folder.mkdir(exist_ok=True)
+
         # Get plain text version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
-            snapshot.url,
+            link.url,
             "--format=text"
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
@@ -74,10 +73,15 @@ def save_mercury(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIM
         except json.JSONDecodeError:
             raise ShellError(cmd, result)
         
+        if article_text.get('failed'):
+            raise ArchiveError('Mercury was not able to get article text from the URL')
+
+        atomic_write(str(output_folder / "content.txt"), article_text["content"])
+
         # Get HTML version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
-            snapshot.url
+            link.url
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:
@@ -85,9 +89,10 @@ def save_mercury(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIM
         except json.JSONDecodeError:
             raise ShellError(cmd, result)
 
-        output_folder.mkdir(exist_ok=True)
+        if article_text.get('failed'):
+            raise ArchiveError('Mercury was not able to get article HTML from the URL')
+
         atomic_write(str(output_folder / "content.html"), article_json.pop("content"))
-        atomic_write(str(output_folder / "content.txt"), article_text["content"])
         atomic_write(str(output_folder / "article.json"), article_json)
 
         # Check for common failure cases

archivebox/extractors/pdf.py

@@ -3,9 +3,7 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -19,12 +17,10 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
-# output = 'output.pdf'
 
 @enforce_types
-def should_save_pdf(snapshot: Model, verwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
-    if is_static_file(snapshot.url):
+def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -35,15 +31,15 @@ def should_save_pdf(snapshot: Model, verwrite: Optional[bool]=False, out_dir: Op
 
 
 @enforce_types
-def save_pdf(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print PDF of site to file using chrome --headless"""
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'output.pdf'
     cmd = [
         *chrome_args(TIMEOUT=timeout),
         '--print-to-pdf',
-        snapshot.url,
+        link.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/readability.py

@@ -6,9 +6,7 @@ from tempfile import NamedTemporaryFile
 from typing import Optional
 import json
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
@@ -25,52 +23,47 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
-
 @enforce_types
-def get_html(snapshot: Model, path: Path) -> str:
+def get_html(link: Link, path: Path) -> str:
     """
     Try to find wget, singlefile and then dom files.
     If none is found, download the url again.
     """
-    canonical = snapshot.canonical_outputs()
+    canonical = link.canonical_outputs()
     abs_path = path.absolute()
     sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
     document = None
     for source in sources:
         try:
-            with open(abs_path / source, "r") as f:
+            with open(abs_path / source, "r", encoding="utf-8") as f:
                 document = f.read()
                 break
         except (FileNotFoundError, TypeError):
             continue
     if document is None:
-        return download_url(snapshot.url)
+        return download_url(link.url)
     else:
         return document
 
-
-# output = 'readability/'
-
 @enforce_types
-def should_save_readability(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
-    out_dir = out_dir or snapshot.link_dir
-    if is_static_file(snapshot.url):
+def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
-    output = Path(out_dir or snapshot.snapshot_dir) / 'readability'
-    if not overwrite and output.exists():
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and (out_dir / 'readability').exists():
         return False
 
-    return SAVE_READABILITY and READABILITY_VERSION
+    return SAVE_READABILITY
 
 
 @enforce_types
-def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download reader friendly version using @mozilla/readability"""
 
-    out_dir = Path(out_dir or snapshot.snapshot_dir)
+    out_dir = Path(out_dir or link.link_dir)
     output_folder = out_dir.absolute() / "readability"
-    output = str(output_folder)
+    output = "readability"
 
     # Readability Docs: https://github.com/mozilla/readability
 
@@ -78,23 +71,31 @@ def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=
     # fake command to show the user so they have something to try debugging if get_html fails
     cmd = [
         CURL_BINARY,
-        snapshot.url
+        link.url
     ]
     readability_content = None
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        document = get_html(snapshot, out_dir)
+        document = get_html(link, out_dir)
         temp_doc = NamedTemporaryFile(delete=False)
         temp_doc.write(document.encode("utf-8"))
         temp_doc.close()
 
+        if not document or len(document) < 10:
+            raise ArchiveError('Readability could not find HTML to parse for article text')
+
         cmd = [
             DEPENDENCIES['READABILITY_BINARY']['path'],
-            temp_doc.name
+            temp_doc.name,
         ]
 
         result = run(cmd, cwd=out_dir, timeout=timeout)
-        result_json = json.loads(result.stdout)
+        try:
+            result_json = json.loads(result.stdout)
+            assert result_json and 'content' in result_json
+        except json.JSONDecodeError:
+            raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)
+
         output_folder.mkdir(exist_ok=True)
         readability_content = result_json.pop("textContent") 
         atomic_write(str(output_folder / "content.html"), result_json.pop("content"))
@@ -119,6 +120,7 @@ def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=
     except (Exception, OSError) as err:
         status = 'failed'
         output = err
+        cmd = [cmd[0], './{singlefile,dom}.html']
     finally:
         timer.end()
 
@@ -128,6 +130,6 @@ def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=
         cmd_version=READABILITY_VERSION,
         output=output,
         status=status,
-        index_texts= [readability_content] if readability_content else [],
+        index_texts=[readability_content] if readability_content else [],
         **timer.stats,  
     )

archivebox/extractors/screenshot.py

@@ -3,9 +3,7 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -20,12 +18,10 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-# output = 'screenshot.png'
 
 @enforce_types
-def should_save_screenshot(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
-    if is_static_file(snapshot.url):
+def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -35,15 +31,15 @@ def should_save_screenshot(snapshot: Model, overwrite: Optional[bool]=False, out
     return SAVE_SCREENSHOT
 
 @enforce_types
-def save_screenshot(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """take screenshot of site using chrome --headless"""
     
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output: ArchiveOutput = 'screenshot.png'
     cmd = [
         *chrome_args(TIMEOUT=timeout),
         '--screenshot',
-        snapshot.url,
+        link.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/singlefile.py

@@ -5,9 +5,7 @@ from pathlib import Path
 from typing import Optional
 import json
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -25,9 +23,8 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_singlefile(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
-    if is_static_file(snapshot.url):
+def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    if is_static_file(link.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -38,10 +35,10 @@ def should_save_singlefile(snapshot: Model, overwrite: Optional[bool]=False, out
 
 
 @enforce_types
-def save_singlefile(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using single-file"""
 
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or Path(link.link_dir)
     output = "singlefile.html"
 
     browser_args = chrome_args(TIMEOUT=0)
@@ -52,8 +49,7 @@ def save_singlefile(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=
         DEPENDENCIES['SINGLEFILE_BINARY']['path'],
         '--browser-executable-path={}'.format(CHROME_BINARY),
         browser_args,
-        snapshot.url,
-        output
+        link.url,
         output,
     ]
 

archivebox/extractors/title.py

@@ -5,9 +5,7 @@ from html.parser import HTMLParser
 from pathlib import Path
 from typing import Optional
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..util import (
     enforce_types,
     download_url,
@@ -61,12 +59,10 @@ class TitleParser(HTMLParser):
             self.inside_title_tag = False
 
 
-# output = '{title}'
-
 @enforce_types
-def should_save_title(snapshot: Model, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
     # if link already has valid title, skip it
-    if not overwrite and snapshot.title and not snapshot.title.lower().startswith('http'):
+    if not overwrite and link.title and not link.title.lower().startswith('http'):
         return False
 
     return SAVE_TITLE
@@ -77,7 +73,7 @@ def extract_title_with_regex(html):
     return output
 
 @enforce_types
-def save_title(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """try to guess the page's title from its content"""
 
     from core.models import Snapshot
@@ -89,12 +85,12 @@ def save_title(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEO
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        snapshot.url,
+        link.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        html = download_url(snapshot.url, timeout=timeout)
+        html = download_url(link.url, timeout=timeout)
         try:
             # try using relatively strict html parser first
             parser = TitleParser()
@@ -108,11 +104,10 @@ def save_title(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEO
         
         # if title is better than the one in the db, update db with new title
         if isinstance(output, str) and output:
-            if not snapshot.title or len(output) >= len(snapshot.title):
-                Snapshot.objects.filter(url=snapshot.url,
-                                        timestamp=snapshot.timestamp)\
+            if not link.title or len(output) >= len(link.title):
+                Snapshot.objects.filter(url=link.url,
+                                        timestamp=link.timestamp)\
                                 .update(title=output)
-                snapshot.title = output
         else:
             # if no content was returned, dont save a title (because it might be a temporary error)
             if not html:

archivebox/extractors/wget.py

@@ -4,11 +4,9 @@ import re
 from pathlib import Path
 
 from typing import Optional
-from datetime import datetime
+from datetime import datetime, timezone
 
-from django.db.models import Model
-
-from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
+from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -36,24 +34,24 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_wget(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
-    output_path = wget_output_path(snapshot)
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
-    if not overwrite output_path and (out_dir / output_path).exists():
+def should_save_wget(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
+    output_path = wget_output_path(link)
+    out_dir = out_dir or Path(link.link_dir)
+    if not overwrite and output_path and (out_dir / output_path).exists():
         return False
 
     return SAVE_WGET
 
 
 @enforce_types
-def save_wget(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using wget"""
 
-    out_dir = out_dir or snapshot.snapshot_dir
+    out_dir = out_dir or link.link_dir
     if SAVE_WARC:
         warc_dir = out_dir / "warc"
         warc_dir.mkdir(exist_ok=True)
-        warc_path = warc_dir / str(int(datetime.now().timestamp()))
+        warc_path = warc_dir / str(int(datetime.now(timezone.utc).timestamp()))
 
     # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
     output: ArchiveOutput = None
@@ -70,14 +68,14 @@ def save_wget(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOU
         *(['--compression=auto'] if WGET_AUTO_COMPRESSION else []),
         *([] if SAVE_WARC else ['--timestamping']),
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),
-        snapshot.url,
+        link.url,
     ]
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
         result = run(cmd, cwd=str(out_dir), timeout=timeout)
-        output = wget_output_path(snapshot)
+        output = wget_output_path(link)
 
         # parse out number of files downloaded from last line of stderr:
         #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
@@ -128,13 +126,13 @@ def save_wget(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOU
 
 
 @enforce_types
-def wget_output_path(snapshot: Model) -> Optional[str]:
+def wget_output_path(link: Link) -> Optional[str]:
     """calculate the path to the wgetted .html file, since wget may
     adjust some paths to be different than the base_url path.
 
     See docs on wget --adjust-extension (-E)
     """
-
+    
     # Wget downloads can save in a number of different ways depending on the url:
     #    https://example.com
     #       > example.com/index.html
@@ -166,8 +164,8 @@ def wget_output_path(snapshot: Model) -> Optional[str]:
     # and there's no way to get the computed output path from wget
     # in order to avoid having to reverse-engineer how they calculate it,
     # we just look in the output folder read the filename wget used from the filesystem
-    full_path = without_fragment(without_query(path(snapshot.url))).strip('/')
-    search_dir = Path(snapshot.snapshot_dir) / domain(snapshot.url).replace(":", "+") / urldecode(full_path)
+    full_path = without_fragment(without_query(path(link.url))).strip('/')
+    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
     for _ in range(4):
         if search_dir.exists():
             if search_dir.is_dir():
@@ -176,7 +174,7 @@ def wget_output_path(snapshot: Model) -> Optional[str]:
                     if re.search(".+\\.[Ss]?[Hh][Tt][Mm][Ll]?$", str(f), re.I | re.M)
                 ]
                 if html_files:
-                    return str(html_files[0].relative_to(snapshot.snapshot_dir))
+                    return str(html_files[0].relative_to(link.link_dir))
 
                 # sometimes wget'd URLs have no ext and return non-html
                 # e.g. /some/example/rss/all -> some RSS XML content)
@@ -185,23 +183,23 @@ def wget_output_path(snapshot: Model) -> Optional[str]:
                 last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
                 for file_present in search_dir.iterdir():
                     if file_present == last_part_of_url:
-                        return str((search_dir / file_present).relative_to(snapshot.snapshot_dir))
+                        return str((search_dir / file_present).relative_to(link.link_dir))
 
         # Move up one directory level
         search_dir = search_dir.parent
 
-        if search_dir == snapshot.snapshot_dir:
+        if str(search_dir) == link.link_dir:
             break
 
     # check for literally any file present that isnt an empty folder
-    domain_dir = Path(domain(snapshot.url).replace(":", "+"))
-    files_within = list((Path(snapshot.snapshot_dir) / domain_dir).glob('**/*.*'))
+    domain_dir = Path(domain(link.url).replace(":", "+"))
+    files_within = list((Path(link.link_dir) / domain_dir).glob('**/*.*'))
     if files_within:
-        return str((domain_dir / files_within[-1]).relative_to(snapshot.snapshot_dir))
+        return str((domain_dir / files_within[-1]).relative_to(link.link_dir))
     
-    # fallback to just the domain dir, dont try to introspect further inside it
-    search_dir = Path(snapshot.snapshot_dir) / domain(snapshot.url).replace(":", "+")
+    # fallback to just the domain dir
+    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
     if search_dir.is_dir():
-        return domain(snapshot.url).replace(":", "+")
+        return domain(link.url).replace(":", "+")
 
     return None

archivebox/index/__init__.py

@@ -9,7 +9,7 @@ from typing import List, Tuple, Dict, Optional, Iterable
 from collections import OrderedDict
 from contextlib import contextmanager
 from urllib.parse import urlparse
-from django.db.models import QuerySet, Q, Model
+from django.db.models import QuerySet, Q
 
 from ..util import (
     scheme,
@@ -38,16 +38,16 @@ from ..logging_util import (
 
 from .schema import Link, ArchiveResult
 from .html import (
-    write_html_snapshot_details,
+    write_html_link_details,
 )
 from .json import (
     pyjson,
-    load_json_snapshot,
-    write_json_snapshot_details,
+    parse_json_link_details, 
+    write_json_link_details,
 )
 from .sql import (
     write_sql_main_index,
-    write_sql_snapshot_details,
+    write_sql_link_details,
 )
 
 from ..search import search_backend_enabled, query_search_index
@@ -55,12 +55,10 @@ from ..search import search_backend_enabled, query_search_index
 ### Link filtering and checking
 
 @enforce_types
-def merge_snapshots(a: Model, b: Model) -> Model:
-    """deterministially merge two snapshots, favoring longer field values over shorter,
+def merge_links(a: Link, b: Link) -> Link:
+    """deterministially merge two links, favoring longer field values over shorter,
     and "cleaner" values over worse ones.
-    TODO: Check if this makes sense with the new setup
     """
-    return a
     assert a.base_url == b.base_url, f'Cannot merge two links with different URLs ({a.base_url} != {b.base_url})'
 
     # longest url wins (because a fuzzy url will always be shorter)
@@ -111,55 +109,55 @@ def merge_snapshots(a: Model, b: Model) -> Model:
             key=lambda result: result.start_ts,
         )))
 
-    return Snapshot(
+    return Link(
         url=url,
         timestamp=timestamp,
         title=title,
         tags=tags,
-        #sources=sources,
-        #history=history,
+        sources=sources,
+        history=history,
     )
 
 
 @enforce_types
-def validate_snapshots(snapshots: List[Model]) -> List[Model]:
+def validate_links(links: Iterable[Link]) -> List[Link]:
     timer = TimedProgress(TIMEOUT * 4)
     try:
-        snapshots = archivable_snapshots(snapshots)  # remove chrome://, about:, mailto: etc.
-        snapshots = sorted_snapshots(snapshots)      # deterministically sort the links based on timestamp, url
-        snapshots = fix_duplicate_snapshots(snapshots)  # merge/dedupe duplicate timestamps & urls
+        links = archivable_links(links)  # remove chrome://, about:, mailto: etc.
+        links = sorted_links(links)      # deterministically sort the links based on timestamp, url
+        links = fix_duplicate_links(links)  # merge/dedupe duplicate timestamps & urls
     finally:
         timer.end()
 
-    return list(snapshots)
+    return list(links)
 
 @enforce_types
-def archivable_snapshots(snapshots: Iterable[Model]) -> Iterable[Model]:
+def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
     """remove chrome://, about:// or other schemed links that cant be archived"""
-    for snapshot in snapshots:
+    for link in links:
         try:
-            urlparse(snapshot.url)
+            urlparse(link.url)
         except ValueError:
             continue
-        if scheme(snapshot.url) not in ('http', 'https', 'ftp'):
+        if scheme(link.url) not in ('http', 'https', 'ftp'):
             continue
-        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(snapshot.url):
+        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(link.url):
             continue
 
-        yield snapshot
+        yield link
 
 
 @enforce_types
-def fix_duplicate_snapshots(sorted_snapshots: Iterable[Model]) -> Iterable[Model]:
+def fix_duplicate_links(sorted_links: Iterable[Link]) -> Iterable[Link]:
     """
     ensures that all non-duplicate links have monotonically increasing timestamps
-    TODO: Review how to do this with the new snapshots refactor
     """
-    return sorted_snapshots
+    # from core.models import Snapshot
+
     unique_urls: OrderedDict[str, Link] = OrderedDict()
 
-    for snapshot in sorted_snapshots:
-        if snapshot.url in unique_urls:
+    for link in sorted_links:
+        if link.url in unique_urls:
             # merge with any other links that share the same url
             link = merge_links(unique_urls[link.url], link)
         unique_urls[link.url] = link
@@ -168,9 +166,9 @@ def fix_duplicate_snapshots(sorted_snapshots: Iterable[Model]) -> Iterable[Model
 
 
 @enforce_types
-def sorted_snapshots(snapshots: Iterable[Model]) -> Iterable[Model]:
-    sort_func = lambda snapshot: (snapshot.timestamp.split('.', 1)[0], snapshot.url)
-    return sorted(snapshots, key=sort_func, reverse=True)
+def sorted_links(links: Iterable[Link]) -> Iterable[Link]:
+    sort_func = lambda link: (link.timestamp.split('.', 1)[0], link.url)
+    return sorted(links, key=sort_func, reverse=True)
 
 
 @enforce_types
@@ -224,14 +222,14 @@ def timed_index_update(out_path: Path):
 
 
 @enforce_types
-def write_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
+def write_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
     """Writes links to sqlite3 file for a given list of links"""
 
-    log_indexing_process_started(len(snapshots))
+    log_indexing_process_started(len(links))
 
     try:
         with timed_index_update(out_dir / SQL_INDEX_FILENAME):
-            write_sql_main_index(snapshots, out_dir=out_dir)
+            write_sql_main_index(links, out_dir=out_dir)
             os.chmod(out_dir / SQL_INDEX_FILENAME, int(OUTPUT_PERMISSIONS, base=8)) # set here because we don't write it with atomic writes
 
     except (KeyboardInterrupt, SystemExit):
@@ -246,9 +244,7 @@ def write_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
 
 @enforce_types
 def load_main_index(out_dir: Path=OUTPUT_DIR, warn: bool=True) -> List[Link]:
-    """
-    Returns all of the snapshots currently in index
-    """
+    """parse and load existing index with any new links from import_path merged in"""
     from core.models import Snapshot
     try:
         return Snapshot.objects.all()
@@ -269,62 +265,88 @@ def load_main_index_meta(out_dir: Path=OUTPUT_DIR) -> Optional[dict]:
 
 
 @enforce_types
-def parse_snapshots_from_source(source_path: str, root_url: Optional[str]=None) -> List[Model]:
+def parse_links_from_source(source_path: str, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], List[Link]]:
 
-    from ..parsers import parse_snapshots
+    from ..parsers import parse_links
 
-    new_links: List[Model] = []
+    new_links: List[Link] = []
 
     # parse and validate the import file
-    raw_snapshots, parser_name = parse_snapshots(source_path, root_url=root_url)
-    new_snapshots = validate_snapshots(raw_snapshots)
+    raw_links, parser_name = parse_links(source_path, root_url=root_url, parser=parser)
+    new_links = validate_links(raw_links)
 
     if parser_name:
-        num_parsed = len(raw_snapshots)
+        num_parsed = len(raw_links)
         log_parsing_finished(num_parsed, parser_name)
 
-    return new_snapshots
+    return new_links
 
 @enforce_types
-def filter_new_urls(snapshots: QuerySet,
-                 new_snapshots: List) -> List:
+def fix_duplicate_links_in_index(snapshots: QuerySet, links: Iterable[Link]) -> Iterable[Link]:
     """
-    Returns a list of Snapshots corresponding to the urls that were not present in the index
+    Given a list of in-memory Links, dedupe and merge them with any conflicting Snapshots in the DB.
     """
-    urls = {snapshot.url: snapshot for snapshot in new_snapshots}
-    filtered_snapshots = snapshots.filter(url__in=urls.keys())
+    unique_urls: OrderedDict[str, Link] = OrderedDict()
 
-    for found_snapshot in filtered_snapshots:
-        urls.pop(found_snapshot.url)
+    for link in links:
+        index_link = snapshots.filter(url=link.url)
+        if index_link:
+            link = merge_links(index_link[0].as_link(), link)
+
+        unique_urls[link.url] = link
+
+    return unique_urls.values()
+
+@enforce_types
+def dedupe_links(snapshots: QuerySet,
+                 new_links: List[Link]) -> List[Link]:
+    """
+    The validation of links happened at a different stage. This method will
+    focus on actual deduplication and timestamp fixing.
+    """
     
-    log_deduping_finished(len(urls.keys()))
+    # merge existing links in out_dir and new links
+    dedup_links = fix_duplicate_links_in_index(snapshots, new_links)
 
-    return list(urls.values())
+    new_links = [
+        link for link in new_links
+        if not snapshots.filter(url=link.url).exists()
+    ]
+
+    dedup_links_dict = {link.url: link for link in dedup_links}
+
+    # Replace links in new_links with the dedup version
+    for i in range(len(new_links)):
+        if new_links[i].url in dedup_links_dict.keys():
+            new_links[i] = dedup_links_dict[new_links[i].url]
+    log_deduping_finished(len(new_links))
+
+    return new_links
 
 ### Link Details Index
 
 @enforce_types
-def write_snapshot_details(snapshot: List[Model], out_dir: Optional[str]=None, skip_sql_index: bool=False) -> None:
-    out_dir = out_dir or snapshot.snapshot_dir
+def write_link_details(link: Link, out_dir: Optional[str]=None, skip_sql_index: bool=False) -> None:
+    out_dir = out_dir or link.link_dir
 
-    write_json_snapshot_details(snapshot, out_dir=out_dir)
-    write_html_snapshot_details(snapshot, out_dir=out_dir)
+    write_json_link_details(link, out_dir=out_dir)
+    write_html_link_details(link, out_dir=out_dir)
     if not skip_sql_index:
-        write_sql_snapshot_details(snapshot)
+        write_sql_link_details(link)
 
 
 @enforce_types
-def load_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> Model:
+def load_link_details(link: Link, out_dir: Optional[str]=None) -> Link:
     """check for an existing link archive in the given directory, 
        and load+merge it into the given link dict
     """
-    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    out_dir = out_dir or link.link_dir
 
-    existing_snapshot = load_json_snapshot(Path(out_dir))
-    if existing_snapshot:
-        return merge_snapshots(existing_snapshot, snapshot)
+    existing_link = parse_json_link_details(out_dir)
+    if existing_link:
+        return merge_links(existing_link, link)
 
-    return snapshot
+    return link
 
 
 
@@ -334,6 +356,7 @@ LINK_FILTERS = {
     'regex': lambda pattern: Q(url__iregex=pattern),
     'domain': lambda pattern: Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}"),
     'tag': lambda pattern: Q(tags__name=pattern),
+    'timestamp': lambda pattern: Q(timestamp=pattern),
 }
 
 @enforce_types
@@ -379,41 +402,56 @@ def snapshot_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type
         return search_filter(snapshots, filter_patterns, filter_type)
 
 
-def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """indexed links without checking archive status or data directory validity"""
-    return {snapshot.snapshot_dir: snapshot for snapshot in snapshots}
+    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    return {
+        link.link_dir: link
+        for link in links
+    }
 
-def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """indexed links that are archived with a valid data directory"""
-    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_archived, snapshots)}
+    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    return {
+        link.link_dir: link
+        for link in filter(is_archived, links)
+    }
 
-def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """indexed links that are unarchived with no data directory or an empty data directory"""
-    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_unarchived, snapshots)}
+    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    return {
+        link.link_dir: link
+        for link in filter(is_unarchived, links)
+    }
 
-def get_present_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_present_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs that actually exist in the archive/ folder"""
-    from core.models import Snapshot
 
     all_folders = {}
 
     for entry in (out_dir / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
-            snapshot = None
+            link = None
             try:
-                snapshot = load_json_snapshot(Path(entry.path))
+                link = parse_json_link_details(entry.path)
             except Exception:
                 pass
 
-            all_folders[entry.name] = snapshot
+            all_folders[entry.name] = link
 
     return all_folders
 
-def get_valid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_valid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs with a valid index matched to the main index and archived content"""
-    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_valid, snapshots)}
+    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
+    return {
+        link.link_dir: link
+        for link in filter(is_valid, links)
+    }
 
-def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs that are invalid for any reason: corrupted/duplicate/orphaned/unrecognized"""
     duplicate = get_duplicate_folders(snapshots, out_dir=OUTPUT_DIR)
     orphaned = get_orphaned_folders(snapshots, out_dir=OUTPUT_DIR)
@@ -422,7 +460,7 @@ def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Option
     return {**duplicate, **orphaned, **corrupted, **unrecognized}
 
 
-def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs that conflict with other directories that have the same link URL or timestamp"""
     by_url = {}
     by_timestamp = {}
@@ -435,92 +473,91 @@ def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Opti
     )
 
     for path in chain(snapshots.iterator(), data_folders):
-        snapshot = None
+        link = None
         if type(path) is not str:
-            path = path.snapshot_dir
+            path = path.as_link().link_dir
 
         try:
-            snapshot = load_json_snapshot(Path(path))
+            link = parse_json_link_details(path)
         except Exception:
             pass
 
-        if snapshot:
-            # snapshot folder has same timestamp as different link folder
-            by_timestamp[snapshot.timestamp] = by_timestamp.get(snapshot.timestamp, 0) + 1
-            if by_timestamp[snapshot.timestamp] > 1:
-                duplicate_folders[path] = snapshot
+        if link:
+            # link folder has same timestamp as different link folder
+            by_timestamp[link.timestamp] = by_timestamp.get(link.timestamp, 0) + 1
+            if by_timestamp[link.timestamp] > 1:
+                duplicate_folders[path] = link
 
             # link folder has same url as different link folder
-            by_url[snapshot.url] = by_url.get(snapshot.url, 0) + 1
-            if by_url[snapshot.url] > 1:
-                duplicate_folders[path] = snapshot
+            by_url[link.url] = by_url.get(link.url, 0) + 1
+            if by_url[link.url] > 1:
+                duplicate_folders[path] = link
     return duplicate_folders
 
-def get_orphaned_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_orphaned_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs that contain a valid index but aren't listed in the main index"""
     orphaned_folders = {}
 
     for entry in (Path(out_dir) / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
-            snapshot = None
+            link = None
             try:
-                snapshot = load_json_snapshot(entry)
+                link = parse_json_link_details(str(entry))
             except Exception:
                 pass
 
-            if snapshot and not snapshots.filter(timestamp=entry.name).exists():
+            if link and not snapshots.filter(timestamp=entry.name).exists():
                 # folder is a valid link data dir with index details, but it's not in the main index
-                orphaned_folders[str(entry)] = snapshot
+                orphaned_folders[str(entry)] = link
 
     return orphaned_folders
 
-def get_corrupted_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_corrupted_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs that don't contain a valid index and aren't listed in the main index"""
     corrupted = {}
     for snapshot in snapshots.iterator():
-        if is_corrupt(snapshot):
-            corrupted[snapshot.snapshot_dir] = snapshot
+        link = snapshot.as_link()
+        if is_corrupt(link):
+            corrupted[link.link_dir] = link
     return corrupted
 
-def get_unrecognized_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
+def get_unrecognized_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
     """dirs that don't contain recognizable archive data and aren't listed in the main index"""
-    unrecognized_folders: Dict[str, Optional[Model]] = {}
+    unrecognized_folders: Dict[str, Optional[Link]] = {}
 
     for entry in (Path(out_dir) / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
             index_exists = (entry / "index.json").exists()
-            snapshot = None
+            link = None
             try:
-                snapshot = load_json_snapshot(entry)
+                link = parse_json_link_details(str(entry))
             except KeyError:
                 # Try to fix index
                 if index_exists:
-                    pass
-                    # TODO: Implement the `guess` bit for snapshots
-                    # try:
+                    try:
                         # Last attempt to repair the detail index
-                        # link_guessed = parse_json_snapshot_details(str(entry), guess=True)
-                        # write_json_snapshot_details(link_guessed, out_dir=str(entry))
-                        # link = parse_json_link_details(str(entry))
-                    # except Exception:
-                    #     pass
+                        link_guessed = parse_json_link_details(str(entry), guess=True)
+                        write_json_link_details(link_guessed, out_dir=str(entry))
+                        link = parse_json_link_details(str(entry))
+                    except Exception:
+                        pass
 
-            if index_exists and snapshot is None:
+            if index_exists and link is None:
                 # index exists but it's corrupted or unparseable
-                unrecognized_folders[str(entry)] = snapshot
+                unrecognized_folders[str(entry)] = link
             
             elif not index_exists:
                 # link details index doesn't exist and the folder isn't in the main index
                 timestamp = entry.name
                 if not snapshots.filter(timestamp=timestamp).exists():
-                    unrecognized_folders[str(entry)] = snapshot
+                    unrecognized_folders[str(entry)] = link
 
     return unrecognized_folders
 
 
-def is_valid(snapshot: Model) -> bool:
-    dir_exists = Path(snapshot.snapshot_dir).exists()
-    index_exists = (Path(snapshot.snapshot_dir) / "index.json").exists()
+def is_valid(link: Link) -> bool:
+    dir_exists = Path(link.link_dir).exists()
+    index_exists = (Path(link.link_dir) / "index.json").exists()
     if not dir_exists:
         # unarchived links are not included in the valid list
         return False
@@ -528,30 +565,29 @@ def is_valid(snapshot: Model) -> bool:
         return False
     if dir_exists and index_exists:
         try:
-            # TODO: review if the `guess` was necessary here
-            parsed_snapshot = load_json_snapshot(snapshot.snapshot_dir)
-            return snapshot.url == parsed_snapshot.url
+            parsed_link = parse_json_link_details(link.link_dir, guess=True)
+            return link.url == parsed_link.url
         except Exception:
             pass
     return False
 
-def is_corrupt(snapshot: Model) -> bool:
-    if not Path(snapshot.snapshot_dir).exists():
+def is_corrupt(link: Link) -> bool:
+    if not Path(link.link_dir).exists():
         # unarchived links are not considered corrupt
         return False
 
-    if is_valid(snapshot):
+    if is_valid(link):
         return False
 
     return True
 
-def is_archived(snapshot: Model) -> bool:
-    return is_valid(snapshot) and snapshot.is_archived
+def is_archived(link: Link) -> bool:
+    return is_valid(link) and link.is_archived
     
-def is_unarchived(snapshot: Model) -> bool:
-    if not Path(snapshot.snapshot_dir).exists():
+def is_unarchived(link: Link) -> bool:
+    if not Path(link.link_dir).exists():
         return True
-    return not snapshot.is_archived
+    return not link.is_archived
 
 
 def fix_invalid_folder_locations(out_dir: Path=OUTPUT_DIR) -> Tuple[List[str], List[str]]:
@@ -561,22 +597,22 @@ def fix_invalid_folder_locations(out_dir: Path=OUTPUT_DIR) -> Tuple[List[str], L
         if entry.is_dir(follow_symlinks=True):
             if (Path(entry.path) / 'index.json').exists():
                 try:
-                    snapshot = load_json_snapshot(Path(entry.path))
+                    link = parse_json_link_details(entry.path)
                 except KeyError:
-                    snapshot = None
-                if not snapshot:
+                    link = None
+                if not link:
                     continue
 
-                if not entry.path.endswith(f'/{snapshot.timestamp}'):
-                    dest = out_dir / ARCHIVE_DIR_NAME / snapshot.timestamp
+                if not entry.path.endswith(f'/{link.timestamp}'):
+                    dest = out_dir / ARCHIVE_DIR_NAME / link.timestamp
                     if dest.exists():
                         cant_fix.append(entry.path)
                     else:
                         shutil.move(entry.path, dest)
                         fixed.append(dest)
                         timestamp = entry.path.rsplit('/', 1)[-1]
-                        assert snapshot.snapshot_dir == entry.path
-                        assert snapshot.timestamp == timestamp
-                        write_json_snapshot_details(snapshot, out_dir=entry.path)
+                        assert link.link_dir == entry.path
+                        assert link.timestamp == timestamp
+                        write_json_link_details(link, out_dir=entry.path)
 
     return fixed, cant_fix

archivebox/index/csv.py

@@ -2,14 +2,12 @@ __package__ = 'archivebox.index'
 
 from typing import List, Optional, Any
 
-from django.db.models import Model
-
 from ..util import enforce_types
 from .schema import Link
 
 
 @enforce_types
-def snapshots_to_csv(snapshots: List[Model],
+def links_to_csv(links: List[Link],
                  cols: Optional[List[str]]=None,
                  header: bool=True,
                  separator: str=',',
@@ -22,8 +20,8 @@ def snapshots_to_csv(snapshots: List[Model],
         header_str = separator.join(col.ljust(ljust) for col in cols)
 
     row_strs = (
-        snapshot.as_csv(cols=cols, ljust=ljust, separator=separator)
-        for snapshot in snapshots
+        link.to_csv(cols=cols, ljust=ljust, separator=separator)
+        for link in links
     )
 
     return '\n'.join((header_str, *row_strs))

archivebox/index/html.py

@@ -1,19 +1,19 @@
 __package__ = 'archivebox.index'
 
-from datetime import datetime
-from typing import List, Optional, Iterator, Mapping
 from pathlib import Path
-
-from django.db.models import Model
-from django.utils.html import format_html, mark_safe
+from datetime import datetime, timezone
 from collections import defaultdict
+from typing import List, Optional, Iterator, Mapping
+
+from django.utils.html import format_html, mark_safe
+from django.core.cache import cache
 
 from .schema import Link
 from ..system import atomic_write
 from ..logging_util import printable_filesize
 from ..util import (
     enforce_types,
-    ts_to_date,
+    ts_to_date_str,
     urlencode,
     htmlencode,
     urldecode,
@@ -21,7 +21,6 @@ from ..util import (
 from ..config import (
     OUTPUT_DIR,
     VERSION,
-    GIT_SHA,
     FOOTER_INFO,
     HTML_INDEX_FILENAME,
     SAVE_ARCHIVE_DOT_ORG,
@@ -48,24 +47,24 @@ def parse_html_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[str]:
     return ()
 
 @enforce_types
-def generate_index_from_snapshots(snapshots: List[Model], with_headers: bool):
+def generate_index_from_links(links: List[Link], with_headers: bool):
     if with_headers:
-        output = main_index_template(snapshots)
+        output = main_index_template(links)
     else:
-        output = main_index_template(snapshots, template=MINIMAL_INDEX_TEMPLATE)
+        output = main_index_template(links, template=MINIMAL_INDEX_TEMPLATE)
     return output
 
 @enforce_types
-def main_index_template(snapshots: List[Model], template: str=MAIN_INDEX_TEMPLATE) -> str:
+def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) -> str:
     """render the template for the entire main index"""
 
     return render_django_template(template, {
         'version': VERSION,
-        'git_sha': GIT_SHA,
-        'num_snapshots': str(len(snapshots)),
-        'date_updated': datetime.now().strftime('%Y-%m-%d'),
-        'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
-        'snapshots': snapshots,
+        'git_sha': VERSION,  # not used anymore, but kept for backwards compatibility
+        'num_links': str(len(links)),
+        'date_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d'),
+        'time_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M'),
+        'links': [link._asdict(extended=True) for link in links],
         'FOOTER_INFO': FOOTER_INFO,
     })
 
@@ -73,36 +72,38 @@ def main_index_template(snapshots: List[Model], template: str=MAIN_INDEX_TEMPLAT
 ### Link Details Index
 
 @enforce_types
-def write_html_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> None:
-    out_dir = out_dir or snapshot.snapshot_dir
+def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
+    out_dir = out_dir or link.link_dir
 
-    rendered_html = snapshot_details_template(snapshot)
+    rendered_html = link_details_template(link)
     atomic_write(str(Path(out_dir) / HTML_INDEX_FILENAME), rendered_html)
 
 
 @enforce_types
-def snapshot_details_template(snapshot: Model) -> str:
+def link_details_template(link: Link) -> str:
 
     from ..extractors.wget import wget_output_path
 
+    link_info = link._asdict(extended=True)
+
     return render_django_template(LINK_DETAILS_TEMPLATE, {
-        **snapshot.as_json(),
-        **snapshot.canonical_outputs(),
+        **link_info,
+        **link_info['canonical'],
         'title': htmlencode(
-            snapshot.title
-            or (snapshot.base_url if snapshot.is_archived else TITLE_LOADING_MSG)
+            link.title
+            or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
         ),
-        'url_str': htmlencode(urldecode(snapshot.base_url)),
+        'url_str': htmlencode(urldecode(link.base_url)),
         'archive_url': urlencode(
-            wget_output_path(snapshot)
-            or (snapshot.domain if snapshot.is_archived else '')
+            wget_output_path(link)
+            or (link.domain if link.is_archived else '')
         ) or 'about:blank',
-        'extension': snapshot.extension or 'html',
-        'tags': snapshot.tags_str() or 'untagged',
-        'size': printable_filesize(snapshot.archive_size) if snapshot.archive_size else 'pending',
-        'status': 'archived' if snapshot.is_archived else 'not yet archived',
-        'status_color': 'success' if snapshot.is_archived else 'danger',
-        'oldest_archive_date': ts_to_date(snapshot.oldest_archive_date),
+        'extension': link.extension or 'html',
+        'tags': link.tags or 'untagged',
+        'size': printable_filesize(link.archive_size) if link.archive_size else 'pending',
+        'status': 'archived' if link.is_archived else 'not yet archived',
+        'status_color': 'success' if link.is_archived else 'danger',
+        'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
         'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
     })
 
@@ -115,70 +116,78 @@ def render_django_template(template: str, context: Mapping[str, str]) -> str:
 
 
 def snapshot_icons(snapshot) -> str:
-    from core.models import EXTRACTORS
+    cache_key = f'{snapshot.id}-{(snapshot.updated or snapshot.added).timestamp()}-snapshot-icons'
+    
+    def calc_snapshot_icons():
+        from core.models import EXTRACTORS
+        # start = datetime.now(timezone.utc)
 
-    # start = datetime.now()
+        archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
+        link = snapshot.as_link()
+        path = link.archive_path
+        canon = link.canonical_outputs()
+        output = ""
+        output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
+        icons = {
+            "singlefile": "❶",
+            "wget": "🆆",
+            "dom": "🅷",
+            "pdf": "📄",
+            "screenshot": "💻",
+            "media": "📼",
+            "git": "🅶",
+            "archive_org": "🏛",
+            "readability": "🆁",
+            "mercury": "🅼",
+            "warc": "📦"
+        }
+        exclude = ["favicon", "title", "headers", "archive_org"]
+        # Missing specific entry for WARC
 
-    archive_results = snapshot.archiveresult_set.filter(status="succeeded")
-    path = snapshot.archive_path
-    canon = snapshot.canonical_outputs()
-    output = ""
-    output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
-    icons = {
-        "singlefile": "❶",
-        "wget": "🆆",
-        "dom": "🅷",
-        "pdf": "📄",
-        "screenshot": "💻",
-        "media": "📼",
-        "git": "🅶",
-        "archive_org": "🏛",
-        "readability": "🆁",
-        "mercury": "🅼",
-        "warc": "📦"
-    }
-    exclude = ["favicon", "title", "headers", "archive_org"]
-    # Missing specific entry for WARC
+        extractor_outputs = defaultdict(lambda: None)
+        for extractor, _ in EXTRACTORS:
+            for result in archive_results:
+                if result.extractor == extractor and result:
+                    extractor_outputs[extractor] = result
 
-    extractor_outputs = defaultdict(lambda: None)
-    for extractor, _ in EXTRACTORS:
-        for result in archive_results:
-            if result.extractor == extractor and result:
-                extractor_outputs[extractor] = result
+        for extractor, _ in EXTRACTORS:
+            if extractor not in exclude:
+                existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                # Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
+                # if existing:
+                #     existing = (Path(path) / existing)
+                #     if existing.is_file():
+                #         existing = True
+                #     elif existing.is_dir():
+                #         existing = any(existing.glob('*.*'))
+                output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
+                                             extractor, icons.get(extractor, "?"))
+            if extractor == "wget":
+                # warc isn't technically it's own extractor, so we have to add it after wget
+                
+                # get from db (faster but less thurthful)
+                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                # get from filesystem (slower but more accurate)
+                # exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
+                output += format_html(output_template, path, canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
 
-    for extractor, _ in EXTRACTORS:
-        if extractor not in exclude:
-            existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-            # Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
-            # if existing:
-            #     existing = (Path(path) / existing)
-            #     if existing.is_file():
-            #         existing = True
-            #     elif existing.is_dir():
-            #         existing = any(existing.glob('*.*'))
-            output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
-                                         extractor, icons.get(extractor, "?"))
-        if extractor == "wget":
-            # warc isn't technically it's own extractor, so we have to add it after wget
-            
-            # get from db (faster but less thurthful)
-            exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-            # get from filesystem (slower but more accurate)
-            # exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
-            output += format_html(output_template, 'warc/', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
+            if extractor == "archive_org":
+                # The check for archive_org is different, so it has to be handled separately
 
-        if extractor == "archive_org":
-            # The check for archive_org is different, so it has to be handled separately
+                # get from db (faster)
+                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                # get from filesystem (slower)
+                # target_path = Path(path) / "archive.org.txt"
+                # exists = target_path.exists()
+                output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
+                                                                                            "archive_org", icons.get("archive_org", "?"))
 
-            # get from db (faster)
-            exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-            # get from filesystem (slower)
-            # target_path = Path(path) / "archive.org.txt"
-            # exists = target_path.exists()
-            output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
-                                                                                        "archive_org", icons.get("archive_org", "?"))
+        result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
+        # end = datetime.now(timezone.utc)
+        # print(((end - start).total_seconds()*1000) // 1, 'ms')
+        return result
 
-    result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
-    # end = datetime.now()
-    # print(((end - start).total_seconds()*1000) // 1, 'ms')
-    return result
+    return cache.get_or_set(cache_key, calc_snapshot_icons)
+    # return calc_snapshot_icons()
+
+   

archivebox/index/json.py

@@ -5,9 +5,8 @@ import sys
 import json as pyjson
 from pathlib import Path
 
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Optional, Iterator, Any, Union
-from django.db.models import Model
 
 from .schema import Link
 from ..system import atomic_write
@@ -16,7 +15,6 @@ from ..config import (
     VERSION,
     OUTPUT_DIR,
     FOOTER_INFO,
-    GIT_SHA,
     DEPENDENCIES,
     JSON_INDEX_FILENAME,
     ARCHIVE_DIR_NAME,
@@ -31,7 +29,7 @@ MAIN_INDEX_HEADER = {
     'meta': {
         'project': 'ArchiveBox',
         'version': VERSION,
-        'git_sha': GIT_SHA,
+        'git_sha': VERSION,  # not used anymore, but kept for backwards compatibility
         'website': 'https://ArchiveBox.io',
         'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
         'source': 'https://github.com/ArchiveBox/ArchiveBox',
@@ -41,18 +39,17 @@ MAIN_INDEX_HEADER = {
 }
 
 @enforce_types
-def generate_json_index_from_snapshots(snapshots: List[Model], with_headers: bool):
-    snapshots_json = [snapshot.as_json() for snapshot in snapshots]
+def generate_json_index_from_links(links: List[Link], with_headers: bool):
     if with_headers:
         output = {
             **MAIN_INDEX_HEADER,
-            'num_links': len(snapshots),
-            'updated': datetime.now(),
+            'num_links': len(links),
+            'updated': datetime.now(timezone.utc),
             'last_run_cmd': sys.argv,
-            'links': snapshots_json,
+            'links': links,
         }
     else:
-        output = snapshots_json
+        output = links
     return to_json(output, indent=4, sort_keys=True)
 
 
@@ -63,7 +60,18 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     index_path = Path(out_dir) / JSON_INDEX_FILENAME
     if index_path.exists():
         with open(index_path, 'r', encoding='utf-8') as f:
-            links = pyjson.load(f)['links']
+            try:
+                links = pyjson.load(f)['links']
+                if links:
+                    Link.from_json(links[0])
+            except Exception as err:
+                print("    {lightyellow}! Found an index.json in the project root but couldn't load links from it: {} {}".format(
+                    err.__class__.__name__,
+                    err,
+                    **ANSI,
+                ))
+                return ()
+
             for link_json in links:
                 try:
                     yield Link.from_json(link_json)
@@ -83,46 +91,41 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
 ### Link Details Index
 
 @enforce_types
-def write_json_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> None:
-    """write a json file with some info about the snapshot"""
+def write_json_link_details(link: Link, out_dir: Optional[str]=None) -> None:
+    """write a json file with some info about the link"""
     
-    out_dir = out_dir or snapshot.snapshot_dir
+    out_dir = out_dir or link.link_dir
     path = Path(out_dir) / JSON_INDEX_FILENAME
-    atomic_write(str(path), snapshot.as_json())
+    atomic_write(str(path), link._asdict(extended=True))
 
 
 @enforce_types
-def load_json_snapshot(out_dir: Path) -> Optional[Model]:
-    """
-    Loads the detail from the local json index
-    """
-    from core.models import Snapshot
-
+def parse_json_link_details(out_dir: Union[Path, str], guess: Optional[bool]=False) -> Optional[Link]:
+    """load the json link index from a given directory"""
     existing_index = Path(out_dir) / JSON_INDEX_FILENAME
     if existing_index.exists():
         with open(existing_index, 'r', encoding='utf-8') as f:
             try:
-                output = pyjson.load(f)
-                output = Snapshot.from_json(output)
-                return output
+                link_json = pyjson.load(f)
+                return Link.from_json(link_json, guess)
             except pyjson.JSONDecodeError:
                 pass
     return None
 
 
 @enforce_types
-def parse_json_snapshot_details(out_dir: Union[Path, str]) -> Iterator[dict]:
-    """read through all the archive data folders and return the parsed snapshots"""
+def parse_json_links_details(out_dir: Union[Path, str]) -> Iterator[Link]:
+    """read through all the archive data folders and return the parsed links"""
 
     for entry in os.scandir(Path(out_dir) / ARCHIVE_DIR_NAME):
         if entry.is_dir(follow_symlinks=True):
             if (Path(entry.path) / 'index.json').exists():
                 try:
-                    snapshot_details = load_json_snapshot(Path(entry.path))
+                    link = parse_json_link_details(entry.path)
                 except KeyError:
-                    snapshot_details = None
-                if snapshot_details:
-                    yield snapshot_details
+                    link = None
+                if link:
+                    yield link
 
 
 

archivebox/index/schema.py

@@ -10,15 +10,16 @@ __package__ = 'archivebox.index'
 
 from pathlib import Path
 
-from datetime import datetime, timedelta
+from datetime import datetime, timezone, timedelta
 
 from typing import List, Dict, Any, Optional, Union
 
 from dataclasses import dataclass, asdict, field, fields
 
+from django.utils.functional import cached_property
 
 from ..system import get_dir_size
-
+from ..util import ts_to_date_str, parse_date
 from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
 
 class ArchiveError(Exception):
@@ -55,11 +56,11 @@ class ArchiveResult:
         assert isinstance(self.end_ts, datetime)
         assert isinstance(self.cmd, list)
         assert all(isinstance(arg, str) and arg for arg in self.cmd)
-        assert self.pwd is None or isinstance(self.pwd, str) and self.pwd
-        assert self.cmd_version is None or isinstance(self.cmd_version, str) and self.cmd_version
+
+        # TODO: replace emptystrings in these three with None / remove them from the DB
+        assert self.pwd is None or isinstance(self.pwd, str)
+        assert self.cmd_version is None or isinstance(self.cmd_version, str)
         assert self.output is None or isinstance(self.output, (str, Exception))
-        if isinstance(self.output, str):
-            assert self.output
 
     @classmethod
     def guess_ts(_cls, dict_info):
@@ -133,7 +134,6 @@ class Link:
     updated: Optional[datetime] = None
     schema: str = 'Link'
 
-
     def __str__(self) -> str:
         return f'[{self.timestamp}] {self.url} "{self.title}"'
 
@@ -190,6 +190,7 @@ class Link:
         }
         if extended:
             info.update({
+                'snapshot_id': self.snapshot_id,
                 'link_dir': self.link_dir,
                 'archive_path': self.archive_path,
                 
@@ -201,6 +202,9 @@ class Link:
                 'basename': self.basename,
                 'extension': self.extension,
                 'is_static': self.is_static,
+                
+                'tags_str': (self.tags or '').strip(','),   # only used to render static index in index/html.py, remove if no longer needed there
+                'icons': None,           # only used to render static index in index/html.py, remove if no longer needed there
 
                 'bookmarked_date': self.bookmarked_date,
                 'updated_date': self.updated_date,
@@ -255,6 +259,11 @@ class Link:
 
         return to_csv(self, cols=cols or self.field_names(), separator=separator, ljust=ljust)
 
+    @cached_property
+    def snapshot_id(self):
+        from core.models import Snapshot
+        return str(Snapshot.objects.only('id').get(url=self.url).id)
+
     @classmethod
     def field_names(cls):
         return [f.name for f in fields(cls)]
@@ -316,13 +325,11 @@ class Link:
     ### Pretty Printing Helpers
     @property
     def bookmarked_date(self) -> Optional[str]:
-        from ..util import ts_to_date
-
-        max_ts = (datetime.now() + timedelta(days=30)).timestamp()
+        max_ts = (datetime.now(timezone.utc) + timedelta(days=30)).timestamp()
 
         if self.timestamp and self.timestamp.replace('.', '').isdigit():
             if 0 < float(self.timestamp) < max_ts:
-                return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
+                return ts_to_date_str(datetime.fromtimestamp(float(self.timestamp)))
             else:
                 return str(self.timestamp)
         return None
@@ -330,13 +337,12 @@ class Link:
 
     @property
     def updated_date(self) -> Optional[str]:
-        from ..util import ts_to_date
-        return ts_to_date(self.updated) if self.updated else None
+        return ts_to_date_str(self.updated) if self.updated else None
 
     @property
     def archive_dates(self) -> List[datetime]:
         return [
-            result.start_ts
+            parse_date(result.start_ts)
             for method in self.history.keys()
                 for result in self.history[method]
         ]
@@ -408,7 +414,6 @@ class Link:
         return latest
 
 
-
     def canonical_outputs(self) -> Dict[str, Optional[str]]:
         """predict the expected output paths that should be present after archiving"""
 

archivebox/index/sql.py

@@ -3,12 +3,11 @@ __package__ = 'archivebox.index'
 from io import StringIO
 from pathlib import Path
 from typing import List, Tuple, Iterator
-from django.db.models import QuerySet, Model
+from django.db.models import QuerySet
 from django.db import transaction
-from datetime import datetime
 
 from .schema import Link
-from ..util import enforce_types
+from ..util import enforce_types, parse_date
 from ..config import OUTPUT_DIR
 
 
@@ -24,55 +23,94 @@ def parse_sql_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     )
 
 @enforce_types
-def remove_from_sql_main_index(snapshots: QuerySet, out_dir: Path=OUTPUT_DIR) -> None:
-    with transaction.atomic():
-        snapshots.delete()
+def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
+    if atomic:
+        with transaction.atomic():
+            return snapshots.delete()
+    return snapshots.delete()
 
 @enforce_types
-def write_snapshot_to_index(snapshot: Model):
-    from core.models import Snapshot
+def write_link_to_sql_index(link: Link):
+    from core.models import Snapshot, ArchiveResult
+    info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}
+    tags = info.pop("tags")
+    if tags is None:
+        tags = []
+
     try:
-        timestamp = Snapshot.objects.get(url=snapshot.url).timestamp
+        info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp
     except Snapshot.DoesNotExist:
-        timestamp = snapshot.timestamp
-        if not timestamp:
-            timestamp = str(datetime.now().timestamp())
-        while Snapshot.objects.filter(timestamp=timestamp).exists():
-            print("the timestamp is: ", timestamp)
-            timestamp = str(float(timestamp) + 1.0)
+        while Snapshot.objects.filter(timestamp=info["timestamp"]).exists():
+            info["timestamp"] = str(float(info["timestamp"]) + 1.0)
+
+        snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info)
+    snapshot.save_tags(tags)
+
+    for extractor, entries in link.history.items():
+        for entry in entries:
+            if isinstance(entry, dict):
+                result, _ = ArchiveResult.objects.get_or_create(
+                    snapshot_id=snapshot.id,
+                    extractor=extractor,
+                    start_ts=parse_date(entry['start_ts']),
+                    defaults={
+                        'end_ts': parse_date(entry['end_ts']),
+                        'cmd': entry['cmd'],
+                        'output': entry['output'],
+                        'cmd_version': entry.get('cmd_version') or 'unknown',
+                        'pwd': entry['pwd'],
+                        'status': entry['status'],
+                    }
+                )
+            else:
+                result, _ = ArchiveResult.objects.update_or_create(
+                    snapshot_id=snapshot.id,
+                    extractor=extractor,
+                    start_ts=parse_date(entry.start_ts),
+                    defaults={
+                        'end_ts': parse_date(entry.end_ts),
+                        'cmd': entry.cmd,
+                        'output': entry.output,
+                        'cmd_version': entry.cmd_version or 'unknown',
+                        'pwd': entry.pwd,
+                        'status': entry.status,
+                    }
+                )
 
-    snapshot.timestamp = timestamp
-    snapshot.save()
     return snapshot
 
 
 @enforce_types
-def write_sql_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
-    with transaction.atomic():
-        for snapshot in snapshots:
-            write_snapshot_to_index(snapshot)
+def write_sql_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
+    for link in links:
+        # with transaction.atomic():
+            # write_link_to_sql_index(link)
+        write_link_to_sql_index(link)
             
 
 @enforce_types
-def write_sql_snapshot_details(snapshot: Model, out_dir: Path=OUTPUT_DIR) -> None:
+def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
     from core.models import Snapshot
 
-    with transaction.atomic():
-        try:
-            snap = Snapshot.objects.get(url=snapshot.url)
-        except Snapshot.DoesNotExist:
-            snap = write_snapshot_to_index(snapshot)
-        snap.title = snapshot.title
+    # with transaction.atomic():
+    #     try:
+    #         snap = Snapshot.objects.get(url=link.url)
+    #     except Snapshot.DoesNotExist:
+    #         snap = write_link_to_sql_index(link)
+    #     snap.title = link.title
+    try:
+        snap = Snapshot.objects.get(url=link.url)
+    except Snapshot.DoesNotExist:
+        snap = write_link_to_sql_index(link)
+    snap.title = link.title
 
-        # TODO: If there are actual tags, this will break
-        #tag_set = (
-        #    set(tag.strip() for tag in (snapshot.tags.all() or '').split(','))
-        #)
-        #tag_list = list(tag_set) or []
+    tag_set = (
+        set(tag.strip() for tag in (link.tags or '').split(','))
+    )
+    tag_list = list(tag_set) or []
 
-        snap.save()
-        #snap.save_tags(tag_list)
-        return snap
+    snap.save()
+    snap.save_tags(tag_list)
 
 
 

archivebox/logging_util.py

@@ -3,26 +3,30 @@ __package__ = 'archivebox'
 import re
 import os
 import sys
+import stat
 import time
 import argparse
 from math import log
 from multiprocessing import Process
 from pathlib import Path
 
-from datetime import datetime
+from datetime import datetime, timezone
 from dataclasses import dataclass
-from typing import Optional, List, Dict, Union, IO, TYPE_CHECKING
+from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .index.schema import Link, ArchiveResult
 
+from .system import get_dir_size
 from .util import enforce_types
 from .config import (
     ConfigDict,
     OUTPUT_DIR,
     PYTHON_ENCODING,
+    VERSION,
     ANSI,
     IS_TTY,
+    IN_DOCKER,
     TERM_WIDTH,
     SHOW_PROGRESS,
     SOURCES_DIR_NAME,
@@ -50,6 +54,37 @@ class RuntimeStats:
 _LAST_RUN_STATS = RuntimeStats()
 
 
+def debug_dict_summary(obj: Dict[Any, Any]) -> None:
+    stderr(' '.join(f'{key}={str(val).ljust(6)}' for key, val in obj.items()))
+
+
+def get_fd_info(fd) -> Dict[str, Any]:
+    NAME = fd.name[1:-1]
+    FILENO = fd.fileno()
+    MODE = os.fstat(FILENO).st_mode
+    IS_TTY = hasattr(fd, 'isatty') and fd.isatty()
+    IS_PIPE = stat.S_ISFIFO(MODE)
+    IS_FILE = stat.S_ISREG(MODE)
+    IS_TERMINAL =  not (IS_PIPE or IS_FILE)
+    IS_LINE_BUFFERED = fd.line_buffering
+    IS_READABLE = fd.readable()
+    return {
+        'NAME': NAME, 'FILENO': FILENO, 'MODE': MODE,
+        'IS_TTY': IS_TTY, 'IS_PIPE': IS_PIPE, 'IS_FILE': IS_FILE,
+        'IS_TERMINAL': IS_TERMINAL, 'IS_LINE_BUFFERED': IS_LINE_BUFFERED,
+        'IS_READABLE': IS_READABLE,
+    }
+    
+
+# # Log debug information about stdin, stdout, and stderr
+# sys.stdout.write('[>&1] this is python stdout\n')
+# sys.stderr.write('[>&2] this is python stderr\n')
+
+# debug_dict_summary(get_fd_info(sys.stdin))
+# debug_dict_summary(get_fd_info(sys.stdout))
+# debug_dict_summary(get_fd_info(sys.stderr))
+
+
 
 class SmartFormatter(argparse.HelpFormatter):
     """Patched formatter that prints newlines in argparse help strings"""
@@ -62,22 +97,40 @@ class SmartFormatter(argparse.HelpFormatter):
 def reject_stdin(caller: str, stdin: Optional[IO]=sys.stdin) -> None:
     """Tell the user they passed stdin to a command that doesn't accept it"""
 
-    if stdin and not stdin.isatty():
-        stdin_raw_text = stdin.read().strip()
+    if not stdin:
+        return None
+
+    if IN_DOCKER:
+        # when TTY is disabled in docker we cant tell if stdin is being piped in or not
+        # if we try to read stdin when its not piped we will hang indefinitely waiting for it
+        return None
+
+    if not stdin.isatty():
+        # stderr('READING STDIN TO REJECT...')
+        stdin_raw_text = stdin.read()
         if stdin_raw_text:
+            # stderr('GOT STDIN!', len(stdin_str))
             stderr(f'[X] The "{caller}" command does not accept stdin.', color='red')
             stderr(f'    Run archivebox "{caller} --help" to see usage and examples.')
             stderr()
             raise SystemExit(1)
+    return None
 
 
 def accept_stdin(stdin: Optional[IO]=sys.stdin) -> Optional[str]:
     """accept any standard input and return it as a string or None"""
+    
     if not stdin:
         return None
-    elif stdin and not stdin.isatty():
-        stdin_str = stdin.read().strip()
-        return stdin_str or None
+
+    if not stdin.isatty():
+        # stderr('READING STDIN TO ACCEPT...')
+        stdin_str = stdin.read()
+
+        if stdin_str:
+            # stderr('GOT STDIN...', len(stdin_str))
+            return stdin_str
+
     return None
 
 
@@ -85,17 +138,19 @@ class TimedProgress:
     """Show a progress bar and measure elapsed time until .end() is called"""
 
     def __init__(self, seconds, prefix=''):
+
         self.SHOW_PROGRESS = SHOW_PROGRESS
         if self.SHOW_PROGRESS:
             self.p = Process(target=progress_bar, args=(seconds, prefix))
             self.p.start()
 
-        self.stats = {'start_ts': datetime.now(), 'end_ts': None}
+        self.stats = {'start_ts': datetime.now(timezone.utc), 'end_ts': None}
 
     def end(self):
         """immediately end progress, clear the progressbar line, and save end_ts"""
 
-        end_ts = datetime.now()
+
+        end_ts = datetime.now(timezone.utc)
         self.stats['end_ts'] = end_ts
         
         if self.SHOW_PROGRESS:
@@ -104,7 +159,10 @@ class TimedProgress:
                 # kill the progress bar subprocess
                 try:
                     self.p.close()   # must be closed *before* its terminnated
-                except:
+                except (KeyboardInterrupt, SystemExit):
+                    print()
+                    raise
+                except BaseException:                                           # lgtm [py/catch-base-exception]
                     pass
                 self.p.terminate()
                 self.p.join()
@@ -170,14 +228,12 @@ def progress_bar(seconds: int, prefix: str='') -> None:
         # sys.stdout.flush()
     except (KeyboardInterrupt, BrokenPipeError):
         print()
-        pass
 
 
 def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str):
-    from .config import VERSION, ANSI
     cmd = ' '.join(('archivebox', subcommand, *subcommand_args))
     stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{reset}'.format(
-        now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+        now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
         VERSION=VERSION,
         cmd=cmd,
         **ANSI,
@@ -189,7 +245,7 @@ def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional
 
 
 def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool):
-    _LAST_RUN_STATS.parse_start_ts = datetime.now()
+    _LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc)
     print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format(
         _LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'),
         len(urls) if isinstance(urls, list) else len(urls.split('\n')),
@@ -202,7 +258,7 @@ def log_source_saved(source_file: str):
     print('    > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1]))
 
 def log_parsing_finished(num_parsed: int, parser_name: str):
-    _LAST_RUN_STATS.parse_end_ts = datetime.now()
+    _LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc)
     print('    > Parsed {} URLs from input ({})'.format(num_parsed, parser_name))
 
 def log_deduping_finished(num_new_links: int):
@@ -216,7 +272,7 @@ def log_crawl_started(new_links):
 ### Indexing Stage
 
 def log_indexing_process_started(num_links: int):
-    start_ts = datetime.now()
+    start_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.index_start_ts = start_ts
     print()
     print('{black}[*] [{}] Writing {} links to main index...{reset}'.format(
@@ -227,23 +283,24 @@ def log_indexing_process_started(num_links: int):
 
 
 def log_indexing_process_finished():
-    end_ts = datetime.now()
+    end_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.index_end_ts = end_ts
 
 
 def log_indexing_started(out_path: str):
     if IS_TTY:
-        sys.stdout.write(f'    > {out_path}')
+        sys.stdout.write(f'    > ./{Path(out_path).relative_to(OUTPUT_DIR)}')
 
 
 def log_indexing_finished(out_path: str):
-    print(f'\r    √ {out_path}')
+    print(f'\r    √ ./{Path(out_path).relative_to(OUTPUT_DIR)}')
 
 
 ### Archiving Stage
 
 def log_archiving_started(num_links: int, resume: Optional[float]=None):
-    start_ts = datetime.now()
+
+    start_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.archiving_start_ts = start_ts
     print()
     if resume:
@@ -261,7 +318,8 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
         ))
 
 def log_archiving_paused(num_links: int, idx: int, timestamp: str):
-    end_ts = datetime.now()
+
+    end_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.archiving_end_ts = end_ts
     print()
     print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
@@ -272,13 +330,14 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str):
         total=num_links,
     ))
     print()
-    print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI))
-    print('        archivebox server  # then visit http://127.0.0.1:8000')
     print('    Continue archiving where you left off by running:')
     print('        archivebox update --resume={}'.format(timestamp))
 
 def log_archiving_finished(num_links: int):
-    end_ts = datetime.now()
+
+    from core.models import Snapshot
+
+    end_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.archiving_end_ts = end_ts
     assert _LAST_RUN_STATS.archiving_start_ts is not None
     seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
@@ -298,12 +357,15 @@ def log_archiving_finished(num_links: int):
     print('    - {} links skipped'.format(_LAST_RUN_STATS.skipped))
     print('    - {} links updated'.format(_LAST_RUN_STATS.succeeded + _LAST_RUN_STATS.failed))
     print('    - {} links had errors'.format(_LAST_RUN_STATS.failed))
-    print()
-    print('    {lightred}Hint:{reset} To manage your archive in a Web UI, run:'.format(**ANSI))
-    print('        archivebox server 0.0.0.0:8000')
+    
+    if Snapshot.objects.count() < 50:
+        print()
+        print('    {lightred}Hint:{reset} To manage your archive in a Web UI, run:'.format(**ANSI))
+        print('        archivebox server 0.0.0.0:8000')
 
 
-def log_snapshot_archiving_started(snapshot: Model, snapshot_dir: str, is_new: bool):
+def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
+
     # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
     #     http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
     #     > output/archive/1478739709
@@ -311,17 +373,17 @@ def log_snapshot_archiving_started(snapshot: Model, snapshot_dir: str, is_new: b
     print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format(
         symbol_color=ANSI['green' if is_new else 'black'],
         symbol='+' if is_new else '√',
-        now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-        title=snapshot.title or snapshot.base_url,
+        now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
+        title=link.title or link.base_url,
         **ANSI,
     ))
-    print('    {blue}{url}{reset}'.format(url=snapshot.url, **ANSI))
+    print('    {blue}{url}{reset}'.format(url=link.url, **ANSI))
     print('    {} {}'.format(
         '>' if is_new else '√',
-        pretty_path(snapshot_dir),
+        pretty_path(link_dir),
     ))
 
-def log_snapshot_archiving_finished(snapshot: Model, snapshot_dir: str, is_new: bool, stats: dict):
+def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats: dict, start_ts: datetime):
     total = sum(stats.values())
 
     if stats['failed'] > 0 :
@@ -331,6 +393,11 @@ def log_snapshot_archiving_finished(snapshot: Model, snapshot_dir: str, is_new:
     else:
         _LAST_RUN_STATS.succeeded += 1
 
+    size = get_dir_size(link_dir)
+    end_ts = datetime.now(timezone.utc)
+    duration = str(end_ts - start_ts).split('.')[0]
+    print('        {black}{} files ({}) in {}s {reset}'.format(size[2], printable_filesize(size[0]), duration, **ANSI))
+
 
 def log_archive_method_started(method: str):
     print('      > {}'.format(method))
@@ -395,49 +462,49 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str):
     ))
     print('    {}'.format(' '.join(filter_patterns or ())))
 
-def log_list_finished(snapshots):
-    from .index.csv import snapshots_to_csv
+def log_list_finished(links):
+    from .index.csv import links_to_csv
     print()
     print('---------------------------------------------------------------------------------------------------')
-    print(snapshots_to_csv(snapshots, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
+    print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
     print('---------------------------------------------------------------------------------------------------')
     print()
 
 
-def log_removal_started(snapshots: List["Snapshot"], yes: bool, delete: bool):
-    print('{lightyellow}[i] Found {} matching URLs to remove.{reset}'.format(len(snapshots), **ANSI))
+def log_removal_started(links: List["Link"], yes: bool, delete: bool):
+    print('{lightyellow}[i] Found {} matching URLs to remove.{reset}'.format(len(links), **ANSI))
     if delete:
-        file_counts = [snapshot.num_outputs for snapshot in snapshots if Path(snapshot.snapshot_dir).exists()]
+        file_counts = [link.num_outputs for link in links if Path(link.link_dir).exists()]
         print(
-            f'    {len(snapshots)} Snapshots will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
+            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
             f'    ({len(file_counts)} data folders with {sum(file_counts)} archived files will be deleted!)'
         )
     else:
         print(
-            '    Matching snapshots will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
+            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
             '    (Pass --delete if you also want to permanently delete the data folders)'
         )
 
     if not yes:
         print()
-        print('{lightyellow}[?] Do you want to proceed with removing these {} snapshots?{reset}'.format(len(snapshots), **ANSI))
+        print('{lightyellow}[?] Do you want to proceed with removing these {} links?{reset}'.format(len(links), **ANSI))
         try:
             assert input('    y/[n]: ').lower() == 'y'
         except (KeyboardInterrupt, EOFError, AssertionError):
             raise SystemExit(0)
 
-def log_removal_finished(all_snapshots: int, to_remove: int):
-    if to_remove == 0:
+def log_removal_finished(all_links: int, to_remove: int):
+    if all_links == 0:
         print()
-        print('{red}[X] No matching snapshots found.{reset}'.format(**ANSI))
+        print('{red}[X] No matching links found.{reset}'.format(**ANSI))
     else:
         print()
-        print('{red}[√] Removed {} out of {} snapshots from the archive index.{reset}'.format(
+        print('{red}[√] Removed {} out of {} links from the archive index.{reset}'.format(
             to_remove,
-            all_snapshots,
+            all_links,
             **ANSI,
         ))
-        print('    Index now contains {} snapshots.'.format(all_snapshots - to_remove))
+        print('    Index now contains {} links.'.format(all_links - to_remove))
 
 
 def log_shell_welcome_msg():

archivebox/parsers/__init__.py

@@ -11,17 +11,17 @@ import re
 from io import StringIO
 
 from typing import IO, Tuple, List, Optional
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path 
 
-from django.db.models import Model
-
 from ..system import atomic_write
 from ..config import (
     ANSI,
     OUTPUT_DIR,
     SOURCES_DIR_NAME,
     TIMEOUT,
+    stderr,
+    hint,
 )
 from ..util import (
     basename,
@@ -33,106 +33,121 @@ from ..util import (
 from ..index.schema import Link
 from ..logging_util import TimedProgress, log_source_saved
 
-from .pocket_html import parse_pocket_html_export
-from .pocket_api import parse_pocket_api_export
-from .pinboard_rss import parse_pinboard_rss_export
-from .wallabag_atom import parse_wallabag_atom_export
-from .shaarli_rss import parse_shaarli_rss_export
-from .medium_rss import parse_medium_rss_export
-from .netscape_html import parse_netscape_html_export
-from .generic_rss import parse_generic_rss_export
-from .generic_json import parse_generic_json_export
-from .generic_html import parse_generic_html_export
-from .generic_txt import parse_generic_txt_export
+from . import pocket_api
+from . import wallabag_atom
+from . import pocket_html
+from . import pinboard_rss
+from . import shaarli_rss
+from . import medium_rss
 
-PARSERS = (
+from . import netscape_html
+from . import generic_rss
+from . import generic_json
+from . import generic_html
+from . import generic_txt
+from . import url_list
+
+
+PARSERS = {
     # Specialized parsers
-    ('Pocket API', parse_pocket_api_export),
-    ('Wallabag ATOM', parse_wallabag_atom_export),
-    ('Pocket HTML', parse_pocket_html_export),
-    ('Pinboard RSS', parse_pinboard_rss_export),
-    ('Shaarli RSS', parse_shaarli_rss_export),
-    ('Medium RSS', parse_medium_rss_export),
-    
-    # General parsers
-    ('Netscape HTML', parse_netscape_html_export),
-    ('Generic RSS', parse_generic_rss_export),
-    ('Generic JSON', parse_generic_json_export),
-    ('Generic HTML', parse_generic_html_export),
+    pocket_api.KEY:     (pocket_api.NAME,       pocket_api.PARSER),
+    wallabag_atom.KEY:  (wallabag_atom.NAME,    wallabag_atom.PARSER),
+    pocket_html.KEY:    (pocket_html.NAME,      pocket_html.PARSER),
+    pinboard_rss.KEY:   (pinboard_rss.NAME,     pinboard_rss.PARSER),
+    shaarli_rss.KEY:    (shaarli_rss.NAME,      shaarli_rss.PARSER),
+    medium_rss.KEY:     (medium_rss.NAME,       medium_rss.PARSER),
 
-    # Fallback parser
-    ('Plain Text', parse_generic_txt_export),
-)
+    # General parsers
+    netscape_html.KEY:  (netscape_html.NAME,    netscape_html.PARSER),
+    generic_rss.KEY:    (generic_rss.NAME,      generic_rss.PARSER),
+    generic_json.KEY:   (generic_json.NAME,     generic_json.PARSER),
+    generic_html.KEY:   (generic_html.NAME,     generic_html.PARSER),
+
+    # Catchall fallback parser
+    generic_txt.KEY:    (generic_txt.NAME,      generic_txt.PARSER),
+
+    # Explicitly specified parsers
+    url_list.KEY:       (url_list.NAME,         url_list.PARSER),
+}
 
 
 @enforce_types
-def parse_snapshots_memory(urls: List[str], root_url: Optional[str]=None):
+def parse_links_memory(urls: List[str], root_url: Optional[str]=None):
     """
     parse a list of URLS without touching the filesystem
     """
-    check_url_parsing_invariants()
 
     timer = TimedProgress(TIMEOUT * 4)
     #urls = list(map(lambda x: x + "\n", urls))
     file = StringIO()
     file.writelines(urls)
     file.name = "io_string"
-    snapshots, parser = run_parser_functions(file, timer, root_url=root_url)
+    links, parser = run_parser_functions(file, timer, root_url=root_url)
     timer.end()
 
     if parser is None:
         return [], 'Failed to parse'
-    return snapshots, parser
+    return links, parser
     
 
 @enforce_types
-def parse_snapshots(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Model], str]:
+def parse_links(source_file: str, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], str]:
     """parse a list of URLs with their metadata from an 
        RSS feed, bookmarks export, or text file
     """
 
-    check_url_parsing_invariants()
-
     timer = TimedProgress(TIMEOUT * 4)
     with open(source_file, 'r', encoding='utf-8') as file:
-        snapshots, parser = run_parser_functions(file, timer, root_url=root_url)
+        links, parser = run_parser_functions(file, timer, root_url=root_url, parser=parser)
 
     timer.end()
     if parser is None:
         return [], 'Failed to parse'
-    return snapshots, parser
+    return links, parser
 
 
-def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None) -> Tuple[List[Model], Optional[str]]:
-    most_snapshots: List[Model] = []
+def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], Optional[str]]:
+    most_links: List[Link] = []
     best_parser_name = None
 
-    for parser_name, parser_func in PARSERS:
+    if parser != "auto":
+        parser_name, parser_func = PARSERS[parser]
+        parsed_links = list(parser_func(to_parse, root_url=root_url))
+        if not parsed_links:
+            stderr()
+            stderr(f'[X] No links found using {parser_name} parser', color='red')
+            hint('Try a different parser or double check the input?')
+            stderr()
+        timer.end()
+        return parsed_links, parser_name
+
+    for parser_id in PARSERS:
+        parser_name, parser_func = PARSERS[parser_id]
         try:
-            parsed_snapshots = list(parser_func(to_parse, root_url=root_url))
-            if not parsed_snapshots:
-                raise Exception('no links found')
+            parsed_links = list(parser_func(to_parse, root_url=root_url))
+            if not parsed_links:
+                raise Exception(f'No links found using {parser_name} parser')
 
             # print(f'[√] Parser {parser_name} succeeded: {len(parsed_links)} links parsed')
-            if len(parsed_snapshots) > len(most_snapshots):
-                most_snapshots = parsed_snapshots
+            if len(parsed_links) > len(most_links):
+                most_links = parsed_links
                 best_parser_name = parser_name
                 
         except Exception as err:                                                # noqa
             # Parsers are tried one by one down the list, and the first one
-            # that succeeds is used. To see why a certain parser was not used
-            # due to error or format incompatibility, uncomment this line:
+            # that succeeds is used. To debug why a certain parser was not used
+            # due to python error or format incompatibility, uncomment this line:
             
             # print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err))
             # raise
             pass
     timer.end()
-    return most_snapshots, best_parser_name
+    return most_links, best_parser_name
 
 
 @enforce_types
 def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str:
-    ts = str(datetime.now().timestamp()).split('.', 1)[0]
+    ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
     source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts))
     atomic_write(source_path, raw_text)
     log_source_saved(source_file=source_path)
@@ -142,7 +157,7 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir:
 @enforce_types
 def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: Path=OUTPUT_DIR) -> str:
     """download a given url's content into output/sources/domain-<timestamp>.txt"""
-    ts = str(datetime.now().timestamp()).split('.', 1)[0]
+    ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
     source_path = str(OUTPUT_DIR / SOURCES_DIR_NAME / filename.format(basename=basename(path), ts=ts))
 
     if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')):
@@ -175,31 +190,48 @@ def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{ba
     return source_path
 
 
-def check_url_parsing_invariants() -> None:
-    """Check that plain text regex URL parsing works as expected"""
-
-    # this is last-line-of-defense to make sure the URL_REGEX isn't
-    # misbehaving, as the consequences could be disastrous and lead to many
-    # incorrect/badly parsed links being added to the archive
-
-    test_urls = '''
-    https://example1.com/what/is/happening.html?what=1#how-about-this=1
-    https://example2.com/what/is/happening/?what=1#how-about-this=1
-    HTtpS://example3.com/what/is/happening/?what=1#how-about-this=1f
-    https://example4.com/what/is/happening.html
-    https://example5.com/
-    https://example6.com
-
-    <test>http://example7.com</test>
-    [https://example8.com/what/is/this.php?what=1]
-    [and http://example9.com?what=1&other=3#and-thing=2]
-    <what>https://example10.com#and-thing=2 "</about>
-    abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def
-    sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi
-    example13.bada
-    and example14.badb
-    <or>htt://example15.badc</that>
-    '''
-    # print('\n'.join(re.findall(URL_REGEX, test_urls)))
-    assert len(re.findall(URL_REGEX, test_urls)) == 12
-
+# Check that plain text regex URL parsing works as expected
+#   this is last-line-of-defense to make sure the URL_REGEX isn't
+#   misbehaving due to some OS-level or environment level quirks (e.g. bad regex lib)
+#   the consequences of bad URL parsing could be disastrous and lead to many
+#   incorrect/badly parsed links being added to the archive, so this is worth the cost of checking
+_test_url_strs = {
+    'example.com': 0,
+    '/example.com': 0,
+    '//example.com': 0,
+    ':/example.com': 0,
+    '://example.com': 0,
+    'htt://example8.com': 0,
+    '/htt://example.com': 0,
+    'https://example': 1,
+    'https://localhost/2345': 1,
+    'https://localhost:1234/123': 1,
+    '://': 0,
+    'https://': 0,
+    'http://': 0,
+    'ftp://': 0,
+    'ftp://example.com': 0,
+    'https://example.com': 1,
+    'https://example.com/': 1,
+    'https://a.example.com': 1,
+    'https://a.example.com/': 1,
+    'https://a.example.com/what/is/happening.html': 1,
+    'https://a.example.com/what/ís/happening.html': 1,
+    'https://a.example.com/what/is/happening.html?what=1&2%20b#höw-about-this=1a': 1,
+    'https://a.example.com/what/is/happéning/?what=1&2%20b#how-aboüt-this=1a': 1,
+    'HTtpS://a.example.com/what/is/happening/?what=1&2%20b#how-about-this=1af&2f%20b': 1,
+    'https://example.com/?what=1#how-about-this=1&2%20baf': 1,
+    'https://example.com?what=1#how-about-this=1&2%20baf': 1,
+    '<test>http://example7.com</test>': 1,
+    '[https://example8.com/what/is/this.php?what=1]': 1,
+    '[and http://example9.com?what=1&other=3#and-thing=2]': 1,
+    '<what>https://example10.com#and-thing=2 "</about>': 1,
+    'abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def': 1,
+    'sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi': 1,
+    '<or>http://examplehttp://15.badc</that>': 2,
+    'https://a.example.com/one.html?url=http://example.com/inside/of/another?=http://': 2,
+    '[https://a.example.com/one.html?url=http://example.com/inside/of/another?=](http://a.example.com)': 3,
+}
+for url_str, num_urls in _test_url_strs.items():
+    assert len(re.findall(URL_REGEX, url_str)) == num_urls, (
+        f'{url_str} does not contain {num_urls} urls')

archivebox/parsers/generic_html.py

@@ -4,10 +4,9 @@ __package__ = 'archivebox.parsers'
 import re
 
 from typing import IO, Iterable, Optional
-from datetime import datetime
-
-from django.db.models import Model
+from datetime import datetime, timezone
 
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -30,9 +29,8 @@ class HrefParser(HTMLParser):
 
 
 @enforce_types
-def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None, **_kwargs) -> Iterable[Model]:
+def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None, **_kwargs) -> Iterable[Link]:
     """Parse Generic HTML for href tags and use only the url (support for title coming later)"""
-    from core.models import Snapshot
 
     html_file.seek(0)
     for line in html_file:
@@ -46,10 +44,15 @@ def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None,
                 url = urljoin(root_url, url)
             
             for archivable_url in re.findall(URL_REGEX, url):
-                yield Snapshot(
+                yield Link(
                     url=htmldecode(archivable_url),
-                    timestamp=str(datetime.now().timestamp()),
+                    timestamp=str(datetime.now(timezone.utc).timestamp()),
                     title=None,
-                    #tags=None,
-                    #sources=[html_file.name],
+                    tags=None,
+                    sources=[html_file.name],
                 )
+
+
+KEY = 'html'
+NAME = 'Generic HTML'
+PARSER = parse_generic_html_export

archivebox/parsers/generic_json.py

@@ -3,10 +3,9 @@ __package__ = 'archivebox.parsers'
 import json
 
 from typing import IO, Iterable
-from datetime import datetime
-
-from django.db.models import Model
+from datetime import datetime, timezone
 
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,9 +13,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)"""
-    from core.models import Snapshot
 
     json_file.seek(0)
     links = json.load(json_file)
@@ -32,7 +30,7 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Model]:
                 raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
 
             # Parse the timestamp
-            ts_str = str(datetime.now().timestamp())
+            ts_str = str(datetime.now(timezone.utc).timestamp())
             if link.get('timestamp'):
                 # chrome/ff histories use a very precise timestamp
                 ts_str = str(link['timestamp'] / 10000000)  
@@ -58,10 +56,15 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Model]:
             elif link.get('name'):
                 title = link['name'].strip()
 
-            yield Snapshot(
+            yield Link(
                 url=htmldecode(url),
                 timestamp=ts_str,
                 title=htmldecode(title) or None,
-                #tags=htmldecode(link.get('tags')) or '',
-                #sources=[json_file.name],
+                tags=htmldecode(link.get('tags')) or '',
+                sources=[json_file.name],
             )
+
+
+KEY = 'json'
+NAME = 'Generic JSON'
+PARSER = parse_generic_json_export

archivebox/parsers/generic_rss.py

@@ -4,8 +4,7 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from django.db.models import Model
-
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -13,9 +12,8 @@ from ..util import (
 )
 
 @enforce_types
-def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse RSS XML-format files into links"""
-    from core.models import Snapshot
 
     rss_file.seek(0)
     items = rss_file.read().split('<item>')
@@ -42,10 +40,15 @@ def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
         time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z")
         title = str_between(get_row('title'), '<![CDATA[', ']]').strip()
 
-        yield Snapshot(
+        yield Link(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            #tags=None,
-            #sources=[rss_file.name],
+            tags=None,
+            sources=[rss_file.name],
         )
+
+
+KEY = 'rss'
+NAME = 'Generic RSS'
+PARSER = parse_generic_rss_export

archivebox/parsers/generic_txt.py

@@ -4,11 +4,10 @@ __description__ = 'Plain Text'
 import re
 
 from typing import IO, Iterable
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
-from django.db.models import Model
-
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -17,10 +16,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Model]:
-    """Parse raw links from each line in a text file"""
-    # TODO: Check if we should add sources list to the database
-    from core.models import Snapshot
+def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
+    """Parse links from a text file, ignoring other text"""
 
     text_file.seek(0)
     for line in text_file.readlines():
@@ -30,12 +27,12 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Model]:
         # if the line is a local file path that resolves, then we can archive it
         try:
             if Path(line).exists():
-                yield Snapshot(
+                yield Link(
                     url=line,
-                    timestamp=str(datetime.now().timestamp()),
+                    timestamp=str(datetime.now(timezone.utc).timestamp()),
                     title=None,
-                    #tags=None,
-                    #sources=[text_file.name],
+                    tags=None,
+                    sources=[text_file.name],
                 )
         except (OSError, PermissionError):
             # nvm, not a valid path...
@@ -43,22 +40,26 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Model]:
 
         # otherwise look for anything that looks like a URL in the line
         for url in re.findall(URL_REGEX, line):
-            yield Snapshot(
+            yield Link(
                 url=htmldecode(url),
-                timestamp=str(datetime.now().timestamp()),
+                timestamp=str(datetime.now(timezone.utc).timestamp()),
                 title=None,
-                #tags=None,
-                #sources=[text_file.name],
+                tags=None,
+                sources=[text_file.name],
             )
 
             # look inside the URL for any sub-urls, e.g. for archive.org links
             # https://web.archive.org/web/20200531203453/https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             # -> https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             for sub_url in re.findall(URL_REGEX, line[1:]):
-                yield Snapshot(
+                yield Link(
                     url=htmldecode(sub_url),
-                    timestamp=str(datetime.now().timestamp()),
+                    timestamp=str(datetime.now(timezone.utc).timestamp()),
                     title=None,
-                    #tags=None,
-                    #sources=[text_file.name],
+                    tags=None,
+                    sources=[text_file.name],
                 )
+
+KEY = 'txt'
+NAME = 'Generic TXT'
+PARSER = parse_generic_txt_export

archivebox/parsers/medium_rss.py

@@ -4,10 +4,9 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from django.db.models import Model
-
 from xml.etree import ElementTree
 
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -15,9 +14,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse Medium RSS feed files into links"""
-    from core.models import Snapshot
 
     rss_file.seek(0)
     root = ElementTree.parse(rss_file).getroot()
@@ -28,10 +26,15 @@ def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
         ts_str = item.find("pubDate").text                              # type: ignore
         time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z")    # type: ignore
         
-        yield Snapshot(
+        yield Link(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            #tags=None,
-            #sources=[rss_file.name],
+            tags=None,
+            sources=[rss_file.name],
         )
+
+
+KEY = 'medium_rss'
+NAME = 'Medium RSS'
+PARSER = parse_medium_rss_export

archivebox/parsers/netscape_html.py

@@ -6,8 +6,7 @@ import re
 from typing import IO, Iterable
 from datetime import datetime
 
-from django.db.models import Model
-
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -15,9 +14,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse netscape-format bookmarks export files (produced by all browsers)"""
-    from core.models import Snapshot
 
     html_file.seek(0)
     pattern = re.compile("<a href=\"(.+?)\" add_date=\"(\\d+)\"[^>]*>(.+)</a>", re.UNICODE | re.IGNORECASE)
@@ -31,11 +29,15 @@ def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Model]
             time = datetime.fromtimestamp(float(match.group(2)))
             title = match.group(3).strip()
 
-            yield Snapshot(
+            yield Link(
                 url=htmldecode(url),
                 timestamp=str(time.timestamp()),
                 title=htmldecode(title) or None,
-                #tags=None,
-                #sources=[html_file.name],
+                tags=None,
+                sources=[html_file.name],
             )
 
+
+KEY = 'netscape_html'
+NAME = 'Netscape HTML'
+PARSER = parse_netscape_html_export

archivebox/parsers/pinboard_rss.py

@@ -2,12 +2,11 @@ __package__ = 'archivebox.parsers'
 
 
 from typing import IO, Iterable
-from datetime import datetime
-
-from django.db.models import Model
+from datetime import datetime, timezone
 
 from xml.etree import ElementTree
 
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -15,9 +14,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse Pinboard RSS feed files into links"""
-    from core.models import Snapshot
 
     rss_file.seek(0)
     root = ElementTree.parse(rss_file).getroot()
@@ -38,12 +36,17 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
         if ts_str:
             time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
         else:
-            time = datetime.now()
+            time = datetime.now(timezone.utc)
 
-        yield Snapshot(
+        yield Link(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            #tags=htmldecode(tags) or None,
-            #sources=[rss_file.name],
+            tags=htmldecode(tags) or None,
+            sources=[rss_file.name],
         )
+
+
+KEY = 'pinboard_rss'
+NAME = 'Pinboard RSS'
+PARSER = parse_pinboard_rss_export

archivebox/parsers/pocket_api.py

@@ -7,11 +7,9 @@ from typing import IO, Iterable, Optional
 from configparser import ConfigParser
 
 from pathlib import Path
-
-from django.db.models import Model
-
 from ..vendor.pocket import Pocket
 
+from ..index.schema import Link
 from ..util import enforce_types
 from ..system import atomic_write
 from ..config import (
@@ -48,21 +46,19 @@ def get_pocket_articles(api: Pocket, since=None, page=0):
         api.last_since = body['since']
 
 
-def snapshot_from_article(article: dict, sources: list):
-    from core.models import Snapshot
-
+def link_from_article(article: dict, sources: list):
     url: str = article['resolved_url'] or article['given_url']
     broken_protocol = _BROKEN_PROTOCOL_RE.match(url)
     if broken_protocol:
         url = url.replace(f'{broken_protocol.group(1)}:/', f'{broken_protocol.group(1)}://')
     title = article['resolved_title'] or article['given_title'] or url
 
-    return Snapshot(
+    return Link(
         url=url,
         timestamp=article['time_read'],
         title=title,
-        #tags=article.get('tags'),
-        #sources=sources
+        tags=article.get('tags'),
+        sources=sources
     )
 
 
@@ -99,7 +95,7 @@ def should_parse_as_pocket_api(text: str) -> bool:
 
 
 @enforce_types
-def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse bookmarks from the Pocket API"""
 
     input_buffer.seek(0)
@@ -112,6 +108,11 @@ def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Model]
             api.last_since = None
     
             for article in get_pocket_articles(api, since=read_since(username)):
-                yield snapshot_from_article(article, sources=[line])
+                yield link_from_article(article, sources=[line])
     
             write_since(username, api.last_since)
+
+
+KEY = 'pocket_api'
+NAME = 'Pocket API'
+PARSER = parse_pocket_api_export

archivebox/parsers/pocket_html.py

@@ -6,8 +6,7 @@ import re
 from typing import IO, Iterable
 from datetime import datetime
 
-from django.db.models import Model
-
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -15,9 +14,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse Pocket-format bookmarks export files (produced by getpocket.com/export/)"""
-    from core.models import Snapshot
 
     html_file.seek(0)
     pattern = re.compile("^\\s*<li><a href=\"(.+)\" time_added=\"(\\d+)\" tags=\"(.*)\">(.+)</a></li>", re.UNICODE)
@@ -31,10 +29,15 @@ def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Model]:
             tags = match.group(3)
             title = match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '')
             
-            yield Snapshot(
+            yield Link(
                 url=htmldecode(url),
                 timestamp=str(time.timestamp()),
                 title=htmldecode(title) or None,
-                #tags=tags or '',
-                #sources=[html_file.name],
+                tags=tags or '',
+                sources=[html_file.name],
             )
+
+
+KEY = 'pocket_html'
+NAME = 'Pocket HTML'
+PARSER = parse_pocket_html_export

archivebox/parsers/shaarli_rss.py

@@ -4,8 +4,7 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from django.db.models import Model
-
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,9 +13,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse Shaarli-specific RSS XML-format files into links"""
-    from core.models import Snapshot
 
     rss_file.seek(0)
     entries = rss_file.read().split('<entry>')[1:]
@@ -43,10 +41,15 @@ def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
         ts_str = str_between(get_row('published'), '<published>', '</published>')
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
 
-        yield Snapshot(
+        yield Link(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            #tags=None,
-            #sources=[rss_file.name],
+            tags=None,
+            sources=[rss_file.name],
         )
+
+
+KEY = 'shaarli_rss'
+NAME = 'Shaarli RSS'
+PARSER = parse_shaarli_rss_export

archivebox/parsers/url_list.py

@@ -0,0 +1,37 @@
+__package__ = 'archivebox.parsers'
+__description__ = 'URL list'
+
+import re
+
+from typing import IO, Iterable
+from datetime import datetime, timezone
+
+from ..index.schema import Link
+from ..util import (
+    enforce_types,
+    URL_REGEX,
+)
+
+
+@enforce_types
+def parse_url_list(text_file: IO[str], **_kwargs) -> Iterable[Link]:
+    """Parse raw URLs from each line in a text file"""
+
+    text_file.seek(0)
+    for line in text_file.readlines():
+        url = line.strip()
+        if (not url) or not re.findall(URL_REGEX, url):
+            continue
+
+        yield Link(
+            url=url,
+            timestamp=str(datetime.now(timezone.utc).timestamp()),
+            title=None,
+            tags=None,
+            sources=[text_file.name],
+        )
+
+
+KEY = 'url_list'
+NAME = 'URL List'
+PARSER = parse_url_list

archivebox/parsers/wallabag_atom.py

@@ -4,8 +4,7 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from django.db.models import Model
-
+from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,9 +13,8 @@ from ..util import (
 
 
 @enforce_types
-def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
+def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse Wallabag Atom files into links"""
-    from core.models import Snapshot
 
     rss_file.seek(0)
     entries = rss_file.read().split('<entry>')[1:]
@@ -50,10 +48,15 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
         except Exception:
             tags = None
 
-        yield Snapshot(
+        yield Link(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            #tags=tags or '',
-            #sources=[rss_file.name],
+            tags=tags or '',
+            sources=[rss_file.name],
         )
+
+
+KEY = 'wallabag_atom'
+NAME = 'Wallabag Atom'
+PARSER = parse_wallabag_atom_export

archivebox/search/__init__.py

@@ -2,7 +2,7 @@ from typing import List, Union
 from pathlib import Path
 from importlib import import_module
 
-from django.db.models import QuerySet, Model
+from django.db.models import QuerySet
 
 from archivebox.index.schema import Link
 from archivebox.util import enforce_types
@@ -28,22 +28,24 @@ def import_backend():
     return backend
 
 @enforce_types
-def write_search_index(snapshot: Model, texts: Union[List[str], None]=None, out_dir: Path=OUTPUT_DIR, skip_text_index: bool=False) -> None:
+def write_search_index(link: Link, texts: Union[List[str], None]=None, out_dir: Path=OUTPUT_DIR, skip_text_index: bool=False) -> None:
     if not indexing_enabled():
         return
 
     if not skip_text_index and texts:
         from core.models import Snapshot
 
+        snap = Snapshot.objects.filter(url=link.url).first()
         backend = import_backend()
-        try:
-            backend.index(snapshot_id=str(snapshot.id), texts=texts)
-        except Exception as err:
-            stderr()
-            stderr(
-                f'[X] The search backend threw an exception={err}:',
+        if snap:
+            try:
+                backend.index(snapshot_id=str(snap.id), texts=texts)
+            except Exception as err:
+                stderr()
+                stderr(
+                    f'[X] The search backend threw an exception={err}:',
                 color='red',
-            )
+                )
 
 @enforce_types
 def query_search_index(query: str, out_dir: Path=OUTPUT_DIR) -> QuerySet:
@@ -83,17 +85,17 @@ def flush_search_index(snapshots: QuerySet):
         )
 
 @enforce_types
-def index_snapshots(snapshots: Union[List[Model],None], out_dir: Path=OUTPUT_DIR):
-    if not snapshots:
+def index_links(links: Union[List[Link],None], out_dir: Path=OUTPUT_DIR):
+    if not links:
         return
 
     from core.models import Snapshot, ArchiveResult
 
-    for snapshot in snapshots:
-        snap = Snapshot.objects.filter(url=snapshot.url).first()
+    for link in links:
+        snap = Snapshot.objects.filter(url=link.url).first()
         if snap: 
             results = ArchiveResult.objects.indexable().filter(snapshot=snap)
-            log_index_started(snapshot.url)
+            log_index_started(link.url)
             try:
                 texts = get_indexable_content(results)
             except Exception as err:
@@ -103,4 +105,4 @@ def index_snapshots(snapshots: Union[List[Model],None], out_dir: Path=OUTPUT_DIR
                     color='red',
                     ) 
             else:
-                write_search_index(snapshot, texts, out_dir=out_dir)
+                write_search_index(link, texts, out_dir=out_dir)

archivebox/search/backends/ripgrep.py

@@ -2,7 +2,7 @@ import re
 from subprocess import run, PIPE
 from typing import List, Generator
 
-from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION
+from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION, SEARCH_BACKEND_TIMEOUT
 from archivebox.util import enforce_types
 
 RG_IGNORE_EXTENSIONS = ('css','js','orig','svg')
@@ -32,7 +32,7 @@ def search(text: str) -> List[str]:
     from core.models import Snapshot
 
     rg_cmd = ['rg', RG_ADD_TYPE, RG_IGNORE_ARGUMENTS, RG_DEFAULT_ARGUMENTS, RG_REGEX_ARGUMENT, text, str(ARCHIVE_DIR)]
-    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=60)
+    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=SEARCH_BACKEND_TIMEOUT)
     file_paths = [p.decode() for p in rg.stdout.splitlines()]
     timestamps = set()
     for path in file_paths:

archivebox/search/backends/sonic.py

@@ -5,15 +5,31 @@ from sonic import IngestClient, SearchClient
 from archivebox.util import enforce_types
 from archivebox.config import SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD, SONIC_BUCKET, SONIC_COLLECTION
 
-MAX_SONIC_TEXT_LENGTH = 2000
+MAX_SONIC_TEXT_TOTAL_LENGTH = 100000000     # dont index more than 100 million characters per text
+MAX_SONIC_TEXT_CHUNK_LENGTH = 2000          # dont index more than 2000 characters per chunk
+MAX_SONIC_ERRORS_BEFORE_ABORT = 5
 
 @enforce_types
 def index(snapshot_id: str, texts: List[str]):
+    error_count = 0
     with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
         for text in texts:
-            chunks = [text[i:i+MAX_SONIC_TEXT_LENGTH] for i in range(0, len(text), MAX_SONIC_TEXT_LENGTH)]
-            for chunk in chunks:
-                ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))
+            chunks = (
+                text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
+                for i in range(
+                    0,
+                    min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
+                    MAX_SONIC_TEXT_CHUNK_LENGTH,
+                )
+            )
+            try:
+                for chunk in chunks:
+                    ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))
+            except Exception as err:
+                print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}')
+                error_count += 1
+                if error_count > MAX_SONIC_ERRORS_BEFORE_ABORT:
+                    raise
 
 @enforce_types
 def search(text: str) -> List[str]:

archivebox/search/utils.py

@@ -16,7 +16,7 @@ def get_file_result_content(res, extra_path, use_pwd=False):
     if extra_path:
         fpath = f'{fpath}/{extra_path}'
 
-    with open(fpath, 'r') as file:
+    with open(fpath, 'r', encoding='utf-8') as file:
         data = file.read()
     if data:
         return [data]
@@ -36,10 +36,10 @@ def get_indexable_content(results: QuerySet):
 
     # TODO: banish this duplication and get these from the extractor file
     if method == 'readability':
-        return get_file_result_content(res, 'content.txt')
+        return get_file_result_content(res, 'content.txt', use_pwd=True)
     elif method == 'singlefile':
-        return get_file_result_content(res,'',use_pwd=True)
+        return get_file_result_content(res, '', use_pwd=True)
     elif method == 'dom':
-        return get_file_result_content(res,'',use_pwd=True)
+        return get_file_result_content(res, '', use_pwd=True)
     elif method == 'wget':
-        return get_file_result_content(res,'',use_pwd=True)
+        return get_file_result_content(res, '', use_pwd=True)

archivebox/system.py

@@ -2,34 +2,76 @@ __package__ = 'archivebox'
 
 
 import os
+import signal
 import shutil
 
 from json import dump
 from pathlib import Path
 from typing import Optional, Union, Set, Tuple
-from subprocess import run as subprocess_run
+from subprocess import _mswindows, PIPE, Popen, CalledProcessError, CompletedProcess, TimeoutExpired
 
 from crontab import CronTab
-from atomicwrites import atomic_write as lib_atomic_write
+from .vendor.atomicwrites import atomic_write as lib_atomic_write
 
 from .util import enforce_types, ExtendedEncoder
 from .config import OUTPUT_PERMISSIONS
 
 
 
-def run(*args, input=None, capture_output=True, text=False, **kwargs):
-    """Patched of subprocess.run to fix blocking io making timeout=innefective"""
+def run(*args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs):
+    """Patched of subprocess.run to kill forked child subprocesses and fix blocking io making timeout=innefective
+        Mostly copied from https://github.com/python/cpython/blob/master/Lib/subprocess.py
+    """
 
     if input is not None:
-        if 'stdin' in kwargs:
+        if kwargs.get('stdin') is not None:
             raise ValueError('stdin and input arguments may not both be used.')
+        kwargs['stdin'] = PIPE
 
     if capture_output:
         if ('stdout' in kwargs) or ('stderr' in kwargs):
             raise ValueError('stdout and stderr arguments may not be used '
                              'with capture_output.')
+        kwargs['stdout'] = PIPE
+        kwargs['stderr'] = PIPE
 
-    return subprocess_run(*args, input=input, capture_output=capture_output, text=text, **kwargs)
+    pgid = None
+    try:
+        with Popen(*args, start_new_session=start_new_session, **kwargs) as process:
+            pgid = os.getpgid(process.pid)
+            try:
+                stdout, stderr = process.communicate(input, timeout=timeout)
+            except TimeoutExpired as exc:
+                process.kill()
+                if _mswindows:
+                    # Windows accumulates the output in a single blocking
+                    # read() call run on child threads, with the timeout
+                    # being done in a join() on those threads.  communicate()
+                    # _after_ kill() is required to collect that and add it
+                    # to the exception.
+                    exc.stdout, exc.stderr = process.communicate()
+                else:
+                    # POSIX _communicate already populated the output so
+                    # far into the TimeoutExpired exception.
+                    process.wait()
+                raise
+            except:  # Including KeyboardInterrupt, communicate handled that.
+                process.kill()
+                # We don't call process.wait() as .__exit__ does that for us.
+                raise
+
+            retcode = process.poll()
+            if check and retcode:
+                raise CalledProcessError(retcode, process.args,
+                                         output=stdout, stderr=stderr)
+    finally:
+        # force kill any straggler subprocesses that were forked from the main proc
+        try:
+            os.killpg(pgid, signal.SIGINT)
+        except Exception:
+            pass
+
+    return CompletedProcess(process.args, retcode, stdout, stderr)
 
 
 @enforce_types
@@ -37,18 +79,19 @@ def atomic_write(path: Union[Path, str], contents: Union[dict, str, bytes], over
     """Safe atomic write to filesystem by writing to temp file + atomic rename"""
 
     mode = 'wb+' if isinstance(contents, bytes) else 'w'
+    encoding = None if isinstance(contents, bytes) else 'utf-8'  # enforce utf-8 on all text writes
 
     # print('\n> Atomic Write:', mode, path, len(contents), f'overwrite={overwrite}')
     try:
-        with lib_atomic_write(path, mode=mode, overwrite=overwrite) as f:
+        with lib_atomic_write(path, mode=mode, overwrite=overwrite, encoding=encoding) as f:
             if isinstance(contents, dict):
                 dump(contents, f, indent=4, sort_keys=True, cls=ExtendedEncoder)
             elif isinstance(contents, (bytes, str)):
                 f.write(contents)
     except OSError as e:
         print(f"[X] OSError: Failed to write {path} with fcntl.F_FULLFSYNC. ({e})")
-        print("    For data integrity, ArchiveBox requires a filesystem that supports atomic writes.")
-        print("    Filesystems and network drives that don't implement FSYNC are incompatible and require workarounds.")
+        print("    You can store the archive/ subfolder on a hard drive or network share that doesn't support support syncronous writes,")
+        print("    but the main folder containing the index.sqlite3 and ArchiveBox.conf files must be on a filesystem that supports FSYNC.")
         raise SystemExit(1)
     os.chmod(path, int(OUTPUT_PERMISSIONS, base=8))
 

archivebox/templates/admin/actions_as_select.html

@@ -1 +0,0 @@
-actions_as_select

archivebox/templates/admin/base.html

@@ -1,246 +1,231 @@
-{% load i18n static %}<!DOCTYPE html>
-{% get_current_language as LANGUAGE_CODE %}{% get_current_language_bidi as LANGUAGE_BIDI %}
+{% load i18n static tz %}
+{% get_current_language as LANGUAGE_CODE %}
+{% get_current_language_bidi as LANGUAGE_BIDI %}
+
+<!DOCTYPE html>
 <html lang="{{ LANGUAGE_CODE|default:"en-us" }}" {% if LANGUAGE_BIDI %}dir="rtl"{% endif %}>
-<head>
-<title>{% block title %}{% endblock %} | ArchiveBox</title>
-<link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
-{% block extrastyle %}{% endblock %}
-{% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">{% endif %}
-{% block extrahead %}{% endblock %}
-{% block responsive %}
-    <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0">
-    <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}">
-    {% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">{% endif %}
-{% endblock %}
-{% block blockbots %}<meta name="robots" content="NONE,NOARCHIVE">{% endblock %}
-<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
-</head>
-{% load i18n %}
-
-<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}"
-  data-admin-utc-offset="{% now "Z" %}">
-
-  <style nonce="{{nonce}}">
-      /* Loading Progress Bar */
-        #progress {
-            position: absolute;
-            z-index: 1000;
-            top: 0px;
-            left: -6px;
-            width: 2%;
-            opacity: 1;
-            height: 2px;
-            background: #1a1a1a;
-            border-radius: 1px;
-            transition: width 4s ease-out, opacity 400ms linear;
-        }
-  
-        @-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
-  </style>
-
-  <script>
-        // Page Loading Bar
-        window.loadStart = function(distance) {
-            var distance = distance || 0;
-            // only add progrstess bar if not already present
-            if (django.jQuery("#loading-bar").length == 0) {
-                django.jQuery("body").add("<div id=\"loading-bar\"></div>");
-            }
-            if (django.jQuery("#progress").length === 0) {
-                django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
-                let last_distance = (distance || (30 + (Math.random() * 30)))
-                django.jQuery("#progress").width(last_distance + "%");
-                setInterval(function() {
-                    last_distance += Math.random()
-                    django.jQuery("#progress").width(last_distance + "%");
-                }, 1000)
-            }
-        };
-  
-        window.loadFinish = function() {
-            django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
-                django.jQuery(this).remove();
-            });
-        };
-        window.loadStart();
-        window.addEventListener('beforeunload', function() {window.loadStart(27)});
-        document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
-  </script>
-
-
-<!-- Container -->
-<div id="container">
-
-    {% if not is_popup %}
-    <!-- Header -->
-    <div id="header">
-        <div id="branding">
-            <h1 id="site-name">
-                <a href="{% url 'Home' %}">
-                    <img src="{% static 'archive.png' %}" id="logo">
-                    ArchiveBox
-                </a>
-            </h1>
-                
-        </div>
-        {% block usertools %}
-        {% if has_permission %}
-        <div id="user-tools">
-            <a href="{% url 'admin:Add' %}">Add ➕</a> /
-            <a href="{% url 'Home' %}">Snapshots</a> /
-            <a href="/admin/core/tag/">Tags</a> /
-            <a href="/admin/auth/user/">Users</a> /
-            <a href="{% url 'Docs' %}">Docs</a>
-             &nbsp; &nbsp;
-            {% block welcome-msg %}
-                {% trans 'User' %}
-                <strong>{% firstof user.get_short_name user.get_username %}</strong> &nbsp; &nbsp;
-            {% endblock %}
-            {% block userlinks %}
-                {% if user.is_active and user.is_staff %}
-                    {% url 'django-admindocs-docroot' as docsroot %}
-                    {% if docsroot %}
-                        <a href="{{ docsroot }}">{% trans 'Documentation' %}</a> /
-                    {% endif %}
-                {% endif %}
-                {% if user.has_usable_password %}
-                <a href="{% url 'admin:password_change' %}">{% trans 'Change password' %}</a> /
-                {% endif %}
-                <a href="{% url 'admin:logout' %}">{% trans 'Log out' %}</a>
-                |
-                <a> <span id="snapshotListView" style="cursor: pointer">☰</span> </a>       
-                <a> <span id="snapshotGridView"style="letter-spacing: -.4em; cursor: pointer;">⣿⣿</span></a> 
-            {% endblock %}
-        </div>
+    <head>
+        <title>{% block title %}Home{% endblock %} | ArchiveBox</title>
+        
+        {% block blockbots %}
+            <meta name="robots" content="NONE,NOARCHIVE">
+        {% endblock %}
+        
+        <link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
+        {% block extrastyle %}{% endblock %}
+        
+        {% if LANGUAGE_BIDI %}
+            <link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">
         {% endif %}
+        
+        {% block responsive %}
+            <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0">
+            <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}">
+            {% if LANGUAGE_BIDI %}
+                <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">
+            {% endif %}
         {% endblock %}
-        {% block nav-global %}{% endblock %}
-    </div>
-        <!-- END Header -->
-        {% block breadcrumbs %}
-        <div class="breadcrumbs">
-        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-        {% if title %} &rsaquo; {{ title }}{% endif %}
+        
+        <link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
+        
+        <script>
+            function selectSnapshotListView(e) {
+                e && e.stopPropagation()
+                e && e.preventDefault()
+                console.log('Switching to Snapshot list view...')
+                localStorage.setItem('preferred_snapshot_view_mode', 'list')
+                window.location = "{% url 'admin:core_snapshot_changelist' %}" + document.location.search
+                return false
+            }
+            function selectSnapshotGridView(e) {
+                e && e.stopPropagation()
+                e && e.preventDefault()
+                console.log('Switching to Snapshot grid view...')
+                localStorage.setItem('preferred_snapshot_view_mode', 'grid')
+                window.location = "{% url 'admin:grid' %}" + document.location.search
+                return false
+            }
+
+            const preferred_view = localStorage.getItem('preferred_snapshot_view_mode') || 'unset'
+            const current_view = (
+                window.location.pathname === "{% url 'admin:core_snapshot_changelist' %}"
+                    ? 'list'
+                    : 'grid')
+            console.log('Preferred snapshot view is:', preferred_view, 'Current view mode is:', current_view)
+            
+            if (preferred_view === 'grid' && current_view !== 'grid') {  
+                selectSnapshotGridView()
+            }
+        </script>
+
+        {% block extrahead %}{% endblock %}
+    </head>
+
+
+    <body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" data-admin-utc-offset="{% now "Z" %}">
+        {% include 'progressbar.html' %}
+
+        <div id="container">
+            {% if not is_popup %}
+                <div id="header">
+                    <div id="branding">
+                        <h1 id="site-name">
+                            <a href="{% url 'Home' %}">
+                                <img src="{% static 'archive.png' %}" id="logo">
+                                ArchiveBox
+                            </a>
+                        </h1>
+                    </div>
+                    {% block usertools %}
+                        {% if has_permission %}
+                            {% include 'navigation.html' %}
+                        {% endif %}
+                    {% endblock %}
+                    {% block nav-global %}{% endblock %}
+                </div>
+
+                {% block breadcrumbs %}
+                    <div class="breadcrumbs">
+                        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+                        {% if title %} &rsaquo; {{ title }}{% endif %}
+                    </div>
+                {% endblock %}
+            {% endif %}
+
+            {% block messages %}
+                {% if messages %}
+                    <ul class="messagelist">
+                        {% for message in messages %}
+                            <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li>
+                        {% endfor %}
+                    </ul>
+                {% endif %}
+            {% endblock messages %}
+
+            <div id="content" class="{% block coltype %}colM{% endblock %}">
+                {% if opts.model_name == 'snapshot' and cl %}
+                    <small id="snapshot-view-mode">
+                        <a href="#list" title="List view" id="snapshot-view-list">☰</a> | 
+                        <a href="#grid" title="Grid view" id="snapshot-view-grid" style="letter-spacing: -.4em;">⣿⣿</a> 
+                    </small>
+                {% endif %}
+                {% block pretitle %}{% endblock %}
+                {% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %}
+                {% block content %}
+                    {% block object-tools %}{% endblock %}
+                    {{ content }}
+                {% endblock %}
+                {% block sidebar %}{% endblock %}
+                <br class="clear">
+            </div>
+
+            {% block footer %}<div id="footer"></div>{% endblock %}
         </div>
-        {% endblock %}
-    {% endif %}
 
-    {% block messages %}
-        {% if messages %}
-        <ul class="messagelist">{% for message in messages %}
-          <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li>
-        {% endfor %}</ul>
-        {% endif %}
-    {% endblock messages %}
+        <script>
+            $ = django.jQuery;
+            $.fn.reverse = [].reverse;
 
-    <!-- Content -->
-    <div id="content" class="{% block coltype %}colM{% endblock %}">
-        {% block pretitle %}{% endblock %}
-        {% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %}
-        {% block content %}
-            {% block object-tools %}{% endblock %}
-            {{ content }}
-        {% endblock %}
-        {% block sidebar %}{% endblock %}
-        <br class="clear">
-    </div>
-    <!-- END Content -->
+            // hide images that fail to load
+            document.querySelector('body').addEventListener('error', function (e) {
+                e.target.style.opacity = 0;
+            }, true)
 
-    {% block footer %}<div id="footer"></div>{% endblock %}
-</div>
-<!-- END Container -->
+            // setup timezone
+            {% get_current_timezone as TIME_ZONE %}
+            window.TIME_ZONE = '{{TIME_ZONE}}'
 
-<script>
-    (function ($) {
-        $.fn.reverse = [].reverse;
+            window.setCookie = function(name, value, days) {
+                let expires = ""
+                if (days) {
+                    const date = new Date()
+                    date.setTime(date.getTime() + (days*24*60*60*1000))
+                    expires = "; expires=" + date.toUTCString()
+                }
+                document.cookie = name + "=" + (value || "")  + expires + "; path=/"
+            }
 
-        function fix_actions() {
-            var container = $('div.actions');
+            function setTimeOffset() {
+                if (window.GMT_OFFSET) return
+                window.GMT_OFFSET = -(new Date).getTimezoneOffset()
+                window.setCookie('GMT_OFFSET', window.GMT_OFFSET, 365)
+            }
 
-            if (container.find('option').length < 10) {
-                container.find('label, button').hide();
+            // change the admin actions button from a dropdown to buttons across
+            function fix_actions() {
+                const container = $('div.actions')
 
-                var buttons = $('<div></div>')
-                    .prependTo(container)
+                // too many actions to turn into buttons
+                if (container.find('select[name=action] option').length >= 11) return
+
+                // hide the empty default option thats just a placeholder with no value
+                container.find('label:nth-child(1), button[value=0]').hide()
+
+                const buttons = $('<div></div>')
+                    .insertAfter('div.actions button[type=submit]')
                     .css('display', 'inline')
                     .addClass('class', 'action-buttons');
 
-                container.find('option:gt(0)').reverse().each(function () {
-                    const name = this.value
+                // for each action in the dropdown, turn it into a button instead
+                container.find('select[name=action] option:gt(0)').each(function () {
+                    const action_type = this.value
                     $('<button>')
-                        .appendTo(buttons)
-                        .attr('name', this.value)
+                        .attr('type', 'button')
+                        .attr('name', action_type)
                         .addClass('button')
                         .text(this.text)
-                        .click(function () {
-                            container.find('select')
-                                .find(':selected').attr('selected', '').end()
-                                .find('[value=' + this.name + ']').attr('selected', 'selected');
-                            $('#changelist-form button[name="index"]').click();
+                        .click(function (e) {
+                            e.preventDefault()
+                            e.stopPropagation()
+
+                            const num_selected = document.querySelector('.action-counter').innerText.split(' ')[0]
+
+                            if (action_type === 'overwrite_snapshots') {
+                                const message = (
+                                    'Are you sure you want to re-archive (overwrite) ' + num_selected + ' Snapshots?\n\n' +
+                                    'This will delete all previously saved files from these Snapshots and re-archive them from scratch.\n\n'
+                                )
+                                if (!window.confirm(message)) return false
+                            }
+                            if (action_type === 'delete_snapshots') {
+                                const message = (
+                                    'Are you sure you want to permanently delete ' + num_selected + ' Snapshots?\n\n' +
+                                    'They will be removed from your index, and all their Snapshot content on disk will be permanently deleted.'
+                                )
+                                if (!window.confirm(message)) return false
+                            }
+
+                            // select the action button from the dropdown
+                            container.find('select[name=action]')
+                                .find('op:selected').removeAttr('selected').end()
+                                .find('[value=' + action_type + ']').attr('selected', 'selected').click()
+                            
+                            // click submit & replace the archivebox logo with a spinner
+                            $('#changelist-form button[name="index"]').click()
                             document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
-                        });
-                });
-            }
-        };
-        
-        function redirectWithQuery(uri){
-            uri_query = uri + document.location.search;
-            window.location = uri_query;
-
-        };
-
-        function selectSnapshotListView(){
-            localStorage.setItem('currentSnapshotView', 'List');
-            redirectWithQuery("{% url 'admin:core_snapshot_changelist' %}");
-        };
-
-        function selectSnapshotGridView(){
-            localStorage.setItem('currentSnapshotView', 'Grid');
-            redirectWithQuery("{% url 'admin:grid' %}");
-        };
-        
-        function setPreferredSnapshotView(view){
-            urlPath = window.location.pathname;
-
-            if((view==="Grid") && urlPath == "{% url 'admin:core_snapshot_changelist' %}"){  
-                selectSnapshotGridView();
+                            return false
+                        })
+                        .appendTo(buttons)
+                })
+                console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
             }
             
-            {% comment %} 
-                else if((view==="List") && urlPath == "{% url 'admin:grid' %}"){
-                selectSnapshotListView();
-            
-            } 
-            {% endcomment %}
-        };
+            function setupSnapshotGridListToggle() {
+                $("#snapshot-view-list").click(selectSnapshotListView)
+                $("#snapshot-view-grid").click(selectSnapshotGridView)
 
-        function setupSnapshotViews() {
-            const preferredSnapshotView = localStorage.getItem('currentSnapshotView');
-            setPreferredSnapshotView(preferredSnapshotView);
-
-            $( document ).ready(function() {
-                
-                $("#snapshotListView").click(function() {
-                    selectSnapshotListView();
-                });  
-                $("#snapshotGridView").click(function() {
-                    selectSnapshotGridView();
-                });  
-
-                $('input:checkbox').change(function(){
-                    if($(this).is(':checked')) 
-                        $(this).parent().parent().parent().parent().addClass('selected-card'); 
+                $('#changelist-form .card input:checkbox').change(function() {
+                    if ($(this).is(':checked'))
+                        $(this).parents('.card').addClass('selected-card')
                     else 
-                        $(this).parent().parent().parent().parent().removeClass('selected-card')
-                });
-
-            });
-        };
-        $(function () {
-            fix_actions();
-            setupSnapshotViews();
-        });
-    })(django.jQuery);
-</script>
-</body>
+                        $(this).parents('.card').removeClass('selected-card')
+                })
+            };
+            $(document).ready(function() {
+                fix_actions()
+                setupSnapshotGridListToggle()
+                setTimeOffset()
+            })
+        </script>
+    </body>
 </html>

archivebox/templates/admin/snapshots_grid.html

@@ -3,160 +3,188 @@
 
 {% block extrastyle %}
 <style>
-* {
-  -webkit-box-sizing: border-box;
-    -moz-box-sizing: border-box;
-    box-sizing: border-box;
-}
+  #changelist-search #searchbar {
+    min-height: 24px;
+  }
+  .cards {
+      padding-top: 10px;
+      display: grid;
+      grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */
+      grid-auto-rows: minmax(200px, auto);
+      grid-gap: 14px 14px;
+  }
 
-a {
-  text-decoration: none;
-  color: orange;
-}
-h2 {
-  color: #000;
-  margin: 2rem 0 .5rem;
-  font-size: 1.25rem;
-  font-weight: 400;
-  {% comment %} text-transform: uppercase; {% endcomment %}
-}
-
-card.img {
-  display: block;
-    border: 0;
-    width: 100%;
-    height: auto;
-}
-
-/***************************           Cards            *******************************/
-
-.cards {
-  display: grid;
-  grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */
-  grid-auto-rows: minmax(200px, auto);
-  grid-gap: 1rem;
-}
-
-.card {
-  /*height: 200px;*/
-  /*background: red;*/
-  border: 2px solid #e7e7e7;
-  border-radius: 4px;
-  -webkit-box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
-  box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
-  display: flex;
-  /* -webkit-box-orient: vertical; */
-  /* -webkit-box-direction: normal; */
-  -ms-flex-direction: column;
-  flex-direction: column;
-  position: relative;
-  color: #5d5e5e;
-} /* li item */
-
-.thumbnail img {
-    height: 100%;
-    box-sizing: border-box;
-    max-width: 100%;
-    max-height: 100%;
-    width: 100%;
-}
-
-.card-content {
-    font-size: .75rem;
-    padding: .5rem;
-    display: flex;
-    -webkit-box-orient: vertical;
-    -webkit-box-direction: normal;
-    -ms-flex-direction: column;
-    flex-direction: column;
-    -webkit-box-flex: 1;
-    -ms-flex: 1;
-    flex: 1;
+  .cards .card {
+      position: relative;
+      max-height: 380px;
+      overflow: hidden;
     
-}
+      background-color: #fffcfc;
+      border: 1px solid #f1efef;
+      border-radius: 4px;
+      box-shadow: 4px 4px 2px 2px rgba(0, 0, 0, 0.01);
+    
+      text-align: center;
+      color: #5d5e5e;
+  }
 
-.card-content h4{
-vertical-align:bottom;
-margin: 1.2em 0 0em 0;
-}
+  .cards .card.selected-card {
+    border: 3px solid #2196f3;
+    box-shadow: 2px 3px 6px 2px rgba(0, 0, 221, 0.14);
+  }
 
-.category {
-  font-size: .75rem;
-  text-transform: uppercase;
-}
-.category {
-    position: absolute;
-    top: 5%;
-    right: 0;
-    color: #fff;
-    background: #e74c3c;
-    padding: 10px 15px;
-    font-size: 10px;
-    font-weight: 600;
-    text-transform: uppercase;
-}
+  .cards .card .card-thumbnail {
+      display: block;
+      width: 100%;
+      height: 345px;
+      overflow: hidden;
+      border-radius: 4px;
+      background-color: #fffcfc;
+  }
 
-.category__01 {
-  background-color: #50c6db;
-  
-}
+  .cards .card .card-thumbnail img {
+      width: 100%;
+      height: auto;
+      border: 0;
+  }
+  .cards .card .card-thumbnail.missing img {
+      opacity: 0.03;
+      width: 20%;
+      height: auto;
+      margin-top: 84px;
+  }
 
-.tags{
-  opacity: 0.8;
-}
+  .cards .card .card-tags {
+      width: 100%;
+  }
+  .cards .card .card-tags span {
+      display: inline-block;
+      padding: 2px 5px;
+      border-radius: 5px;
+      opacity: 0.95;
+      background-color: #bfdfff;
+      color: #679ac2;
+      font-size: 12px;
+      margin-bottom: 3px;
+  }
 
-footer {
-  border-top: 2px solid #e7e7e7;
-  {% comment %} margin: .5rem 0 0; {% endcomment %}
-  {% comment %} min-height: 30px; {% endcomment %}
-  font-size: .5rem;
-}
-.post-meta {
-  padding: .3rem;
-}
+  .cards .card .card-footer {
+      width: 100%;
+      position: absolute;
+      bottom: 0;
+      text-align: center;
+  }
+  .cards .card .card-title {
+      padding: 4px 5px;
+      background-color: #fffcfc;
+      /*height: 50px;
+      vertical-align: middle;
+      line-height: 50px;*/
+  }
+  .cards .card .card-title h4 {
+      color: initial;
+      display: block;
+      vertical-align: middle;
+      line-height: normal;
+      margin: 0px;
+      padding: 5px 0px;
+      font-size: 13.5px;
+      font-weight: 400;
+      word-break: break-all;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      max-height: 46px;
+  }
+  .cards .card .card-title h4 .title-text {
+    user-select: all;
+  }
+  .cards .card .card-title .link-favicon {
+      height: 15px;
+      margin: 2px;
+      vertical-align: -5px;
+      display: inline-block;
+  }
 
-.comments {
-  margin-left: .5rem;
-}
-
-.selected-card{
-  border: 5px solid #ffaa31;
-}
+  .cards .card .card-info {
+      padding: 2px 4px;
+      /*border-top: 1px solid #ddd;*/
+      background-color: #fffcfc;
+      font-size: 11px;
+      color: #333;
+  }
+  .cards .card .card-info input[type=checkbox] {
+      float: right;
+      width: 18px;
+      height: 18px;
+  }
+  .cards .card .card-info label {
+      display: inline-block;
+      height: 20px;
+      width: 145px;
+      font-size: 11px;
+  }
+  .cards .card .card-info .timestamp {
+      font-weight: 600;
+  }
+  .cards .card .card-footer code {
+      display: inline-block;
+      width: 100%;
+      margin-top: 2px;
+      font-size: 10px;
+      opacity: 0.4;
+      user-select: all;
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+  }
 
 </style>
 
 {% endblock %}
 
 {% block content %}
-    <section class="cards">
+  <section class="cards">
     {% for obj in results %}
-      <article class="card">
-          <picture class="thumbnail">
-            <a href="/{{obj.archive_path}}/index.html">
-              <img class="category__01" src="{% snapshot_image obj%}" alt="" /> 
-            </a>
-          </picture>
-          <div class="card-content">
-            {% if obj.tags_str %}
-            <p class="category category__01 tags">{{obj.tags_str}}</p>
-            {% endif %}
-            {% if obj.title  %}
+      <div class="card">
+          <div class="card-info">
             <a href="{% url 'admin:core_snapshot_change' obj.id %}">
-              <h4>{{obj.title|truncatechars:55 }}</h4>
+              <span class="timestamp">{{obj.added}}</span>
             </a>
-            {% endif %}
-            {% comment %} <p> TEXT If needed.</p> {% endcomment %}
-        </div><!-- .card-content -->
-        <footer>
-            <div class="post-meta">
-                <span style="float:right;"><input type="checkbox" name="_selected_action" value="{{obj.pk}}" class="action-select"></span>
-                <span class="timestamp">&#128337 {{obj.added}}</span>
-                <span class="comments">📖{{obj.num_outputs}}</span>
-                <span>🗄️{{ obj.archive_size | file_size }}</span>
-            </div>
-        </footer> 
-      </article>
+            <label>
+              <span class="num_outputs">📄 &nbsp; {{obj.num_outputs}}</span> &nbsp; &nbsp;
+              <span>🗄&nbsp; {{ obj.archive_size | file_size }}</span>
+              <input type="checkbox" name="_selected_action" value="{{obj.pk}}"/>
+            </label>
+          </div>
+          <a href="/{{obj.archive_path}}/index.html" class="card-thumbnail {% if not obj.thumbnail_url %}missing{% endif %}">
+            <img src="{{obj.thumbnail_url|default:'/static/spinner.gif' }}" alt="{{obj.title|default:'Not yet archived...'}}" /> 
+          </a>
+          <div class="card-footer">
+              {% if obj.tags_str %}
+                <div class="card-tags">
+                  {% for tag in obj.tags_str|split:',' %}
+                    <span>
+                      {{tag}}
+                    </span>
+                  {% endfor %}
+                </div>
+              {% endif %}
+              <div class="card-title" title="{{obj.title}}">
+                <a href="/{{obj.archive_path}}/index.html">
+                  <h4>
+                    {% if obj.is_archived %}
+                        <img src="/{{obj.archive_path}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
+                    {% else %}
+                        <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
+                    {% endif %}
+                    <span class="title-text">{{obj.title|default:'Pending...' }}</span>
+                  </h4>
+                </a>
+                <code title="{{obj.url}}">{{obj.url}}</code>
+              </div>
+          </div> 
+      </div>
     {% endfor %}
-    </section>
-
+  </section>
+  <br/>
 {% endblock %}

archivebox/templates/core/add.html

@@ -15,7 +15,7 @@
 {% endblock %}
 
 {% block body %}
-    <div style="max-width: 550px; margin: auto; float: none">
+    <div style="max-width: 1440px; margin: auto; float: none">
         <br/><br/>
         {% if stdout %}
             <h1>Add new URLs to your archive: results</h1>
@@ -38,16 +38,7 @@
             </form>
             <br/><br/><br/>
             <center id="delay-warning" style="display: none">
-                <b><i>This page will be unresponsive until the process is completely finished.</i></b>
-                <br/><br/>
-                <div>
-                    Warning: it may take several minutes to finish adding!<br/>
-                    <br/>
-                    Progress will be displayed in the <code>archivebox server</code> stdout,<br/>
-                    and on this page once the archiving process completes.<br/>
-                    <br/>
-                    <small>(it's safe to leave this page, adding will continue in the background)</small>
-                </div>
+                <small>(it's safe to leave this page, adding will continue in the background)</small>
             </center>
             {% if absolute_add_path %}
             <center id="bookmarklet">
@@ -58,7 +49,7 @@
             <script>
                 document.getElementById('add-form').addEventListener('submit', function(event) {
                     setTimeout(function() {
-                        document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>(see terminal for progress)</center>'
+                        document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for progress...</center>'
                         document.getElementById('delay-warning').style.display = 'block'
                     }, 200)
                     return true

archivebox/templates/core/base.html

@@ -1,67 +1,36 @@
-{% load admin_urls %}
-{% load static %}
+{% load static tz admin_urls %}
 
 <!DOCTYPE html>
 <html lang="en">
+    <head>
+        <title>Archived Sites</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
 
-<head>
-    <title>Archived Sites</title>
-    <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
-
-    <link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
-    <link rel="stylesheet" type="text/css" href="{% static 'admin.css' %}">
-    <link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
-    <link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}" />
-            {% block extra_head %}
-            {% endblock %}
-    <script src="{% static 'jquery.min.js' %}"></script>
-    <script src="{% static 'jquery.dataTables.min.js' %}"></script>
-    <script>
-        document.addEventListener('error', function (e) {
-            e.target.style.opacity = 0;
-        }, true)
-        jQuery(document).ready(function () {
-            jQuery('#table-bookmarks').DataTable({
-                searching: false,
-                paging: false,
-                stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
-                dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
-                    order: [[0, 'desc']],
-                    iDisplayLength: 100,
-                });
-            });
-        </script>
-        <base href="{% url 'Home' %}">
+        <link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
+        <link rel="stylesheet" href="{% static 'admin.css' %}">
+        <link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
+        
+        <script src="{% static 'jquery.min.js' %}"></script>
+        {% block extra_head %}
+        {% endblock %}
     </head>
     <body>
         <div id="container">
             <div id="header">
                 <div id="branding">
                     <h1 id="site-name">
-                        <a href="{% url 'public-index' %}" class="header-archivebox" title="Last updated: {{updated}}">
+                        <a href="{% url 'public-index' %}" class="header-archivebox">
                             <img src="{% static 'archive.png' %}" alt="Logo" style="height: 30px"/>
                             ArchiveBox
                         </a>
                     </h1>
                 </div>
-                <div id="user-tools">
-                    <a href="/add/">➕&nbsp; Add</a> &nbsp;/&nbsp;
-                    <a href="/">Snapshots</a> &nbsp;/&nbsp;
-                    <a href="/admin/">Admin</a> &nbsp;/&nbsp;
-                    <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-                    {% if user.is_authenticated %}
-                        &nbsp; &nbsp;
-                        User
-                        <strong>{% firstof user.get_short_name user.get_username %}</strong> &nbsp; &nbsp;
-                        {% if user.has_usable_password %}
-                            <a href="{% url 'admin:password_change' %}">Change password</a> /
-                        {% endif %}
-                        <a href="{% url 'admin:logout' %}">Log out</a>
-                    {% endif %}
-                </div>
+                {% include 'navigation.html' %}
+               
             </div>
             <div id="content" class="flex">
                 {% block body %}
+
                 {% endblock %}
             </div>
             {% block footer %}
@@ -70,7 +39,7 @@
                     <center>
                         <small>
                             Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a> version
-                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases" title="Releases">v{{VERSION}}</a>.
+                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{VERSION}}" title="Releases">v{{VERSION}}</a>.
                             <br/><br/>
                             {{FOOTER_INFO}}
                         </small>
@@ -79,6 +48,35 @@
                 </footer>
             {% endblock %}
         </div>
+        <script>
+            // hide images that fail to load
+            document.querySelector('body').addEventListener('error', function (e) {
+                e.target.style.opacity = 0;
+            }, true)
+
+            // setup timezone
+            {% get_current_timezone as TIME_ZONE %}
+            window.TIME_ZONE = '{{TIME_ZONE}}'
+
+            window.setCookie = function(name, value, days) {
+                let expires = ""
+                if (days) {
+                    const date = new Date()
+                    date.setTime(date.getTime() + (days*24*60*60*1000))
+                    expires = "; expires=" + date.toUTCString()
+                }
+                document.cookie = name + "=" + (value || "")  + expires + "; path=/"
+            }
+
+            function setTimeOffset() {
+                if (window.GMT_OFFSET) return
+                window.GMT_OFFSET = -(new Date).getTimezoneOffset()
+                window.setCookie('GMT_OFFSET', window.GMT_OFFSET, 365)
+            }
+
+            jQuery(document).ready(function () {
+                setTimeOffset();
+            });
+        </script>
     </body>
-    
-    </html>
+</html>

archivebox/templates/core/index_row.html

@@ -1,59 +1,44 @@
-{% load static %}
+{% load static tz core_tags %}
 
 <tr>
-{% comment %}
-    <!-- from upstream, may need to merge these two -->
-    <td title="{{snapshot.timestamp}}"> {% if snapshot.bookmarked_date  %} {{ snapshot.bookmarked_date }} {% else %} {{ snapshot.added }} {% endif %} </td>
-    <td class="title-col">
-        {% if snapshot.is_archived %}
-            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="archive/{{snapshot.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async"></a>
-        {% else %}
-            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async"></a>
-        {% endif %}
-            <a href="archive/{{snapshot.timestamp}}/{{snapshot.canonical_outputs.wget_path}}" title="{{snapshot.title}}">
-                <span data-title-for="{{snapshot.url}}" data-archived="{{snapshot.is_archived}}">{{snapshot.title|default:'Loading...'}}</span>
-                <small style="float:right">{% if snapshot.tags_str != None %} {{snapshot.tags_str|default:''}} {% else %} untagged {% endif %}</small>
-            </a>
+    <td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.added.timestamp}}">
+        {{ link.added|localtime }}
     </td>
-    <td>
-        <a href="archive/{{snapshot.timestamp}}/index.html">📄 
-            <span data-number-for="{{snapshot.url}}" title="Fetching any missing files...">{% if snapshot.icons  %} {{snapshot.icons}} {% else %} {{ snapshot.num_outputs}} {% endif %}<img src="{% static 'spinner.gif' %}" class="files-spinner" decoding="async"/></span>
-        </a>
-    </td>
-   <td style="text-align:left"><a href="{{snapshot.url}}">{{snapshot.url}}</a></td>
-{% endcomment %}
-
-    <td title="{{snapshot.timestamp}}"> {% if snapshot.bookmarked_date  %} {{ snapshot.bookmarked_date }} {% else %} {{ snapshot.added }} {% endif %} </td>
-    <td class="title-col" style="opacity: {% if snapshot.title %}1{% else %}0.3{% endif %}">
-        {% if snapshot.is_archived %}
-            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="archive/{{snapshot.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async"></a>
+    <td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}" title="{{link.title|default:'Not yet archived...'}}">
+        {% if link.is_archived %}
+            <a href="/archive/{{link.timestamp}}/index.html"><img src="/archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
         {% else %}
-            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async" style="height: 15px"></a>
+            <a href="/archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
         {% endif %}
 
-        <a href="archive/{{snapshot.timestamp}}/index.html" title="{{snapshot.title|default:'Not yet archived...'}}">
-            <span data-title-for="{{snapshot.url}}" data-archived="{{snapshot.is_archived}}">{{snapshot.title|default:'Loading...'}}</span>
-            {% if snapshot.tags_str %}
-                <span class="tags" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
-                    {% if snapshot.tags_str != None %}
-                        {{snapshot.tags_str|default:''}}
-                    {% else %}
-                        {{ snapshot.tags|default:'' }}
-                    {% endif %}
-                </span>
+        <a href="/archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
+            <span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">
+                {{link.title|default:'Loading...'|truncatechars:128}}
+            </span>
+            {% if link.tags_str %}
+                {% for tag in link.tags_str|split:',' %}
+                    <span class="tag" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
+                        {{tag}}
+                    </span>
+                {% endfor %}
             {% endif %}
         </a>
     </td>
     <td>
-        <span data-number-for="{{snapshot.url}}" title="Fetching any missing files...">
-            {% if snapshot.icons  %}
-                {{snapshot.icons}} <small style="float:right; opacity: 0.5">{{snapshot.num_outputs}}</small>
+        <span data-number-for="{{link.url}}" title="Fetching any missing files...">
+            {% if link.icons  %}
+                {{link.icons}}&nbsp; <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
             {% else %}
-                <a href="archive/{{snapshot.timestamp}}/index.html">📄 
-                    {{snapshot.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
+                <a href="/archive/{{link.timestamp}}/index.html">
+                    📄 &nbsp;
+                    {{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
                 </a>
             {% endif %}
         </span>
     </td>
-   <td style="text-align:left"><a href="{{snapshot.url}}">{{snapshot.url}}</a></td>
+    <td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; title="{{link.url}}">
+        <a href="{{link.url}}">
+            {{link.url}}
+        </a>
+    </td>
 </tr>

archivebox/templates/core/minimal_index.html

@@ -1,30 +1,24 @@
 <!DOCTYPE html>
 <html lang="en">
-  <head>
-    <title>Archived Sites</title>
-    <meta
-      charset="utf-8"
-      name="viewport"
-      content="width=device-width, initial-scale=1"
-    />
-  </head>
-  <body data-status="{{status}}">
-    <table id="table-bookmarks">
-      <thead>
-        <tr class="thead-tr">
-          <th style="width: 100px">Bookmarked</th>
-          <th style="width: 26vw">Saved Link ({{num_links}})</th>
-          <th style="width: 50px">Files</th>
-          <th style="width: 16vw; whitespace: nowrap; overflow-x: hidden">
-            Original URL
-          </th>
-        </tr>
-      </thead>
-      <tbody>
-        {% for snapshot in snapshots %}
-          {% include "index_row.html" with snapshot=snapshot %}
-        {% endfor %}
-      </tbody>
-    </table>
-  </body>
+    <head>
+        <title>Archived Sites</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+    </head>
+    <body>
+        <table id="table-bookmarks">
+            <thead>
+                <tr class="thead-tr">
+                    <th style="width: 100px;">Bookmarked</th>
+                    <th style="width: 26vw;">Saved Link ({{num_links}})</th>
+                    <th style="width: 50px">Files</th>
+                    <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for link in links %}
+                    {% include "index_row.html" with link=link %} 
+                {% endfor %}
+            </tbody>
+        </table>
+    </body>
 </html>

archivebox/templates/core/navigation.html

@@ -0,0 +1,26 @@
+{% load i18n static %}
+
+<div id="user-tools">
+    <a href="{% url 'admin:Add' %}">Add ➕</a> &nbsp; &nbsp;
+    <a href="{% url 'Home' %}">Snapshots</a> |
+    <a href="/admin/core/tag/">Tags</a> |
+    <a href="/admin/core/archiveresult/?o=-1">Log</a> &nbsp; &nbsp;
+    <a href="{% url 'Docs' %}">Docs</a> | 
+    <a href="{% url 'public-index' %}">Public</a> | 
+    <a href="/admin/">Admin</a>
+     &nbsp; &nbsp;
+    {% if user.is_authenticated %}
+        {% block welcome-msg %}
+            {% trans 'User' %}
+            <strong>{% firstof user.get_short_name user.get_username %}</strong> &nbsp; &nbsp;
+        {% endblock %}
+        {% block userlinks %}
+            {% if user.has_usable_password %}
+                <a href="{% url 'admin:password_change' %}">Account</a> /
+            {% endif %}
+            <a href="{% url 'admin:logout' %}">{% trans 'Log out' %}</a>
+        {% endblock %}
+    {% else %}
+        <a href="{% url 'admin:login' %}">{% trans 'Log in' %}</a>
+    {% endif %}
+</div>

archivebox/templates/core/progressbar.html

@@ -0,0 +1,45 @@
+<style>
+  /* Loading Progress Bar */
+    #progress {
+        position: absolute;
+        z-index: 1000;
+        top: 0px;
+        left: -6px;
+        width: 2%;
+        opacity: 1;
+        height: 2px;
+        background: #1a1a1a;
+        border-radius: 1px;
+        transition: width 4s ease-out, opacity 400ms linear;
+    }
+
+    @-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
+</style>
+<script>
+    // Page Loading Bar
+    window.loadStart = function(distance) {
+        var distance = distance || 0;
+        // only add progrstess bar if not already present
+        if (django.jQuery("#loading-bar").length == 0) {
+            django.jQuery("body").add("<div id=\"loading-bar\"></div>");
+        }
+        if (django.jQuery("#progress").length === 0) {
+            django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
+            let last_distance = (distance || (30 + (Math.random() * 30)))
+            django.jQuery("#progress").width(last_distance + "%");
+            setInterval(function() {
+                last_distance += Math.random()
+                django.jQuery("#progress").width(last_distance + "%");
+            }, 1000)
+        }
+    };
+
+    window.loadFinish = function() {
+        django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
+            django.jQuery(this).remove();
+        });
+    };
+    window.loadStart();
+    window.addEventListener('beforeunload', function() {window.loadStart(27)});
+    document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
+</script>

archivebox/templates/core/public_index.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% load static %}
+{% load static tz %}
 
 {% block body %}
     <div id="toolbar">
@@ -14,43 +14,49 @@
                        onclick="location.href='{% url 'public-index' %}'"
                        style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
                 </input>
+                &nbsp;
+                &nbsp;
+                {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
+                &nbsp;
+               (Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})
             </div>
         </form>
     </div>
-    <table id="table-bookmarks">
-        <thead>
-            <tr>
-                <th style="width: 100px;">Bookmarked</th>
-                <th style="width: 26vw;">Snapshot ({{object_list|length}})</th>
-                <th style="width: 140px">Files</th>
-                <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
-            </tr>
-        </thead>
+    <div style="width: 100%; overflow-x: auto;">
+        <table id="table-bookmarks" style="width: 100%; table-layout: fixed">
+            <thead>
+                <tr>
+                    <th style="width: 132px">Bookmarked</th>
+                    <th>Snapshot ({{page_obj.paginator.count}})</th>
+                    <th style="width: 280px">Files</th>
+                    <th>Original URL</th>
+                </tr>
+            </thead>
             <tbody>
                 {% for link in object_list %}
                     {% include 'index_row.html' with link=link  %}
                 {% endfor %}
             </tbody>
         </table>
-        <center>
-            <span class="step-links">
-                {% if page_obj.has_previous %}
-                    <a href="{% url 'public-index' %}?page=1">&laquo; first</a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
-                {% endif %}
-        
-                <span class="current">
-                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
-                </span>
-        
-                {% if page_obj.has_next %}
-                    <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
-                {% endif %}
-            </span>
+    </div>
+    <br/>
+    <center>
+        Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
+        <br/>
+        <span class="step-links">
+            {% if page_obj.has_previous %}
+                <a href="{% url 'public-index' %}?page=1">&laquo; first</a> &nbsp;
+                <a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
+                &nbsp;
+            {% endif %}
     
+            <span class="current">
+                Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
+            </span>
+        
             {% if page_obj.has_next %}
-                <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
+                &nbsp;
+                <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a> &nbsp;
                 <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
             {% endif %}
         </span>

archivebox/templates/core/snapshot.html

@@ -1,8 +1,19 @@
+{% load tz core_tags %}
+
 <!DOCTYPE html>
 <html lang="en">
     <head>
         <title>{{title}}</title>
         <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+        <style>
+            /*!
+            * Bootstrap v4.0.0 (https://getbootstrap.com)
+            * Copyright 2011-2018 The Bootstrap Authors
+            * Copyright 2011-2018 Twitter, Inc.
+            * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+            */
+            :root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}@-ms-viewport{width:device-width}article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}dfn{font-style:italic}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent;-webkit-text-decoration-skip:objects}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg:not(:root){overflow:hidden}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-family:inherit;font-weight:500;line-height:1.2;color:inherit}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-cLine truncated
+        </style>
         <style>
             html, body {
                 width: 100%;
@@ -11,7 +22,6 @@
             }
             header {
                 background-color: #aa1e55;
-                padding-bottom: 12px;
             }
             small {
                 font-weight: 200;
@@ -25,15 +35,15 @@
                 min-height: 40px;
                 margin: 0px;
                 text-align: center;
-                color: white;
-                font-size: calc(11px + 0.84vw);
+                color: #f6f6f6;
+                font-size: calc(10px + 0.84vw);
                 font-weight: 200;
-                padding: 4px 4px;
+                padding: 3px 4px;
                 background-color: #aa1e55;
             }
             .nav > div {
                 min-height: 30px;
-                line-height: 1.3;
+                line-height: 1.2;
             }
             .header-top a {
                 text-decoration: none;
@@ -59,9 +69,14 @@
             .header-archivebox img:hover {
                 opacity: 0.5;
             }
-            .header-url small {
+            header small code {
                 white-space: nowrap;
                 font-weight: 200;
+                display: block;
+                margin-top: -1px;
+                font-size: 13px;
+                opacity: 0.8;
+                user-select: all;
             }
             .header-url img {
                 height: 20px;
@@ -81,28 +96,38 @@
             .info-row .alert {
                 margin-bottom: 0px;
             }
+            .row.header-bottom {
+                margin-left: -10px;
+                margin-right: -10px;
+            }
+            .header-bottom .col-lg-2 {
+                padding-left: 4px;
+                padding-right: 4px;
+            }
+
             .header-bottom-frames .card {
-                overflow: hidden;
                 box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
-                margin-top: 10px;
+                margin-bottom: 5px;
                 border: 1px solid rgba(0,0,0,3);
-                border-radius: 14px;
+                border-radius: 10px;
                 background-color: black;
+                overflow: hidden;
             }
             .card h4 {
                 font-size: 1.4vw;
             }
             .card-body {
-                font-size: 15px;
+                font-size: 14px;
                 padding: 13px 10px;
-                padding-bottom: 6px;
+                padding-bottom: 1px;
                 /* padding-left: 3px; */
                 /* padding-right: 3px; */
                 /* padding-bottom: 3px; */
-                line-height: 1.1;
+                line-height: 1;
                 word-wrap: break-word;
                 max-height: 102px;
                 overflow: hidden;
+                text-overflow: ellipsis;
                 background-color: #1a1a1a;
                 color: #d3d3d3;
             }
@@ -137,22 +162,12 @@
                 border-top: 3px solid #aa1e55;
             }
             .card.selected-card {
-                border: 1px solid orange;
+                border: 2px solid orange;
                 box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
             }
             .iframe-large {
                 height: calc(100% - 40px);
             }
-            .pdf-frame {
-                transform: none;
-                width: 100%;
-                height: 160px;
-                margin-top: -60px;
-                margin-bottom: 0px;
-                transform: scale(1.1);
-                width: 100%;
-                margin-left: -10%;
-            }
             img.external {
                 height: 30px;
                 margin-right: -10px;
@@ -176,7 +191,7 @@
             }
             .header-bottom {
                 border-top: 1px solid rgba(170, 30, 85, 0.9);
-                padding-bottom: 12px;
+                padding-bottom: 1px;
                 border-bottom: 5px solid rgb(170, 30, 85);
                 margin-bottom: -1px;
 
@@ -206,10 +221,11 @@
             }
             .info-chunk {
                 width: auto;
-                display:inline-block;
+                display: inline-block;
                 text-align: center;
-                margin: 10px 10px;
+                margin: 8px 4px;
                 vertical-align: top;
+                font-size: 14px;
             }
             .info-chunk .badge {
                 margin-top: 5px;
@@ -217,13 +233,12 @@
             .header-bottom-frames .card-title {
                 width: 100%;
                 text-align: center;
-                font-size: 18px;
-                margin-bottom: 5px;
+                font-size: 17px;
+                margin-bottom: 0px;
                 display: inline-block;
                 color: #d3d3d3;
                 font-weight: 200;
-                vertical-align: 0px;
-                margin-top: -6px;
+                vertical-align: 3px;
             }
             .header-bottom-frames .card-text {
                 width: 100%;
@@ -263,14 +278,12 @@
                 }
             }
         </style>
-        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
     </head>
     <body>
         <header>
             <div class="header-top container-fluid">
                 <div class="row nav">
-                    <div class="col-lg-2" style="line-height: 64px;">
-
+                    <div class="col-lg-2" style="line-height: 50px; vertical-align: middle">
                         <a href="../../index.html" class="header-archivebox" title="Go to Main Index...">
                             <img src="../../static/archive.png" alt="Archive Icon">
                             ArchiveBox
@@ -279,13 +292,12 @@
                     <div class="col-lg-8">
                         <img src="favicon.ico" onerror="this.style.opacity=0" alt="Favicon">
                         &nbsp;&nbsp;
-                        {{title}}
+                        {{title|safe}}
                         &nbsp;&nbsp;
                         <a href="#" class="header-toggle">▾</a>
-                        <br/>
                         <small>
                             <a href="{{url}}" class="header-url" title="{{url}}">
-                                {{url_str}}
+                                <code>{{url}}</code>
                             </a>
                         </small>
                     </div>
@@ -294,27 +306,25 @@
             <div class="header-bottom container-fluid">
                 <div class="row header-bottom-info">
                     <div class="col-lg-4">
-                        <div title="Date bookmarked or imported" class="info-chunk">
+                        <div title="Date bookmarked or imported" class="info-chunk" title="UTC Timezone {{timestamp}}">
                             <h5>Added</h5>
                             {{bookmarked_date}}
                         </div>
-                        <div title="Date first archived" class="info-chunk">
+                        <div title="Date first archived" class="info-chunk" title="UTC Timezone">
                             <h5>First Archived</h5>
                             {{oldest_archive_date}}
                         </div>
-                        <div title="Date last checked" class="info-chunk">
+                        <div title="Date last checked" class="info-chunk" title="UTC Timezone">
                             <h5>Last Checked</h5>
                             {{updated_date}}
                         </div>
                     </div>
                     <div class="col-lg-4">
-                        <div class="info-chunk">
-                            <h5>Type</h5>
-                            <div class="badge badge-default">{{extension}}</div>
-                        </div>
-                        <div class="info-chunk">
-                            <h5>Tags</h5>
-                            <div class="badge badge-warning">{{tags}}</div> 
+                        <div class="info-chunk" style="max-width: 280px">
+                            <h5>Tags <small title="Auto-guessed content type">({{extension}})</small></h5>
+                            {% for tag in tags_str|split:',' %}
+                                <div class="badge badge-info" style="word-break: break-all;">{{tag}}</div>
+                            {% endfor %}
                         </div>
                         <div class="info-chunk">
                             <h5>Status</h5>
@@ -322,11 +332,11 @@
                         </div>
                         <div class="info-chunk">
                             <h5>Saved</h5>
-                            ✅  {{num_outputs}}
+                            ✅&nbsp; {{num_outputs}}
                         </div>
                         <div class="info-chunk">
                             <h5>Errors</h5>
-                            ❌  {{num_failures}}
+                            ❌&nbsp; {{num_failures}}
                         </div>
                         <div class="info-chunk">
                             <h5>Size</h5>
@@ -335,20 +345,21 @@
                     </div>
                     <div class="col-lg-4">
                         <div class="info-chunk">
-                            <h5>🗃 Files</h5>
+                            <h5>🗃&nbsp; Snapshot: <a href="/admin/core/snapshot/{{snapshot_id}}/change/"><code style="color: rgba(255,255,255,0.6); font-weight: 200; font-size: 12px; background-color: #1a1a1a"><b>[{{timestamp}}]</b> <small>{{snapshot_id|truncatechars:24}}</small></code></a></h5>
                             <a href="index.json" title="JSON summary of archived link.">JSON</a> | 
                             <a href="warc/" title="Any WARC archives for the page">WARC</a> | 
                             <a href="media/" title="Audio, Video, and Subtitle files.">Media</a> | 
                             <a href="git/" title="Any git repos at the url">Git</a> | 
-                            <a href="favicon.ico" title="Any git repos at the url">Favicon</a> | 
-                            <a href="." title="Webserver-provided index of files directory.">See all...</a>
+                            <a href="/admin/core/snapshot/?id__startswith={{snapshot_id}}" title="Go to the Snapshot admin to update, overwrite, or delete this Snapshot">Actions</a> | 
+                            <a href="/admin/core/snapshot/{{snapshot_id}}/change/" title="Edit this snapshot in the Admin UI">Admin</a> | 
+                            <a href="." title="Webserver-provided index of files directory.">See all files...</a><br/>
                         </div>
                     </div>
                 </div>
                 <div class="row header-bottom-frames">
                     <div class="col-lg-2">
                         <div class="card selected-card">
-                            <iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{singlefile_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./singlefile.html</code></p>
@@ -359,7 +370,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top pdf-frame" src="{{pdf_path}}" scrolling="no"></iframe>
+                            <iframe class="card-img-top pdf-frame" src="{{pdf_path}}#toolbar=0" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{pdf_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./output.pdf</code></p>
@@ -381,7 +392,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                          <iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                          <iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                           <div class="card-body">
                                 <a href="{{archive_url}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./{{domain}}</code></p>
@@ -393,30 +404,30 @@
                     {% if SAVE_ARCHIVE_DOT_ORG %}
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{archive_org_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>🌐 web.archive.org/web/...</code></p>
                                 </a>
-                                <a href="{{archive_org_path}}" target="preview"><h4 class="card-title">Archive.Org</h4></a>
+                                <a href="{{archive_org_path}}" target="preview" id="archive_dot_org-btn"><h4 class="card-title">Archive.Org</h4></a>
                           </div>
                         </div>
                     </div>
                     {% endif %}
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>🌐 {{domain}}</code></p>
                                 </a>
-                                <a href="{{url}}" target="preview"><h4 class="card-title">Original</h4></a>
+                                <a href="{{url}}" target="preview" id="original-btn"><h4 class="card-title">Original</h4></a>
                           </div>
                         </div>
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{headers_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./headers.json</code></p>
@@ -427,7 +438,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{dom_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./output.html</code></p>
@@ -438,7 +449,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{readability_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./readability/content.html</code></p>
@@ -450,7 +461,7 @@
                     <br/>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{mercury_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./mercury/content.html</code></p>
@@ -461,7 +472,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{media_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./media/*.mp4</code></p>
@@ -472,7 +483,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{git_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./git/*.git</code></p>
@@ -484,15 +495,24 @@
                 </div>
             </div>
         </header>
-        <iframe sandbox="allow-same-origin allow-scripts allow-forms" class="full-page-iframe" src="{{singlefile_path}}" name="preview"></iframe>
+        <iframe sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" class="full-page-iframe" src="{{singlefile_path}}" name="preview"></iframe>
     
-        <script
-              src="https://code.jquery.com/jquery-3.2.1.slim.min.js"
-              integrity="sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g="
-              crossorigin="anonymous"></script>
-        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/js/bootstrap.min.js" integrity="sha384-vBWWzlZJ8ea9aCX4pEW3rVHjgjt7zpkNpZk+02D9phzyeVkE+jo0ieGizqPLForn" crossorigin="anonymous"></script>
+        <script>
+            /*! jQuery v3.2.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/parseXML,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-event/ajax,-effects,-effects/Tween,-effects/animatedSelector | (c) JS Foundation and other contributors | jquery.org/license */
+            !function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var q="3.2.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/parseXML,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-event/ajax,-effects,-effects/Tween,-effects/animatedSelector",r=function(a,b){return new r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return f.call(this)},get:function(a){return null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var b=r.merge(this.constructor(),a);return b.prevObject=this,b},each:function(a){return r.each(this,a)},map:function(a){return this.pushStack(r.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(f.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(a<0?b:0);return this.pushStack(c>=0&&c<b?[this[c]]:[])},end:function(){return this.prevObject||this.constructor()},push:h,sort:c.sort,splice:c.splice},r.extend=r.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||r.isFunction(g)||(g={}),h===i&&(g=this,h--);h<i;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(r.isPlainObject(d)||(e=Array.isArray(d)))?(e?(e=!1,f=c&&Array.isArray(c)?c:[]):f=c&&r.isPlainObject(c)?c:{},g[b]=r.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},r.extend({expando:"jQuery"+(q+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===r.type(a)},isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){var b=r.type(a);return("number"===b||"string"===b)&&!isNaN(a-parseFloat(a))},isPlainObject:function(a){var b,c;return!(!a||"[object Object]"!==k.call(a))&&(!(b=e(a))||(c=l.call(b,"constructor")&&b.constructor,"function"==typeof c&&m.call(c)===n))},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?j[k.call(a)]||"object":typeof a},globalEval:function(a){p(a)},camelCase:function(a){return a.replace(t,"ms-").replace(u,v)},each:function(a,b){var c,d=0;if(w(a)){for(c=a.length;d<c;d++)if(b.call(a[d],d,a[d])===!1)break}else for(d in a)if(b.call(a[d],d,a[d])===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(s,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(w(Object(a))?r.merge(c,"string"==typeof a?[a]:a):h.call(c,a)),c},inArray:function(a,b,c){return null==b?-1:i.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;d<c;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;f<g;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,e,f=0,h=[];if(w(a))for(d=a.length;f<d;f++)e=b(a[f],f,c),null!=e&&h.push(e);else for(f in a)e=b(a[f],f,c),null!=e&&h.push(e);return g.apply([],h)},guid:1,proxy:function(a,b){var c,d,e;if("string"==typeof b&&(c=a[b],b=a,a=c),r.isFunction(a))return d=f.call(arguments,2),e=function(){return a.apply(b||this,d.concat(f.call(arguments)))},e.guid=a.guid=a.guid||r.guid++,e},now:Date.now,support:o}),"function"==typeof Symbol&&(r.fn[Symbol.iterator]=c[Symbol.iterator]),r.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(a,b){j["[object "+b+"]"]=b.toLowerCase()});function w(a){var b=!!a&&"length"in a&&a.length,c=r.type(a);return"function"!==c&&!r.isWindow(a)&&("array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a)}var x=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var c=0,d=a.length;c<d;c++)if(a[c]===b)return c;return-1},J="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",K="[\\x20\\t\\r\\n\\f]",L="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",M="\\["+K+"*("+L+")(?:"+K+"*([*^$|!~]?=)"+K+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+L+"))|)"+K+"*\\]",N=":("+L+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+M+")*)|.*)\\)|)",O=new RegExp(K+"+","g"),P=new RegExp("Line truncated
+        </script>
 
         <script>
+            function getPreviewTypeFromPath(link) {
+                if (link.id == 'original-btn') {
+                    return 'original'
+                }
+                if (link.id == 'archive_dot_org-btn') {
+                    return 'archive_dot_org'
+                }
+                return link.pathname.split('/').filter(a => a.length).slice(-1)[0].toLowerCase()
+            }
+
             // show selected file in iframe when preview card is clicked
             jQuery('.card').on('click', function(e) {
                 jQuery('.selected-card').removeClass('selected-card')
@@ -502,11 +522,26 @@
                 if (e.currentTarget.href.endsWith('.pdf')) {
                     jQuery('.full-page-iframe')[0].removeAttribute('sandbox')
                 } else {
-                    jQuery('.full-page-iframe')[0].sandbox = "allow-same-origin allow-scripts allow-forms"
+                    jQuery('.full-page-iframe')[0].sandbox = "allow-same-origin allow-scripts allow-forms allow-top-navigation-by-user-activation"
                 }
+                window.location.hash = getPreviewTypeFromPath(e.currentTarget)
                 return true
             })
 
+            // check URL for hash e.g. #git and load relevant preview
+            jQuery(document).ready(function() {
+                if (window.location.hash) {
+                    for (const link of jQuery('a[target=preview]')) {
+                        console.log(link.pathname)
+                        if (getPreviewTypeFromPath(link) == window.location.hash.slice(1).toLowerCase()) {
+                            jQuery(link).closest('.card').click()
+                            jQuery(link).click()
+                            link.click()
+                        }
+                    }
+                }
+            })
+
             // un-sandbox iframes showing pdfs (required to display pdf viewer)
             jQuery('iframe').map(function() {
                 if (this.src.endsWith('.pdf')) {

archivebox/templates/core/static_index.html

@@ -209,7 +209,7 @@
             <div class="header-top container-fluid">
                 <div class="row nav">
                     <div class="col-sm-2">
-                        <a href="/" class="header-archivebox" title="Last updated: {{updated}}">
+                        <a href="/" class="header-archivebox">
                             <img src="{% static 'archive.png' %}" alt="Logo"/>
                             ArchiveBox: Index
                         </a>
@@ -227,14 +227,14 @@
             <thead>
                 <tr>
                     <th style="width: 100px;">Bookmarked</th>
-                    <th style="width: 26vw;">Saved Link ({{num_snapshots}})</th>
+                    <th style="width: 26vw;">Saved Link ({{num_links}})</th>
                     <th style="width: 50px">Files</th>
                     <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
                 </tr>
             </thead>
             <tbody>
-                {% for snapshot in snapshots %}
-                   {% include 'index_row.html' with snapshot=snapshot %}
+                {% for link in links %}
+                   {% include 'index_row.html' with link=link %}
                 {% endfor %}
             </tbody>
         </table>
@@ -243,7 +243,7 @@
             <center>
                 <small>
                     Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
-                    version <a href="https://github.com/ArchiveBox/ArchiveBox/tree/v{{version}}" title="Git commit">v{{version}}</a> &nbsp; | &nbsp; 
+                    version <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{version}}" title="View source code and release info">v{{version}}</a> &nbsp; | &nbsp; 
                     Download index as <a href="index.json" title="JSON summary of archived links.">JSON</a>
                     <br/><br/>
                     {{FOOTER_INFO}}

archivebox/templates/static/add.css

@@ -39,10 +39,10 @@ header {
   margin: 10px;
   padding: 10px;
   font-size: 15px;
-  background-color: #f5dd5d;
+  background-color: #8bc34a;
 }
 #stdout {
-  background-color: #ded;
+  background-color: #fbfbfb;
   padding: 10px 10px;
   border-radius: 4px;
   white-space: normal;

archivebox/templates/static/admin.css

@@ -1,3 +1,9 @@
+* {
+    -webkit-box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    box-sizing: border-box;
+}
+
 #logo {
     height: 30px;
     vertical-align: -6px;
@@ -36,6 +42,23 @@ div.breadcrumbs {
     padding: 6px 15px;
 }
 
+#toolbar #searchbar {
+    height: 25px;
+}
+
+#snapshot-view-mode {
+    float: right;
+    margin-bottom: -40px;
+    display: inline-block;
+    margin-top: 3px;
+    margin-right: 10px;
+    font-size: 14px;
+    opacity: 0.8;
+}
+#snapshot-view-mode a {
+    color: #ccc;
+}
+
 body.model-snapshot.change-list div.breadcrumbs,
 body.model-snapshot.change-list #content .object-tools {
     display: none;
@@ -92,6 +115,14 @@ body.model-snapshot.change-list #content .object-tools {
     background: none;
     margin-right: 0px;
     width: auto;
+    max-height: 40px;
+    overflow: hidden;
+    display: block;
+}
+@media (max-width: 1000px) {
+    #content #changelist .actions {
+        max-height: 200px;
+    }
 }
 
 #content #changelist .actions .button {
@@ -116,20 +147,45 @@ body.model-snapshot.change-list #content .object-tools {
     background-color:lightseagreen;
     color: #333;
 }
+#content #changelist .actions .button[name=resnapshot_snapshot] {
+    background-color: #9ee54b;
+    color: #333;
+}
 #content #changelist .actions .button[name=overwrite_snapshots] {
     background-color: #ffaa31;
     color: #333;
+    margin-left: 10px;
 }
 #content #changelist .actions .button[name=delete_snapshots] {
     background-color: #f91f74;
     color: rgb(255 248 252 / 64%);
 }
+#content #changelist .actions .button[name=add_tags] {
+}
+#content #changelist .actions .button[name=remove_tags] {
+    margin-right: 25px;
+}
 
+#content #changelist .actions .select2-selection {
+    max-height: 25px;
+}
+#content #changelist .actions .select2-container--admin-autocomplete.select2-container {
+    width: auto !important;
+    min-width: 90px;
+}
+#content #changelist .actions .select2-selection__rendered .select2-selection__choice {
+    margin-top: 3px;
+}
 
 #content #changelist-filter h2 {
     border-radius: 4px 4px 0px 0px;
 }
 
+#changelist .paginator {
+    border-top: 0px;
+    border-bottom: 0px;
+}
+
 @media (min-width: 767px) {
     #content #changelist-filter {
         top: 35px;
@@ -157,7 +213,7 @@ body.model-snapshot.change-list #content .object-tools {
 
 #content a img.favicon {
     height: 20px;
-    width: 20px;
+    max-width: 28px;
     vertical-align: -5px;
     padding-right: 6px;
 }
@@ -177,7 +233,7 @@ body.model-snapshot.change-list #content .object-tools {
 
 #content th.field-added, #content td.field-updated {
     word-break: break-word;
-    min-width: 128px;
+    min-width: 135px;
     white-space: normal;
 }
 
@@ -237,3 +293,40 @@ body.model-snapshot.change-list #content .object-tools {
     opacity: 0.1;
     filter: grayscale(100%);
 }
+
+
+#result_list tbody td.field-cmd_str pre,
+#result_list tbody td.field-output_str pre {
+    max-width: 22vw;
+    word-wrap: anywhere;
+    white-space: break-spaces;
+    max-height: 40px;
+    overflow: hidden;
+    margin: 2px;
+    background-color: rgba(0,0,0,0.05);
+    padding: 1px 4px 16px 8px;
+    border-radius: 4px;
+}
+
+#result_list tbody td.field-extractor {
+    font-weight: 800;
+    font-variant: small-caps;
+}
+
+#result_list tbody td.field-status {
+    font-variant: small-caps;
+}
+
+.inline-group .tabular td.original p {
+    margin-top: -33px;
+}
+
+tbody .output-link {
+    float: right;
+    margin-bottom: -25px;
+    margin-right: -3px;
+    margin-top: -4px;
+    opacity: 0.4;
+    box-shadow:   4px 4px 4px rgba(0,0,0,0.1);
+}
+tbody .output-link:hover {opacity: 1;}