Merge branch 'dev'

add command: --parser option (fixes #235)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,8 +1,8 @@
 ---
 name: 🐞 Bug report
 about: Create a report to help us improve
-title: 'Bugfix: ...'
-labels: 'changes: bugfixes'
+title: 'Bug: ...'
+labels: 'bug'
 assignees: ''
 
 ---

.github/workflows/docker.yml

@@ -1,10 +1,13 @@
 name: Build Docker image
 
 on:
-  on: workflow_dispatch
-  release:
-    types:
-      - created
+  workflow_dispatch:
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+  # pull_request:
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -53,13 +56,16 @@ jobs:
       
       - name: Collect Docker tags
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: crazy-max/ghaction-docker-meta@v2
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          tag-sha: true
-          tag-semver: |
-            {{version}}
-            {{major}}.{{minor}}
+          flavor: |
+              latest=auto
+          tags: |
+              type=ref,event=branch
+              type=semver,pattern={{version}}
+              type=semver,pattern={{major}}.{{minor}}
+              type=sha
       
       - name: Build and push
         id: docker_build
@@ -72,7 +78,7 @@ jobs:
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
-          platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/pip.yml

@@ -39,6 +39,12 @@ jobs:
           archivebox version
           archivebox status
 
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
+
       # - name: Commit built package
       #   run: |
       #     cd pip_dist/
@@ -54,8 +60,3 @@ jobs:
       #     branch: ${{ github.ref }}
       #     directory: pip_dist
 
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}

.gitmodules

@@ -23,3 +23,6 @@
 [submodule "archivebox/vendor/django-taggit"]
 	path = archivebox/vendor/django-taggit
 	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites

Dockerfile

@@ -50,19 +50,13 @@ RUN apt-get update -qq \
         fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
     && rm -rf /var/lib/apt/lists/*
 
-# Install apt development dependencies
-# RUN apt-get install -qq \
-#     && apt-get install -qq -y --no-install-recommends \
-#         python3 python3-dev python3-pip python3-venv python3-all \
-#         dh-python debhelper devscripts dput software-properties-common \
-#         python3-distutils python3-setuptools python3-wheel python3-stdeb
-
 # Install Node environment
 RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
     && echo 'deb https://deb.nodesource.com/node_15.x buster main' >> /etc/apt/sources.list \
     && apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
         nodejs \
+    # && npm install -g npm \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Node dependencies
@@ -78,16 +72,27 @@ WORKDIR "$CODE_DIR"
 ENV PATH="${PATH}:$VENV_PATH/bin"
 RUN python -m venv --clear --symlinks "$VENV_PATH" \
     && pip install --upgrade --quiet pip setuptools
-ADD ./pip_dist/archivebox.egg-info/requires.txt "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt"
+ADD "./setup.py" "$CODE_DIR/"
+ADD "./package.json" "$CODE_DIR/archivebox/"
 RUN apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
         build-essential python-dev python3-dev \
-    && grep -B 1000 -E '^$' "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin \
-    && pip install --quiet "sonic-client==0.0.5" \
+    && echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \
+    && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
+    && pip install --quiet -r /tmp/requirements.txt \
     && apt-get purge -y build-essential python-dev python3-dev \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
+# Install apt development dependencies
+# RUN apt-get install -qq \
+#     && apt-get install -qq -y --no-install-recommends \
+#         python3 python3-dev python3-pip python3-venv python3-all \
+#         dh-python debhelper devscripts dput software-properties-common \
+#         python3-distutils python3-setuptools python3-wheel python3-stdeb
+# RUN python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.extras_require["dev"]))' > /tmp/dev_requirements.txt \
+    # && pip install --quiet -r /tmp/dev_requirements.txt
+
 # Install ArchiveBox Python package and its dependencies
 WORKDIR "$CODE_DIR"
 ADD . "$CODE_DIR"
@@ -113,5 +118,8 @@ RUN /app/bin/docker_entrypoint.sh archivebox version
 VOLUME "$DATA_DIR"
 EXPOSE 8000
 
+HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+    CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1
+
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

MANIFEST.in

@@ -2,3 +2,5 @@ graft archivebox
 global-exclude .DS_Store
 global-exclude __pycache__
 global-exclude *.pyc
+
+prune tests/

archivebox/cli/__init__.py

@@ -8,18 +8,17 @@ import argparse
 from typing import Optional, Dict, List, IO, Union
 from pathlib import Path
 
-from ..config import OUTPUT_DIR
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations
 
 from importlib import import_module
 
 CLI_DIR = Path(__file__).resolve().parent
 
 # these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')
-main_cmds = ('init', 'info', 'config')
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')
-
-fake_db = ("oneshot",)
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
 
 display_first = (*meta_cmds, *main_cmds, *archive_cmds)
 
@@ -61,9 +60,21 @@ def run_subcommand(subcommand: str,
                    pwd: Union[Path, str, None]=None) -> None:
     """Run a given ArchiveBox subcommand with the given list of args"""
 
+    subcommand_args = subcommand_args or []
+
     if subcommand not in meta_cmds:
         from ..config import setup_django
-        setup_django(in_memory_db=subcommand in fake_db, check_db=subcommand in archive_cmds)
+
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+
+        if cmd_requires_db:
+            check_data_folder(pwd)
+
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
+
+        if cmd_requires_db:
+            check_migrations()
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore

archivebox/cli/archivebox_add.py

@@ -10,6 +10,7 @@ from typing import List, Optional, IO
 
 from ..main import add
 from ..util import docstring
+from ..parsers import PARSERS
 from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr
 
@@ -22,6 +23,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         add_help=True,
         formatter_class=SmartFormatter,
     )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
     parser.add_argument(
         '--update-all', #'-n',
         action='store_true',
@@ -73,9 +80,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
               This does not take precedence over the configuration",
         default=""
     )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        help="Parser used to read inputted URLs.",
+        default="auto",
+        choices=["auto", *PARSERS.keys()],
+    )
     command = parser.parse_args(args or ())
     urls = command.urls
-    stdin_urls = accept_stdin(stdin)
+
+    stdin_urls = ''
+    if not urls:
+        stdin_urls = accept_stdin(stdin)
+
     if (stdin_urls and urls) or (not stdin and not urls):
         stderr(
             '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
@@ -85,11 +103,13 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     add(
         urls=stdin_urls or urls,
         depth=command.depth,
+        tag=command.tag,
         update_all=command.update_all,
         index_only=command.index_only,
         overwrite=command.overwrite,
         init=command.init,
         extractors=command.extract,
+        parser=command.parser,
         out_dir=pwd or OUTPUT_DIR,
     )
 

archivebox/cli/archivebox_config.py

@@ -45,7 +45,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='KEY or KEY=VALUE formatted config values to get or set',
     )
     command = parser.parse_args(args or ())
-    config_options_str = accept_stdin(stdin)
+
+    config_options_str = ''
+    if not command.config_options:
+        config_options_str = accept_stdin(stdin)
 
     config(
         config_options_str=config_options_str,

archivebox/cli/archivebox_init.py

@@ -27,11 +27,23 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Ignore unrecognized files in current directory and initialize anyway',
     )
+    parser.add_argument(
+        '--quick', '-q',
+        action='store_true',
+        help='Run any updates or migrations without rechecking all snapshot dirs',
+    )
+    parser.add_argument(
+        '--setup', #'-s',
+        action='store_true',
+        help='Automatically install dependencies and extras used for archiving',
+    )
     command = parser.parse_args(args or ())
     reject_stdin(__command__, stdin)
 
     init(
         force=command.force,
+        quick=command.quick,
+        setup=command.setup,
         out_dir=pwd or OUTPUT_DIR,
     )
     

archivebox/cli/archivebox_list.py

@@ -12,6 +12,7 @@ from ..main import list_all
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
+    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -23,7 +24,7 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
-from ..logging_util import SmartFormatter, accept_stdin, stderr
+from ..logging_util import SmartFormatter, reject_stdin, stderr
 
 
 @docstring(list_all.__doc__)
@@ -44,7 +45,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     group.add_argument(
         '--json', #'-j',
         action='store_true',
-        help="Print the output in JSON format with all columns included.",
+        help="Print the output in JSON format with all columns included",
     )
     group.add_argument(
         '--html',
@@ -59,19 +60,19 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     parser.add_argument(
         '--sort', #'-s',
         type=str,
-        help="List the links sorted using the given key, e.g. timestamp or updated.",
+        help="List the links sorted using the given key, e.g. timestamp or updated",
         default=None,
     )
     parser.add_argument(
         '--before', #'-b',
         type=float,
-        help="List only links bookmarked before the given timestamp.",
+        help="List only links bookmarked before (less than) the given timestamp",
         default=None,
     )
     parser.add_argument(
         '--after', #'-a',
         type=float,
-        help="List only links bookmarked after the given timestamp.",
+        help="List only links bookmarked after (greater than or equal to) the given timestamp",
         default=None,
     )
     parser.add_argument(
@@ -96,9 +97,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type',
+        '--filter-type', '-t',
         type=str,
-        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
+        choices=(*LINK_FILTERS.keys(), 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -107,20 +108,19 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         nargs='*',
         type=str,
         default=None,
-        help='List only URLs matching these filter patterns.'
+        help='List only URLs matching these filter patterns'
     )
     command = parser.parse_args(args or ())
-    filter_patterns_str = accept_stdin(stdin)
+    reject_stdin(stdin)
 
     if command.with_headers and not (command.json or command.html or command.csv):
         stderr(
-            '[X] --with-headers can only be used with --json, --html or --csv options.\n',
+            '[X] --with-headers can only be used with --json, --html or --csv options\n',
             color='red',
         )
         raise SystemExit(2)
 
     matching_folders = list_all(
-        filter_patterns_str=filter_patterns_str,
         filter_patterns=command.filter_patterns,
         filter_type=command.filter_type,
         status=command.status,

archivebox/cli/archivebox_oneshot.py

@@ -50,8 +50,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help= "Path to save the single archive folder to, e.g. ./example.com_archive"
     )
     command = parser.parse_args(args or ())
+    stdin_url = None
     url = command.url
-    stdin_url = accept_stdin(stdin)
+    if not url:
+        stdin_url = accept_stdin(stdin)
+
     if (stdin_url and url) or (not stdin and not url):
         stderr(
             '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',

archivebox/cli/archivebox_remove.py

@@ -61,7 +61,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         help='URLs matching this filter pattern will be removed from the index.'
     )
     command = parser.parse_args(args or ())
-    filter_str = accept_stdin(stdin)
+    
+    filter_str = None
+    if not command.filter_patterns:
+        filter_str = accept_stdin(stdin)
 
     remove(
         filter_str=filter_str,

archivebox/cli/archivebox_schedule.py

@@ -44,7 +44,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         type=int,
         choices=[0, 1],
         default=0,
-        help='Depth to archive to [0] or 1, see "add" command help for more info.',
+        help='Depth to archive to [0] or 1, see "add" command help for more info',
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
     )
     group.add_argument(
         '--clear', # '-c'
@@ -88,6 +93,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         quiet=command.quiet,
         every=command.every,
         depth=command.depth,
+        overwrite=command.overwrite,
         import_path=command.import_path,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_server.py

@@ -38,10 +38,20 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         action='store_true',
         help='Enable DEBUG=True mode with more verbose errors',
     )
+    parser.add_argument(
+        '--nothreading',
+        action='store_true',
+        help='Force runserver to run in single-threaded mode',
+    )
     parser.add_argument(
         '--init',
         action='store_true',
-        help='Run archivebox init before starting the server',
+        help='Run a full archivebox init/upgrade before starting the server',
+    )
+    parser.add_argument(
+        '--quick-init', '-i',
+        action='store_true',
+        help='Run quick archivebox init/upgrade before starting the server',
     )
     parser.add_argument(
         '--createsuperuser',
@@ -52,10 +62,11 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     reject_stdin(__command__, stdin)
     
     server(
-        runserver_args=command.runserver_args,
+        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
         reload=command.reload,
         debug=command.debug,
         init=command.init,
+        quick_init=command.quick_init,
         createsuperuser=command.createsuperuser,
         out_dir=pwd or OUTPUT_DIR,
     )

archivebox/cli/archivebox_setup.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox setup'
+
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import setup
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+
+
+@docstring(setup.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=setup.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    # parser.add_argument(
+    #     '--force', # '-f',
+    #     action='store_true',
+    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
+    # )
+    command = parser.parse_args(args or ())   # noqa
+    reject_stdin(__command__, stdin)
+
+    setup(
+        # force=command.force,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_update.py

@@ -12,6 +12,7 @@ from ..main import update
 from ..util import docstring
 from ..config import OUTPUT_DIR
 from ..index import (
+    LINK_FILTERS,
     get_indexed_folders,
     get_archived_folders,
     get_unarchived_folders,
@@ -89,9 +90,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         )
     )
     parser.add_argument(
-        '--filter-type',
+        '--filter-type', '-t',
         type=str,
-        choices=('exact', 'substring', 'domain', 'regex', 'tag', 'search'),
+        choices=(*LINK_FILTERS.keys(), 'search'),
         default='exact',
         help='Type of pattern matching to use when filtering URLs',
     )
@@ -110,7 +111,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         default=""
     )
     command = parser.parse_args(args or ())
-    filter_patterns_str = accept_stdin(stdin)
+
+    filter_patterns_str = None
+    if not command.filter_patterns:
+        filter_patterns_str = accept_stdin(stdin)
 
     update(
         resume=command.resume,

archivebox/cli/tests.py

@@ -75,23 +75,23 @@ def output_hidden(show_failing=True):
         yield
         return
 
-    sys.stdout = open('stdout.txt', 'w+')
-    sys.stderr = open('stderr.txt', 'w+')
+    sys.stdout = open('stdout.txt', 'w+', encoding='utf-8')
+    sys.stderr = open('stderr.txt', 'w+', encoding='utf-8')
     try:
         yield
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
-    except:
+    except Exception:
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = stdout
         sys.stderr = stderr
         if show_failing:
-            with open('stdout.txt', 'r') as f:
+            with open('stdout.txt', 'r', encoding='utf-8') as f:
                 print(f.read())
-            with open('stderr.txt', 'r') as f:
+            with open('stderr.txt', 'r', encoding='utf-8') as f:
                 print(f.read())
         raise
     finally:
@@ -116,7 +116,7 @@ class TestInit(unittest.TestCase):
         assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
 
     def test_conflicting_init(self):
-        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+') as f:
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
             f.write('test')
 
         try:
@@ -132,7 +132,7 @@ class TestInit(unittest.TestCase):
         try:
             load_main_index(out_dir=OUTPUT_DIR)
             assert False, 'load_main_index should raise an exception when no index is present'
-        except:
+        except Exception:
             pass
 
     def test_no_dirty_state(self):
@@ -161,7 +161,7 @@ class TestAdd(unittest.TestCase):
 
     def test_add_arg_file(self):
         test_file = Path(OUTPUT_DIR) / 'test.txt'
-        with open(test_file, 'w+') as f:
+        with open(test_file, 'w+', encoding='utf') as f:
             f.write(test_urls)
 
         with output_hidden():
@@ -216,7 +216,7 @@ class TestRemove(unittest.TestCase):
             with output_hidden(show_failing=False):
                 archivebox_remove.main(['--yes', '--delete', 'https://doesntexist.com'])
             assert False, 'Should raise if no URLs match'
-        except:
+        except Exception:
             pass
 
 

archivebox/config.py

@@ -29,10 +29,12 @@ import json
 import getpass
 import platform
 import shutil
+import sqlite3
 import django
 
 from hashlib import md5
 from pathlib import Path
+from datetime import datetime, timezone
 from typing import Optional, Type, Tuple, Dict, Union, List
 from subprocess import run, PIPE, DEVNULL
 from configparser import ConfigParser
@@ -77,6 +79,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'PUBLIC_SNAPSHOTS':         {'type': bool,  'default': True},
         'PUBLIC_ADD_VIEW':          {'type': bool,  'default': False},
         'FOOTER_INFO':              {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
+        'SNAPSHOTS_PER_PAGE':       {'type': int,   'default': 40},
+        'CUSTOM_TEMPLATES_DIR':     {'type': str,   'default': None},
+        'TIME_ZONE':                {'type': str,   'default': 'UTC'},
     },
 
     'ARCHIVE_METHOD_TOGGLES': {
@@ -99,30 +104,35 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
 
     'ARCHIVE_METHOD_OPTIONS': {
         'RESOLUTION':               {'type': str,   'default': '1440,2000', 'aliases': ('SCREENSHOT_RESOLUTION',)},
-        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com'},
+        'GIT_DOMAINS':              {'type': str,   'default': 'github.com,bitbucket.org,gitlab.com,gist.github.com'},
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
+        'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'CURL_USER_AGENT':          {'type': str,   'default': 'ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': 'ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36'},
+        'CURL_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.61 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
 
         'CHROME_HEADLESS':          {'type': bool,  'default': True},
         'CHROME_SANDBOX':           {'type': bool,  'default': lambda c: not c['IN_DOCKER']},
-        'YOUTUBEDL_ARGS':           {'type': list,  'default': ['--write-description',
+        'YOUTUBEDL_ARGS':           {'type': list,  'default': lambda c: [
+                                                                '--write-description',
                                                                 '--write-info-json',
                                                                 '--write-annotations',
                                                                 '--write-thumbnail',
                                                                 '--no-call-home',
+                                                                '--write-sub',
                                                                 '--all-subs',
+                                                                '--write-auto-sub',
+                                                                '--convert-subs=srt',
                                                                 '--yes-playlist',
                                                                 '--continue',
                                                                 '--ignore-errors',
                                                                 '--geo-bypass',
                                                                 '--add-metadata',
-                                                                '--max-filesize=750m',
+                                                                '--max-filesize={}'.format(c['MEDIA_MAX_SIZE']),
                                                                 ]},
                                                                     
 
@@ -152,6 +162,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         # SONIC
         'SONIC_COLLECTION':         {'type': str,   'default': 'archivebox'},
         'SONIC_BUCKET':             {'type': str,   'default': 'snapshots'},
+        'SEARCH_BACKEND_TIMEOUT':   {'type': int,   'default': 90},
     },
 
     'DEPENDENCY_CONFIG': {
@@ -169,9 +180,9 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CURL_BINARY':              {'type': str,   'default': 'curl'},
         'GIT_BINARY':               {'type': str,   'default': 'git'},
         'WGET_BINARY':              {'type': str,   'default': 'wget'},
-        'SINGLEFILE_BINARY':        {'type': str,   'default': 'single-file'},
-        'READABILITY_BINARY':       {'type': str,   'default': 'readability-extractor'},
-        'MERCURY_BINARY':           {'type': str,   'default': 'mercury-parser'},
+        'SINGLEFILE_BINARY':        {'type': str,   'default': lambda c: bin_path('single-file')},
+        'READABILITY_BINARY':       {'type': str,   'default': lambda c: bin_path('readability-extractor')},
+        'MERCURY_BINARY':           {'type': str,   'default': lambda c: bin_path('mercury-parser')},
         'YOUTUBEDL_BINARY':         {'type': str,   'default': 'youtube-dl'},
         'NODE_BINARY':              {'type': str,   'default': 'node'},
         'RIPGREP_BINARY':           {'type': str,   'default': 'rg'},
@@ -263,7 +274,38 @@ STATICFILE_EXTENSIONS = {
     # html, htm, shtml, xhtml, xml, aspx, php, cgi
 }
 
-
+# When initializing archivebox in a new directory, we check to make sure the dir is
+# actually empty so that we dont clobber someone's home directory or desktop by accident.
+# These files are exceptions to the is_empty check when we're trying to init a new dir,
+# as they could be from a previous archivebox version, system artifacts, dependencies, etc.
+ALLOWED_IN_OUTPUT_DIR = {
+    '.gitignore',
+    'lost+found',
+    '.DS_Store',
+    '.venv',
+    'venv',
+    'virtualenv',
+    '.virtualenv',
+    'node_modules',
+    'package.json',
+    'package-lock.json',
+    'yarn.lock',
+    'static',
+    'sonic',
+    ARCHIVE_DIR_NAME,
+    SOURCES_DIR_NAME,
+    LOGS_DIR_NAME,
+    SQL_INDEX_FILENAME,
+    f'{SQL_INDEX_FILENAME}-wal',
+    f'{SQL_INDEX_FILENAME}-shm',
+    JSON_INDEX_FILENAME,
+    HTML_INDEX_FILENAME,
+    ROBOTS_TXT_FILENAME,
+    FAVICON_FILENAME,
+    CONFIG_FILENAME,
+    f'{CONFIG_FILENAME}.bak',
+    'static_index.json',
+}
 
 ############################## Derived Config ##################################
 
@@ -275,6 +317,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
 
     'PACKAGE_DIR':              {'default': lambda c: Path(__file__).resolve().parent},
     'TEMPLATES_DIR':            {'default': lambda c: c['PACKAGE_DIR'] / TEMPLATES_DIR_NAME},
+    'CUSTOM_TEMPLATES_DIR':     {'default': lambda c: c['CUSTOM_TEMPLATES_DIR'] and Path(c['CUSTOM_TEMPLATES_DIR'])},
 
     'OUTPUT_DIR':               {'default': lambda c: Path(c['OUTPUT_DIR']).resolve() if c['OUTPUT_DIR'] else Path(os.curdir).resolve()},
     'ARCHIVE_DIR':              {'default': lambda c: c['OUTPUT_DIR'] / ARCHIVE_DIR_NAME},
@@ -285,9 +328,8 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'CHROME_USER_DATA_DIR':     {'default': lambda c: find_chrome_data_dir() if c['CHROME_USER_DATA_DIR'] is None else (Path(c['CHROME_USER_DATA_DIR']).resolve() if c['CHROME_USER_DATA_DIR'] else None)},   # None means unset, so we autodetect it with find_chrome_Data_dir(), but emptystring '' means user manually set it to '', and we should store it as None
     'URL_BLACKLIST_PTN':        {'default': lambda c: c['URL_BLACKLIST'] and re.compile(c['URL_BLACKLIST'] or '', re.IGNORECASE | re.UNICODE | re.MULTILINE)},
 
-    'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0]},
-    'VERSION':                  {'default': lambda c: json.loads((Path(c['PACKAGE_DIR']) / 'package.json').read_text().strip())['version']},
-    'GIT_SHA':                  {'default': lambda c: c['VERSION'].split('+')[-1] or 'unknown'},
+    'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0] or bin_path('archivebox')},
+    'VERSION':                  {'default': lambda c: json.loads((Path(c['PACKAGE_DIR']) / 'package.json').read_text(encoding='utf-8').strip())['version']},
 
     'PYTHON_BINARY':            {'default': lambda c: sys.executable},
     'PYTHON_ENCODING':          {'default': lambda c: sys.stdout.encoding.upper()},
@@ -331,8 +373,8 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'SAVE_MEDIA':               {'default': lambda c: c['USE_YOUTUBEDL'] and c['SAVE_MEDIA']},
     'YOUTUBEDL_ARGS':           {'default': lambda c: c['YOUTUBEDL_ARGS'] or []},
 
-    'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
-    'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] if c['CHROME_BINARY'] else find_chrome_binary()},
+    'CHROME_BINARY':            {'default': lambda c: c['CHROME_BINARY'] or find_chrome_binary()},
+    'USE_CHROME':               {'default': lambda c: c['USE_CHROME'] and c['CHROME_BINARY'] and (c['SAVE_PDF'] or c['SAVE_SCREENSHOT'] or c['SAVE_DOM'] or c['SAVE_SINGLEFILE'])},
     'CHROME_VERSION':           {'default': lambda c: bin_version(c['CHROME_BINARY']) if c['USE_CHROME'] else None},
     
     'SAVE_PDF':                 {'default': lambda c: c['USE_CHROME'] and c['SAVE_PDF']},
@@ -459,7 +501,7 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
     config_file.optionxform = str
     config_file.read(config_path)
 
-    with open(config_path, 'r') as old:
+    with open(config_path, 'r', encoding='utf-8') as old:
         atomic_write(f'{config_path}.bak', old.read())
 
     find_section = lambda key: [name for name, opts in CONFIG_SCHEMA.items() if key in opts][0]
@@ -480,32 +522,33 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
 
     if (not existing_secret_key) or ('not a valid secret' in existing_secret_key):
         from django.utils.crypto import get_random_string
-        chars = 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.'
+        chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
         random_secret_key = get_random_string(50, chars)
         if 'SERVER_CONFIG' in config_file:
             config_file['SERVER_CONFIG']['SECRET_KEY'] = random_secret_key
         else:
             config_file['SERVER_CONFIG'] = {'SECRET_KEY': random_secret_key}
 
-    with open(config_path, 'w+') as new:
+    with open(config_path, 'w+', encoding='utf-8') as new:
         config_file.write(new)
     
     try:
         # validate the config by attempting to re-parse it
         CONFIG = load_all_config()
-        return {
-            key.upper(): CONFIG.get(key.upper())
-            for key in config.keys()
-        }
-    except:
+    except BaseException:                                                       # lgtm [py/catch-base-exception]
         # something went horribly wrong, rever to the previous version
-        with open(f'{config_path}.bak', 'r') as old:
+        with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
             atomic_write(config_path, old.read())
 
+        raise
+
     if Path(f'{config_path}.bak').exists():
         os.remove(f'{config_path}.bak')
-
-    return {}
+    
+    return {
+        key.upper(): CONFIG.get(key.upper())
+        for key in config.keys()
+    }
 
    
 
@@ -542,7 +585,7 @@ def load_config(defaults: ConfigDefaultDict,
             stderr('    For config documentation and examples see:')
             stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration')
             stderr()
-            raise
+            # raise
             raise SystemExit(2)
     
     return extended_config
@@ -703,6 +746,11 @@ def get_code_locations(config: ConfigDict) -> SimpleConfigValueDict:
             'enabled': True,
             'is_valid': (config['TEMPLATES_DIR'] / 'static').exists(),
         },
+        'CUSTOM_TEMPLATES_DIR': {
+            'path': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).resolve(),
+            'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
+            'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
+        },
         # 'NODE_MODULES_DIR': {
         #     'path': ,
         #     'enabled': ,
@@ -980,9 +1028,9 @@ def check_dependencies(config: ConfigDict=CONFIG, show_help: bool=True) -> None:
                     info['version'] or 'unable to detect version',
                 )
             )
-            if dependency in ('SINGLEFILE_BINARY', 'READABILITY_BINARY', 'MERCURY_BINARY'):
-                hint(('npm install --prefix . "git+https://github.com/ArchiveBox/ArchiveBox.git"',
-                    f'or archivebox config --set SAVE_{dependency.rsplit("_", 1)[0]}=False to silence this warning',
+            if dependency in ('YOUTUBEDL_BINARY', 'CHROME_BINARY', 'SINGLEFILE_BINARY', 'READABILITY_BINARY', 'MERCURY_BINARY'):
+                hint(('To install all packages automatically run: archivebox setup',
+                    f'or to disable it and silence this warning: archivebox config --set SAVE_{dependency.rsplit("_", 1)[0]}=False',
                     ''), prefix='      ')
         stderr('')
 
@@ -1017,8 +1065,8 @@ def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=C
     output_dir = out_dir or config['OUTPUT_DIR']
     assert isinstance(output_dir, (str, Path))
 
-    sql_index_exists = (Path(output_dir) / SQL_INDEX_FILENAME).exists()
-    if not sql_index_exists:
+    archive_dir_exists = (Path(output_dir) / ARCHIVE_DIR_NAME).exists()
+    if not archive_dir_exists:
         stderr('[X] No archivebox index found in the current directory.', color='red')
         stderr(f'    {output_dir}', color='lightyellow')
         stderr()
@@ -1030,26 +1078,22 @@ def check_data_folder(out_dir: Union[str, Path, None]=None, config: ConfigDict=C
         stderr('        archivebox init')
         raise SystemExit(2)
 
+def check_migrations(out_dir: Union[str, Path, None]=None, config: ConfigDict=CONFIG):
+    output_dir = out_dir or config['OUTPUT_DIR']
     from .index.sql import list_migrations
 
     pending_migrations = [name for status, name in list_migrations() if not status]
 
-    if (not sql_index_exists) or pending_migrations:
-        if sql_index_exists:
-            pending_operation = f'apply the {len(pending_migrations)} pending migrations'
-        else:
-            pending_operation = 'generate the new SQL main index'
-
+    if pending_migrations:
         stderr('[X] This collection was created with an older version of ArchiveBox and must be upgraded first.', color='lightyellow')
         stderr(f'    {output_dir}')
         stderr()
-        stderr(f'    To upgrade it to the latest version and {pending_operation} run:')
+        stderr(f'    To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:')
         stderr('        archivebox init')
         raise SystemExit(3)
 
-    sources_dir = Path(output_dir) / SOURCES_DIR_NAME
-    if not sources_dir.exists():
-        sources_dir.mkdir()
+    (Path(output_dir) / SOURCES_DIR_NAME).mkdir(exist_ok=True)
+    (Path(output_dir) / LOGS_DIR_NAME).mkdir(exist_ok=True)
 
 
 
@@ -1061,24 +1105,72 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
     assert isinstance(output_dir, Path) and isinstance(config['PACKAGE_DIR'], Path)
 
     try:
-        import django
+        from django.core.management import call_command
+
         sys.path.append(str(config['PACKAGE_DIR']))
         os.environ.setdefault('OUTPUT_DIR', str(output_dir))
         assert (config['PACKAGE_DIR'] / 'core' / 'settings.py').exists(), 'settings.py was not found at archivebox/core/settings.py'
         os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
 
+        # Check to make sure JSON extension is available in our Sqlite3 instance
+        try:
+            cursor = sqlite3.connect(':memory:').cursor()
+            cursor.execute('SELECT JSON(\'{"a": "b"}\')')
+        except sqlite3.OperationalError as exc:
+            stderr(f'[X] Your SQLite3 version is missing the required JSON1 extension: {exc}', color='red')
+            hint([
+                'Upgrade your Python version or install the extension manually:',
+                'https://code.djangoproject.com/wiki/JSON1Extension'
+            ])
+
         if in_memory_db:
-            # Put the db in memory and run migrations in case any command requires it
-            from django.core.management import call_command
+            # some commands (e.g. oneshot) dont store a long-lived sqlite3 db file on disk.
+            # in those cases we create a temporary in-memory db and run the migrations
+            # immediately to get a usable in-memory-database at startup
             os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
             django.setup()
             call_command("migrate", interactive=False, verbosity=0)
         else:
+            # Otherwise use default sqlite3 file-based database and initialize django
+            # without running migrations automatically (user runs them manually by calling init)
             django.setup()
+            
+
+        from django.conf import settings
+
+        # log startup message to the error log
+        with open(settings.ERROR_LOG, "a+", encoding='utf-8') as f:
+            command = ' '.join(sys.argv)
+            ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
+            f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
+
 
         if check_db:
+            # Enable WAL mode in sqlite3
+            from django.db import connection
+            with connection.cursor() as cursor:
+                current_mode = cursor.execute("PRAGMA journal_mode")
+                if current_mode != 'wal':
+                    cursor.execute("PRAGMA journal_mode=wal;")
+
+            # Create cache table in DB if needed
+            try:
+                from django.core.cache import cache
+                cache.get('test', None)
+            except django.db.utils.OperationalError:
+                call_command("createcachetable", verbosity=0)
+
+
+            # if archivebox gets imported multiple times, we have to close
+            # the sqlite3 whenever we init from scratch to avoid multiple threads
+            # sharing the same connection by accident
+            from django.db import connections
+            for conn in connections.all():
+                conn.close_if_unusable_or_obsolete()
+
             sql_index_path = Path(output_dir) / SQL_INDEX_FILENAME
             assert sql_index_path.exists(), (
-                f'No database file {SQL_INDEX_FILENAME} found in OUTPUT_DIR: {config["OUTPUT_DIR"]}')
+                f'No database file {SQL_INDEX_FILENAME} found in: {config["OUTPUT_DIR"]} (Are you in an ArchiveBox collection directory?)')
+
     except KeyboardInterrupt:
         raise SystemExit(2)

archivebox/core/admin.py

@@ -1,7 +1,9 @@
 __package__ = 'archivebox.core'
 
 from io import StringIO
+from pathlib import Path
 from contextlib import redirect_stdout
+from datetime import datetime, timezone
 
 from django.contrib import admin
 from django.urls import path
@@ -13,15 +15,15 @@ from django import forms
 
 from ..util import htmldecode, urldecode, ansi_to_html
 
-from core.models import Snapshot, Tag
-from core.forms import AddLinkForm, TagField
+from core.models import Snapshot, ArchiveResult, Tag
+from core.forms import AddLinkForm
 
 from core.mixins import SearchResultsAdminMixin
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
-from config import OUTPUT_DIR
+from config import OUTPUT_DIR, SNAPSHOTS_PER_PAGE
 from extractors import archive_links
 
 # Admin URLs
@@ -36,95 +38,126 @@ from extractors import archive_links
 
 # TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
 
-def update_snapshots(modeladmin, request, queryset):
-    archive_links([
-        snapshot.as_link()
-        for snapshot in queryset
-    ], out_dir=OUTPUT_DIR)
-update_snapshots.short_description = "Archive"
 
-def update_titles(modeladmin, request, queryset):
-    archive_links([
-        snapshot.as_link()
-        for snapshot in queryset
-    ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
-update_titles.short_description = "Pull title"
+class ArchiveResultInline(admin.TabularInline):
+    model = ArchiveResult
 
-def overwrite_snapshots(modeladmin, request, queryset):
-    archive_links([
-        snapshot.as_link()
-        for snapshot in queryset
-    ], overwrite=True, out_dir=OUTPUT_DIR)
-overwrite_snapshots.short_description = "Re-archive (overwrite)"
+class TagInline(admin.TabularInline):
+    model = Snapshot.tags.through
 
-def verify_snapshots(modeladmin, request, queryset):
-    for snapshot in queryset:
-        print(snapshot.timestamp, snapshot.url, snapshot.is_archived, snapshot.archive_size, len(snapshot.history))
+from django.contrib.admin.helpers import ActionForm
+from django.contrib.admin.widgets import AutocompleteSelectMultiple
 
-verify_snapshots.short_description = "Check"
+class AutocompleteTags:
+    model = Tag
+    search_fields = ['name']
 
-def delete_snapshots(modeladmin, request, queryset):
-    remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
-
-delete_snapshots.short_description = "Delete"
+class AutocompleteTagsAdminStub:
+    name = 'admin'
 
 
-class SnapshotAdminForm(forms.ModelForm):
-    tags = TagField(required=False)
+class SnapshotActionForm(ActionForm):
+    tags = forms.ModelMultipleChoiceField(
+        queryset=Tag.objects.all(),
+        required=False,
+        widget=AutocompleteSelectMultiple(
+            AutocompleteTags(),
+            AutocompleteTagsAdminStub(),
+        ),
+    )
 
-    class Meta:
-        model = Snapshot
-        fields = "__all__"
-
-    def save(self, commit=True):
-        # Based on: https://stackoverflow.com/a/49933068/3509554
-
-        # Get the unsave instance
-        instance = forms.ModelForm.save(self, False)
-        tags = self.cleaned_data.pop("tags")
-
-        #update save_m2m
-        def new_save_m2m():
-            instance.save_tags(tags)
-
-        # Do we need to save all changes now?
-        self.save_m2m = new_save_m2m
-        if commit:
-            instance.save()
-
-        return instance
+    # TODO: allow selecting actions for specific extractors? is this useful?
+    # EXTRACTOR_CHOICES = [
+    #     (name, name.title())
+    #     for name, _, _ in get_default_archive_methods()
+    # ]
+    # extractor = forms.ChoiceField(
+    #     choices=EXTRACTOR_CHOICES,
+    #     required=False,
+    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
+    # )
 
 
 class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
-    list_display = ('added', 'title_str', 'url_str', 'files', 'size')
-    sort_fields = ('title_str', 'url_str', 'added')
-    readonly_fields = ('id', 'url', 'timestamp', 'num_outputs', 'is_archived', 'url_hash', 'added', 'updated')
-    search_fields = ['url__icontains', 'timestamp', 'title', 'tags__name']
-    fields = (*readonly_fields, 'title', 'tags')
-    list_filter = ('added', 'updated', 'tags')
+    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'added', 'files')
+    readonly_fields = ('info', 'bookmarked', 'added', 'updated')
+    search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
+    fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
+    list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
     ordering = ['-added']
-    actions = [delete_snapshots, overwrite_snapshots, update_snapshots, update_titles, verify_snapshots]
-    actions_template = 'admin/actions_as_select.html'
-    form = SnapshotAdminForm
-    list_per_page = 40
+    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
+    autocomplete_fields = ['tags']
+    inlines = [ArchiveResultInline]
+    list_per_page = SNAPSHOTS_PER_PAGE
+
+    action_form = SnapshotActionForm
 
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
-            path('grid/', self.admin_site.admin_view(self.grid_view),name='grid')
+            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
         ]
         return custom_urls + urls
 
     def get_queryset(self, request):
+        self.request = request
         return super().get_queryset(request).prefetch_related('tags')
 
     def tag_list(self, obj):
         return ', '.join(obj.tags.values_list('name', flat=True))
 
-    def id_str(self, obj):
+    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
+    # def action(self, obj):
+    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
+    #     # action: update_snapshots
+    #     # select_across: 0
+    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
+    #     return format_html(
+    #         '''
+    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
+    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
+    #                 <input type="hidden" name="_selected_action" value="{}">
+    #                 <button name="update_snapshots">Check</button>
+    #                 <button name="update_titles">Pull title + favicon</button>
+    #                 <button name="update_snapshots">Update</button>
+    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
+    #                 <button name="delete_snapshots">Permanently delete</button>
+    #             </form>
+    #         ''',
+    #         csrf.get_token(self.request),
+    #         obj.id,
+    #     )
+
+    def info(self, obj):
         return format_html(
-            '<code style="font-size: 10px">{}</code>',
-            obj.url_hash[:8],
+            '''
+            UUID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
+            Timestamp: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
+            URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            Archived: {} ({} files {}) &nbsp; &nbsp;
+            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
+            Status code: {} &nbsp; &nbsp;
+            Server: {} &nbsp; &nbsp;
+            Content type: {} &nbsp; &nbsp;
+            Extension: {} &nbsp; &nbsp;
+            <br/><br/>
+            <a href="/archive/{}">View Snapshot index ➡️</a> &nbsp; &nbsp;
+            <a href="/admin/core/snapshot/?id__exact={}">View actions ⚙️</a>
+            ''',
+            obj.id,
+            obj.timestamp,
+            obj.url_hash,
+            '✅' if obj.is_archived else '❌',
+            obj.num_outputs,
+            self.size(obj),
+            f'/archive/{obj.timestamp}/favicon.ico',
+            obj.status_code or '?',
+            obj.headers and obj.headers.get('Server') or '?',
+            obj.headers and obj.headers.get('Content-Type') or '?',
+            obj.extension or '?',
+            obj.timestamp,
+            obj.id,
         )
 
     def title_str(self, obj):
@@ -151,8 +184,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     def files(self, obj):
         return snapshot_icons(obj)
 
+    files.admin_order_field = 'updated'
+    files.short_description = 'Files Saved'
+
     def size(self, obj):
-        archive_size = obj.archive_size
+        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
         if archive_size:
             size_txt = printable_filesize(archive_size)
             if archive_size > 52428800:
@@ -165,14 +201,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             size_txt,
         )
 
+    size.admin_order_field = 'archiveresult__count'
+
     def url_str(self, obj):
         return format_html(
-            '<a href="{}"><code>{}</code></a>',
+            '<a href="{}"><code style="user-select: all;">{}</code></a>',
+            obj.url,
             obj.url,
-            obj.url.split('://www.', 1)[-1].split('://', 1)[-1][:64],
         )
 
-    def grid_view(self, request):
+    def grid_view(self, request, extra_context=None):
 
         # cl = self.get_changelist_instance(request)
 
@@ -183,35 +221,153 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
         # Monkey patch here plus core_tags.py
         self.change_list_template = 'private_index_grid.html'
-        self.list_per_page = 20
+        self.list_per_page = SNAPSHOTS_PER_PAGE
         self.list_max_show_all = self.list_per_page
 
         # Call monkey patched view
-        rendered_response = self.changelist_view(request)
+        rendered_response = self.changelist_view(request, extra_context=extra_context)
 
         # Restore values
-        self.change_list_template =  saved_change_list_template
+        self.change_list_template = saved_change_list_template
         self.list_per_page = saved_list_per_page
         self.list_max_show_all = saved_list_max_show_all
 
         return rendered_response
+
+    # for debugging, uncomment this to print all requests:
+    # def changelist_view(self, request, extra_context=None):
+    #     print('[*] Got request', request.method, request.POST)
+    #     return super().changelist_view(request, extra_context=None)
+
+    def update_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], out_dir=OUTPUT_DIR)
+    update_snapshots.short_description = "Pull"
+
+    def update_titles(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, methods=('title','favicon'), out_dir=OUTPUT_DIR)
+    update_titles.short_description = "⬇️ Title"
+
+    def resnapshot_snapshot(self, request, queryset):
+        for snapshot in queryset:
+            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
+            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
+            add(new_url, tag=snapshot.tags_str())
+    resnapshot_snapshot.short_description = "Re-Snapshot"
+
+    def overwrite_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, out_dir=OUTPUT_DIR)
+    overwrite_snapshots.short_description = "Reset"
+
+    def delete_snapshots(self, request, queryset):
+        remove(snapshots=queryset, yes=True, delete=True, out_dir=OUTPUT_DIR)
+
+    delete_snapshots.short_description = "Delete"
+
+    def add_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[+] Adding tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.add(*tags)
+
+    add_tags.short_description = "+"
+
+    def remove_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[-] Removing tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.remove(*tags)
+
+    remove_tags.short_description = "–"
+
         
 
-    id_str.short_description = 'ID'
     title_str.short_description = 'Title'
     url_str.short_description = 'Original URL'
 
-    id_str.admin_order_field = 'id'
     title_str.admin_order_field = 'title'
     url_str.admin_order_field = 'url'
 
+
+
 class TagAdmin(admin.ModelAdmin):
-    list_display = ('slug', 'name', 'id')
+    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
     sort_fields = ('id', 'name', 'slug')
-    readonly_fields = ('id',)
+    readonly_fields = ('id', 'num_snapshots', 'snapshots')
     search_fields = ('id', 'name', 'slug')
     fields = (*readonly_fields, 'name', 'slug')
+    actions = ['delete_selected']
+    ordering = ['-id']
 
+    def num_snapshots(self, obj):
+        return format_html(
+            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
+            obj.id,
+            obj.snapshot_set.count(),
+        )
+
+    def snapshots(self, obj):
+        total_count = obj.snapshot_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
+                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.id,
+                snap.timestamp,
+                snap.url,
+            )
+            for snap in obj.snapshot_set.order_by('-updated')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={obj.id}">and {total_count-10} more...<a>' if obj.snapshot_set.count() > 10 else ''))
+
+
+class ArchiveResultAdmin(admin.ModelAdmin):
+    list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
+    sort_fields = ('start_ts', 'extractor', 'status')
+    readonly_fields = ('id', 'uuid', 'snapshot_str', 'tags_str')
+    search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
+    autocomplete_fields = ['snapshot']
+
+    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
+    ordering = ['-start_ts']
+    list_per_page = SNAPSHOTS_PER_PAGE
+
+    def snapshot_str(self, obj):
+        return format_html(
+            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
+            '<small>{}</small>',
+            obj.snapshot.timestamp,
+            obj.snapshot.timestamp,
+            obj.snapshot.url[:128],
+        )
+
+    def tags_str(self, obj):
+        return obj.snapshot.tags_str()
+
+    def cmd_str(self, obj):
+        return format_html(
+            '<pre>{}</pre>',
+            ' '.join(obj.cmd) if isinstance(obj.cmd, list) else str(obj.cmd),
+        )
+
+    def output_str(self, obj):
+        return format_html(
+            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
+            obj.snapshot.timestamp,
+            obj.output if (obj.status == 'succeeded') and obj.extractor not in ('title', 'archive_org') else 'index.html',
+            obj.output,
+        )
+
+    tags_str.short_description = 'tags'
+    snapshot_str.short_description = 'snapshot'
 
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = 'ArchiveBox'
@@ -266,4 +422,5 @@ admin.site = ArchiveBoxAdmin()
 admin.site.register(get_user_model())
 admin.site.register(Snapshot, SnapshotAdmin)
 admin.site.register(Tag, TagAdmin)
+admin.site.register(ArchiveResult, ArchiveResultAdmin)
 admin.site.disable_action('delete_selected')

archivebox/core/apps.py

@@ -3,3 +3,4 @@ from django.apps import AppConfig
 
 class CoreConfig(AppConfig):
     name = 'core'
+    default_auto_field = 'django.db.models.UUIDField'

archivebox/core/forms.py

@@ -3,9 +3,14 @@ __package__ = 'archivebox.core'
 from django import forms
 
 from ..util import URL_REGEX
+from ..parsers import PARSERS
 from ..vendor.taggit_utils import edit_string_for_tags, parse_tags
 
-CHOICES = (
+PARSER_CHOICES = [
+    (parser_key, parser[0])
+    for parser_key, parser in PARSERS.items()
+]
+DEPTH_CHOICES = (
     ('0', 'depth = 0 (archive just these URLs)'),
     ('1', 'depth = 1 (archive these URLs and all URLs one hop away)'),
 )
@@ -20,7 +25,9 @@ ARCHIVE_METHODS = [
 
 class AddLinkForm(forms.Form):
     url = forms.RegexField(label="URLs (one per line)", regex=URL_REGEX, min_length='6', strip=True, widget=forms.Textarea, required=True)
-    depth = forms.ChoiceField(label="Archive depth", choices=CHOICES, widget=forms.RadioSelect, initial='0')
+    parser = forms.ChoiceField(label="URLs format", choices=[('auto', 'Auto-detect parser'), *PARSER_CHOICES], initial='auto')
+    tag = forms.CharField(label="Tags (comma separated tag1,tag2,tag3)", strip=True, required=False)
+    depth = forms.ChoiceField(label="Archive depth", choices=DEPTH_CHOICES, initial='0', widget=forms.RadioSelect(attrs={"class": "depth-selection"}))
     archive_methods = forms.MultipleChoiceField(
         label="Archive methods (select at least 1, otherwise all will be used by default)",
         required=False,

archivebox/core/middleware.py

@@ -0,0 +1,37 @@
+__package__ = 'archivebox.core'
+
+from django.utils import timezone
+
+from ..config import PUBLIC_SNAPSHOTS
+
+
+def detect_timezone(request, activate: bool=True):
+    gmt_offset = (request.COOKIES.get('GMT_OFFSET') or '').strip()
+    tz = None
+    if gmt_offset.replace('-', '').isdigit():
+        tz = timezone.get_fixed_timezone(int(gmt_offset))
+        if activate:
+            timezone.activate(tz)
+    # print('GMT_OFFSET', gmt_offset, tz)
+    return tz
+
+
+def TimezoneMiddleware(get_response):
+    def middleware(request):
+        detect_timezone(request, activate=True)
+        return get_response(request)
+
+    return middleware
+
+
+def CacheControlMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        if '/archive/' in request.path or '/static/' in request.path:
+            policy = 'public' if PUBLIC_SNAPSHOTS else 'private'
+            response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
+            # print('Set Cache-Control header to', response['Cache-Control'])
+        return response
+
+    return middleware

archivebox/core/migrations/0006_auto_20201012_1520.py

@@ -17,7 +17,7 @@ def forwards_func(apps, schema_editor):
         tag_set.discard("")
 
         for tag in tag_set:
-            to_add, _ = TagModel.objects.get_or_create(name=tag, slug=slugify(tag))
+            to_add, _ = TagModel.objects.get_or_create(name=tag, defaults={'slug': slugify(tag)})
             snapshot.tags.add(to_add)
 
 

archivebox/core/migrations/0007_archiveresult.py

@@ -36,8 +36,25 @@ def forwards_func(apps, schema_editor):
 
         for extractor in history:
             for result in history[extractor]:
-                ArchiveResult.objects.create(extractor=extractor, snapshot=snapshot, cmd=result["cmd"], cmd_version=result["cmd_version"] or 'unknown', 
-                start_ts=result["start_ts"], end_ts=result["end_ts"], status=result["status"], pwd=result["pwd"], output=result["output"])
+                try:
+                    ArchiveResult.objects.create(
+                        extractor=extractor,
+                        snapshot=snapshot,
+                        pwd=result["pwd"],
+                        cmd=result.get("cmd") or [],
+                        cmd_version=result.get("cmd_version") or 'unknown',
+                        start_ts=result["start_ts"],
+                        end_ts=result["end_ts"],
+                        status=result["status"],
+                        output=result.get("output") or 'null',
+                    )
+                except Exception as e:
+                    print(
+                        '    ! Skipping import due to missing/invalid index.json:',
+                        out_dir,
+                        e,
+                        '(open an issue with this index.json for help)',
+                    )
 
 
 def verify_json_index_integrity(snapshot):

archivebox/core/migrations/0009_auto_20210216_1038.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-16 10:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0008_auto_20210105_1421'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='updated',
+            field=models.DateTimeField(auto_now=True, db_index=True, null=True),
+        ),
+    ]

archivebox/core/migrations/0010_auto_20210216_1055.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-16 10:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_auto_20210216_1038'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='start_ts',
+            field=models.DateTimeField(db_index=True),
+        ),
+    ]

archivebox/core/migrations/0011_auto_20210216_1331.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.1.3 on 2021-02-16 13:31
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0010_auto_20210216_1055'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='extractor',
+            field=models.CharField(choices=[('title', 'title'), ('favicon', 'favicon'), ('headers', 'headers'), ('singlefile', 'singlefile'), ('pdf', 'pdf'), ('screenshot', 'screenshot'), ('dom', 'dom'), ('wget', 'wget'), ('readability', 'readability'), ('mercury', 'mercury'), ('git', 'git'), ('media', 'media'), ('archive_org', 'archive_org')], max_length=32),
+        ),
+    ]

archivebox/core/migrations/0012_auto_20210216_1425.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2021-02-16 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0011_auto_20210216_1331'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='cmd_version',
+            field=models.CharField(blank=True, default=None, max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='output',
+            field=models.CharField(max_length=1024),
+        ),
+    ]

archivebox/core/migrations/0013_auto_20210218_0729.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 07:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0012_auto_20210216_1425'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=256, null=True),
+        ),
+    ]

archivebox/core/migrations/0014_auto_20210218_0729.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 07:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_auto_20210218_0729'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=1024, null=True),
+        ),
+    ]

archivebox/core/migrations/0015_auto_20210218_0730.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 07:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0014_auto_20210218_0729'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=512, null=True),
+        ),
+    ]

archivebox/core/migrations/0016_auto_20210218_1204.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-18 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0015_auto_20210218_0730'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='tags',
+            field=models.ManyToManyField(blank=True, to='core.Tag'),
+        ),
+    ]

archivebox/core/migrations/0017_auto_20210219_0211.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-02-19 02:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0016_auto_20210218_1204'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(blank=True, max_length=100, unique=True, verbose_name='slug'),
+        ),
+    ]

archivebox/core/migrations/0018_auto_20210327_0952.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2021-03-27 09:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0017_auto_20210219_0211'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=100, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(blank=True, max_length=100, unique=True),
+        ),
+    ]

archivebox/core/migrations/0019_auto_20210401_0654.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.3 on 2021-04-01 06:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0018_auto_20210327_0952'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='url',
+            field=models.URLField(db_index=True, unique=True),
+        ),
+    ]

archivebox/core/migrations/0020_auto_20210410_1031.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.8 on 2021-04-10 10:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0019_auto_20210401_0654'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.AutoField(primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

archivebox/core/mixins.py

@@ -2,22 +2,20 @@ from django.contrib import messages
 
 from archivebox.search import query_search_index
 
-class SearchResultsAdminMixin(object):
-    def get_search_results(self, request, queryset, search_term):
-        ''' Enhances the search queryset with results from the search backend.
-        '''
-        qs, use_distinct = \
-            super(SearchResultsAdminMixin, self).get_search_results(
-                request, queryset, search_term)
+class SearchResultsAdminMixin:
+    def get_search_results(self, request, queryset, search_term: str):
+        """Enhances the search queryset with results from the search backend"""
+        
+        qs, use_distinct = super().get_search_results(request, queryset, search_term)
 
         search_term = search_term.strip()
         if not search_term:
             return qs, use_distinct
         try:
             qsearch = query_search_index(search_term)
+            qs = qs | qsearch
         except Exception as err:
+            print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
             messages.add_message(request, messages.WARNING, f'Error from the search backend, only showing results from default admin search fields - Error: {err}')
-        else:
-            qs = queryset & qsearch
-        finally:
-            return qs, use_distinct
+        
+        return qs, use_distinct

archivebox/core/models.py

@@ -1,14 +1,25 @@
 __package__ = 'archivebox.core'
 
-import uuid
 
-from django.db import models, transaction
+import uuid
+import json
+
+from pathlib import Path
+from typing import Optional, List
+
+from django.db import models
 from django.utils.functional import cached_property
 from django.utils.text import slugify
+from django.core.cache import cache
+from django.urls import reverse
 from django.db.models import Case, When, Value, IntegerField
+from django.contrib.auth.models import User   # noqa
 
-from ..util import parse_date
+from ..config import ARCHIVE_DIR, ARCHIVE_DIR_NAME
+from ..system import get_dir_size
+from ..util import parse_date, base_url, hashurl
 from ..index.schema import Link
+from ..index.html import snapshot_icons
 from ..extractors import get_default_archive_methods, ARCHIVE_METHODS_INDEXING_PRECEDENCE
 
 EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
@@ -29,8 +40,13 @@ class Tag(models.Model):
     """
     Based on django-taggit model
     """
-    name = models.CharField(verbose_name="name", unique=True, blank=False, max_length=100)
-    slug = models.SlugField(verbose_name="slug", unique=True, max_length=100)
+    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
+
+    name = models.CharField(unique=True, blank=False, max_length=100)
+
+    # slug is autoset on save from name, never set it manually
+    slug = models.SlugField(unique=True, blank=True, max_length=100)
+
 
     class Meta:
         verbose_name = "Tag"
@@ -49,20 +65,21 @@ class Tag(models.Model):
         if self._state.adding and not self.slug:
             self.slug = self.slugify(self.name)
 
-            with transaction.atomic():
-                slugs = set(
-                    type(self)
-                    ._default_manager.filter(slug__startswith=self.slug)
-                    .values_list("slug", flat=True)
-                )
+            # if name is different but slug conficts with another tags slug, append a counter
+            # with transaction.atomic():
+            slugs = set(
+                type(self)
+                ._default_manager.filter(slug__startswith=self.slug)
+                .values_list("slug", flat=True)
+            )
 
-                i = None
-                while True:
-                    slug = self.slugify(self.name, i)
-                    if slug not in slugs:
-                        self.slug = slug
-                        return super().save(*args, **kwargs)
-                    i = 1 if i is None else i+1
+            i = None
+            while True:
+                slug = self.slugify(self.name, i)
+                if slug not in slugs:
+                    self.slug = slug
+                    return super().save(*args, **kwargs)
+                i = 1 if i is None else i+1
         else:
             return super().save(*args, **kwargs)
 
@@ -70,14 +87,14 @@ class Tag(models.Model):
 class Snapshot(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
 
-    url = models.URLField(unique=True)
+    url = models.URLField(unique=True, db_index=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True)
 
-    title = models.CharField(max_length=128, null=True, blank=True, db_index=True)
+    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
 
     added = models.DateTimeField(auto_now_add=True, db_index=True)
-    updated = models.DateTimeField(null=True, blank=True, db_index=True)
-    tags = models.ManyToManyField(Tag)
+    updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
+    tags = models.ManyToManyField(Tag, blank=True)
 
     keys = ('url', 'timestamp', 'title', 'tags', 'updated')
 
@@ -109,13 +126,32 @@ class Snapshot(models.Model):
         from ..index import load_link_details
         return load_link_details(self.as_link())
 
-    def tags_str(self) -> str:
-        return ','.join(self.tags.order_by('name').values_list('name', flat=True))
+    def tags_str(self, nocache=True) -> str:
+        cache_key = f'{self.id}-{(self.updated or self.added).timestamp()}-tags'
+        calc_tags_str = lambda: ','.join(self.tags.order_by('name').values_list('name', flat=True))
+        if nocache:
+            tags_str = calc_tags_str()
+            cache.set(cache_key, tags_str)
+            return tags_str
+        return cache.get_or_set(cache_key, calc_tags_str)
+
+    def icons(self) -> str:
+        return snapshot_icons(self)
+
+    @cached_property
+    def extension(self) -> str:
+        from ..util import extension
+        return extension(self.url)
 
     @cached_property
     def bookmarked(self):
         return parse_date(self.timestamp)
 
+    @cached_property
+    def bookmarked_date(self):
+        # TODO: remove this
+        return self.bookmarked
+
     @cached_property
     def is_archived(self):
         return self.as_link().is_archived
@@ -126,42 +162,95 @@ class Snapshot(models.Model):
 
     @cached_property
     def url_hash(self):
-        return self.as_link().url_hash
+        return hashurl(self.url)
 
     @cached_property
     def base_url(self):
-        return self.as_link().base_url
+        return base_url(self.url)
 
     @cached_property
     def link_dir(self):
-        return self.as_link().link_dir
+        return str(ARCHIVE_DIR / self.timestamp)
 
     @cached_property
     def archive_path(self):
-        return self.as_link().archive_path
+        return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp)
 
     @cached_property
     def archive_size(self):
-        return self.as_link().archive_size
+        cache_key = f'{str(self.id)[:12]}-{(self.updated or self.added).timestamp()}-size'
+
+        def calc_dir_size():
+            try:
+                return get_dir_size(self.link_dir)[0]
+            except Exception:
+                return 0
+
+        return cache.get_or_set(cache_key, calc_dir_size)
 
     @cached_property
-    def history(self):
+    def thumbnail_url(self) -> Optional[str]:
+        result = self.archiveresult_set.filter(
+            extractor='screenshot',
+            status='succeeded'
+        ).only('output').last()
+        if result:
+            return reverse('Snapshot', args=[f'{str(self.timestamp)}/{result.output}'])
+        return None
+
+    @cached_property
+    def headers(self) -> Optional[dict]:
+        try:
+            return json.loads((Path(self.link_dir) / 'headers.json').read_text(encoding='utf-8').strip())
+        except Exception:
+            pass
+        return None
+
+    @cached_property
+    def status_code(self) -> Optional[str]:
+        return self.headers and self.headers.get('Status-Code')
+
+    @cached_property
+    def history(self) -> dict:
         # TODO: use ArchiveResult for this instead of json
         return self.as_link_with_details().history
 
     @cached_property
-    def latest_title(self):
-        if ('title' in self.history
-            and self.history['title']
-            and (self.history['title'][-1].status == 'succeeded')
-            and self.history['title'][-1].output.strip()):
-            return self.history['title'][-1].output.strip()
+    def latest_title(self) -> Optional[str]:
+        if self.title:
+            return self.title   # whoopdedoo that was easy
+        
+        try:
+            # take longest successful title from ArchiveResult db history
+            return sorted(
+                self.archiveresult_set\
+                    .filter(extractor='title', status='succeeded', output__isnull=False)\
+                    .values_list('output', flat=True),
+                key=lambda r: len(r),
+            )[-1]
+        except IndexError:
+            pass
+
+        try:
+            # take longest successful title from Link json index file history
+            return sorted(
+                (
+                    result.output.strip()
+                    for result in self.history['title']
+                    if result.status == 'succeeded' and result.output.strip()
+                ),
+                key=lambda r: len(r),
+            )[-1]
+        except (KeyError, IndexError):
+            pass
+
         return None
 
-    def save_tags(self, tags=()):
+    def save_tags(self, tags: List[str]=()) -> None:
         tags_id = []
         for tag in tags:
-            tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
+            if tag.strip():
+                tags_id.append(Tag.objects.get_or_create(name=tag)[0].id)
         self.tags.clear()
         self.tags.add(*tags_id)
 
@@ -178,15 +267,18 @@ class ArchiveResultManager(models.Manager):
 
 
 class ArchiveResult(models.Model):
+    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
+    uuid = models.UUIDField(default=uuid.uuid4, editable=False)
+
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
+    extractor = models.CharField(choices=EXTRACTORS, max_length=32)
     cmd = JSONField()
     pwd = models.CharField(max_length=256)
-    cmd_version = models.CharField(max_length=32, default=None, null=True, blank=True)
-    output = models.CharField(max_length=512)
-    start_ts = models.DateTimeField()
+    cmd_version = models.CharField(max_length=128, default=None, null=True, blank=True)
+    output = models.CharField(max_length=1024)
+    start_ts = models.DateTimeField(db_index=True)
     end_ts = models.DateTimeField()
     status = models.CharField(max_length=16, choices=STATUS_CHOICES)
-    extractor = models.CharField(choices=EXTRACTORS, max_length=32)
 
     objects = ArchiveResultManager()
 

archivebox/core/settings.py

@@ -2,21 +2,26 @@ __package__ = 'archivebox.core'
 
 import os
 import sys
+import re
+import logging
+import tempfile
 
 from pathlib import Path
 from django.utils.crypto import get_random_string
 
-from ..config import (                                                          # noqa: F401
+from ..config import (
     DEBUG,
     SECRET_KEY,
     ALLOWED_HOSTS,
     PACKAGE_DIR,
     TEMPLATES_DIR_NAME,
+    CUSTOM_TEMPLATES_DIR,
     SQL_INDEX_FILENAME,
     OUTPUT_DIR,
+    LOGS_DIR,
+    TIME_ZONE,
 )
 
-
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@@ -50,18 +55,54 @@ INSTALLED_APPS = [
 
 
 MIDDLEWARE = [
+    'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
+    'core.middleware.CacheControlMiddleware',
 ]
 
 AUTHENTICATION_BACKENDS = [
     'django.contrib.auth.backends.ModelBackend',
 ]
 
+# only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode)
+DEBUG_TOOLBAR = DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv)
+if DEBUG_TOOLBAR:
+    try:
+        import debug_toolbar   # noqa
+        DEBUG_TOOLBAR = True
+    except ImportError:
+        DEBUG_TOOLBAR = False
+
+if DEBUG_TOOLBAR:
+    INSTALLED_APPS = [*INSTALLED_APPS, 'debug_toolbar']
+    INTERNAL_IPS = ['0.0.0.0', '127.0.0.1', '*']
+    DEBUG_TOOLBAR_CONFIG = {
+        "SHOW_TOOLBAR_CALLBACK": lambda request: True,
+        "RENDER_PANELS": True,
+    }
+    DEBUG_TOOLBAR_PANELS = [
+        'debug_toolbar.panels.history.HistoryPanel',
+        'debug_toolbar.panels.versions.VersionsPanel',
+        'debug_toolbar.panels.timer.TimerPanel',
+        'debug_toolbar.panels.settings.SettingsPanel',
+        'debug_toolbar.panels.headers.HeadersPanel',
+        'debug_toolbar.panels.request.RequestPanel',
+        'debug_toolbar.panels.sql.SQLPanel',
+        'debug_toolbar.panels.staticfiles.StaticFilesPanel',
+        # 'debug_toolbar.panels.templates.TemplatesPanel',
+        'debug_toolbar.panels.cache.CachePanel',
+        'debug_toolbar.panels.signals.SignalsPanel',
+        'debug_toolbar.panels.logging.LoggingPanel',
+        'debug_toolbar.panels.redirects.RedirectsPanel',
+        'debug_toolbar.panels.profiling.ProfilingPanel',
+        'djdt_flamegraph.FlamegraphPanel',
+    ]
+    MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
 
 ################################################################################
 ### Staticfile and Template Settings
@@ -70,10 +111,12 @@ AUTHENTICATION_BACKENDS = [
 STATIC_URL = '/static/'
 
 STATICFILES_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
 ]
 
 TEMPLATE_DIRS = [
+    *([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
     str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
@@ -107,6 +150,23 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': DATABASE_NAME,
+        'OPTIONS': {
+            'timeout': 60,
+            'check_same_thread': False,
+        },
+        'TIME_ZONE': 'UTC',
+        # DB setup is sometimes modified at runtime by setup_django() in config.py
+    }
+}
+
+CACHE_BACKEND = 'django.core.cache.backends.locmem.LocMemCache'
+# CACHE_BACKEND = 'django.core.cache.backends.db.DatabaseCache'
+# CACHE_BACKEND = 'django.core.cache.backends.dummy.DummyCache'
+
+CACHES = {
+    'default': {
+        'BACKEND': CACHE_BACKEND,
+        'LOCATION': 'django_cache_default',
     }
 }
 
@@ -117,12 +177,13 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 ### Security Settings
 ################################################################################
 
-SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.')
+SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')
 
 ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
 
 SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
 
 CSRF_COOKIE_SECURE = False
 SESSION_COOKIE_SECURE = False
@@ -131,6 +192,8 @@ SESSION_COOKIE_AGE = 1209600  # 2 weeks
 SESSION_EXPIRE_AT_BROWSER_CLOSE = False
 SESSION_SAVE_EVERY_REQUEST = True
 
+SESSION_ENGINE = "django.contrib.sessions.backends.db"
+
 AUTH_PASSWORD_VALIDATORS = [
     {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
     {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'},
@@ -156,10 +219,84 @@ if IS_SHELL:
 ################################################################################
 
 LANGUAGE_CODE = 'en-us'
-TIME_ZONE = 'UTC'
-USE_I18N = False
-USE_L10N = False
-USE_TZ = False
-
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
 DATETIME_FORMAT = 'Y-m-d g:iA'
 SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
+TIME_ZONE = TIME_ZONE                            # noqa
+
+from django.conf.locale.en import formats as en_formats
+
+en_formats.DATETIME_FORMAT = DATETIME_FORMAT
+en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
+
+
+################################################################################
+### Logging Settings
+################################################################################
+
+IGNORABLE_404_URLS = [
+    re.compile(r'apple-touch-icon.*\.png$'),
+    re.compile(r'favicon\.ico$'),
+    re.compile(r'robots\.txt$'),
+    re.compile(r'.*\.(css|js)\.map$'),
+]
+
+class NoisyRequestsFilter(logging.Filter):
+    def filter(self, record):
+        logline = record.getMessage()
+
+        # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
+        for ignorable_url_pattern in IGNORABLE_404_URLS:
+            ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
+            if ignorable_log_pattern.match(logline):
+                return 0
+
+        # ignore staticfile requests that 200 or 30*
+        ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M)
+        if ignoreable_200_log_pattern.match(logline):
+            return 0
+
+        return 1
+
+if LOGS_DIR.exists():
+    ERROR_LOG = (LOGS_DIR / 'errors.log')
+else:
+    # meh too many edge cases here around creating log dir w/ correct permissions
+    # cant be bothered, just trash the log and let them figure it out via stdout/stderr
+    ERROR_LOG = tempfile.NamedTemporaryFile().name
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+        'logfile': {
+            'level': 'ERROR',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': ERROR_LOG,
+            'maxBytes': 1024 * 1024 * 25,  # 25 MB
+            'backupCount': 10,
+        },
+    },
+    'filters': {
+        'noisyrequestsfilter': {
+            '()': NoisyRequestsFilter,
+        }
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['console', 'logfile'],
+            'level': 'INFO',
+            'filters': ['noisyrequestsfilter'],
+        },
+        'django.server': {
+            'handlers': ['console', 'logfile'],
+            'level': 'INFO',
+            'filters': ['noisyrequestsfilter'],
+        }
+    },
+}

archivebox/core/templatetags/core_tags.py

@@ -1,22 +1,15 @@
 from django import template
-from django.urls import reverse
 from django.contrib.admin.templatetags.base import InclusionAdminNode
-from django.templatetags.static import static
 
 
 from typing import Union
 
-from core.models import ArchiveResult
 
 register = template.Library()
 
-@register.simple_tag
-def snapshot_image(snapshot):
-    result = ArchiveResult.objects.filter(snapshot=snapshot, extractor='screenshot', status='succeeded').first()
-    if result:
-        return reverse('Snapshot', args=[f'{str(snapshot.timestamp)}/{result.output}'])
-    
-    return static('archive.png')
+@register.filter(name='split')
+def split(value, separator: str=','):
+    return (value or '').split(separator)
 
 @register.filter
 def file_size(num_bytes: Union[int, float]) -> str:

archivebox/core/urls.py

@@ -2,6 +2,7 @@ from django.contrib import admin
 
 from django.urls import path, include
 from django.views import static
+from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.conf import settings
 from django.views.generic.base import RedirectView
 
@@ -13,8 +14,8 @@ from core.views import HomepageView, SnapshotView, PublicIndexView, AddView
 urlpatterns = [
     path('public/', PublicIndexView.as_view(), name='public-index'),
 
-    path('robots.txt', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'robots.txt'}),
-    path('favicon.ico', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'favicon.ico'}),
+    path('robots.txt', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'robots.txt'}),
+    path('favicon.ico', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'favicon.ico'}),
 
     path('docs/', RedirectView.as_view(url='https://github.com/ArchiveBox/ArchiveBox/wiki'), name='Docs'),
 
@@ -35,35 +36,43 @@ urlpatterns = [
     path('index.json', static.serve, {'document_root': settings.OUTPUT_DIR, 'path': 'index.json'}),
     path('', HomepageView.as_view(), name='Home'),
 ]
+urlpatterns += staticfiles_urlpatterns()
 
-    # # Proposed UI URLs spec
-    # path('',                 HomepageView)
-    # path('/add',             AddView)
-    # path('/public',          PublicIndexView)
-    # path('/snapshot/:slug',  SnapshotView)
-    
-    # path('/admin',           admin.site.urls)
-    # path('/accounts',        django.contrib.auth.urls)
+if settings.DEBUG_TOOLBAR:
+    import debug_toolbar
+    urlpatterns += [
+        path('__debug__/', include(debug_toolbar.urls)),
+    ]
 
-    # # Prposed REST API spec
-    # # :slugs can be uuid, short_uuid, or any of the unique index_fields
-    # path('api/v1/'),
-    # path('api/v1/core/'                      [GET])
-    # path('api/v1/core/snapshot/',            [GET, POST, PUT]),
-    # path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
-    # path('api/v1/core/archiveresult',        [GET, POST, PUT]),
-    # path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
-    # path('api/v1/core/tag/',                 [GET, POST, PUT]),
-    # path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
 
-    # path('api/v1/cli/',                      [GET])
-    # path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
+# # Proposed FUTURE URLs spec
+# path('',                 HomepageView)
+# path('/add',             AddView)
+# path('/public',          PublicIndexView)
+# path('/snapshot/:slug',  SnapshotView)
 
-    # path('api/v1/extractors/',                    [GET])
-    # path('api/v1/extractors/:extractor/',         [GET]),
-    # path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
+# path('/admin',           admin.site.urls)
+# path('/accounts',        django.contrib.auth.urls)
 
-    # future, just an idea:
-    # path('api/v1/scheduler/',                [GET])
-    # path('api/v1/scheduler/task/',           [GET, POST, PUT]),
-    # path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),
+# # Prposed REST API spec
+# # :slugs can be uuid, short_uuid, or any of the unique index_fields
+# path('api/v1/'),
+# path('api/v1/core/'                      [GET])
+# path('api/v1/core/snapshot/',            [GET, POST, PUT]),
+# path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
+# path('api/v1/core/archiveresult',        [GET, POST, PUT]),
+# path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
+# path('api/v1/core/tag/',                 [GET, POST, PUT]),
+# path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
+
+# path('api/v1/cli/',                      [GET])
+# path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
+
+# path('api/v1/extractors/',                    [GET])
+# path('api/v1/extractors/:extractor/',         [GET]),
+# path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
+
+# future, just an idea:
+# path('api/v1/scheduler/',                [GET])
+# path('api/v1/scheduler/task/',           [GET, POST, PUT]),
+# path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),

archivebox/core/views.py

@@ -4,8 +4,8 @@ from io import StringIO
 from contextlib import redirect_stdout
 
 from django.shortcuts import render, redirect
-
-from django.http import HttpResponse
+from django.http import HttpResponse, Http404
+from django.utils.html import format_html, mark_safe
 from django.views import View, static
 from django.views.generic.list import ListView
 from django.views.generic import FormView
@@ -22,10 +22,11 @@ from ..config import (
     PUBLIC_ADD_VIEW,
     VERSION,
     FOOTER_INFO,
+    SNAPSHOTS_PER_PAGE,
 )
-from main import add
+from ..main import add
 from ..util import base_url, ansi_to_html
-from ..index.html import snapshot_icons
+from ..search import query_search_index
 
 
 class HomepageView(View):
@@ -43,10 +44,6 @@ class SnapshotView(View):
     # render static html index from filesystem archive/<timestamp>/index.html
 
     def get(self, request, path):
-        # missing trailing slash -> redirect to index
-        if '/' not in path:
-            return redirect(f'{path}/index.html')
-
         if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
             return redirect(f'/admin/login/?next={request.path}')
 
@@ -55,47 +52,164 @@ class SnapshotView(View):
         except (IndexError, ValueError):
             slug, archivefile = path.split('/', 1)[0], 'index.html'
 
-        all_pages = list(Snapshot.objects.all())
-
         # slug is a timestamp
-        by_ts = {page.timestamp: page for page in all_pages}
-        try:
-            # print('SERVING STATICFILE', by_ts[slug].link_dir, request.path, path)
-            response = static.serve(request, archivefile, document_root=by_ts[slug].link_dir, show_indexes=True)
-            response["Link"] = f'<{by_ts[slug].url}>; rel="canonical"'
-            return response
-        except KeyError:
-            pass
+        if slug.replace('.','').isdigit():
 
-        # slug is a hash
-        by_hash = {page.url_hash: page for page in all_pages}
-        try:
-            timestamp = by_hash[slug].timestamp
-            return redirect(f'/archive/{timestamp}/{archivefile}')
-        except KeyError:
-            pass
+            # missing trailing slash -> redirect to index
+            if '/' not in path:
+                return redirect(f'{path}/index.html')
 
+            try:
+                try:
+                    snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
+                    response = static.serve(request, archivefile, document_root=snapshot.link_dir, show_indexes=True)
+                    response["Link"] = f'<{snapshot.url}>; rel="canonical"'
+                    return response
+                except Snapshot.DoesNotExist:
+                    if Snapshot.objects.filter(timestamp__startswith=slug).exists():
+                        raise Snapshot.MultipleObjectsReturned
+                    else:
+                        raise
+            except Snapshot.DoesNotExist:
+                # Snapshot does not exist
+                return HttpResponse(
+                    format_html(
+                        (
+                            '<center><br/><br/><br/>'
+                            'No Snapshot directories match the given timestamp or UUID: <code>{}</code><br/><br/>'
+                            'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
+                            '</center>'
+                        ),
+                        slug,
+                        path,
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
+            except Snapshot.MultipleObjectsReturned:
+                snapshot_hrefs = mark_safe('<br/>').join(
+                    format_html(
+                        '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                        snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                        snap.timestamp,
+                        snap.timestamp,
+                        snap.url,
+                        snap.title or '',
+                    )
+                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
+                )
+                return HttpResponse(
+                    format_html(
+                        (
+                            'Multiple Snapshots match the given timestamp/UUID <code>{}</code><br/><pre>'
+                        ),
+                        slug,
+                    ) + snapshot_hrefs + format_html(
+                        (
+                            '</pre><br/>'
+                            'Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'
+                        )
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
+            except Http404:
+                # Snapshot dir exists but file within does not e.g. 124235.324234/screenshot.png
+                return HttpResponse(
+                    format_html(
+                        (
+                            '<center><br/><br/><br/>'
+                            f'Snapshot <a href="/archive/{snapshot.timestamp}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a> exists in DB, but resource <b><code>{snapshot.timestamp}/'
+                            '{}'
+                            f'</code></b> does not exist in <a href="/archive/{snapshot.timestamp}/" target="_top">snapshot dir</a> yet.<br/><br/>'
+                            'Maybe this resource type is not availabe for this Snapshot,<br/>or the archiving process has not completed yet?<br/>'
+                            f'<pre><code># run this cmd to finish archiving this Snapshot<br/>archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
+                            '<div class="text-align: left; width: 100%; max-width: 400px">'
+                            '<i><b>Next steps:</i></b><br/>'
+                            f'- list all the <a href="/archive/{snapshot.timestamp}/" target="_top">Snapshot files <code>.*</code></a><br/>'
+                            f'- view the <a href="/archive/{snapshot.timestamp}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
+                            f'- go to the <a href="/admin/core/snapshot/{snapshot.id}/change/" target="_top">Snapshot admin</a> to edit<br/>'
+                            f'- go to the <a href="/admin/core/snapshot/?id__startswith={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
+                            '- or return to <a href="/" target="_top">the main index...</a></div>'
+                            '</center>'
+                        ),
+                        archivefile,
+                    ),
+                    content_type="text/html",
+                    status=404,
+                )
         # slug is a URL
-        by_url = {page.base_url: page for page in all_pages}
         try:
-            # TODO: add multiple snapshot support by showing index of all snapshots
-            # for given url instead of redirecting to timestamp index
-            timestamp = by_url[base_url(path)].timestamp
-            return redirect(f'/archive/{timestamp}/index.html')
-        except KeyError:
-            pass
-
-        return HttpResponse(
-            'No archived link matches the given timestamp or hash.',
-            content_type="text/plain",
-            status=404,
-        )
+            try:
+                # try exact match on full url first
+                snapshot = Snapshot.objects.get(
+                    Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
+                )
+            except Snapshot.DoesNotExist:
+                # fall back to match on exact base_url
+                try:
+                    snapshot = Snapshot.objects.get(
+                        Q(url='http://' + base_url(path)) | Q(url='https://' + base_url(path))
+                    )
+                except Snapshot.DoesNotExist:
+                    # fall back to matching base_url as prefix
+                    snapshot = Snapshot.objects.get(
+                        Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
+                    )
+            return redirect(f'/archive/{snapshot.timestamp}/index.html')
+        except Snapshot.DoesNotExist:
+            return HttpResponse(
+                format_html(
+                    (
+                        '<center><br/><br/><br/>'
+                        'No Snapshots match the given url: <code>{}</code><br/><br/><br/>'
+                        'Return to the <a href="/" target="_top">Main Index</a>, or:<br/><br/>'
+                        '+ <i><a href="/add/?url={}" target="_top">Add a new Snapshot for <code>{}</code></a><br/><br/></i>'
+                        '</center>'
+                    ),
+                    base_url(path),
+                    path if '://' in path else f'https://{path}',
+                    path,
+                ),
+                content_type="text/html",
+                status=404,
+            )
+        except Snapshot.MultipleObjectsReturned:
+            snapshot_hrefs = mark_safe('<br/>').join(
+                format_html(
+                    '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                    snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                    snap.timestamp,
+                    snap.timestamp,
+                    snap.url,
+                    snap.title or '',
+                )
+                for snap in Snapshot.objects.filter(
+                    Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
+                ).only('url', 'timestamp', 'title', 'added').order_by('-added')
+            )
+            return HttpResponse(
+                format_html(
+                    (
+                        'Multiple Snapshots match the given URL <code>{}</code><br/><pre>'
+                    ),
+                    base_url(path),
+                ) + snapshot_hrefs + format_html(
+                    (
+                        '</pre><br/>'
+                        'Choose a Snapshot to proceed or go back to the <a href="/" target="_top">Main Index</a>'
+                    )
+                ),
+                content_type="text/html",
+                status=404,
+            )
+        
 
 class PublicIndexView(ListView):
     template_name = 'public_index.html'
     model = Snapshot
-    paginate_by = 100
-    ordering = ['title']
+    paginate_by = SNAPSHOTS_PER_PAGE
+    ordering = ['-added']
 
     def get_context_data(self, **kwargs):
         return {
@@ -105,12 +219,14 @@ class PublicIndexView(ListView):
         }
 
     def get_queryset(self, **kwargs): 
-        qs = super().get_queryset(**kwargs) 
+        qs = super().get_queryset(**kwargs)
         query = self.request.GET.get('q')
-        if query:
+        if query and query.strip():
             qs = qs.filter(Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query))
-        for snapshot in qs:
-            snapshot.icons = snapshot_icons(snapshot)
+            try:
+                qs = qs | query_search_index(query)
+            except Exception as err:
+                print(f'[!] Error while using search backend: {err.__class__.__name__} {err}')
         return qs
 
     def get(self, *args, **kwargs):
@@ -130,9 +246,9 @@ class AddView(UserPassesTestMixin, FormView):
         if self.request.method == 'GET':
             url = self.request.GET.get('url', None)
             if url:
-                return {'url': url}
-        else:
-            return super().get_initial()
+                return {'url': url if '://' in url else f'https://{url}'}
+        
+        return super().get_initial()
 
     def test_func(self):
         return PUBLIC_ADD_VIEW or self.request.user.is_authenticated
@@ -145,16 +261,21 @@ class AddView(UserPassesTestMixin, FormView):
             'absolute_add_path': self.request.build_absolute_uri(self.request.path),
             'VERSION': VERSION,
             'FOOTER_INFO': FOOTER_INFO,
+            'stdout': '',
         }
 
     def form_valid(self, form):
         url = form.cleaned_data["url"]
         print(f'[+] Adding URL: {url}')
+        parser = form.cleaned_data["parser"]
+        tag = form.cleaned_data["tag"]
         depth = 0 if form.cleaned_data["depth"] == "0" else 1
         extractors = ','.join(form.cleaned_data["archive_methods"])
         input_kwargs = {
             "urls": url,
+            "tag": tag,
             "depth": depth,
+            "parser": parser,
             "update_all": False,
             "out_dir": OUTPUT_DIR,
         }

archivebox/core/wsgi.py

@@ -7,10 +7,10 @@ For more information on this file, see
 https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
 """
 
-import os
+
+from archivebox.config import setup_django
+setup_django(in_memory_db=False, check_db=True)
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.settings')
-
 application = get_wsgi_application()

archivebox/extractors/__init__.py

@@ -4,7 +4,7 @@ import os
 from pathlib import Path
 
 from typing import Optional, List, Iterable, Union
-from datetime import datetime
+from datetime import datetime, timezone
 from django.db.models import QuerySet
 
 from ..index.schema import Link
@@ -44,16 +44,16 @@ def get_default_archive_methods():
     return [
         ('title', should_save_title, save_title),
         ('favicon', should_save_favicon, save_favicon),
-        ('wget', should_save_wget, save_wget),
+        ('headers', should_save_headers, save_headers),
         ('singlefile', should_save_singlefile, save_singlefile),
         ('pdf', should_save_pdf, save_pdf),
         ('screenshot', should_save_screenshot, save_screenshot),
         ('dom', should_save_dom, save_dom),
-        ('readability', should_save_readability, save_readability), #keep readability below wget and singlefile, as it depends on them
+        ('wget', should_save_wget, save_wget),
+        ('readability', should_save_readability, save_readability),  # keep readability below wget and singlefile, as it depends on them
         ('mercury', should_save_mercury, save_mercury),
         ('git', should_save_git, save_git),
         ('media', should_save_media, save_media),
-        ('headers', should_save_headers, save_headers),
         ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
@@ -94,8 +94,9 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         link = load_link_details(link, out_dir=out_dir)
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
         log_link_archiving_started(link, out_dir, is_new)
-        link = link.overwrite(updated=datetime.now())
+        link = link.overwrite(updated=datetime.now(timezone.utc))
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
+        start_ts = datetime.now(timezone.utc)
 
         for method_name, should_run, method_function in ARCHIVE_METHODS:
             try:
@@ -115,6 +116,13 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                     ArchiveResult.objects.create(snapshot=snapshot, extractor=method_name, cmd=result.cmd, cmd_version=result.cmd_version,
                                                  output=result.output, pwd=result.pwd, start_ts=result.start_ts, end_ts=result.end_ts, status=result.status)
 
+
+                    # bump the updated time on the main Snapshot here, this is critical
+                    # to be able to cache summaries of the ArchiveResults for a given
+                    # snapshot without having to load all the results from the DB each time.
+                    # (we use {Snapshot.id}-{Snapshot.updated} as the cache key and assume
+                    # ArchiveResults are unchanged as long as the updated timestamp is unchanged)
+                    snapshot.save()
                 else:
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
                     stats['skipped'] += 1
@@ -135,7 +143,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
 
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
 
-        log_link_archiving_finished(link, link.link_dir, is_new, stats)
+        log_link_archiving_finished(link, link.link_dir, is_new, stats, start_ts)
 
     except KeyboardInterrupt:
         try:

archivebox/extractors/archive_org.py

@@ -31,7 +31,7 @@ def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwr
 
     out_dir = out_dir or Path(link.link_dir)
     if not overwrite and (out_dir / 'archive.org.txt').exists():
-        # if open(path, 'r').read().strip() != 'None':
+        # if open(path, 'r', encoding='utf-8').read().strip() != 'None':
         return False
 
     return SAVE_ARCHIVE_DOT_ORG

archivebox/extractors/media.py

@@ -70,11 +70,24 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
     finally:
         timer.end()
 
+    # add video description and subtitles to full-text index
+    index_texts = [
+        text_file.read_text(encoding='utf-8').strip()
+        for text_file in (
+            *output_path.glob('*.description'),
+            *output_path.glob('*.srt'),
+            *output_path.glob('*.vtt'),
+            *output_path.glob('*.lrc'),
+            *output_path.glob('*.lrc'),
+        )
+    ]
+
     return ArchiveResult(
         cmd=cmd,
         pwd=str(out_dir),
         cmd_version=YOUTUBEDL_VERSION,
         output=output,
         status=status,
+        index_texts=index_texts,
         **timer.stats,
     )

archivebox/extractors/mercury.py

@@ -54,11 +54,13 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
 
     out_dir = Path(out_dir or link.link_dir)
     output_folder = out_dir.absolute() / "mercury"
-    output = str(output_folder)
+    output = "mercury"
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
+        output_folder.mkdir(exist_ok=True)
+
         # Get plain text version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
@@ -71,6 +73,11 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         except json.JSONDecodeError:
             raise ShellError(cmd, result)
         
+        if article_text.get('failed'):
+            raise ArchiveError('Mercury was not able to get article text from the URL')
+
+        atomic_write(str(output_folder / "content.txt"), article_text["content"])
+
         # Get HTML version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
@@ -82,9 +89,10 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         except json.JSONDecodeError:
             raise ShellError(cmd, result)
 
-        output_folder.mkdir(exist_ok=True)
+        if article_text.get('failed'):
+            raise ArchiveError('Mercury was not able to get article HTML from the URL')
+
         atomic_write(str(output_folder / "content.html"), article_json.pop("content"))
-        atomic_write(str(output_folder / "content.txt"), article_text["content"])
         atomic_write(str(output_folder / "article.json"), article_json)
 
         # Check for common failure cases

archivebox/extractors/readability.py

@@ -35,7 +35,7 @@ def get_html(link: Link, path: Path) -> str:
     document = None
     for source in sources:
         try:
-            with open(abs_path / source, "r") as f:
+            with open(abs_path / source, "r", encoding="utf-8") as f:
                 document = f.read()
                 break
         except (FileNotFoundError, TypeError):
@@ -63,7 +63,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
 
     out_dir = Path(out_dir or link.link_dir)
     output_folder = out_dir.absolute() / "readability"
-    output = str(output_folder)
+    output = "readability"
 
     # Readability Docs: https://github.com/mozilla/readability
 
@@ -81,13 +81,21 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         temp_doc.write(document.encode("utf-8"))
         temp_doc.close()
 
+        if not document or len(document) < 10:
+            raise ArchiveError('Readability could not find HTML to parse for article text')
+
         cmd = [
             DEPENDENCIES['READABILITY_BINARY']['path'],
-            temp_doc.name
+            temp_doc.name,
         ]
 
         result = run(cmd, cwd=out_dir, timeout=timeout)
-        result_json = json.loads(result.stdout)
+        try:
+            result_json = json.loads(result.stdout)
+            assert result_json and 'content' in result_json
+        except json.JSONDecodeError:
+            raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr)
+
         output_folder.mkdir(exist_ok=True)
         readability_content = result_json.pop("textContent") 
         atomic_write(str(output_folder / "content.html"), result_json.pop("content"))
@@ -112,6 +120,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
     except (Exception, OSError) as err:
         status = 'failed'
         output = err
+        cmd = [cmd[0], './{singlefile,dom}.html']
     finally:
         timer.end()
 
@@ -121,6 +130,6 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
         cmd_version=READABILITY_VERSION,
         output=output,
         status=status,
-        index_texts= [readability_content] if readability_content else [],
+        index_texts=[readability_content] if readability_content else [],
         **timer.stats,  
     )

archivebox/extractors/wget.py

@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 
 from typing import Optional
-from datetime import datetime
+from datetime import datetime, timezone
 
 from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
@@ -51,7 +51,7 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
     if SAVE_WARC:
         warc_dir = out_dir / "warc"
         warc_dir.mkdir(exist_ok=True)
-        warc_path = warc_dir / str(int(datetime.now().timestamp()))
+        warc_path = warc_dir / str(int(datetime.now(timezone.utc).timestamp()))
 
     # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
     output: ArchiveOutput = None

archivebox/index/__init__.py

@@ -265,14 +265,14 @@ def load_main_index_meta(out_dir: Path=OUTPUT_DIR) -> Optional[dict]:
 
 
 @enforce_types
-def parse_links_from_source(source_path: str, root_url: Optional[str]=None) -> Tuple[List[Link], List[Link]]:
+def parse_links_from_source(source_path: str, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], List[Link]]:
 
     from ..parsers import parse_links
 
     new_links: List[Link] = []
 
     # parse and validate the import file
-    raw_links, parser_name = parse_links(source_path, root_url=root_url)
+    raw_links, parser_name = parse_links(source_path, root_url=root_url, parser=parser)
     new_links = validate_links(raw_links)
 
     if parser_name:
@@ -356,6 +356,7 @@ LINK_FILTERS = {
     'regex': lambda pattern: Q(url__iregex=pattern),
     'domain': lambda pattern: Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}"),
     'tag': lambda pattern: Q(tags__name=pattern),
+    'timestamp': lambda pattern: Q(timestamp=pattern),
 }
 
 @enforce_types

archivebox/index/html.py

@@ -1,18 +1,19 @@
 __package__ = 'archivebox.index'
 
-from datetime import datetime
-from typing import List, Optional, Iterator, Mapping
 from pathlib import Path
+from datetime import datetime, timezone
+from collections import defaultdict
+from typing import List, Optional, Iterator, Mapping
 
 from django.utils.html import format_html, mark_safe
-from collections import defaultdict
+from django.core.cache import cache
 
 from .schema import Link
 from ..system import atomic_write
 from ..logging_util import printable_filesize
 from ..util import (
     enforce_types,
-    ts_to_date,
+    ts_to_date_str,
     urlencode,
     htmlencode,
     urldecode,
@@ -20,7 +21,6 @@ from ..util import (
 from ..config import (
     OUTPUT_DIR,
     VERSION,
-    GIT_SHA,
     FOOTER_INFO,
     HTML_INDEX_FILENAME,
     SAVE_ARCHIVE_DOT_ORG,
@@ -60,10 +60,10 @@ def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) ->
 
     return render_django_template(template, {
         'version': VERSION,
-        'git_sha': GIT_SHA,
+        'git_sha': VERSION,  # not used anymore, but kept for backwards compatibility
         'num_links': str(len(links)),
-        'date_updated': datetime.now().strftime('%Y-%m-%d'),
-        'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
+        'date_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d'),
+        'time_updated': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M'),
         'links': [link._asdict(extended=True) for link in links],
         'FOOTER_INFO': FOOTER_INFO,
     })
@@ -103,7 +103,7 @@ def link_details_template(link: Link) -> str:
         'size': printable_filesize(link.archive_size) if link.archive_size else 'pending',
         'status': 'archived' if link.is_archived else 'not yet archived',
         'status_color': 'success' if link.is_archived else 'danger',
-        'oldest_archive_date': ts_to_date(link.oldest_archive_date),
+        'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
         'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
     })
 
@@ -116,71 +116,78 @@ def render_django_template(template: str, context: Mapping[str, str]) -> str:
 
 
 def snapshot_icons(snapshot) -> str:
-    from core.models import EXTRACTORS
+    cache_key = f'{snapshot.id}-{(snapshot.updated or snapshot.added).timestamp()}-snapshot-icons'
+    
+    def calc_snapshot_icons():
+        from core.models import EXTRACTORS
+        # start = datetime.now(timezone.utc)
 
-    # start = datetime.now()
+        archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
+        link = snapshot.as_link()
+        path = link.archive_path
+        canon = link.canonical_outputs()
+        output = ""
+        output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
+        icons = {
+            "singlefile": "❶",
+            "wget": "🆆",
+            "dom": "🅷",
+            "pdf": "📄",
+            "screenshot": "💻",
+            "media": "📼",
+            "git": "🅶",
+            "archive_org": "🏛",
+            "readability": "🆁",
+            "mercury": "🅼",
+            "warc": "📦"
+        }
+        exclude = ["favicon", "title", "headers", "archive_org"]
+        # Missing specific entry for WARC
 
-    archive_results = snapshot.archiveresult_set.filter(status="succeeded")
-    link = snapshot.as_link()
-    path = link.archive_path
-    canon = link.canonical_outputs()
-    output = ""
-    output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
-    icons = {
-        "singlefile": "❶",
-        "wget": "🆆",
-        "dom": "🅷",
-        "pdf": "📄",
-        "screenshot": "💻",
-        "media": "📼",
-        "git": "🅶",
-        "archive_org": "🏛",
-        "readability": "🆁",
-        "mercury": "🅼",
-        "warc": "📦"
-    }
-    exclude = ["favicon", "title", "headers", "archive_org"]
-    # Missing specific entry for WARC
+        extractor_outputs = defaultdict(lambda: None)
+        for extractor, _ in EXTRACTORS:
+            for result in archive_results:
+                if result.extractor == extractor and result:
+                    extractor_outputs[extractor] = result
 
-    extractor_outputs = defaultdict(lambda: None)
-    for extractor, _ in EXTRACTORS:
-        for result in archive_results:
-            if result.extractor == extractor and result:
-                extractor_outputs[extractor] = result
+        for extractor, _ in EXTRACTORS:
+            if extractor not in exclude:
+                existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                # Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
+                # if existing:
+                #     existing = (Path(path) / existing)
+                #     if existing.is_file():
+                #         existing = True
+                #     elif existing.is_dir():
+                #         existing = any(existing.glob('*.*'))
+                output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
+                                             extractor, icons.get(extractor, "?"))
+            if extractor == "wget":
+                # warc isn't technically it's own extractor, so we have to add it after wget
+                
+                # get from db (faster but less thurthful)
+                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                # get from filesystem (slower but more accurate)
+                # exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
+                output += format_html(output_template, path, canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
 
-    for extractor, _ in EXTRACTORS:
-        if extractor not in exclude:
-            existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-            # Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)
-            # if existing:
-            #     existing = (Path(path) / existing)
-            #     if existing.is_file():
-            #         existing = True
-            #     elif existing.is_dir():
-            #         existing = any(existing.glob('*.*'))
-            output += format_html(output_template, path, canon[f"{extractor}_path"], str(bool(existing)),
-                                         extractor, icons.get(extractor, "?"))
-        if extractor == "wget":
-            # warc isn't technically it's own extractor, so we have to add it after wget
-            
-            # get from db (faster but less thurthful)
-            exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-            # get from filesystem (slower but more accurate)
-            # exists = list((Path(path) / canon["warc_path"]).glob("*.warc.gz"))
-            output += format_html(output_template, 'warc/', canon["warc_path"], str(bool(exists)), "warc", icons.get("warc", "?"))
+            if extractor == "archive_org":
+                # The check for archive_org is different, so it has to be handled separately
 
-        if extractor == "archive_org":
-            # The check for archive_org is different, so it has to be handled separately
+                # get from db (faster)
+                exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
+                # get from filesystem (slower)
+                # target_path = Path(path) / "archive.org.txt"
+                # exists = target_path.exists()
+                output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
+                                                                                            "archive_org", icons.get("archive_org", "?"))
 
-            # get from db (faster)
-            exists = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
-            # get from filesystem (slower)
-            # target_path = Path(path) / "archive.org.txt"
-            # exists = target_path.exists()
-            output += '<a href="{}" class="exists-{}" title="{}">{}</a> '.format(canon["archive_org_path"], str(exists),
-                                                                                        "archive_org", icons.get("archive_org", "?"))
+        result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
+        # end = datetime.now(timezone.utc)
+        # print(((end - start).total_seconds()*1000) // 1, 'ms')
+        return result
 
-    result = format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}<span>', mark_safe(output))
-    # end = datetime.now()
-    # print(((end - start).total_seconds()*1000) // 1, 'ms')
-    return result
+    return cache.get_or_set(cache_key, calc_snapshot_icons)
+    # return calc_snapshot_icons()
+
+   

archivebox/index/json.py

@@ -5,7 +5,7 @@ import sys
 import json as pyjson
 from pathlib import Path
 
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Optional, Iterator, Any, Union
 
 from .schema import Link
@@ -15,7 +15,6 @@ from ..config import (
     VERSION,
     OUTPUT_DIR,
     FOOTER_INFO,
-    GIT_SHA,
     DEPENDENCIES,
     JSON_INDEX_FILENAME,
     ARCHIVE_DIR_NAME,
@@ -30,7 +29,7 @@ MAIN_INDEX_HEADER = {
     'meta': {
         'project': 'ArchiveBox',
         'version': VERSION,
-        'git_sha': GIT_SHA,
+        'git_sha': VERSION,  # not used anymore, but kept for backwards compatibility
         'website': 'https://ArchiveBox.io',
         'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
         'source': 'https://github.com/ArchiveBox/ArchiveBox',
@@ -45,7 +44,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
         output = {
             **MAIN_INDEX_HEADER,
             'num_links': len(links),
-            'updated': datetime.now(),
+            'updated': datetime.now(timezone.utc),
             'last_run_cmd': sys.argv,
             'links': links,
         }
@@ -61,7 +60,18 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     index_path = Path(out_dir) / JSON_INDEX_FILENAME
     if index_path.exists():
         with open(index_path, 'r', encoding='utf-8') as f:
-            links = pyjson.load(f)['links']
+            try:
+                links = pyjson.load(f)['links']
+                if links:
+                    Link.from_json(links[0])
+            except Exception as err:
+                print("    {lightyellow}! Found an index.json in the project root but couldn't load links from it: {} {}".format(
+                    err.__class__.__name__,
+                    err,
+                    **ANSI,
+                ))
+                return ()
+
             for link_json in links:
                 try:
                     yield Link.from_json(link_json)

archivebox/index/schema.py

@@ -10,15 +10,16 @@ __package__ = 'archivebox.index'
 
 from pathlib import Path
 
-from datetime import datetime, timedelta
+from datetime import datetime, timezone, timedelta
 
 from typing import List, Dict, Any, Optional, Union
 
 from dataclasses import dataclass, asdict, field, fields
 
+from django.utils.functional import cached_property
 
 from ..system import get_dir_size
-
+from ..util import ts_to_date_str, parse_date
 from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
 
 class ArchiveError(Exception):
@@ -55,11 +56,11 @@ class ArchiveResult:
         assert isinstance(self.end_ts, datetime)
         assert isinstance(self.cmd, list)
         assert all(isinstance(arg, str) and arg for arg in self.cmd)
-        assert self.pwd is None or isinstance(self.pwd, str) and self.pwd
-        assert self.cmd_version is None or isinstance(self.cmd_version, str) and self.cmd_version
+
+        # TODO: replace emptystrings in these three with None / remove them from the DB
+        assert self.pwd is None or isinstance(self.pwd, str)
+        assert self.cmd_version is None or isinstance(self.cmd_version, str)
         assert self.output is None or isinstance(self.output, (str, Exception))
-        if isinstance(self.output, str):
-            assert self.output
 
     @classmethod
     def guess_ts(_cls, dict_info):
@@ -133,7 +134,6 @@ class Link:
     updated: Optional[datetime] = None
     schema: str = 'Link'
 
-
     def __str__(self) -> str:
         return f'[{self.timestamp}] {self.url} "{self.title}"'
 
@@ -190,6 +190,7 @@ class Link:
         }
         if extended:
             info.update({
+                'snapshot_id': self.snapshot_id,
                 'link_dir': self.link_dir,
                 'archive_path': self.archive_path,
                 
@@ -201,6 +202,9 @@ class Link:
                 'basename': self.basename,
                 'extension': self.extension,
                 'is_static': self.is_static,
+                
+                'tags_str': (self.tags or '').strip(','),   # only used to render static index in index/html.py, remove if no longer needed there
+                'icons': None,           # only used to render static index in index/html.py, remove if no longer needed there
 
                 'bookmarked_date': self.bookmarked_date,
                 'updated_date': self.updated_date,
@@ -255,6 +259,11 @@ class Link:
 
         return to_csv(self, cols=cols or self.field_names(), separator=separator, ljust=ljust)
 
+    @cached_property
+    def snapshot_id(self):
+        from core.models import Snapshot
+        return str(Snapshot.objects.only('id').get(url=self.url).id)
+
     @classmethod
     def field_names(cls):
         return [f.name for f in fields(cls)]
@@ -316,13 +325,11 @@ class Link:
     ### Pretty Printing Helpers
     @property
     def bookmarked_date(self) -> Optional[str]:
-        from ..util import ts_to_date
-
-        max_ts = (datetime.now() + timedelta(days=30)).timestamp()
+        max_ts = (datetime.now(timezone.utc) + timedelta(days=30)).timestamp()
 
         if self.timestamp and self.timestamp.replace('.', '').isdigit():
             if 0 < float(self.timestamp) < max_ts:
-                return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
+                return ts_to_date_str(datetime.fromtimestamp(float(self.timestamp)))
             else:
                 return str(self.timestamp)
         return None
@@ -330,13 +337,12 @@ class Link:
 
     @property
     def updated_date(self) -> Optional[str]:
-        from ..util import ts_to_date
-        return ts_to_date(self.updated) if self.updated else None
+        return ts_to_date_str(self.updated) if self.updated else None
 
     @property
     def archive_dates(self) -> List[datetime]:
         return [
-            result.start_ts
+            parse_date(result.start_ts)
             for method in self.history.keys()
                 for result in self.history[method]
         ]

archivebox/index/sql.py

@@ -7,7 +7,7 @@ from django.db.models import QuerySet
 from django.db import transaction
 
 from .schema import Link
-from ..util import enforce_types
+from ..util import enforce_types, parse_date
 from ..config import OUTPUT_DIR
 
 
@@ -23,13 +23,15 @@ def parse_sql_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
     )
 
 @enforce_types
-def remove_from_sql_main_index(snapshots: QuerySet, out_dir: Path=OUTPUT_DIR) -> None:
-    with transaction.atomic():
-        snapshots.delete()
+def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
+    if atomic:
+        with transaction.atomic():
+            return snapshots.delete()
+    return snapshots.delete()
 
 @enforce_types
 def write_link_to_sql_index(link: Link):
-    from core.models import Snapshot
+    from core.models import Snapshot, ArchiveResult
     info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}
     tags = info.pop("tags")
     if tags is None:
@@ -41,36 +43,74 @@ def write_link_to_sql_index(link: Link):
         while Snapshot.objects.filter(timestamp=info["timestamp"]).exists():
             info["timestamp"] = str(float(info["timestamp"]) + 1.0)
 
-    snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info)
+        snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info)
     snapshot.save_tags(tags)
+
+    for extractor, entries in link.history.items():
+        for entry in entries:
+            if isinstance(entry, dict):
+                result, _ = ArchiveResult.objects.get_or_create(
+                    snapshot_id=snapshot.id,
+                    extractor=extractor,
+                    start_ts=parse_date(entry['start_ts']),
+                    defaults={
+                        'end_ts': parse_date(entry['end_ts']),
+                        'cmd': entry['cmd'],
+                        'output': entry['output'],
+                        'cmd_version': entry.get('cmd_version') or 'unknown',
+                        'pwd': entry['pwd'],
+                        'status': entry['status'],
+                    }
+                )
+            else:
+                result, _ = ArchiveResult.objects.update_or_create(
+                    snapshot_id=snapshot.id,
+                    extractor=extractor,
+                    start_ts=parse_date(entry.start_ts),
+                    defaults={
+                        'end_ts': parse_date(entry.end_ts),
+                        'cmd': entry.cmd,
+                        'output': entry.output,
+                        'cmd_version': entry.cmd_version or 'unknown',
+                        'pwd': entry.pwd,
+                        'status': entry.status,
+                    }
+                )
+
     return snapshot
 
 
 @enforce_types
 def write_sql_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
-    with transaction.atomic():
-        for link in links:
-            write_link_to_sql_index(link)
+    for link in links:
+        # with transaction.atomic():
+            # write_link_to_sql_index(link)
+        write_link_to_sql_index(link)
             
 
 @enforce_types
 def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
     from core.models import Snapshot
 
-    with transaction.atomic():
-        try:
-            snap = Snapshot.objects.get(url=link.url)
-        except Snapshot.DoesNotExist:
-            snap = write_link_to_sql_index(link)
-        snap.title = link.title
+    # with transaction.atomic():
+    #     try:
+    #         snap = Snapshot.objects.get(url=link.url)
+    #     except Snapshot.DoesNotExist:
+    #         snap = write_link_to_sql_index(link)
+    #     snap.title = link.title
+    try:
+        snap = Snapshot.objects.get(url=link.url)
+    except Snapshot.DoesNotExist:
+        snap = write_link_to_sql_index(link)
+    snap.title = link.title
 
-        tag_set = (
-            set(tag.strip() for tag in (link.tags or '').split(','))
-        )
-        tag_list = list(tag_set) or []
+    tag_set = (
+        set(tag.strip() for tag in (link.tags or '').split(','))
+    )
+    tag_list = list(tag_set) or []
 
-        snap.save()
-        snap.save_tags(tag_list)
+    snap.save()
+    snap.save_tags(tag_list)
 
 
 

archivebox/logging_util.py

@@ -3,26 +3,30 @@ __package__ = 'archivebox'
 import re
 import os
 import sys
+import stat
 import time
 import argparse
 from math import log
 from multiprocessing import Process
 from pathlib import Path
 
-from datetime import datetime
+from datetime import datetime, timezone
 from dataclasses import dataclass
-from typing import Optional, List, Dict, Union, IO, TYPE_CHECKING
+from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .index.schema import Link, ArchiveResult
 
+from .system import get_dir_size
 from .util import enforce_types
 from .config import (
     ConfigDict,
     OUTPUT_DIR,
     PYTHON_ENCODING,
+    VERSION,
     ANSI,
     IS_TTY,
+    IN_DOCKER,
     TERM_WIDTH,
     SHOW_PROGRESS,
     SOURCES_DIR_NAME,
@@ -50,6 +54,37 @@ class RuntimeStats:
 _LAST_RUN_STATS = RuntimeStats()
 
 
+def debug_dict_summary(obj: Dict[Any, Any]) -> None:
+    stderr(' '.join(f'{key}={str(val).ljust(6)}' for key, val in obj.items()))
+
+
+def get_fd_info(fd) -> Dict[str, Any]:
+    NAME = fd.name[1:-1]
+    FILENO = fd.fileno()
+    MODE = os.fstat(FILENO).st_mode
+    IS_TTY = hasattr(fd, 'isatty') and fd.isatty()
+    IS_PIPE = stat.S_ISFIFO(MODE)
+    IS_FILE = stat.S_ISREG(MODE)
+    IS_TERMINAL =  not (IS_PIPE or IS_FILE)
+    IS_LINE_BUFFERED = fd.line_buffering
+    IS_READABLE = fd.readable()
+    return {
+        'NAME': NAME, 'FILENO': FILENO, 'MODE': MODE,
+        'IS_TTY': IS_TTY, 'IS_PIPE': IS_PIPE, 'IS_FILE': IS_FILE,
+        'IS_TERMINAL': IS_TERMINAL, 'IS_LINE_BUFFERED': IS_LINE_BUFFERED,
+        'IS_READABLE': IS_READABLE,
+    }
+    
+
+# # Log debug information about stdin, stdout, and stderr
+# sys.stdout.write('[>&1] this is python stdout\n')
+# sys.stderr.write('[>&2] this is python stderr\n')
+
+# debug_dict_summary(get_fd_info(sys.stdin))
+# debug_dict_summary(get_fd_info(sys.stdout))
+# debug_dict_summary(get_fd_info(sys.stderr))
+
+
 
 class SmartFormatter(argparse.HelpFormatter):
     """Patched formatter that prints newlines in argparse help strings"""
@@ -62,22 +97,40 @@ class SmartFormatter(argparse.HelpFormatter):
 def reject_stdin(caller: str, stdin: Optional[IO]=sys.stdin) -> None:
     """Tell the user they passed stdin to a command that doesn't accept it"""
 
-    if stdin and not stdin.isatty():
-        stdin_raw_text = stdin.read().strip()
+    if not stdin:
+        return None
+
+    if IN_DOCKER:
+        # when TTY is disabled in docker we cant tell if stdin is being piped in or not
+        # if we try to read stdin when its not piped we will hang indefinitely waiting for it
+        return None
+
+    if not stdin.isatty():
+        # stderr('READING STDIN TO REJECT...')
+        stdin_raw_text = stdin.read()
         if stdin_raw_text:
+            # stderr('GOT STDIN!', len(stdin_str))
             stderr(f'[X] The "{caller}" command does not accept stdin.', color='red')
             stderr(f'    Run archivebox "{caller} --help" to see usage and examples.')
             stderr()
             raise SystemExit(1)
+    return None
 
 
 def accept_stdin(stdin: Optional[IO]=sys.stdin) -> Optional[str]:
     """accept any standard input and return it as a string or None"""
+    
     if not stdin:
         return None
-    elif stdin and not stdin.isatty():
-        stdin_str = stdin.read().strip()
-        return stdin_str or None
+
+    if not stdin.isatty():
+        # stderr('READING STDIN TO ACCEPT...')
+        stdin_str = stdin.read()
+
+        if stdin_str:
+            # stderr('GOT STDIN...', len(stdin_str))
+            return stdin_str
+
     return None
 
 
@@ -85,17 +138,19 @@ class TimedProgress:
     """Show a progress bar and measure elapsed time until .end() is called"""
 
     def __init__(self, seconds, prefix=''):
+
         self.SHOW_PROGRESS = SHOW_PROGRESS
         if self.SHOW_PROGRESS:
             self.p = Process(target=progress_bar, args=(seconds, prefix))
             self.p.start()
 
-        self.stats = {'start_ts': datetime.now(), 'end_ts': None}
+        self.stats = {'start_ts': datetime.now(timezone.utc), 'end_ts': None}
 
     def end(self):
         """immediately end progress, clear the progressbar line, and save end_ts"""
 
-        end_ts = datetime.now()
+
+        end_ts = datetime.now(timezone.utc)
         self.stats['end_ts'] = end_ts
         
         if self.SHOW_PROGRESS:
@@ -104,7 +159,10 @@ class TimedProgress:
                 # kill the progress bar subprocess
                 try:
                     self.p.close()   # must be closed *before* its terminnated
-                except:
+                except (KeyboardInterrupt, SystemExit):
+                    print()
+                    raise
+                except BaseException:                                           # lgtm [py/catch-base-exception]
                     pass
                 self.p.terminate()
                 self.p.join()
@@ -170,14 +228,12 @@ def progress_bar(seconds: int, prefix: str='') -> None:
         # sys.stdout.flush()
     except (KeyboardInterrupt, BrokenPipeError):
         print()
-        pass
 
 
 def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional[str], pwd: str):
-    from .config import VERSION, ANSI
     cmd = ' '.join(('archivebox', subcommand, *subcommand_args))
     stderr('{black}[i] [{now}] ArchiveBox v{VERSION}: {cmd}{reset}'.format(
-        now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+        now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
         VERSION=VERSION,
         cmd=cmd,
         **ANSI,
@@ -189,7 +245,7 @@ def log_cli_command(subcommand: str, subcommand_args: List[str], stdin: Optional
 
 
 def log_importing_started(urls: Union[str, List[str]], depth: int, index_only: bool):
-    _LAST_RUN_STATS.parse_start_ts = datetime.now()
+    _LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc)
     print('{green}[+] [{}] Adding {} links to index (crawl depth={}){}...{reset}'.format(
         _LAST_RUN_STATS.parse_start_ts.strftime('%Y-%m-%d %H:%M:%S'),
         len(urls) if isinstance(urls, list) else len(urls.split('\n')),
@@ -202,7 +258,7 @@ def log_source_saved(source_file: str):
     print('    > Saved verbatim input to {}/{}'.format(SOURCES_DIR_NAME, source_file.rsplit('/', 1)[-1]))
 
 def log_parsing_finished(num_parsed: int, parser_name: str):
-    _LAST_RUN_STATS.parse_end_ts = datetime.now()
+    _LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc)
     print('    > Parsed {} URLs from input ({})'.format(num_parsed, parser_name))
 
 def log_deduping_finished(num_new_links: int):
@@ -216,7 +272,7 @@ def log_crawl_started(new_links):
 ### Indexing Stage
 
 def log_indexing_process_started(num_links: int):
-    start_ts = datetime.now()
+    start_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.index_start_ts = start_ts
     print()
     print('{black}[*] [{}] Writing {} links to main index...{reset}'.format(
@@ -227,23 +283,24 @@ def log_indexing_process_started(num_links: int):
 
 
 def log_indexing_process_finished():
-    end_ts = datetime.now()
+    end_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.index_end_ts = end_ts
 
 
 def log_indexing_started(out_path: str):
     if IS_TTY:
-        sys.stdout.write(f'    > {out_path}')
+        sys.stdout.write(f'    > ./{Path(out_path).relative_to(OUTPUT_DIR)}')
 
 
 def log_indexing_finished(out_path: str):
-    print(f'\r    √ {out_path}')
+    print(f'\r    √ ./{Path(out_path).relative_to(OUTPUT_DIR)}')
 
 
 ### Archiving Stage
 
 def log_archiving_started(num_links: int, resume: Optional[float]=None):
-    start_ts = datetime.now()
+
+    start_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.archiving_start_ts = start_ts
     print()
     if resume:
@@ -261,7 +318,8 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
         ))
 
 def log_archiving_paused(num_links: int, idx: int, timestamp: str):
-    end_ts = datetime.now()
+
+    end_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.archiving_end_ts = end_ts
     print()
     print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
@@ -272,13 +330,14 @@ def log_archiving_paused(num_links: int, idx: int, timestamp: str):
         total=num_links,
     ))
     print()
-    print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI))
-    print('        archivebox server  # then visit http://127.0.0.1:8000')
     print('    Continue archiving where you left off by running:')
     print('        archivebox update --resume={}'.format(timestamp))
 
 def log_archiving_finished(num_links: int):
-    end_ts = datetime.now()
+
+    from core.models import Snapshot
+
+    end_ts = datetime.now(timezone.utc)
     _LAST_RUN_STATS.archiving_end_ts = end_ts
     assert _LAST_RUN_STATS.archiving_start_ts is not None
     seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
@@ -298,12 +357,15 @@ def log_archiving_finished(num_links: int):
     print('    - {} links skipped'.format(_LAST_RUN_STATS.skipped))
     print('    - {} links updated'.format(_LAST_RUN_STATS.succeeded + _LAST_RUN_STATS.failed))
     print('    - {} links had errors'.format(_LAST_RUN_STATS.failed))
-    print()
-    print('    {lightred}Hint:{reset} To manage your archive in a Web UI, run:'.format(**ANSI))
-    print('        archivebox server 0.0.0.0:8000')
+    
+    if Snapshot.objects.count() < 50:
+        print()
+        print('    {lightred}Hint:{reset} To manage your archive in a Web UI, run:'.format(**ANSI))
+        print('        archivebox server 0.0.0.0:8000')
 
 
 def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
+
     # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
     #     http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
     #     > output/archive/1478739709
@@ -311,7 +373,7 @@ def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
     print('\n[{symbol_color}{symbol}{reset}] [{symbol_color}{now}{reset}] "{title}"'.format(
         symbol_color=ANSI['green' if is_new else 'black'],
         symbol='+' if is_new else '√',
-        now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+        now=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
         title=link.title or link.base_url,
         **ANSI,
     ))
@@ -321,7 +383,7 @@ def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
         pretty_path(link_dir),
     ))
 
-def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats: dict):
+def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats: dict, start_ts: datetime):
     total = sum(stats.values())
 
     if stats['failed'] > 0 :
@@ -331,6 +393,11 @@ def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats
     else:
         _LAST_RUN_STATS.succeeded += 1
 
+    size = get_dir_size(link_dir)
+    end_ts = datetime.now(timezone.utc)
+    duration = str(end_ts - start_ts).split('.')[0]
+    print('        {black}{} files ({}) in {}s {reset}'.format(size[2], printable_filesize(size[0]), duration, **ANSI))
+
 
 def log_archive_method_started(method: str):
     print('      > {}'.format(method))

archivebox/main.py

@@ -27,6 +27,7 @@ from .parsers import (
 from .index.schema import Link
 from .util import enforce_types                         # type: ignore
 from .system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
+from .system import run as run_shell
 from .index import (
     load_main_index,
     parse_links_from_source,
@@ -67,23 +68,24 @@ from .config import (
     ConfigDict,
     ANSI,
     IS_TTY,
+    DEBUG,
     IN_DOCKER,
     USER,
+    PYTHON_BINARY,
     ARCHIVEBOX_BINARY,
     ONLY_NEW,
     OUTPUT_DIR,
     SOURCES_DIR,
     ARCHIVE_DIR,
     LOGS_DIR,
+    PACKAGE_DIR,
     CONFIG_FILE,
     ARCHIVE_DIR_NAME,
-    SOURCES_DIR_NAME,
-    LOGS_DIR_NAME,
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,
     SQL_INDEX_FILENAME,
-    ROBOTS_TXT_FILENAME,
-    FAVICON_FILENAME,
+    ALLOWED_IN_OUTPUT_DIR,
+    SEARCH_BACKEND_ENGINE,
     check_dependencies,
     check_data_folder,
     write_config_file,
@@ -92,10 +94,22 @@ from .config import (
     EXTERNAL_LOCATIONS,
     DATA_LOCATIONS,
     DEPENDENCIES,
+    USE_CHROME,
+    CHROME_BINARY,
+    CHROME_VERSION,
+    YOUTUBEDL_BINARY,
+    YOUTUBEDL_VERSION,
+    SINGLEFILE_VERSION,
+    READABILITY_VERSION,
+    MERCURY_VERSION,
+    USE_YOUTUBEDL,
+    USE_NODE,
+    NODE_VERSION,
     load_all_config,
     CONFIG,
     USER_CONFIG,
     get_real_name,
+    setup_django,
 )
 from .logging_util import (
     TERM_WIDTH,
@@ -115,25 +129,7 @@ from .logging_util import (
 
 from .search import flush_search_index, index_links
 
-ALLOWED_IN_OUTPUT_DIR = {
-    'lost+found',
-    '.DS_Store',
-    '.venv',
-    'venv',
-    'virtualenv',
-    '.virtualenv',
-    'node_modules',
-    'package-lock.json',
-    'static',
-    ARCHIVE_DIR_NAME,
-    SOURCES_DIR_NAME,
-    LOGS_DIR_NAME,
-    SQL_INDEX_FILENAME,
-    JSON_INDEX_FILENAME,
-    HTML_INDEX_FILENAME,
-    ROBOTS_TXT_FILENAME,
-    FAVICON_FILENAME,
-}
+
 
 @enforce_types
 def help(out_dir: Path=OUTPUT_DIR) -> None:
@@ -214,9 +210,23 @@ def version(quiet: bool=False,
     if quiet:
         print(VERSION)
     else:
+        # ArchiveBox v0.5.6
+        # Cpython Linux Linux-4.19.121-linuxkit-x86_64-with-glibc2.28 x86_64 (in Docker) (in TTY)
         print('ArchiveBox v{}'.format(VERSION))
         p = platform.uname()
-        print(sys.implementation.name.title(), p.system, platform.platform(), p.machine, '(in Docker)' if IN_DOCKER else '(not in Docker)')
+        print(
+            sys.implementation.name.title(),
+            p.system,
+            platform.platform(),
+            p.machine,
+        )
+        print(
+            f'IN_DOCKER={IN_DOCKER}',
+            f'DEBUG={DEBUG}',
+            f'IS_TTY={IS_TTY}',
+            f'TZ={os.environ.get("TZ", "UTC")}',
+            f'SEARCH_BACKEND_ENGINE={SEARCH_BACKEND_ENGINE}',
+        )
         print()
 
         print('{white}[i] Dependency versions:{reset}'.format(**ANSI))
@@ -261,26 +271,27 @@ def run(subcommand: str,
 
 
 @enforce_types
-def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
+def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
+    
     from core.models import Snapshot
-    Path(out_dir).mkdir(exist_ok=True)
+
+    out_dir.mkdir(exist_ok=True)
     is_empty = not len(set(os.listdir(out_dir)) - ALLOWED_IN_OUTPUT_DIR)
 
-    if (Path(out_dir) / JSON_INDEX_FILENAME).exists():
+    if (out_dir / JSON_INDEX_FILENAME).exists():
         stderr("[!] This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.", color="lightyellow")
-        stderr("    You can run `archivebox list --json --with-headers > index.json` to manually generate it.", color="lightyellow")
+        stderr("    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.", color="lightyellow")
 
-    existing_index = (Path(out_dir) / SQL_INDEX_FILENAME).exists()
+    existing_index = (out_dir / SQL_INDEX_FILENAME).exists()
 
     if is_empty and not existing_index:
-        print('{green}[+] Initializing a new ArchiveBox collection in this folder...{reset}'.format(**ANSI))
-        print(f'    {out_dir}')
-        print('{green}------------------------------------------------------------------{reset}'.format(**ANSI))
+        print('{green}[+] Initializing a new ArchiveBox v{} collection...{reset}'.format(VERSION, **ANSI))
+        print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     elif existing_index:
-        print('{green}[*] Updating existing ArchiveBox collection in this folder...{reset}'.format(**ANSI))
-        print(f'    {out_dir}')
-        print('{green}------------------------------------------------------------------{reset}'.format(**ANSI))
+        # TODO: properly detect and print the existing version in current index as well
+        print('{green}[^] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
+        print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     else:
         if force:
             stderr('[!] This folder appears to already have files in it, but no index.sqlite3 is present.', color='lightyellow')
@@ -301,30 +312,25 @@ def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
     else:
         print('\n{green}[+] Building archive folder structure...{reset}'.format(**ANSI))
     
+    print(f'    + ./{ARCHIVE_DIR.relative_to(OUTPUT_DIR)}, ./{SOURCES_DIR.relative_to(OUTPUT_DIR)}, ./{LOGS_DIR.relative_to(OUTPUT_DIR)}...')
     Path(SOURCES_DIR).mkdir(exist_ok=True)
-    print(f'    √ {SOURCES_DIR}')
-    
     Path(ARCHIVE_DIR).mkdir(exist_ok=True)
-    print(f'    √ {ARCHIVE_DIR}')
-
     Path(LOGS_DIR).mkdir(exist_ok=True)
-    print(f'    √ {LOGS_DIR}')
-
+    print(f'    + ./{CONFIG_FILE.relative_to(OUTPUT_DIR)}...')
     write_config_file({}, out_dir=out_dir)
-    print(f'    √ {CONFIG_FILE}')
-    if (Path(out_dir) / SQL_INDEX_FILENAME).exists():
-        print('\n{green}[*] Verifying main SQL index and running migrations...{reset}'.format(**ANSI))
+
+    if (out_dir / SQL_INDEX_FILENAME).exists():
+        print('\n{green}[*] Verifying main SQL index and running any migrations needed...{reset}'.format(**ANSI))
     else:
-        print('\n{green}[+] Building main SQL index and running migrations...{reset}'.format(**ANSI))
+        print('\n{green}[+] Building main SQL index and running initial migrations...{reset}'.format(**ANSI))
     
-    DATABASE_FILE = Path(out_dir) / SQL_INDEX_FILENAME
-    print(f'    √ {DATABASE_FILE}')
-    print()
+    DATABASE_FILE = out_dir / SQL_INDEX_FILENAME
     for migration_line in apply_migrations(out_dir):
         print(f'    {migration_line}')
 
-
     assert DATABASE_FILE.exists()
+    print()
+    print(f'    √ ./{DATABASE_FILE.relative_to(OUTPUT_DIR)}')
     
     # from django.contrib.auth.models import User
     # if IS_TTY and not User.objects.filter(is_superuser=True).exists():
@@ -332,7 +338,7 @@ def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
     #     call_command("createsuperuser", interactive=True)
 
     print()
-    print('{green}[*] Collecting links from any existing indexes and archive folders...{reset}'.format(**ANSI))
+    print('{green}[*] Checking links from indexes and archive folders (safe to Ctrl+C)...{reset}'.format(**ANSI))
 
     all_links = Snapshot.objects.none()
     pending_links: Dict[str, Link] = {}
@@ -341,73 +347,88 @@ def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
         all_links = load_main_index(out_dir=out_dir, warn=False)
         print('    √ Loaded {} links from existing main index.'.format(all_links.count()))
 
-    # Links in data folders that dont match their timestamp
-    fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir)
-    if fixed:
-        print('    {lightyellow}√ Fixed {} data directory locations that didn\'t match their link timestamps.{reset}'.format(len(fixed), **ANSI))
-    if cant_fix:
-        print('    {lightyellow}! Could not fix {} data directory locations due to conflicts with existing folders.{reset}'.format(len(cant_fix), **ANSI))
+    if quick:
+        print('    > Skipping full snapshot directory check (quick mode)')
+    else:
+        try:
+            # Links in data folders that dont match their timestamp
+            fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir)
+            if fixed:
+                print('    {lightyellow}√ Fixed {} data directory locations that didn\'t match their link timestamps.{reset}'.format(len(fixed), **ANSI))
+            if cant_fix:
+                print('    {lightyellow}! Could not fix {} data directory locations due to conflicts with existing folders.{reset}'.format(len(cant_fix), **ANSI))
 
-    # Links in JSON index but not in main index
-    orphaned_json_links = {
-        link.url: link
-        for link in parse_json_main_index(out_dir)
-        if not all_links.filter(url=link.url).exists()
-    }
-    if orphaned_json_links:
-        pending_links.update(orphaned_json_links)
-        print('    {lightyellow}√ Added {} orphaned links from existing JSON index...{reset}'.format(len(orphaned_json_links), **ANSI))
+            # Links in JSON index but not in main index
+            orphaned_json_links = {
+                link.url: link
+                for link in parse_json_main_index(out_dir)
+                if not all_links.filter(url=link.url).exists()
+            }
+            if orphaned_json_links:
+                pending_links.update(orphaned_json_links)
+                print('    {lightyellow}√ Added {} orphaned links from existing JSON index...{reset}'.format(len(orphaned_json_links), **ANSI))
 
-    # Links in data dir indexes but not in main index
-    orphaned_data_dir_links = {
-        link.url: link
-        for link in parse_json_links_details(out_dir)
-        if not all_links.filter(url=link.url).exists()
-    }
-    if orphaned_data_dir_links:
-        pending_links.update(orphaned_data_dir_links)
-        print('    {lightyellow}√ Added {} orphaned links from existing archive directories.{reset}'.format(len(orphaned_data_dir_links), **ANSI))
+            # Links in data dir indexes but not in main index
+            orphaned_data_dir_links = {
+                link.url: link
+                for link in parse_json_links_details(out_dir)
+                if not all_links.filter(url=link.url).exists()
+            }
+            if orphaned_data_dir_links:
+                pending_links.update(orphaned_data_dir_links)
+                print('    {lightyellow}√ Added {} orphaned links from existing archive directories.{reset}'.format(len(orphaned_data_dir_links), **ANSI))
 
-    # Links in invalid/duplicate data dirs
-    invalid_folders = {
-        folder: link
-        for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items()
-    }
-    if invalid_folders:
-        print('    {lightyellow}! Skipped adding {} invalid link data directories.{reset}'.format(len(invalid_folders), **ANSI))
-        print('        X ' + '\n        X '.join(f'{folder} {link}' for folder, link in invalid_folders.items()))
-        print()
-        print('    {lightred}Hint:{reset} For more information about the link data directories that were skipped, run:'.format(**ANSI))
-        print('        archivebox status')
-        print('        archivebox list --status=invalid')
+            # Links in invalid/duplicate data dirs
+            invalid_folders = {
+                folder: link
+                for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items()
+            }
+            if invalid_folders:
+                print('    {lightyellow}! Skipped adding {} invalid link data directories.{reset}'.format(len(invalid_folders), **ANSI))
+                print('        X ' + '\n        X '.join(f'./{Path(folder).relative_to(OUTPUT_DIR)} {link}' for folder, link in invalid_folders.items()))
+                print()
+                print('    {lightred}Hint:{reset} For more information about the link data directories that were skipped, run:'.format(**ANSI))
+                print('        archivebox status')
+                print('        archivebox list --status=invalid')
 
+        except (KeyboardInterrupt, SystemExit):
+            stderr()
+            stderr('[x] Stopped checking archive directories due to Ctrl-C/SIGTERM', color='red')
+            stderr('    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.')
+            stderr()
+            stderr('    {lightred}Hint:{reset} In the future you can run a quick init without checking dirs like so:'.format(**ANSI))
+            stderr('        archivebox init --quick')
+            raise SystemExit(1)
+        
+        write_main_index(list(pending_links.values()), out_dir=out_dir)
 
-    write_main_index(list(pending_links.values()), out_dir=out_dir)
-
-    print('\n{green}------------------------------------------------------------------{reset}'.format(**ANSI))
+    print('\n{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     if existing_index:
         print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**ANSI))
     else:
-        print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links), **ANSI))
-    print()
-    print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI))
-    print('        archivebox server  # then visit http://127.0.0.1:8000')
-    print()
-    print('    To add new links, you can run:')
-    print("        archivebox add ~/some/path/or/url/to/list_of_links.txt")
-    print()
-    print('    For more usage and examples, run:')
-    print('        archivebox help')
+        print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links) + len(pending_links), **ANSI))
 
-    json_index = Path(out_dir) / JSON_INDEX_FILENAME
-    html_index = Path(out_dir) / HTML_INDEX_FILENAME
+    json_index = out_dir / JSON_INDEX_FILENAME
+    html_index = out_dir / HTML_INDEX_FILENAME
     index_name = f"{date.today()}_index_old"
     if json_index.exists():
         json_index.rename(f"{index_name}.json")
     if html_index.exists():
         html_index.rename(f"{index_name}.html")
 
+    if setup:
+        run_subcommand('setup', pwd=out_dir)
 
+    if Snapshot.objects.count() < 25:     # hide the hints for experienced users
+        print()
+        print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI))
+        print('        archivebox server  # then visit http://127.0.0.1:8000')
+        print()
+        print('    To add new links, you can run:')
+        print("        archivebox add ~/some/path/or/url/to/list_of_links.txt")
+        print()
+        print('    For more usage and examples, run:')
+        print('        archivebox help')
 
 @enforce_types
 def status(out_dir: Path=OUTPUT_DIR) -> None:
@@ -529,15 +550,20 @@ def oneshot(url: str, extractors: str="", out_dir: Path=OUTPUT_DIR):
 
 @enforce_types
 def add(urls: Union[str, List[str]],
+        tag: str='',
         depth: int=0,
         update_all: bool=not ONLY_NEW,
         index_only: bool=False,
         overwrite: bool=False,
+        # duplicate: bool=False,  # TODO: reuse the logic from admin.py resnapshot to allow adding multiple snapshots by appending timestamp automatically
         init: bool=False,
         extractors: str="",
+        parser: str="auto",
         out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Add a new URL or list of URLs to your archive"""
 
+    from core.models import Tag
+
     assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)'
 
     extractors = extractors.split(",") if extractors else []
@@ -559,7 +585,7 @@ def add(urls: Union[str, List[str]],
         # save verbatim args to sources
         write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir)
     
-    new_links += parse_links_from_source(write_ahead_log, root_url=None)
+    new_links += parse_links_from_source(write_ahead_log, root_url=None, parser=parser)
 
     # If we're going one level deeper, download each link and look for more links
     new_links_depth = []
@@ -570,26 +596,48 @@ def add(urls: Union[str, List[str]],
             new_links_depth += parse_links_from_source(downloaded_file, root_url=new_link.url)
 
     imported_links = list({link.url: link for link in (new_links + new_links_depth)}.values())
+    
     new_links = dedupe_links(all_links, imported_links)
 
     write_main_index(links=new_links, out_dir=out_dir)
     all_links = load_main_index(out_dir=out_dir)
 
     if index_only:
-        return all_links
+        # mock archive all the links using the fake index_only extractor method in order to update their state
+        if overwrite:
+            archive_links(imported_links, overwrite=overwrite, methods=['index_only'], out_dir=out_dir)
+        else:
+            archive_links(new_links, overwrite=False, methods=['index_only'], out_dir=out_dir)
+    else:
+        # fully run the archive extractor methods for each link
+        archive_kwargs = {
+            "out_dir": out_dir,
+        }
+        if extractors:
+            archive_kwargs["methods"] = extractors
+
+        if update_all:
+            archive_links(all_links, overwrite=overwrite, **archive_kwargs)
+        elif overwrite:
+            archive_links(imported_links, overwrite=True, **archive_kwargs)
+        elif new_links:
+            archive_links(new_links, overwrite=False, **archive_kwargs)
+
+
+    # add any tags to imported links
+    tags = [
+        Tag.objects.get_or_create(name=name.strip())[0]
+        for name in tag.split(',')
+        if name.strip()
+    ]
+    if tags:
+        for link in imported_links:
+            snapshot = link.as_snapshot()
+            snapshot.tags.add(*tags)
+            snapshot.tags_str(nocache=True)
+            snapshot.save()
+        # print(f'    √ Tagged {len(imported_links)} Snapshots with {len(tags)} tags {tags_str}')
 
-    # Run the archive methods for each link
-    archive_kwargs = {
-        "out_dir": out_dir,
-    }
-    if extractors:
-        archive_kwargs["methods"] = extractors
-    if update_all:
-        archive_links(all_links, overwrite=overwrite, **archive_kwargs)
-    elif overwrite:
-        archive_links(imported_links, overwrite=True, **archive_kwargs)
-    elif new_links:
-        archive_links(new_links, overwrite=False, **archive_kwargs)
 
     return all_links
 
@@ -809,11 +857,15 @@ def list_links(snapshots: Optional[QuerySet]=None,
         all_snapshots = load_main_index(out_dir=out_dir)
 
     if after is not None:
-        all_snapshots = all_snapshots.filter(timestamp__lt=after)
+        all_snapshots = all_snapshots.filter(timestamp__gte=after)
     if before is not None:
-        all_snapshots = all_snapshots.filter(timestamp__gt=before)
+        all_snapshots = all_snapshots.filter(timestamp__lt=before)
     if filter_patterns:
         all_snapshots = snapshot_filter(all_snapshots, filter_patterns, filter_type)
+
+    if not all_snapshots:
+        stderr('[!] No Snapshots matched your filters:', filter_patterns, f'({filter_type})', color='lightyellow')
+
     return all_snapshots
 
 @enforce_types
@@ -841,6 +893,118 @@ def list_folders(links: List[Link],
     except KeyError:
         raise ValueError('Status not recognized.')
 
+@enforce_types
+def setup(out_dir: Path=OUTPUT_DIR) -> None:
+    """Automatically install all ArchiveBox dependencies and extras"""
+
+    if not (out_dir / ARCHIVE_DIR_NAME).exists():
+        run_subcommand('init', stdin=None, pwd=out_dir)
+
+    setup_django(out_dir=out_dir, check_db=True)
+    from core.models import User
+
+    if not User.objects.filter(is_superuser=True).exists():
+        stderr('\n[+] Creating new admin user for the Web UI...', color='green')
+        run_subcommand('manage', subcommand_args=['createsuperuser'], pwd=out_dir)
+
+    stderr('\n[+] Installing enabled ArchiveBox dependencies automatically...', color='green')
+
+    stderr('\n    Installing YOUTUBEDL_BINARY automatically using pip...')
+    if USE_YOUTUBEDL:
+        if YOUTUBEDL_VERSION:
+            print(f'{YOUTUBEDL_VERSION} is already installed', YOUTUBEDL_BINARY)
+        else:
+            try:
+                run_shell([
+                    PYTHON_BINARY, '-m', 'pip',
+                    'install',
+                    '--upgrade',
+                    '--no-cache-dir',
+                    '--no-warn-script-location',
+                    'youtube_dl',
+                ], capture_output=False, cwd=out_dir)
+                pkg_path = run_shell([
+                    PYTHON_BINARY, '-m', 'pip',
+                    'show',
+                    'youtube_dl',
+                ], capture_output=True, text=True, cwd=out_dir).stdout.split('Location: ')[-1].split('\n', 1)[0]
+                NEW_YOUTUBEDL_BINARY = Path(pkg_path) / 'youtube_dl' / '__main__.py'
+                os.chmod(NEW_YOUTUBEDL_BINARY, 0o777)
+                assert NEW_YOUTUBEDL_BINARY.exists(), f'youtube_dl must exist inside {pkg_path}'
+                config(f'YOUTUBEDL_BINARY={NEW_YOUTUBEDL_BINARY}', set=True, out_dir=out_dir)
+            except BaseException as e:
+                stderr(f'[X] Failed to install python packages: {e}', color='red')
+                raise SystemExit(1)
+
+    stderr('\n    Installing CHROME_BINARY automatically using playwright...')
+    if USE_CHROME:
+        if CHROME_VERSION:
+            print(f'{CHROME_VERSION} is already installed', CHROME_BINARY)
+        else:
+            try:
+                run_shell([
+                    PYTHON_BINARY, '-m', 'pip',
+                    'install',
+                    '--upgrade',
+                    '--no-cache-dir',
+                    '--no-warn-script-location',
+                    'playwright',
+                ], capture_output=False, cwd=out_dir)
+                run_shell([PYTHON_BINARY, '-m', 'playwright', 'install', 'chromium'], capture_output=False, cwd=out_dir)
+                proc = run_shell([PYTHON_BINARY, '-c', 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)'], capture_output=True, text=True, cwd=out_dir)
+                NEW_CHROME_BINARY = proc.stdout.decode().strip() if isinstance(proc.stdout, bytes) else proc.stdout.strip()
+                assert NEW_CHROME_BINARY and len(NEW_CHROME_BINARY), 'CHROME_BINARY must contain a path'
+                config(f'CHROME_BINARY={NEW_CHROME_BINARY}', set=True, out_dir=out_dir)
+            except BaseException as e:
+                stderr(f'[X] Failed to install chromium using playwright: {e.__class__.__name__} {e}', color='red')
+                raise SystemExit(1)
+
+    stderr('\n    Installing SINGLEFILE_BINARY, READABILITY_BINARY, MERCURY_BINARY automatically using npm...')
+    if USE_NODE:
+        if not NODE_VERSION:
+            stderr('[X] You must first install node using your system package manager', color='red')
+            hint([
+                'curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -',
+                'or to disable all node-based modules run: archivebox config --set USE_NODE=False',
+            ])
+            raise SystemExit(1)
+
+        if all((SINGLEFILE_VERSION, READABILITY_VERSION, MERCURY_VERSION)):
+            print('SINGLEFILE_BINARY, READABILITY_BINARY, and MERCURURY_BINARY are already installed')
+        else:
+            try:
+                # clear out old npm package locations
+                paths = (
+                    out_dir / 'package.json',
+                    out_dir / 'package_lock.json',
+                    out_dir / 'node_modules',
+                )
+                for path in paths:
+                    if path.is_dir():
+                        shutil.rmtree(path, ignore_errors=True)
+                    elif path.is_file():
+                        os.remove(path)
+
+                shutil.copyfile(PACKAGE_DIR / 'package.json', out_dir / 'package.json')
+                run_shell([
+                    'npm',
+                    'install',
+                    '--prefix', str(out_dir),
+                    '--force',
+                    '--no-save',
+                    '--no-audit',
+                    '--no-fund',
+                    '--loglevel', 'error',
+                ], capture_output=False, cwd=out_dir)
+                os.remove(out_dir / 'package.json')
+            except BaseException as e:
+                stderr(f'[X] Failed to install npm packages: {e}', color='red')
+                hint(f'Try deleting {out_dir}/node_modules and running it again')
+                raise SystemExit(1)
+
+    stderr('\n[√] Set up ArchiveBox and its dependencies successfully.', color='green')
+    
+    run_shell([ARCHIVEBOX_BINARY, '--version'], capture_output=False, cwd=out_dir)
 
 @enforce_types
 def config(config_options_str: Optional[str]=None,
@@ -924,7 +1088,7 @@ def config(config_options_str: Optional[str]=None,
             stderr()
             stderr('[X] These options failed to set (check for typos):', color='red')
             stderr('    {}'.format('\n    '.join(failed_options)))
-        raise SystemExit(bool(failed_options))
+            raise SystemExit(1)
     elif reset:
         stderr('[X] This command is not implemented yet.', color='red')
         stderr('    Please manually remove the relevant lines from your config file:')
@@ -947,13 +1111,14 @@ def schedule(add: bool=False,
              quiet: bool=False,
              every: Optional[str]=None,
              depth: int=0,
+             overwrite: bool=False,
              import_path: Optional[str]=None,
              out_dir: Path=OUTPUT_DIR):
     """Set ArchiveBox to regularly import URLs at specific times using cron"""
     
     check_data_folder(out_dir=out_dir)
 
-    (Path(out_dir) / LOGS_DIR_NAME).mkdir(exist_ok=True)
+    Path(LOGS_DIR).mkdir(exist_ok=True)
 
     cron = CronTab(user=True)
     cron = dedupe_cron_jobs(cron)
@@ -967,15 +1132,20 @@ def schedule(add: bool=False,
 
     if every or add:
         every = every or 'day'
-        quoted = lambda s: f'"{s}"' if s and ' ' in str(s) else str(s)
+        quoted = lambda s: f'"{s}"' if (s and ' ' in str(s)) else str(s)
         cmd = [
             'cd',
             quoted(out_dir),
             '&&',
             quoted(ARCHIVEBOX_BINARY),
-            *(['add', f'--depth={depth}', f'"{import_path}"'] if import_path else ['update']),
-            '>',
-            quoted(Path(LOGS_DIR) / 'archivebox.log'),
+            *([
+                'add',
+                *(['--overwrite'] if overwrite else []),
+                f'--depth={depth}',
+                f'"{import_path}"',
+            ] if import_path else ['update']),
+            '>>',
+            quoted(Path(LOGS_DIR) / 'schedule.log'),
             '2>&1',
 
         ]
@@ -990,8 +1160,8 @@ def schedule(add: bool=False,
             stderr('{red}[X] Got invalid timeperiod for cron task.{reset}'.format(**ANSI))
             stderr('    It must be one of minute/hour/day/month')
             stderr('    or a quoted cron-format schedule like:')
-            stderr('        archivebox init --every=day https://example.com/some/rss/feed.xml')
-            stderr('        archivebox init --every="0/5 * * * *" https://example.com/some/rss/feed.xml')
+            stderr('        archivebox init --every=day --depth=1 https://example.com/some/rss/feed.xml')
+            stderr('        archivebox init --every="0/5 * * * *" --depth=1 https://example.com/some/rss/feed.xml')
             raise SystemExit(1)
 
         cron = dedupe_cron_jobs(cron)
@@ -1017,7 +1187,7 @@ def schedule(add: bool=False,
         else:
             stderr('{red}[X] There are no ArchiveBox cron jobs scheduled for your user ({}).{reset}'.format(USER, **ANSI))
             stderr('    To schedule a new job, run:')
-            stderr('        archivebox schedule --every=[timeperiod] https://example.com/some/rss/feed.xml')
+            stderr('        archivebox schedule --every=[timeperiod] --depth=1 https://example.com/some/rss/feed.xml')
         raise SystemExit(0)
 
     cron = CronTab(user=True)
@@ -1027,7 +1197,7 @@ def schedule(add: bool=False,
     if foreground or run_all:
         if not existing_jobs:
             stderr('{red}[X] You must schedule some jobs first before running in foreground mode.{reset}'.format(**ANSI))
-            stderr('    archivebox schedule --every=hour https://example.com/some/rss/feed.xml')
+            stderr('    archivebox schedule --every=hour --depth=1 https://example.com/some/rss/feed.xml')
             raise SystemExit(1)
 
         print('{green}[*] Running {} ArchiveBox jobs in foreground task scheduler...{reset}'.format(len(existing_jobs), **ANSI))
@@ -1035,7 +1205,7 @@ def schedule(add: bool=False,
             try:
                 for job in existing_jobs:
                     sys.stdout.write(f'  > {job.command.split("/archivebox ")[0].split(" && ")[0]}\n')
-                    sys.stdout.write(f'    > {job.command.split("/archivebox ")[-1].split(" > ")[0]}')
+                    sys.stdout.write(f'    > {job.command.split("/archivebox ")[-1].split(" >> ")[0]}')
                     sys.stdout.flush()
                     job.run()
                     sys.stdout.write(f'\r    √ {job.command.split("/archivebox ")[-1]}\n')
@@ -1046,7 +1216,7 @@ def schedule(add: bool=False,
         if foreground:
             try:
                 for job in existing_jobs:
-                    print(f'  > {job.command.split("/archivebox ")[-1].split(" > ")[0]}')
+                    print(f'  > {job.command.split("/archivebox ")[-1].split(" >> ")[0]}')
                 for result in cron.run_scheduler():
                     print(result)
             except KeyboardInterrupt:
@@ -1059,6 +1229,7 @@ def server(runserver_args: Optional[List[str]]=None,
            reload: bool=False,
            debug: bool=False,
            init: bool=False,
+           quick_init: bool=False,
            createsuperuser: bool=False,
            out_dir: Path=OUTPUT_DIR) -> None:
     """Run the ArchiveBox HTTP server"""
@@ -1067,9 +1238,14 @@ def server(runserver_args: Optional[List[str]]=None,
     
     if init:
         run_subcommand('init', stdin=None, pwd=out_dir)
+        print()
+    elif quick_init:
+        run_subcommand('init', subcommand_args=['--quick'], stdin=None, pwd=out_dir)
+        print()
 
     if createsuperuser:
         run_subcommand('manage', subcommand_args=['createsuperuser'], pwd=out_dir)
+        print()
 
     # setup config for django runserver
     from . import config
@@ -1081,12 +1257,9 @@ def server(runserver_args: Optional[List[str]]=None,
     from django.core.management import call_command
     from django.contrib.auth.models import User
 
-    admin_user = User.objects.filter(is_superuser=True).order_by('date_joined').only('username').last()
-
     print('{green}[+] Starting ArchiveBox webserver...{reset}'.format(**ANSI))
-    if admin_user:
-        hint('The admin username is{lightblue} {}{reset}\n'.format(admin_user.username, **ANSI))
-    else:
+    print('    > Logging errors to ./logs/errors.log')
+    if not User.objects.filter(is_superuser=True).exists():
         print('{lightyellow}[!] No admin users exist yet, you will not be able to edit links in the UI.{reset}'.format(**ANSI))
         print()
         print('    To create an admin user, run:')
@@ -1104,7 +1277,6 @@ def server(runserver_args: Optional[List[str]]=None,
     config.SHOW_PROGRESS = False
     config.DEBUG = config.DEBUG or debug
 
-
     call_command("runserver", *runserver_args)
 
 

archivebox/parsers/__init__.py

@@ -11,7 +11,7 @@ import re
 from io import StringIO
 
 from typing import IO, Tuple, List, Optional
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path 
 
 from ..system import atomic_write
@@ -20,6 +20,8 @@ from ..config import (
     OUTPUT_DIR,
     SOURCES_DIR_NAME,
     TIMEOUT,
+    stderr,
+    hint,
 )
 from ..util import (
     basename,
@@ -31,36 +33,42 @@ from ..util import (
 from ..index.schema import Link
 from ..logging_util import TimedProgress, log_source_saved
 
-from .pocket_html import parse_pocket_html_export
-from .pocket_api import parse_pocket_api_export
-from .pinboard_rss import parse_pinboard_rss_export
-from .wallabag_atom import parse_wallabag_atom_export
-from .shaarli_rss import parse_shaarli_rss_export
-from .medium_rss import parse_medium_rss_export
-from .netscape_html import parse_netscape_html_export
-from .generic_rss import parse_generic_rss_export
-from .generic_json import parse_generic_json_export
-from .generic_html import parse_generic_html_export
-from .generic_txt import parse_generic_txt_export
+from . import pocket_api
+from . import wallabag_atom
+from . import pocket_html
+from . import pinboard_rss
+from . import shaarli_rss
+from . import medium_rss
 
-PARSERS = (
+from . import netscape_html
+from . import generic_rss
+from . import generic_json
+from . import generic_html
+from . import generic_txt
+from . import url_list
+
+
+PARSERS = {
     # Specialized parsers
-    ('Pocket API', parse_pocket_api_export),
-    ('Wallabag ATOM', parse_wallabag_atom_export),
-    ('Pocket HTML', parse_pocket_html_export),
-    ('Pinboard RSS', parse_pinboard_rss_export),
-    ('Shaarli RSS', parse_shaarli_rss_export),
-    ('Medium RSS', parse_medium_rss_export),
-    
-    # General parsers
-    ('Netscape HTML', parse_netscape_html_export),
-    ('Generic RSS', parse_generic_rss_export),
-    ('Generic JSON', parse_generic_json_export),
-    ('Generic HTML', parse_generic_html_export),
+    pocket_api.KEY:     (pocket_api.NAME,       pocket_api.PARSER),
+    wallabag_atom.KEY:  (wallabag_atom.NAME,    wallabag_atom.PARSER),
+    pocket_html.KEY:    (pocket_html.NAME,      pocket_html.PARSER),
+    pinboard_rss.KEY:   (pinboard_rss.NAME,     pinboard_rss.PARSER),
+    shaarli_rss.KEY:    (shaarli_rss.NAME,      shaarli_rss.PARSER),
+    medium_rss.KEY:     (medium_rss.NAME,       medium_rss.PARSER),
 
-    # Fallback parser
-    ('Plain Text', parse_generic_txt_export),
-)
+    # General parsers
+    netscape_html.KEY:  (netscape_html.NAME,    netscape_html.PARSER),
+    generic_rss.KEY:    (generic_rss.NAME,      generic_rss.PARSER),
+    generic_json.KEY:   (generic_json.NAME,     generic_json.PARSER),
+    generic_html.KEY:   (generic_html.NAME,     generic_html.PARSER),
+
+    # Catchall fallback parser
+    generic_txt.KEY:    (generic_txt.NAME,      generic_txt.PARSER),
+
+    # Explicitly specified parsers
+    url_list.KEY:       (url_list.NAME,         url_list.PARSER),
+}
 
 
 @enforce_types
@@ -68,7 +76,6 @@ def parse_links_memory(urls: List[str], root_url: Optional[str]=None):
     """
     parse a list of URLS without touching the filesystem
     """
-    check_url_parsing_invariants()
 
     timer = TimedProgress(TIMEOUT * 4)
     #urls = list(map(lambda x: x + "\n", urls))
@@ -84,16 +91,14 @@ def parse_links_memory(urls: List[str], root_url: Optional[str]=None):
     
 
 @enforce_types
-def parse_links(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Link], str]:
+def parse_links(source_file: str, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], str]:
     """parse a list of URLs with their metadata from an 
        RSS feed, bookmarks export, or text file
     """
 
-    check_url_parsing_invariants()
-
     timer = TimedProgress(TIMEOUT * 4)
     with open(source_file, 'r', encoding='utf-8') as file:
-        links, parser = run_parser_functions(file, timer, root_url=root_url)
+        links, parser = run_parser_functions(file, timer, root_url=root_url, parser=parser)
 
     timer.end()
     if parser is None:
@@ -101,15 +106,27 @@ def parse_links(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Li
     return links, parser
 
 
-def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None) -> Tuple[List[Link], Optional[str]]:
+def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None, parser: str="auto") -> Tuple[List[Link], Optional[str]]:
     most_links: List[Link] = []
     best_parser_name = None
 
-    for parser_name, parser_func in PARSERS:
+    if parser != "auto":
+        parser_name, parser_func = PARSERS[parser]
+        parsed_links = list(parser_func(to_parse, root_url=root_url))
+        if not parsed_links:
+            stderr()
+            stderr(f'[X] No links found using {parser_name} parser', color='red')
+            hint('Try a different parser or double check the input?')
+            stderr()
+        timer.end()
+        return parsed_links, parser_name
+
+    for parser_id in PARSERS:
+        parser_name, parser_func = PARSERS[parser_id]
         try:
             parsed_links = list(parser_func(to_parse, root_url=root_url))
             if not parsed_links:
-                raise Exception('no links found')
+                raise Exception(f'No links found using {parser_name} parser')
 
             # print(f'[√] Parser {parser_name} succeeded: {len(parsed_links)} links parsed')
             if len(parsed_links) > len(most_links):
@@ -118,8 +135,8 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None)
                 
         except Exception as err:                                                # noqa
             # Parsers are tried one by one down the list, and the first one
-            # that succeeds is used. To see why a certain parser was not used
-            # due to error or format incompatibility, uncomment this line:
+            # that succeeds is used. To debug why a certain parser was not used
+            # due to python error or format incompatibility, uncomment this line:
             
             # print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err))
             # raise
@@ -130,7 +147,7 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None)
 
 @enforce_types
 def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir: Path=OUTPUT_DIR) -> str:
-    ts = str(datetime.now().timestamp()).split('.', 1)[0]
+    ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
     source_path = str(out_dir / SOURCES_DIR_NAME / filename.format(ts=ts))
     atomic_write(source_path, raw_text)
     log_source_saved(source_file=source_path)
@@ -140,7 +157,7 @@ def save_text_as_source(raw_text: str, filename: str='{ts}-stdin.txt', out_dir:
 @enforce_types
 def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{basename}.txt', out_dir: Path=OUTPUT_DIR) -> str:
     """download a given url's content into output/sources/domain-<timestamp>.txt"""
-    ts = str(datetime.now().timestamp()).split('.', 1)[0]
+    ts = str(datetime.now(timezone.utc).timestamp()).split('.', 1)[0]
     source_path = str(OUTPUT_DIR / SOURCES_DIR_NAME / filename.format(basename=basename(path), ts=ts))
 
     if any(path.startswith(s) for s in ('http://', 'https://', 'ftp://')):
@@ -173,31 +190,48 @@ def save_file_as_source(path: str, timeout: int=TIMEOUT, filename: str='{ts}-{ba
     return source_path
 
 
-def check_url_parsing_invariants() -> None:
-    """Check that plain text regex URL parsing works as expected"""
-
-    # this is last-line-of-defense to make sure the URL_REGEX isn't
-    # misbehaving, as the consequences could be disastrous and lead to many
-    # incorrect/badly parsed links being added to the archive
-
-    test_urls = '''
-    https://example1.com/what/is/happening.html?what=1#how-about-this=1
-    https://example2.com/what/is/happening/?what=1#how-about-this=1
-    HTtpS://example3.com/what/is/happening/?what=1#how-about-this=1f
-    https://example4.com/what/is/happening.html
-    https://example5.com/
-    https://example6.com
-
-    <test>http://example7.com</test>
-    [https://example8.com/what/is/this.php?what=1]
-    [and http://example9.com?what=1&other=3#and-thing=2]
-    <what>https://example10.com#and-thing=2 "</about>
-    abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def
-    sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi
-    example13.bada
-    and example14.badb
-    <or>htt://example15.badc</that>
-    '''
-    # print('\n'.join(re.findall(URL_REGEX, test_urls)))
-    assert len(re.findall(URL_REGEX, test_urls)) == 12
-
+# Check that plain text regex URL parsing works as expected
+#   this is last-line-of-defense to make sure the URL_REGEX isn't
+#   misbehaving due to some OS-level or environment level quirks (e.g. bad regex lib)
+#   the consequences of bad URL parsing could be disastrous and lead to many
+#   incorrect/badly parsed links being added to the archive, so this is worth the cost of checking
+_test_url_strs = {
+    'example.com': 0,
+    '/example.com': 0,
+    '//example.com': 0,
+    ':/example.com': 0,
+    '://example.com': 0,
+    'htt://example8.com': 0,
+    '/htt://example.com': 0,
+    'https://example': 1,
+    'https://localhost/2345': 1,
+    'https://localhost:1234/123': 1,
+    '://': 0,
+    'https://': 0,
+    'http://': 0,
+    'ftp://': 0,
+    'ftp://example.com': 0,
+    'https://example.com': 1,
+    'https://example.com/': 1,
+    'https://a.example.com': 1,
+    'https://a.example.com/': 1,
+    'https://a.example.com/what/is/happening.html': 1,
+    'https://a.example.com/what/ís/happening.html': 1,
+    'https://a.example.com/what/is/happening.html?what=1&2%20b#höw-about-this=1a': 1,
+    'https://a.example.com/what/is/happéning/?what=1&2%20b#how-aboüt-this=1a': 1,
+    'HTtpS://a.example.com/what/is/happening/?what=1&2%20b#how-about-this=1af&2f%20b': 1,
+    'https://example.com/?what=1#how-about-this=1&2%20baf': 1,
+    'https://example.com?what=1#how-about-this=1&2%20baf': 1,
+    '<test>http://example7.com</test>': 1,
+    '[https://example8.com/what/is/this.php?what=1]': 1,
+    '[and http://example9.com?what=1&other=3#and-thing=2]': 1,
+    '<what>https://example10.com#and-thing=2 "</about>': 1,
+    'abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def': 1,
+    'sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi': 1,
+    '<or>http://examplehttp://15.badc</that>': 2,
+    'https://a.example.com/one.html?url=http://example.com/inside/of/another?=http://': 2,
+    '[https://a.example.com/one.html?url=http://example.com/inside/of/another?=](http://a.example.com)': 3,
+}
+for url_str, num_urls in _test_url_strs.items():
+    assert len(re.findall(URL_REGEX, url_str)) == num_urls, (
+        f'{url_str} does not contain {num_urls} urls')

archivebox/parsers/generic_html.py

@@ -4,7 +4,7 @@ __package__ = 'archivebox.parsers'
 import re
 
 from typing import IO, Iterable, Optional
-from datetime import datetime
+from datetime import datetime, timezone
 
 from ..index.schema import Link
 from ..util import (
@@ -46,8 +46,13 @@ def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None,
             for archivable_url in re.findall(URL_REGEX, url):
                 yield Link(
                     url=htmldecode(archivable_url),
-                    timestamp=str(datetime.now().timestamp()),
+                    timestamp=str(datetime.now(timezone.utc).timestamp()),
                     title=None,
                     tags=None,
                     sources=[html_file.name],
                 )
+
+
+KEY = 'html'
+NAME = 'Generic HTML'
+PARSER = parse_generic_html_export

archivebox/parsers/generic_json.py

@@ -3,7 +3,7 @@ __package__ = 'archivebox.parsers'
 import json
 
 from typing import IO, Iterable
-from datetime import datetime
+from datetime import datetime, timezone
 
 from ..index.schema import Link
 from ..util import (
@@ -30,7 +30,7 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
                 raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]')
 
             # Parse the timestamp
-            ts_str = str(datetime.now().timestamp())
+            ts_str = str(datetime.now(timezone.utc).timestamp())
             if link.get('timestamp'):
                 # chrome/ff histories use a very precise timestamp
                 ts_str = str(link['timestamp'] / 10000000)  
@@ -63,3 +63,8 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
                 tags=htmldecode(link.get('tags')) or '',
                 sources=[json_file.name],
             )
+
+
+KEY = 'json'
+NAME = 'Generic JSON'
+PARSER = parse_generic_json_export

archivebox/parsers/generic_rss.py

@@ -47,3 +47,8 @@ def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags=None,
             sources=[rss_file.name],
         )
+
+
+KEY = 'rss'
+NAME = 'Generic RSS'
+PARSER = parse_generic_rss_export

archivebox/parsers/generic_txt.py

@@ -4,7 +4,7 @@ __description__ = 'Plain Text'
 import re
 
 from typing import IO, Iterable
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 from ..index.schema import Link
@@ -17,7 +17,7 @@ from ..util import (
 
 @enforce_types
 def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
-    """Parse raw links from each line in a text file"""
+    """Parse links from a text file, ignoring other text"""
 
     text_file.seek(0)
     for line in text_file.readlines():
@@ -29,7 +29,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
             if Path(line).exists():
                 yield Link(
                     url=line,
-                    timestamp=str(datetime.now().timestamp()),
+                    timestamp=str(datetime.now(timezone.utc).timestamp()),
                     title=None,
                     tags=None,
                     sources=[text_file.name],
@@ -42,7 +42,7 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
         for url in re.findall(URL_REGEX, line):
             yield Link(
                 url=htmldecode(url),
-                timestamp=str(datetime.now().timestamp()),
+                timestamp=str(datetime.now(timezone.utc).timestamp()),
                 title=None,
                 tags=None,
                 sources=[text_file.name],
@@ -54,8 +54,12 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
             for sub_url in re.findall(URL_REGEX, line[1:]):
                 yield Link(
                     url=htmldecode(sub_url),
-                    timestamp=str(datetime.now().timestamp()),
+                    timestamp=str(datetime.now(timezone.utc).timestamp()),
                     title=None,
                     tags=None,
                     sources=[text_file.name],
                 )
+
+KEY = 'txt'
+NAME = 'Generic TXT'
+PARSER = parse_generic_txt_export

archivebox/parsers/medium_rss.py

@@ -33,3 +33,8 @@ def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags=None,
             sources=[rss_file.name],
         )
+
+
+KEY = 'medium_rss'
+NAME = 'Medium RSS'
+PARSER = parse_medium_rss_export

archivebox/parsers/netscape_html.py

@@ -37,3 +37,7 @@ def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
                 sources=[html_file.name],
             )
 
+
+KEY = 'netscape_html'
+NAME = 'Netscape HTML'
+PARSER = parse_netscape_html_export

archivebox/parsers/pinboard_rss.py

@@ -2,7 +2,7 @@ __package__ = 'archivebox.parsers'
 
 
 from typing import IO, Iterable
-from datetime import datetime
+from datetime import datetime, timezone
 
 from xml.etree import ElementTree
 
@@ -36,7 +36,7 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         if ts_str:
             time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
         else:
-            time = datetime.now()
+            time = datetime.now(timezone.utc)
 
         yield Link(
             url=htmldecode(url),
@@ -45,3 +45,8 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags=htmldecode(tags) or None,
             sources=[rss_file.name],
         )
+
+
+KEY = 'pinboard_rss'
+NAME = 'Pinboard RSS'
+PARSER = parse_pinboard_rss_export

archivebox/parsers/pocket_api.py

@@ -111,3 +111,8 @@ def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
                 yield link_from_article(article, sources=[line])
     
             write_since(username, api.last_since)
+
+
+KEY = 'pocket_api'
+NAME = 'Pocket API'
+PARSER = parse_pocket_api_export

archivebox/parsers/pocket_html.py

@@ -36,3 +36,8 @@ def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
                 tags=tags or '',
                 sources=[html_file.name],
             )
+
+
+KEY = 'pocket_html'
+NAME = 'Pocket HTML'
+PARSER = parse_pocket_html_export

archivebox/parsers/shaarli_rss.py

@@ -48,3 +48,8 @@ def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags=None,
             sources=[rss_file.name],
         )
+
+
+KEY = 'shaarli_rss'
+NAME = 'Shaarli RSS'
+PARSER = parse_shaarli_rss_export

archivebox/parsers/url_list.py

@@ -0,0 +1,37 @@
+__package__ = 'archivebox.parsers'
+__description__ = 'URL list'
+
+import re
+
+from typing import IO, Iterable
+from datetime import datetime, timezone
+
+from ..index.schema import Link
+from ..util import (
+    enforce_types,
+    URL_REGEX,
+)
+
+
+@enforce_types
+def parse_url_list(text_file: IO[str], **_kwargs) -> Iterable[Link]:
+    """Parse raw URLs from each line in a text file"""
+
+    text_file.seek(0)
+    for line in text_file.readlines():
+        url = line.strip()
+        if (not url) or not re.findall(URL_REGEX, url):
+            continue
+
+        yield Link(
+            url=url,
+            timestamp=str(datetime.now(timezone.utc).timestamp()),
+            title=None,
+            tags=None,
+            sources=[text_file.name],
+        )
+
+
+KEY = 'url_list'
+NAME = 'URL List'
+PARSER = parse_url_list

archivebox/parsers/wallabag_atom.py

@@ -55,3 +55,8 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags=tags or '',
             sources=[rss_file.name],
         )
+
+
+KEY = 'wallabag_atom'
+NAME = 'Wallabag Atom'
+PARSER = parse_wallabag_atom_export

archivebox/search/backends/ripgrep.py

@@ -2,7 +2,7 @@ import re
 from subprocess import run, PIPE
 from typing import List, Generator
 
-from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION
+from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION, SEARCH_BACKEND_TIMEOUT
 from archivebox.util import enforce_types
 
 RG_IGNORE_EXTENSIONS = ('css','js','orig','svg')
@@ -32,7 +32,7 @@ def search(text: str) -> List[str]:
     from core.models import Snapshot
 
     rg_cmd = ['rg', RG_ADD_TYPE, RG_IGNORE_ARGUMENTS, RG_DEFAULT_ARGUMENTS, RG_REGEX_ARGUMENT, text, str(ARCHIVE_DIR)]
-    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=60)
+    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=SEARCH_BACKEND_TIMEOUT)
     file_paths = [p.decode() for p in rg.stdout.splitlines()]
     timestamps = set()
     for path in file_paths:

archivebox/search/backends/sonic.py

@@ -5,15 +5,31 @@ from sonic import IngestClient, SearchClient
 from archivebox.util import enforce_types
 from archivebox.config import SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD, SONIC_BUCKET, SONIC_COLLECTION
 
-MAX_SONIC_TEXT_LENGTH = 2000
+MAX_SONIC_TEXT_TOTAL_LENGTH = 100000000     # dont index more than 100 million characters per text
+MAX_SONIC_TEXT_CHUNK_LENGTH = 2000          # dont index more than 2000 characters per chunk
+MAX_SONIC_ERRORS_BEFORE_ABORT = 5
 
 @enforce_types
 def index(snapshot_id: str, texts: List[str]):
+    error_count = 0
     with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
         for text in texts:
-            chunks = [text[i:i+MAX_SONIC_TEXT_LENGTH] for i in range(0, len(text), MAX_SONIC_TEXT_LENGTH)]
-            for chunk in chunks:
-                ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))
+            chunks = (
+                text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
+                for i in range(
+                    0,
+                    min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
+                    MAX_SONIC_TEXT_CHUNK_LENGTH,
+                )
+            )
+            try:
+                for chunk in chunks:
+                    ingestcl.push(SONIC_COLLECTION, SONIC_BUCKET, snapshot_id, str(chunk))
+            except Exception as err:
+                print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}')
+                error_count += 1
+                if error_count > MAX_SONIC_ERRORS_BEFORE_ABORT:
+                    raise
 
 @enforce_types
 def search(text: str) -> List[str]:

archivebox/search/utils.py

@@ -16,7 +16,7 @@ def get_file_result_content(res, extra_path, use_pwd=False):
     if extra_path:
         fpath = f'{fpath}/{extra_path}'
 
-    with open(fpath, 'r') as file:
+    with open(fpath, 'r', encoding='utf-8') as file:
         data = file.read()
     if data:
         return [data]
@@ -36,10 +36,10 @@ def get_indexable_content(results: QuerySet):
 
     # TODO: banish this duplication and get these from the extractor file
     if method == 'readability':
-        return get_file_result_content(res, 'content.txt')
+        return get_file_result_content(res, 'content.txt', use_pwd=True)
     elif method == 'singlefile':
-        return get_file_result_content(res,'',use_pwd=True)
+        return get_file_result_content(res, '', use_pwd=True)
     elif method == 'dom':
-        return get_file_result_content(res,'',use_pwd=True)
+        return get_file_result_content(res, '', use_pwd=True)
     elif method == 'wget':
-        return get_file_result_content(res,'',use_pwd=True)
+        return get_file_result_content(res, '', use_pwd=True)

archivebox/system.py

@@ -2,34 +2,76 @@ __package__ = 'archivebox'
 
 
 import os
+import signal
 import shutil
 
 from json import dump
 from pathlib import Path
 from typing import Optional, Union, Set, Tuple
-from subprocess import run as subprocess_run
+from subprocess import _mswindows, PIPE, Popen, CalledProcessError, CompletedProcess, TimeoutExpired
 
 from crontab import CronTab
-from atomicwrites import atomic_write as lib_atomic_write
+from .vendor.atomicwrites import atomic_write as lib_atomic_write
 
 from .util import enforce_types, ExtendedEncoder
 from .config import OUTPUT_PERMISSIONS
 
 
 
-def run(*args, input=None, capture_output=True, text=False, **kwargs):
-    """Patched of subprocess.run to fix blocking io making timeout=innefective"""
+def run(*args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs):
+    """Patched of subprocess.run to kill forked child subprocesses and fix blocking io making timeout=innefective
+        Mostly copied from https://github.com/python/cpython/blob/master/Lib/subprocess.py
+    """
 
     if input is not None:
-        if 'stdin' in kwargs:
+        if kwargs.get('stdin') is not None:
             raise ValueError('stdin and input arguments may not both be used.')
+        kwargs['stdin'] = PIPE
 
     if capture_output:
         if ('stdout' in kwargs) or ('stderr' in kwargs):
             raise ValueError('stdout and stderr arguments may not be used '
                              'with capture_output.')
+        kwargs['stdout'] = PIPE
+        kwargs['stderr'] = PIPE
 
-    return subprocess_run(*args, input=input, capture_output=capture_output, text=text, **kwargs)
+    pgid = None
+    try:
+        with Popen(*args, start_new_session=start_new_session, **kwargs) as process:
+            pgid = os.getpgid(process.pid)
+            try:
+                stdout, stderr = process.communicate(input, timeout=timeout)
+            except TimeoutExpired as exc:
+                process.kill()
+                if _mswindows:
+                    # Windows accumulates the output in a single blocking
+                    # read() call run on child threads, with the timeout
+                    # being done in a join() on those threads.  communicate()
+                    # _after_ kill() is required to collect that and add it
+                    # to the exception.
+                    exc.stdout, exc.stderr = process.communicate()
+                else:
+                    # POSIX _communicate already populated the output so
+                    # far into the TimeoutExpired exception.
+                    process.wait()
+                raise
+            except:  # Including KeyboardInterrupt, communicate handled that.
+                process.kill()
+                # We don't call process.wait() as .__exit__ does that for us.
+                raise
+
+            retcode = process.poll()
+            if check and retcode:
+                raise CalledProcessError(retcode, process.args,
+                                         output=stdout, stderr=stderr)
+    finally:
+        # force kill any straggler subprocesses that were forked from the main proc
+        try:
+            os.killpg(pgid, signal.SIGINT)
+        except Exception:
+            pass
+
+    return CompletedProcess(process.args, retcode, stdout, stderr)
 
 
 @enforce_types
@@ -37,18 +79,19 @@ def atomic_write(path: Union[Path, str], contents: Union[dict, str, bytes], over
     """Safe atomic write to filesystem by writing to temp file + atomic rename"""
 
     mode = 'wb+' if isinstance(contents, bytes) else 'w'
+    encoding = None if isinstance(contents, bytes) else 'utf-8'  # enforce utf-8 on all text writes
 
     # print('\n> Atomic Write:', mode, path, len(contents), f'overwrite={overwrite}')
     try:
-        with lib_atomic_write(path, mode=mode, overwrite=overwrite) as f:
+        with lib_atomic_write(path, mode=mode, overwrite=overwrite, encoding=encoding) as f:
             if isinstance(contents, dict):
                 dump(contents, f, indent=4, sort_keys=True, cls=ExtendedEncoder)
             elif isinstance(contents, (bytes, str)):
                 f.write(contents)
     except OSError as e:
         print(f"[X] OSError: Failed to write {path} with fcntl.F_FULLFSYNC. ({e})")
-        print("    For data integrity, ArchiveBox requires a filesystem that supports atomic writes.")
-        print("    Filesystems and network drives that don't implement FSYNC are incompatible and require workarounds.")
+        print("    You can store the archive/ subfolder on a hard drive or network share that doesn't support support syncronous writes,")
+        print("    but the main folder containing the index.sqlite3 and ArchiveBox.conf files must be on a filesystem that supports FSYNC.")
         raise SystemExit(1)
     os.chmod(path, int(OUTPUT_PERMISSIONS, base=8))
 

archivebox/templates/admin/actions_as_select.html

@@ -1 +0,0 @@
-actions_as_select

archivebox/templates/admin/base.html

@@ -1,246 +1,231 @@
-{% load i18n static %}<!DOCTYPE html>
-{% get_current_language as LANGUAGE_CODE %}{% get_current_language_bidi as LANGUAGE_BIDI %}
+{% load i18n static tz %}
+{% get_current_language as LANGUAGE_CODE %}
+{% get_current_language_bidi as LANGUAGE_BIDI %}
+
+<!DOCTYPE html>
 <html lang="{{ LANGUAGE_CODE|default:"en-us" }}" {% if LANGUAGE_BIDI %}dir="rtl"{% endif %}>
-<head>
-<title>{% block title %}{% endblock %} | ArchiveBox</title>
-<link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
-{% block extrastyle %}{% endblock %}
-{% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">{% endif %}
-{% block extrahead %}{% endblock %}
-{% block responsive %}
-    <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0">
-    <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}">
-    {% if LANGUAGE_BIDI %}<link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">{% endif %}
-{% endblock %}
-{% block blockbots %}<meta name="robots" content="NONE,NOARCHIVE">{% endblock %}
-<link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
-</head>
-{% load i18n %}
-
-<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}"
-  data-admin-utc-offset="{% now "Z" %}">
-
-  <style nonce="{{nonce}}">
-      /* Loading Progress Bar */
-        #progress {
-            position: absolute;
-            z-index: 1000;
-            top: 0px;
-            left: -6px;
-            width: 2%;
-            opacity: 1;
-            height: 2px;
-            background: #1a1a1a;
-            border-radius: 1px;
-            transition: width 4s ease-out, opacity 400ms linear;
-        }
-  
-        @-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
-  </style>
-
-  <script>
-        // Page Loading Bar
-        window.loadStart = function(distance) {
-            var distance = distance || 0;
-            // only add progrstess bar if not already present
-            if (django.jQuery("#loading-bar").length == 0) {
-                django.jQuery("body").add("<div id=\"loading-bar\"></div>");
-            }
-            if (django.jQuery("#progress").length === 0) {
-                django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
-                let last_distance = (distance || (30 + (Math.random() * 30)))
-                django.jQuery("#progress").width(last_distance + "%");
-                setInterval(function() {
-                    last_distance += Math.random()
-                    django.jQuery("#progress").width(last_distance + "%");
-                }, 1000)
-            }
-        };
-  
-        window.loadFinish = function() {
-            django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
-                django.jQuery(this).remove();
-            });
-        };
-        window.loadStart();
-        window.addEventListener('beforeunload', function() {window.loadStart(27)});
-        document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
-  </script>
-
-
-<!-- Container -->
-<div id="container">
-
-    {% if not is_popup %}
-    <!-- Header -->
-    <div id="header">
-        <div id="branding">
-            <h1 id="site-name">
-                <a href="{% url 'Home' %}">
-                    <img src="{% static 'archive.png' %}" id="logo">
-                    ArchiveBox
-                </a>
-            </h1>
-                
-        </div>
-        {% block usertools %}
-        {% if has_permission %}
-        <div id="user-tools">
-            <a href="{% url 'admin:Add' %}">Add ➕</a> /
-            <a href="{% url 'Home' %}">Snapshots</a> /
-            <a href="/admin/core/tag/">Tags</a> /
-            <a href="/admin/auth/user/">Users</a> /
-            <a href="{% url 'Docs' %}">Docs</a>
-             &nbsp; &nbsp;
-            {% block welcome-msg %}
-                {% trans 'User' %}
-                <strong>{% firstof user.get_short_name user.get_username %}</strong> &nbsp; &nbsp;
-            {% endblock %}
-            {% block userlinks %}
-                {% if user.is_active and user.is_staff %}
-                    {% url 'django-admindocs-docroot' as docsroot %}
-                    {% if docsroot %}
-                        <a href="{{ docsroot }}">{% trans 'Documentation' %}</a> /
-                    {% endif %}
-                {% endif %}
-                {% if user.has_usable_password %}
-                <a href="{% url 'admin:password_change' %}">{% trans 'Change password' %}</a> /
-                {% endif %}
-                <a href="{% url 'admin:logout' %}">{% trans 'Log out' %}</a>
-                |
-                <a> <span id="snapshotListView" style="cursor: pointer">☰</span> </a>       
-                <a> <span id="snapshotGridView"style="letter-spacing: -.4em; cursor: pointer;">⣿⣿</span></a> 
-            {% endblock %}
-        </div>
+    <head>
+        <title>{% block title %}Home{% endblock %} | ArchiveBox</title>
+        
+        {% block blockbots %}
+            <meta name="robots" content="NONE,NOARCHIVE">
+        {% endblock %}
+        
+        <link rel="stylesheet" type="text/css" href="{% block stylesheet %}{% static "admin/css/base.css" %}{% endblock %}">
+        {% block extrastyle %}{% endblock %}
+        
+        {% if LANGUAGE_BIDI %}
+            <link rel="stylesheet" type="text/css" href="{% block stylesheet_rtl %}{% static "admin/css/rtl.css" %}{% endblock %}">
         {% endif %}
+        
+        {% block responsive %}
+            <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1.0, maximum-scale=1.0">
+            <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive.css" %}">
+            {% if LANGUAGE_BIDI %}
+                <link rel="stylesheet" type="text/css" href="{% static "admin/css/responsive_rtl.css" %}">
+            {% endif %}
         {% endblock %}
-        {% block nav-global %}{% endblock %}
-    </div>
-        <!-- END Header -->
-        {% block breadcrumbs %}
-        <div class="breadcrumbs">
-        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-        {% if title %} &rsaquo; {{ title }}{% endif %}
+        
+        <link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
+        
+        <script>
+            function selectSnapshotListView(e) {
+                e && e.stopPropagation()
+                e && e.preventDefault()
+                console.log('Switching to Snapshot list view...')
+                localStorage.setItem('preferred_snapshot_view_mode', 'list')
+                window.location = "{% url 'admin:core_snapshot_changelist' %}" + document.location.search
+                return false
+            }
+            function selectSnapshotGridView(e) {
+                e && e.stopPropagation()
+                e && e.preventDefault()
+                console.log('Switching to Snapshot grid view...')
+                localStorage.setItem('preferred_snapshot_view_mode', 'grid')
+                window.location = "{% url 'admin:grid' %}" + document.location.search
+                return false
+            }
+
+            const preferred_view = localStorage.getItem('preferred_snapshot_view_mode') || 'unset'
+            const current_view = (
+                window.location.pathname === "{% url 'admin:core_snapshot_changelist' %}"
+                    ? 'list'
+                    : 'grid')
+            console.log('Preferred snapshot view is:', preferred_view, 'Current view mode is:', current_view)
+            
+            if (preferred_view === 'grid' && current_view !== 'grid') {  
+                selectSnapshotGridView()
+            }
+        </script>
+
+        {% block extrahead %}{% endblock %}
+    </head>
+
+
+    <body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" data-admin-utc-offset="{% now "Z" %}">
+        {% include 'progressbar.html' %}
+
+        <div id="container">
+            {% if not is_popup %}
+                <div id="header">
+                    <div id="branding">
+                        <h1 id="site-name">
+                            <a href="{% url 'Home' %}">
+                                <img src="{% static 'archive.png' %}" id="logo">
+                                ArchiveBox
+                            </a>
+                        </h1>
+                    </div>
+                    {% block usertools %}
+                        {% if has_permission %}
+                            {% include 'navigation.html' %}
+                        {% endif %}
+                    {% endblock %}
+                    {% block nav-global %}{% endblock %}
+                </div>
+
+                {% block breadcrumbs %}
+                    <div class="breadcrumbs">
+                        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+                        {% if title %} &rsaquo; {{ title }}{% endif %}
+                    </div>
+                {% endblock %}
+            {% endif %}
+
+            {% block messages %}
+                {% if messages %}
+                    <ul class="messagelist">
+                        {% for message in messages %}
+                            <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li>
+                        {% endfor %}
+                    </ul>
+                {% endif %}
+            {% endblock messages %}
+
+            <div id="content" class="{% block coltype %}colM{% endblock %}">
+                {% if opts.model_name == 'snapshot' and cl %}
+                    <small id="snapshot-view-mode">
+                        <a href="#list" title="List view" id="snapshot-view-list">☰</a> | 
+                        <a href="#grid" title="Grid view" id="snapshot-view-grid" style="letter-spacing: -.4em;">⣿⣿</a> 
+                    </small>
+                {% endif %}
+                {% block pretitle %}{% endblock %}
+                {% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %}
+                {% block content %}
+                    {% block object-tools %}{% endblock %}
+                    {{ content }}
+                {% endblock %}
+                {% block sidebar %}{% endblock %}
+                <br class="clear">
+            </div>
+
+            {% block footer %}<div id="footer"></div>{% endblock %}
         </div>
-        {% endblock %}
-    {% endif %}
 
-    {% block messages %}
-        {% if messages %}
-        <ul class="messagelist">{% for message in messages %}
-          <li{% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message|capfirst }}</li>
-        {% endfor %}</ul>
-        {% endif %}
-    {% endblock messages %}
+        <script>
+            $ = django.jQuery;
+            $.fn.reverse = [].reverse;
 
-    <!-- Content -->
-    <div id="content" class="{% block coltype %}colM{% endblock %}">
-        {% block pretitle %}{% endblock %}
-        {% block content_title %}{# {% if title %}<h1>{{ title }}</h1>{% endif %} #}{% endblock %}
-        {% block content %}
-            {% block object-tools %}{% endblock %}
-            {{ content }}
-        {% endblock %}
-        {% block sidebar %}{% endblock %}
-        <br class="clear">
-    </div>
-    <!-- END Content -->
+            // hide images that fail to load
+            document.querySelector('body').addEventListener('error', function (e) {
+                e.target.style.opacity = 0;
+            }, true)
 
-    {% block footer %}<div id="footer"></div>{% endblock %}
-</div>
-<!-- END Container -->
+            // setup timezone
+            {% get_current_timezone as TIME_ZONE %}
+            window.TIME_ZONE = '{{TIME_ZONE}}'
 
-<script>
-    (function ($) {
-        $.fn.reverse = [].reverse;
+            window.setCookie = function(name, value, days) {
+                let expires = ""
+                if (days) {
+                    const date = new Date()
+                    date.setTime(date.getTime() + (days*24*60*60*1000))
+                    expires = "; expires=" + date.toUTCString()
+                }
+                document.cookie = name + "=" + (value || "")  + expires + "; path=/"
+            }
 
-        function fix_actions() {
-            var container = $('div.actions');
+            function setTimeOffset() {
+                if (window.GMT_OFFSET) return
+                window.GMT_OFFSET = -(new Date).getTimezoneOffset()
+                window.setCookie('GMT_OFFSET', window.GMT_OFFSET, 365)
+            }
 
-            if (container.find('option').length < 10) {
-                container.find('label, button').hide();
+            // change the admin actions button from a dropdown to buttons across
+            function fix_actions() {
+                const container = $('div.actions')
 
-                var buttons = $('<div></div>')
-                    .prependTo(container)
+                // too many actions to turn into buttons
+                if (container.find('select[name=action] option').length >= 11) return
+
+                // hide the empty default option thats just a placeholder with no value
+                container.find('label:nth-child(1), button[value=0]').hide()
+
+                const buttons = $('<div></div>')
+                    .insertAfter('div.actions button[type=submit]')
                     .css('display', 'inline')
                     .addClass('class', 'action-buttons');
 
-                container.find('option:gt(0)').reverse().each(function () {
-                    const name = this.value
+                // for each action in the dropdown, turn it into a button instead
+                container.find('select[name=action] option:gt(0)').each(function () {
+                    const action_type = this.value
                     $('<button>')
-                        .appendTo(buttons)
-                        .attr('name', this.value)
+                        .attr('type', 'button')
+                        .attr('name', action_type)
                         .addClass('button')
                         .text(this.text)
-                        .click(function () {
-                            container.find('select')
-                                .find(':selected').attr('selected', '').end()
-                                .find('[value=' + this.name + ']').attr('selected', 'selected');
-                            $('#changelist-form button[name="index"]').click();
+                        .click(function (e) {
+                            e.preventDefault()
+                            e.stopPropagation()
+
+                            const num_selected = document.querySelector('.action-counter').innerText.split(' ')[0]
+
+                            if (action_type === 'overwrite_snapshots') {
+                                const message = (
+                                    'Are you sure you want to re-archive (overwrite) ' + num_selected + ' Snapshots?\n\n' +
+                                    'This will delete all previously saved files from these Snapshots and re-archive them from scratch.\n\n'
+                                )
+                                if (!window.confirm(message)) return false
+                            }
+                            if (action_type === 'delete_snapshots') {
+                                const message = (
+                                    'Are you sure you want to permanently delete ' + num_selected + ' Snapshots?\n\n' +
+                                    'They will be removed from your index, and all their Snapshot content on disk will be permanently deleted.'
+                                )
+                                if (!window.confirm(message)) return false
+                            }
+
+                            // select the action button from the dropdown
+                            container.find('select[name=action]')
+                                .find('op:selected').removeAttr('selected').end()
+                                .find('[value=' + action_type + ']').attr('selected', 'selected').click()
+                            
+                            // click submit & replace the archivebox logo with a spinner
+                            $('#changelist-form button[name="index"]').click()
                             document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
-                        });
-                });
-            }
-        };
-        
-        function redirectWithQuery(uri){
-            uri_query = uri + document.location.search;
-            window.location = uri_query;
-
-        };
-
-        function selectSnapshotListView(){
-            localStorage.setItem('currentSnapshotView', 'List');
-            redirectWithQuery("{% url 'admin:core_snapshot_changelist' %}");
-        };
-
-        function selectSnapshotGridView(){
-            localStorage.setItem('currentSnapshotView', 'Grid');
-            redirectWithQuery("{% url 'admin:grid' %}");
-        };
-        
-        function setPreferredSnapshotView(view){
-            urlPath = window.location.pathname;
-
-            if((view==="Grid") && urlPath == "{% url 'admin:core_snapshot_changelist' %}"){  
-                selectSnapshotGridView();
+                            return false
+                        })
+                        .appendTo(buttons)
+                })
+                console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
             }
             
-            {% comment %} 
-                else if((view==="List") && urlPath == "{% url 'admin:grid' %}"){
-                selectSnapshotListView();
-            
-            } 
-            {% endcomment %}
-        };
+            function setupSnapshotGridListToggle() {
+                $("#snapshot-view-list").click(selectSnapshotListView)
+                $("#snapshot-view-grid").click(selectSnapshotGridView)
 
-        function setupSnapshotViews() {
-            const preferredSnapshotView = localStorage.getItem('currentSnapshotView');
-            setPreferredSnapshotView(preferredSnapshotView);
-
-            $( document ).ready(function() {
-                
-                $("#snapshotListView").click(function() {
-                    selectSnapshotListView();
-                });  
-                $("#snapshotGridView").click(function() {
-                    selectSnapshotGridView();
-                });  
-
-                $('input:checkbox').change(function(){
-                    if($(this).is(':checked')) 
-                        $(this).parent().parent().parent().parent().addClass('selected-card'); 
+                $('#changelist-form .card input:checkbox').change(function() {
+                    if ($(this).is(':checked'))
+                        $(this).parents('.card').addClass('selected-card')
                     else 
-                        $(this).parent().parent().parent().parent().removeClass('selected-card')
-                });
-
-            });
-        };
-        $(function () {
-            fix_actions();
-            setupSnapshotViews();
-        });
-    })(django.jQuery);
-</script>
-</body>
+                        $(this).parents('.card').removeClass('selected-card')
+                })
+            };
+            $(document).ready(function() {
+                fix_actions()
+                setupSnapshotGridListToggle()
+                setTimeOffset()
+            })
+        </script>
+    </body>
 </html>

archivebox/templates/admin/snapshots_grid.html

@@ -3,160 +3,188 @@
 
 {% block extrastyle %}
 <style>
-* {
-  -webkit-box-sizing: border-box;
-    -moz-box-sizing: border-box;
-    box-sizing: border-box;
-}
+  #changelist-search #searchbar {
+    min-height: 24px;
+  }
+  .cards {
+      padding-top: 10px;
+      display: grid;
+      grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */
+      grid-auto-rows: minmax(200px, auto);
+      grid-gap: 14px 14px;
+  }
 
-a {
-  text-decoration: none;
-  color: orange;
-}
-h2 {
-  color: #000;
-  margin: 2rem 0 .5rem;
-  font-size: 1.25rem;
-  font-weight: 400;
-  {% comment %} text-transform: uppercase; {% endcomment %}
-}
-
-card.img {
-  display: block;
-    border: 0;
-    width: 100%;
-    height: auto;
-}
-
-/***************************           Cards            *******************************/
-
-.cards {
-  display: grid;
-  grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); /* see notes below */
-  grid-auto-rows: minmax(200px, auto);
-  grid-gap: 1rem;
-}
-
-.card {
-  /*height: 200px;*/
-  /*background: red;*/
-  border: 2px solid #e7e7e7;
-  border-radius: 4px;
-  -webkit-box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
-  box-shadow: 0 2px 2px rgba(0, 0, 0, 0.15);
-  display: flex;
-  /* -webkit-box-orient: vertical; */
-  /* -webkit-box-direction: normal; */
-  -ms-flex-direction: column;
-  flex-direction: column;
-  position: relative;
-  color: #5d5e5e;
-} /* li item */
-
-.thumbnail img {
-    height: 100%;
-    box-sizing: border-box;
-    max-width: 100%;
-    max-height: 100%;
-    width: 100%;
-}
-
-.card-content {
-    font-size: .75rem;
-    padding: .5rem;
-    display: flex;
-    -webkit-box-orient: vertical;
-    -webkit-box-direction: normal;
-    -ms-flex-direction: column;
-    flex-direction: column;
-    -webkit-box-flex: 1;
-    -ms-flex: 1;
-    flex: 1;
+  .cards .card {
+      position: relative;
+      max-height: 380px;
+      overflow: hidden;
     
-}
+      background-color: #fffcfc;
+      border: 1px solid #f1efef;
+      border-radius: 4px;
+      box-shadow: 4px 4px 2px 2px rgba(0, 0, 0, 0.01);
+    
+      text-align: center;
+      color: #5d5e5e;
+  }
 
-.card-content h4{
-vertical-align:bottom;
-margin: 1.2em 0 0em 0;
-}
+  .cards .card.selected-card {
+    border: 3px solid #2196f3;
+    box-shadow: 2px 3px 6px 2px rgba(0, 0, 221, 0.14);
+  }
 
-.category {
-  font-size: .75rem;
-  text-transform: uppercase;
-}
-.category {
-    position: absolute;
-    top: 5%;
-    right: 0;
-    color: #fff;
-    background: #e74c3c;
-    padding: 10px 15px;
-    font-size: 10px;
-    font-weight: 600;
-    text-transform: uppercase;
-}
+  .cards .card .card-thumbnail {
+      display: block;
+      width: 100%;
+      height: 345px;
+      overflow: hidden;
+      border-radius: 4px;
+      background-color: #fffcfc;
+  }
 
-.category__01 {
-  background-color: #50c6db;
-  
-}
+  .cards .card .card-thumbnail img {
+      width: 100%;
+      height: auto;
+      border: 0;
+  }
+  .cards .card .card-thumbnail.missing img {
+      opacity: 0.03;
+      width: 20%;
+      height: auto;
+      margin-top: 84px;
+  }
 
-.tags{
-  opacity: 0.8;
-}
+  .cards .card .card-tags {
+      width: 100%;
+  }
+  .cards .card .card-tags span {
+      display: inline-block;
+      padding: 2px 5px;
+      border-radius: 5px;
+      opacity: 0.95;
+      background-color: #bfdfff;
+      color: #679ac2;
+      font-size: 12px;
+      margin-bottom: 3px;
+  }
 
-footer {
-  border-top: 2px solid #e7e7e7;
-  {% comment %} margin: .5rem 0 0; {% endcomment %}
-  {% comment %} min-height: 30px; {% endcomment %}
-  font-size: .5rem;
-}
-.post-meta {
-  padding: .3rem;
-}
+  .cards .card .card-footer {
+      width: 100%;
+      position: absolute;
+      bottom: 0;
+      text-align: center;
+  }
+  .cards .card .card-title {
+      padding: 4px 5px;
+      background-color: #fffcfc;
+      /*height: 50px;
+      vertical-align: middle;
+      line-height: 50px;*/
+  }
+  .cards .card .card-title h4 {
+      color: initial;
+      display: block;
+      vertical-align: middle;
+      line-height: normal;
+      margin: 0px;
+      padding: 5px 0px;
+      font-size: 13.5px;
+      font-weight: 400;
+      word-break: break-all;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      max-height: 46px;
+  }
+  .cards .card .card-title h4 .title-text {
+    user-select: all;
+  }
+  .cards .card .card-title .link-favicon {
+      height: 15px;
+      margin: 2px;
+      vertical-align: -5px;
+      display: inline-block;
+  }
 
-.comments {
-  margin-left: .5rem;
-}
-
-.selected-card{
-  border: 5px solid #ffaa31;
-}
+  .cards .card .card-info {
+      padding: 2px 4px;
+      /*border-top: 1px solid #ddd;*/
+      background-color: #fffcfc;
+      font-size: 11px;
+      color: #333;
+  }
+  .cards .card .card-info input[type=checkbox] {
+      float: right;
+      width: 18px;
+      height: 18px;
+  }
+  .cards .card .card-info label {
+      display: inline-block;
+      height: 20px;
+      width: 145px;
+      font-size: 11px;
+  }
+  .cards .card .card-info .timestamp {
+      font-weight: 600;
+  }
+  .cards .card .card-footer code {
+      display: inline-block;
+      width: 100%;
+      margin-top: 2px;
+      font-size: 10px;
+      opacity: 0.4;
+      user-select: all;
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+  }
 
 </style>
 
 {% endblock %}
 
 {% block content %}
-    <section class="cards">
+  <section class="cards">
     {% for obj in results %}
-      <article class="card">
-          <picture class="thumbnail">
-            <a href="/{{obj.archive_path}}/index.html">
-              <img class="category__01" src="{% snapshot_image obj%}" alt="" /> 
-            </a>
-          </picture>
-          <div class="card-content">
-            {% if obj.tags_str %}
-            <p class="category category__01 tags">{{obj.tags_str}}</p>
-            {% endif %}
-            {% if obj.title  %}
+      <div class="card">
+          <div class="card-info">
             <a href="{% url 'admin:core_snapshot_change' obj.id %}">
-              <h4>{{obj.title|truncatechars:55 }}</h4>
+              <span class="timestamp">{{obj.added}}</span>
             </a>
-            {% endif %}
-            {% comment %} <p> TEXT If needed.</p> {% endcomment %}
-        </div><!-- .card-content -->
-        <footer>
-            <div class="post-meta">
-                <span style="float:right;"><input type="checkbox" name="_selected_action" value="{{obj.pk}}" class="action-select"></span>
-                <span class="timestamp">&#128337 {{obj.added}}</span>
-                <span class="comments">📖{{obj.num_outputs}}</span>
-                <span>🗄️{{ obj.archive_size | file_size }}</span>
-            </div>
-        </footer> 
-      </article>
+            <label>
+              <span class="num_outputs">📄 &nbsp; {{obj.num_outputs}}</span> &nbsp; &nbsp;
+              <span>🗄&nbsp; {{ obj.archive_size | file_size }}</span>
+              <input type="checkbox" name="_selected_action" value="{{obj.pk}}"/>
+            </label>
+          </div>
+          <a href="/{{obj.archive_path}}/index.html" class="card-thumbnail {% if not obj.thumbnail_url %}missing{% endif %}">
+            <img src="{{obj.thumbnail_url|default:'/static/spinner.gif' }}" alt="{{obj.title|default:'Not yet archived...'}}" /> 
+          </a>
+          <div class="card-footer">
+              {% if obj.tags_str %}
+                <div class="card-tags">
+                  {% for tag in obj.tags_str|split:',' %}
+                    <span>
+                      {{tag}}
+                    </span>
+                  {% endfor %}
+                </div>
+              {% endif %}
+              <div class="card-title" title="{{obj.title}}">
+                <a href="/{{obj.archive_path}}/index.html">
+                  <h4>
+                    {% if obj.is_archived %}
+                        <img src="/{{obj.archive_path}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
+                    {% else %}
+                        <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async"/>
+                    {% endif %}
+                    <span class="title-text">{{obj.title|default:'Pending...' }}</span>
+                  </h4>
+                </a>
+                <code title="{{obj.url}}">{{obj.url}}</code>
+              </div>
+          </div> 
+      </div>
     {% endfor %}
-    </section>
-
+  </section>
+  <br/>
 {% endblock %}

archivebox/templates/core/add.html

@@ -15,7 +15,7 @@
 {% endblock %}
 
 {% block body %}
-    <div style="max-width: 550px; margin: auto; float: none">
+    <div style="max-width: 1440px; margin: auto; float: none">
         <br/><br/>
         {% if stdout %}
             <h1>Add new URLs to your archive: results</h1>
@@ -38,16 +38,7 @@
             </form>
             <br/><br/><br/>
             <center id="delay-warning" style="display: none">
-                <b><i>This page will be unresponsive until the process is completely finished.</i></b>
-                <br/><br/>
-                <div>
-                    Warning: it may take several minutes to finish adding!<br/>
-                    <br/>
-                    Progress will be displayed in the <code>archivebox server</code> stdout,<br/>
-                    and on this page once the archiving process completes.<br/>
-                    <br/>
-                    <small>(it's safe to leave this page, adding will continue in the background)</small>
-                </div>
+                <small>(it's safe to leave this page, adding will continue in the background)</small>
             </center>
             {% if absolute_add_path %}
             <center id="bookmarklet">
@@ -58,7 +49,7 @@
             <script>
                 document.getElementById('add-form').addEventListener('submit', function(event) {
                     setTimeout(function() {
-                        document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>(see terminal for progress)</center>'
+                        document.getElementById('add-form').innerHTML = '<center><h3>Adding URLs to index and running archive methods...<h3><br/><div class="loader"></div><br/>Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for progress...</center>'
                         document.getElementById('delay-warning').style.display = 'block'
                     }, 200)
                     return true

archivebox/templates/core/base.html

@@ -1,67 +1,36 @@
-{% load admin_urls %}
-{% load static %}
+{% load static tz admin_urls %}
 
 <!DOCTYPE html>
 <html lang="en">
+    <head>
+        <title>Archived Sites</title>
+        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
 
-<head>
-    <title>Archived Sites</title>
-    <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
-
-    <link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
-    <link rel="stylesheet" type="text/css" href="{% static 'admin.css' %}">
-    <link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
-    <link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}" />
-            {% block extra_head %}
-            {% endblock %}
-    <script src="{% static 'jquery.min.js' %}"></script>
-    <script src="{% static 'jquery.dataTables.min.js' %}"></script>
-    <script>
-        document.addEventListener('error', function (e) {
-            e.target.style.opacity = 0;
-        }, true)
-        jQuery(document).ready(function () {
-            jQuery('#table-bookmarks').DataTable({
-                searching: false,
-                paging: false,
-                stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
-                dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
-                    order: [[0, 'desc']],
-                    iDisplayLength: 100,
-                });
-            });
-        </script>
-        <base href="{% url 'Home' %}">
+        <link rel="stylesheet" href="{% static 'admin/css/base.css' %}">
+        <link rel="stylesheet" href="{% static 'admin.css' %}">
+        <link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
+        
+        <script src="{% static 'jquery.min.js' %}"></script>
+        {% block extra_head %}
+        {% endblock %}
     </head>
     <body>
         <div id="container">
             <div id="header">
                 <div id="branding">
                     <h1 id="site-name">
-                        <a href="{% url 'public-index' %}" class="header-archivebox" title="Last updated: {{updated}}">
+                        <a href="{% url 'public-index' %}" class="header-archivebox">
                             <img src="{% static 'archive.png' %}" alt="Logo" style="height: 30px"/>
                             ArchiveBox
                         </a>
                     </h1>
                 </div>
-                <div id="user-tools">
-                    <a href="/add/">➕&nbsp; Add</a> &nbsp;/&nbsp;
-                    <a href="/">Snapshots</a> &nbsp;/&nbsp;
-                    <a href="/admin/">Admin</a> &nbsp;/&nbsp;
-                    <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-                    {% if user.is_authenticated %}
-                        &nbsp; &nbsp;
-                        User
-                        <strong>{% firstof user.get_short_name user.get_username %}</strong> &nbsp; &nbsp;
-                        {% if user.has_usable_password %}
-                            <a href="{% url 'admin:password_change' %}">Change password</a> /
-                        {% endif %}
-                        <a href="{% url 'admin:logout' %}">Log out</a>
-                    {% endif %}
-                </div>
+                {% include 'navigation.html' %}
+               
             </div>
             <div id="content" class="flex">
                 {% block body %}
+
                 {% endblock %}
             </div>
             {% block footer %}
@@ -70,7 +39,7 @@
                     <center>
                         <small>
                             Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a> version
-                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases" title="Releases">v{{VERSION}}</a>.
+                            <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{VERSION}}" title="Releases">v{{VERSION}}</a>.
                             <br/><br/>
                             {{FOOTER_INFO}}
                         </small>
@@ -79,6 +48,35 @@
                 </footer>
             {% endblock %}
         </div>
+        <script>
+            // hide images that fail to load
+            document.querySelector('body').addEventListener('error', function (e) {
+                e.target.style.opacity = 0;
+            }, true)
+
+            // setup timezone
+            {% get_current_timezone as TIME_ZONE %}
+            window.TIME_ZONE = '{{TIME_ZONE}}'
+
+            window.setCookie = function(name, value, days) {
+                let expires = ""
+                if (days) {
+                    const date = new Date()
+                    date.setTime(date.getTime() + (days*24*60*60*1000))
+                    expires = "; expires=" + date.toUTCString()
+                }
+                document.cookie = name + "=" + (value || "")  + expires + "; path=/"
+            }
+
+            function setTimeOffset() {
+                if (window.GMT_OFFSET) return
+                window.GMT_OFFSET = -(new Date).getTimezoneOffset()
+                window.setCookie('GMT_OFFSET', window.GMT_OFFSET, 365)
+            }
+
+            jQuery(document).ready(function () {
+                setTimeOffset();
+            });
+        </script>
     </body>
-    
-    </html>
+</html>

archivebox/templates/core/index_row.html

@@ -1,37 +1,44 @@
-{% load static %}
+{% load static tz core_tags %}
 
 <tr>
-    <td title="{{link.timestamp}}"> {% if link.bookmarked_date  %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} </td>
-    <td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}">
+    <td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.added.timestamp}}">
+        {{ link.added|localtime }}
+    </td>
+    <td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}" title="{{link.title|default:'Not yet archived...'}}">
         {% if link.is_archived %}
-            <a href="archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
+            <a href="/archive/{{link.timestamp}}/index.html"><img src="/archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
         {% else %}
-            <a href="archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
+            <a href="/archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
         {% endif %}
 
-        <a href="archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
-            <span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">{{link.title|default:'Loading...'}}</span>
+        <a href="/archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
+            <span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">
+                {{link.title|default:'Loading...'|truncatechars:128}}
+            </span>
             {% if link.tags_str %}
-                <span class="tags" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
-                    {% if link.tags_str != None %}
-                        {{link.tags_str|default:''}}
-                    {% else %}
-                        {{ link.tags|default:'' }}
-                    {% endif %}
-                </span>
+                {% for tag in link.tags_str|split:',' %}
+                    <span class="tag" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
+                        {{tag}}
+                    </span>
+                {% endfor %}
             {% endif %}
         </a>
     </td>
     <td>
         <span data-number-for="{{link.url}}" title="Fetching any missing files...">
             {% if link.icons  %}
-                {{link.icons}} <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
+                {{link.icons}}&nbsp; <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
             {% else %}
-                <a href="archive/{{link.timestamp}}/index.html">📄 
+                <a href="/archive/{{link.timestamp}}/index.html">
+                    📄 &nbsp;
                     {{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
                 </a>
             {% endif %}
         </span>
     </td>
-   <td style="text-align:left"><a href="{{link.url}}">{{link.url}}</a></td>
+    <td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; title="{{link.url}}">
+        <a href="{{link.url}}">
+            {{link.url}}
+        </a>
+    </td>
 </tr>

archivebox/templates/core/minimal_index.html

@@ -4,7 +4,7 @@
         <title>Archived Sites</title>
         <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
     </head>
-    <body data-status="{{status}}">
+    <body>
         <table id="table-bookmarks">
             <thead>
                 <tr class="thead-tr">

archivebox/templates/core/navigation.html

@@ -0,0 +1,26 @@
+{% load i18n static %}
+
+<div id="user-tools">
+    <a href="{% url 'admin:Add' %}">Add ➕</a> &nbsp; &nbsp;
+    <a href="{% url 'Home' %}">Snapshots</a> |
+    <a href="/admin/core/tag/">Tags</a> |
+    <a href="/admin/core/archiveresult/?o=-1">Log</a> &nbsp; &nbsp;
+    <a href="{% url 'Docs' %}">Docs</a> | 
+    <a href="{% url 'public-index' %}">Public</a> | 
+    <a href="/admin/">Admin</a>
+     &nbsp; &nbsp;
+    {% if user.is_authenticated %}
+        {% block welcome-msg %}
+            {% trans 'User' %}
+            <strong>{% firstof user.get_short_name user.get_username %}</strong> &nbsp; &nbsp;
+        {% endblock %}
+        {% block userlinks %}
+            {% if user.has_usable_password %}
+                <a href="{% url 'admin:password_change' %}">Account</a> /
+            {% endif %}
+            <a href="{% url 'admin:logout' %}">{% trans 'Log out' %}</a>
+        {% endblock %}
+    {% else %}
+        <a href="{% url 'admin:login' %}">{% trans 'Log in' %}</a>
+    {% endif %}
+</div>

archivebox/templates/core/progressbar.html

@@ -0,0 +1,45 @@
+<style>
+  /* Loading Progress Bar */
+    #progress {
+        position: absolute;
+        z-index: 1000;
+        top: 0px;
+        left: -6px;
+        width: 2%;
+        opacity: 1;
+        height: 2px;
+        background: #1a1a1a;
+        border-radius: 1px;
+        transition: width 4s ease-out, opacity 400ms linear;
+    }
+
+    @-moz-keyframes bugfix { from { padding-right: 1px ; } to { padding-right: 0; } }
+</style>
+<script>
+    // Page Loading Bar
+    window.loadStart = function(distance) {
+        var distance = distance || 0;
+        // only add progrstess bar if not already present
+        if (django.jQuery("#loading-bar").length == 0) {
+            django.jQuery("body").add("<div id=\"loading-bar\"></div>");
+        }
+        if (django.jQuery("#progress").length === 0) {
+            django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
+            let last_distance = (distance || (30 + (Math.random() * 30)))
+            django.jQuery("#progress").width(last_distance + "%");
+            setInterval(function() {
+                last_distance += Math.random()
+                django.jQuery("#progress").width(last_distance + "%");
+            }, 1000)
+        }
+    };
+
+    window.loadFinish = function() {
+        django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
+            django.jQuery(this).remove();
+        });
+    };
+    window.loadStart();
+    window.addEventListener('beforeunload', function() {window.loadStart(27)});
+    document.addEventListener('DOMContentLoaded', function() {window.loadFinish()});
+</script>

archivebox/templates/core/public_index.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% load static %}
+{% load static tz %}
 
 {% block body %}
     <div id="toolbar">
@@ -14,43 +14,49 @@
                        onclick="location.href='{% url 'public-index' %}'"
                        style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
                 </input>
+                &nbsp;
+                &nbsp;
+                {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
+                &nbsp;
+               (Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})
             </div>
         </form>
     </div>
-    <table id="table-bookmarks">
-        <thead>
-            <tr>
-                <th style="width: 100px;">Bookmarked</th>
-                <th style="width: 26vw;">Snapshot ({{object_list|length}})</th>
-                <th style="width: 140px">Files</th>
-                <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
-            </tr>
-        </thead>
+    <div style="width: 100%; overflow-x: auto;">
+        <table id="table-bookmarks" style="width: 100%; table-layout: fixed">
+            <thead>
+                <tr>
+                    <th style="width: 132px">Bookmarked</th>
+                    <th>Snapshot ({{page_obj.paginator.count}})</th>
+                    <th style="width: 280px">Files</th>
+                    <th>Original URL</th>
+                </tr>
+            </thead>
             <tbody>
                 {% for link in object_list %}
                     {% include 'index_row.html' with link=link  %}
                 {% endfor %}
             </tbody>
         </table>
-        <center>
-            <span class="step-links">
-                {% if page_obj.has_previous %}
-                    <a href="{% url 'public-index' %}?page=1">&laquo; first</a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
-                {% endif %}
-        
-                <span class="current">
-                    Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
-                </span>
-        
-                {% if page_obj.has_next %}
-                    <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
-                    <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
-                {% endif %}
-            </span>
+    </div>
+    <br/>
+    <center>
+        Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
+        <br/>
+        <span class="step-links">
+            {% if page_obj.has_previous %}
+                <a href="{% url 'public-index' %}?page=1">&laquo; first</a> &nbsp;
+                <a href="{% url 'public-index' %}?page={{ page_obj.previous_page_number }}">previous</a>
+                &nbsp;
+            {% endif %}
     
+            <span class="current">
+                Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
+            </span>
+        
             {% if page_obj.has_next %}
-                <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a>
+                &nbsp;
+                <a href="{% url 'public-index' %}?page={{ page_obj.next_page_number }}">next </a> &nbsp;
                 <a href="{% url 'public-index' %}?page={{ page_obj.paginator.num_pages }}">last &raquo;</a>
             {% endif %}
         </span>

archivebox/templates/core/snapshot.html

@@ -1,8 +1,19 @@
+{% load tz core_tags %}
+
 <!DOCTYPE html>
 <html lang="en">
     <head>
         <title>{{title}}</title>
         <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
+        <style>
+            /*!
+            * Bootstrap v4.0.0 (https://getbootstrap.com)
+            * Copyright 2011-2018 The Bootstrap Authors
+            * Copyright 2011-2018 Twitter, Inc.
+            * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+            */
+            :root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}@-ms-viewport{width:device-width}article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}dfn{font-style:italic}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent;-webkit-text-decoration-skip:objects}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg:not(:root){overflow:hidden}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-family:inherit;font-weight:500;line-height:1.2;color:inherit}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-cLine truncated
+        </style>
         <style>
             html, body {
                 width: 100%;
@@ -11,7 +22,6 @@
             }
             header {
                 background-color: #aa1e55;
-                padding-bottom: 12px;
             }
             small {
                 font-weight: 200;
@@ -25,15 +35,15 @@
                 min-height: 40px;
                 margin: 0px;
                 text-align: center;
-                color: white;
-                font-size: calc(11px + 0.84vw);
+                color: #f6f6f6;
+                font-size: calc(10px + 0.84vw);
                 font-weight: 200;
-                padding: 4px 4px;
+                padding: 3px 4px;
                 background-color: #aa1e55;
             }
             .nav > div {
                 min-height: 30px;
-                line-height: 1.3;
+                line-height: 1.2;
             }
             .header-top a {
                 text-decoration: none;
@@ -59,9 +69,14 @@
             .header-archivebox img:hover {
                 opacity: 0.5;
             }
-            .header-url small {
+            header small code {
                 white-space: nowrap;
                 font-weight: 200;
+                display: block;
+                margin-top: -1px;
+                font-size: 13px;
+                opacity: 0.8;
+                user-select: all;
             }
             .header-url img {
                 height: 20px;
@@ -81,28 +96,38 @@
             .info-row .alert {
                 margin-bottom: 0px;
             }
+            .row.header-bottom {
+                margin-left: -10px;
+                margin-right: -10px;
+            }
+            .header-bottom .col-lg-2 {
+                padding-left: 4px;
+                padding-right: 4px;
+            }
+
             .header-bottom-frames .card {
-                overflow: hidden;
                 box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
-                margin-top: 10px;
+                margin-bottom: 5px;
                 border: 1px solid rgba(0,0,0,3);
-                border-radius: 14px;
+                border-radius: 10px;
                 background-color: black;
+                overflow: hidden;
             }
             .card h4 {
                 font-size: 1.4vw;
             }
             .card-body {
-                font-size: 15px;
+                font-size: 14px;
                 padding: 13px 10px;
-                padding-bottom: 6px;
+                padding-bottom: 1px;
                 /* padding-left: 3px; */
                 /* padding-right: 3px; */
                 /* padding-bottom: 3px; */
-                line-height: 1.1;
+                line-height: 1;
                 word-wrap: break-word;
                 max-height: 102px;
                 overflow: hidden;
+                text-overflow: ellipsis;
                 background-color: #1a1a1a;
                 color: #d3d3d3;
             }
@@ -137,22 +162,12 @@
                 border-top: 3px solid #aa1e55;
             }
             .card.selected-card {
-                border: 1px solid orange;
+                border: 2px solid orange;
                 box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
             }
             .iframe-large {
                 height: calc(100% - 40px);
             }
-            .pdf-frame {
-                transform: none;
-                width: 100%;
-                height: 160px;
-                margin-top: -60px;
-                margin-bottom: 0px;
-                transform: scale(1.1);
-                width: 100%;
-                margin-left: -10%;
-            }
             img.external {
                 height: 30px;
                 margin-right: -10px;
@@ -176,7 +191,7 @@
             }
             .header-bottom {
                 border-top: 1px solid rgba(170, 30, 85, 0.9);
-                padding-bottom: 12px;
+                padding-bottom: 1px;
                 border-bottom: 5px solid rgb(170, 30, 85);
                 margin-bottom: -1px;
 
@@ -206,10 +221,11 @@
             }
             .info-chunk {
                 width: auto;
-                display:inline-block;
+                display: inline-block;
                 text-align: center;
-                margin: 10px 10px;
+                margin: 8px 4px;
                 vertical-align: top;
+                font-size: 14px;
             }
             .info-chunk .badge {
                 margin-top: 5px;
@@ -217,13 +233,12 @@
             .header-bottom-frames .card-title {
                 width: 100%;
                 text-align: center;
-                font-size: 18px;
-                margin-bottom: 5px;
+                font-size: 17px;
+                margin-bottom: 0px;
                 display: inline-block;
                 color: #d3d3d3;
                 font-weight: 200;
-                vertical-align: 0px;
-                margin-top: -6px;
+                vertical-align: 3px;
             }
             .header-bottom-frames .card-text {
                 width: 100%;
@@ -263,14 +278,12 @@
                 }
             }
         </style>
-        <link rel="stylesheet" href="../../static/bootstrap.min.css">
     </head>
     <body>
         <header>
             <div class="header-top container-fluid">
                 <div class="row nav">
-                    <div class="col-lg-2" style="line-height: 64px;">
-
+                    <div class="col-lg-2" style="line-height: 50px; vertical-align: middle">
                         <a href="../../index.html" class="header-archivebox" title="Go to Main Index...">
                             <img src="../../static/archive.png" alt="Archive Icon">
                             ArchiveBox
@@ -279,13 +292,12 @@
                     <div class="col-lg-8">
                         <img src="favicon.ico" onerror="this.style.opacity=0" alt="Favicon">
                         &nbsp;&nbsp;
-                        {{title}}
+                        {{title|safe}}
                         &nbsp;&nbsp;
                         <a href="#" class="header-toggle">▾</a>
-                        <br/>
                         <small>
                             <a href="{{url}}" class="header-url" title="{{url}}">
-                                {{url_str}}
+                                <code>{{url}}</code>
                             </a>
                         </small>
                     </div>
@@ -294,27 +306,25 @@
             <div class="header-bottom container-fluid">
                 <div class="row header-bottom-info">
                     <div class="col-lg-4">
-                        <div title="Date bookmarked or imported" class="info-chunk">
+                        <div title="Date bookmarked or imported" class="info-chunk" title="UTC Timezone {{timestamp}}">
                             <h5>Added</h5>
                             {{bookmarked_date}}
                         </div>
-                        <div title="Date first archived" class="info-chunk">
+                        <div title="Date first archived" class="info-chunk" title="UTC Timezone">
                             <h5>First Archived</h5>
                             {{oldest_archive_date}}
                         </div>
-                        <div title="Date last checked" class="info-chunk">
+                        <div title="Date last checked" class="info-chunk" title="UTC Timezone">
                             <h5>Last Checked</h5>
                             {{updated_date}}
                         </div>
                     </div>
                     <div class="col-lg-4">
-                        <div class="info-chunk">
-                            <h5>Type</h5>
-                            <div class="badge badge-default">{{extension}}</div>
-                        </div>
-                        <div class="info-chunk">
-                            <h5>Tags</h5>
-                            <div class="badge badge-warning">{{tags}}</div> 
+                        <div class="info-chunk" style="max-width: 280px">
+                            <h5>Tags <small title="Auto-guessed content type">({{extension}})</small></h5>
+                            {% for tag in tags_str|split:',' %}
+                                <div class="badge badge-info" style="word-break: break-all;">{{tag}}</div>
+                            {% endfor %}
                         </div>
                         <div class="info-chunk">
                             <h5>Status</h5>
@@ -322,11 +332,11 @@
                         </div>
                         <div class="info-chunk">
                             <h5>Saved</h5>
-                            ✅  {{num_outputs}}
+                            ✅&nbsp; {{num_outputs}}
                         </div>
                         <div class="info-chunk">
                             <h5>Errors</h5>
-                            ❌  {{num_failures}}
+                            ❌&nbsp; {{num_failures}}
                         </div>
                         <div class="info-chunk">
                             <h5>Size</h5>
@@ -335,20 +345,21 @@
                     </div>
                     <div class="col-lg-4">
                         <div class="info-chunk">
-                            <h5>🗃 Files</h5>
+                            <h5>🗃&nbsp; Snapshot: <a href="/admin/core/snapshot/{{snapshot_id}}/change/"><code style="color: rgba(255,255,255,0.6); font-weight: 200; font-size: 12px; background-color: #1a1a1a"><b>[{{timestamp}}]</b> <small>{{snapshot_id|truncatechars:24}}</small></code></a></h5>
                             <a href="index.json" title="JSON summary of archived link.">JSON</a> | 
                             <a href="warc/" title="Any WARC archives for the page">WARC</a> | 
                             <a href="media/" title="Audio, Video, and Subtitle files.">Media</a> | 
                             <a href="git/" title="Any git repos at the url">Git</a> | 
-                            <a href="favicon.ico" title="Any git repos at the url">Favicon</a> | 
-                            <a href="." title="Webserver-provided index of files directory.">See all...</a>
+                            <a href="/admin/core/snapshot/?id__startswith={{snapshot_id}}" title="Go to the Snapshot admin to update, overwrite, or delete this Snapshot">Actions</a> | 
+                            <a href="/admin/core/snapshot/{{snapshot_id}}/change/" title="Edit this snapshot in the Admin UI">Admin</a> | 
+                            <a href="." title="Webserver-provided index of files directory.">See all files...</a><br/>
                         </div>
                     </div>
                 </div>
                 <div class="row header-bottom-frames">
                     <div class="col-lg-2">
                         <div class="card selected-card">
-                            <iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{singlefile_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{singlefile_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./singlefile.html</code></p>
@@ -359,7 +370,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top pdf-frame" src="{{pdf_path}}" scrolling="no"></iframe>
+                            <iframe class="card-img-top pdf-frame" src="{{pdf_path}}#toolbar=0" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{pdf_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./output.pdf</code></p>
@@ -381,7 +392,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                          <iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                          <iframe class="card-img-top" src="{{archive_url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                           <div class="card-body">
                                 <a href="{{archive_url}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./{{domain}}</code></p>
@@ -393,30 +404,30 @@
                     {% if SAVE_ARCHIVE_DOT_ORG %}
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{archive_org_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{archive_org_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>🌐 web.archive.org/web/...</code></p>
                                 </a>
-                                <a href="{{archive_org_path}}" target="preview"><h4 class="card-title">Archive.Org</h4></a>
+                                <a href="{{archive_org_path}}" target="preview" id="archive_dot_org-btn"><h4 class="card-title">Archive.Org</h4></a>
                           </div>
                         </div>
                     </div>
                     {% endif %}
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{url}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{url}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>🌐 {{domain}}</code></p>
                                 </a>
-                                <a href="{{url}}" target="preview"><h4 class="card-title">Original</h4></a>
+                                <a href="{{url}}" target="preview" id="original-btn"><h4 class="card-title">Original</h4></a>
                           </div>
                         </div>
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{headers_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{headers_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./headers.json</code></p>
@@ -427,7 +438,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{dom_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{dom_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./output.html</code></p>
@@ -438,7 +449,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{readability_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{readability_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./readability/content.html</code></p>
@@ -450,7 +461,7 @@
                     <br/>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{mercury_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{mercury_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./mercury/content.html</code></p>
@@ -461,7 +472,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{media_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{media_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./media/*.mp4</code></p>
@@ -472,7 +483,7 @@
                     </div>
                     <div class="col-lg-2">
                         <div class="card">
-                            <iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-scripts allow-forms" scrolling="no"></iframe>
+                            <iframe class="card-img-top" src="{{git_path}}" sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" scrolling="no" loading="lazy"></iframe>
                             <div class="card-body">
                                 <a href="{{git_path}}" title="Open in new tab..." target="_blank" rel="noopener">
                                     <p class="card-text"><code>./git/*.git</code></p>
@@ -484,15 +495,24 @@
                 </div>
             </div>
         </header>
-        <iframe sandbox="allow-same-origin allow-scripts allow-forms" class="full-page-iframe" src="{{singlefile_path}}" name="preview"></iframe>
+        <iframe sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms" class="full-page-iframe" src="{{singlefile_path}}" name="preview"></iframe>
     
-        <script
-              src="https://code.jquery.com/jquery-3.2.1.slim.min.js"
-              integrity="sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g="
-              crossorigin="anonymous"></script>
-        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/js/bootstrap.min.js" integrity="sha384-vBWWzlZJ8ea9aCX4pEW3rVHjgjt7zpkNpZk+02D9phzyeVkE+jo0ieGizqPLForn" crossorigin="anonymous"></script>
+        <script>
+            /*! jQuery v3.2.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/parseXML,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-event/ajax,-effects,-effects/Tween,-effects/animatedSelector | (c) JS Foundation and other contributors | jquery.org/license */
+            !function(a,b){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use strict";var c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function p(a,b){b=b||d;var c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var q="3.2.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/parseXML,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-event/ajax,-effects,-effects/Tween,-effects/animatedSelector",r=function(a,b){return new r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return f.call(this)},get:function(a){return null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var b=r.merge(this.constructor(),a);return b.prevObject=this,b},each:function(a){return r.each(this,a)},map:function(a){return this.pushStack(r.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(f.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(a<0?b:0);return this.pushStack(c>=0&&c<b?[this[c]]:[])},end:function(){return this.prevObject||this.constructor()},push:h,sort:c.sort,splice:c.splice},r.extend=r.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||r.isFunction(g)||(g={}),h===i&&(g=this,h--);h<i;h++)if(null!=(a=arguments[h]))for(b in a)c=g[b],d=a[b],g!==d&&(j&&d&&(r.isPlainObject(d)||(e=Array.isArray(d)))?(e?(e=!1,f=c&&Array.isArray(c)?c:[]):f=c&&r.isPlainObject(c)?c:{},g[b]=r.extend(j,f,d)):void 0!==d&&(g[b]=d));return g},r.extend({expando:"jQuery"+(q+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===r.type(a)},isWindow:function(a){return null!=a&&a===a.window},isNumeric:function(a){var b=r.type(a);return("number"===b||"string"===b)&&!isNaN(a-parseFloat(a))},isPlainObject:function(a){var b,c;return!(!a||"[object Object]"!==k.call(a))&&(!(b=e(a))||(c=l.call(b,"constructor")&&b.constructor,"function"==typeof c&&m.call(c)===n))},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?j[k.call(a)]||"object":typeof a},globalEval:function(a){p(a)},camelCase:function(a){return a.replace(t,"ms-").replace(u,v)},each:function(a,b){var c,d=0;if(w(a)){for(c=a.length;d<c;d++)if(b.call(a[d],d,a[d])===!1)break}else for(d in a)if(b.call(a[d],d,a[d])===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(s,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(w(Object(a))?r.merge(c,"string"==typeof a?[a]:a):h.call(c,a)),c},inArray:function(a,b,c){return null==b?-1:i.call(b,a,c)},merge:function(a,b){for(var c=+b.length,d=0,e=a.length;d<c;d++)a[e++]=b[d];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;f<g;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,e,f=0,h=[];if(w(a))for(d=a.length;f<d;f++)e=b(a[f],f,c),null!=e&&h.push(e);else for(f in a)e=b(a[f],f,c),null!=e&&h.push(e);return g.apply([],h)},guid:1,proxy:function(a,b){var c,d,e;if("string"==typeof b&&(c=a[b],b=a,a=c),r.isFunction(a))return d=f.call(arguments,2),e=function(){return a.apply(b||this,d.concat(f.call(arguments)))},e.guid=a.guid=a.guid||r.guid++,e},now:Date.now,support:o}),"function"==typeof Symbol&&(r.fn[Symbol.iterator]=c[Symbol.iterator]),r.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(a,b){j["[object "+b+"]"]=b.toLowerCase()});function w(a){var b=!!a&&"length"in a&&a.length,c=r.type(a);return"function"!==c&&!r.isWindow(a)&&("array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a)}var x=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var c=0,d=a.length;c<d;c++)if(a[c]===b)return c;return-1},J="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",K="[\\x20\\t\\r\\n\\f]",L="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",M="\\["+K+"*("+L+")(?:"+K+"*([*^$|!~]?=)"+K+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+L+"))|)"+K+"*\\]",N=":("+L+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+M+")*)|.*)\\)|)",O=new RegExp(K+"+","g"),P=new RegExp("Line truncated
+        </script>
 
         <script>
+            function getPreviewTypeFromPath(link) {
+                if (link.id == 'original-btn') {
+                    return 'original'
+                }
+                if (link.id == 'archive_dot_org-btn') {
+                    return 'archive_dot_org'
+                }
+                return link.pathname.split('/').filter(a => a.length).slice(-1)[0].toLowerCase()
+            }
+
             // show selected file in iframe when preview card is clicked
             jQuery('.card').on('click', function(e) {
                 jQuery('.selected-card').removeClass('selected-card')
@@ -502,11 +522,26 @@
                 if (e.currentTarget.href.endsWith('.pdf')) {
                     jQuery('.full-page-iframe')[0].removeAttribute('sandbox')
                 } else {
-                    jQuery('.full-page-iframe')[0].sandbox = "allow-same-origin allow-scripts allow-forms"
+                    jQuery('.full-page-iframe')[0].sandbox = "allow-same-origin allow-scripts allow-forms allow-top-navigation-by-user-activation"
                 }
+                window.location.hash = getPreviewTypeFromPath(e.currentTarget)
                 return true
             })
 
+            // check URL for hash e.g. #git and load relevant preview
+            jQuery(document).ready(function() {
+                if (window.location.hash) {
+                    for (const link of jQuery('a[target=preview]')) {
+                        console.log(link.pathname)
+                        if (getPreviewTypeFromPath(link) == window.location.hash.slice(1).toLowerCase()) {
+                            jQuery(link).closest('.card').click()
+                            jQuery(link).click()
+                            link.click()
+                        }
+                    }
+                }
+            })
+
             // un-sandbox iframes showing pdfs (required to display pdf viewer)
             jQuery('iframe').map(function() {
                 if (this.src.endsWith('.pdf')) {

archivebox/templates/core/static_index.html

@@ -209,7 +209,7 @@
             <div class="header-top container-fluid">
                 <div class="row nav">
                     <div class="col-sm-2">
-                        <a href="/" class="header-archivebox" title="Last updated: {{updated}}">
+                        <a href="/" class="header-archivebox">
                             <img src="{% static 'archive.png' %}" alt="Logo"/>
                             ArchiveBox: Index
                         </a>
@@ -243,7 +243,7 @@
             <center>
                 <small>
                     Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
-                    version <a href="https://github.com/ArchiveBox/ArchiveBox/tree/v{{version}}" title="Git commit">v{{version}}</a> &nbsp; | &nbsp; 
+                    version <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{version}}" title="View source code and release info">v{{version}}</a> &nbsp; | &nbsp; 
                     Download index as <a href="index.json" title="JSON summary of archived links.">JSON</a>
                     <br/><br/>
                     {{FOOTER_INFO}}

archivebox/templates/static/add.css

@@ -39,10 +39,10 @@ header {
   margin: 10px;
   padding: 10px;
   font-size: 15px;
-  background-color: #f5dd5d;
+  background-color: #8bc34a;
 }
 #stdout {
-  background-color: #ded;
+  background-color: #fbfbfb;
   padding: 10px 10px;
   border-radius: 4px;
   white-space: normal;

archivebox/templates/static/admin.css

@@ -1,3 +1,9 @@
+* {
+    -webkit-box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    box-sizing: border-box;
+}
+
 #logo {
     height: 30px;
     vertical-align: -6px;
@@ -36,6 +42,23 @@ div.breadcrumbs {
     padding: 6px 15px;
 }
 
+#toolbar #searchbar {
+    height: 25px;
+}
+
+#snapshot-view-mode {
+    float: right;
+    margin-bottom: -40px;
+    display: inline-block;
+    margin-top: 3px;
+    margin-right: 10px;
+    font-size: 14px;
+    opacity: 0.8;
+}
+#snapshot-view-mode a {
+    color: #ccc;
+}
+
 body.model-snapshot.change-list div.breadcrumbs,
 body.model-snapshot.change-list #content .object-tools {
     display: none;
@@ -92,6 +115,14 @@ body.model-snapshot.change-list #content .object-tools {
     background: none;
     margin-right: 0px;
     width: auto;
+    max-height: 40px;
+    overflow: hidden;
+    display: block;
+}
+@media (max-width: 1000px) {
+    #content #changelist .actions {
+        max-height: 200px;
+    }
 }
 
 #content #changelist .actions .button {
@@ -116,20 +147,45 @@ body.model-snapshot.change-list #content .object-tools {
     background-color:lightseagreen;
     color: #333;
 }
+#content #changelist .actions .button[name=resnapshot_snapshot] {
+    background-color: #9ee54b;
+    color: #333;
+}
 #content #changelist .actions .button[name=overwrite_snapshots] {
     background-color: #ffaa31;
     color: #333;
+    margin-left: 10px;
 }
 #content #changelist .actions .button[name=delete_snapshots] {
     background-color: #f91f74;
     color: rgb(255 248 252 / 64%);
 }
+#content #changelist .actions .button[name=add_tags] {
+}
+#content #changelist .actions .button[name=remove_tags] {
+    margin-right: 25px;
+}
 
+#content #changelist .actions .select2-selection {
+    max-height: 25px;
+}
+#content #changelist .actions .select2-container--admin-autocomplete.select2-container {
+    width: auto !important;
+    min-width: 90px;
+}
+#content #changelist .actions .select2-selection__rendered .select2-selection__choice {
+    margin-top: 3px;
+}
 
 #content #changelist-filter h2 {
     border-radius: 4px 4px 0px 0px;
 }
 
+#changelist .paginator {
+    border-top: 0px;
+    border-bottom: 0px;
+}
+
 @media (min-width: 767px) {
     #content #changelist-filter {
         top: 35px;
@@ -157,7 +213,7 @@ body.model-snapshot.change-list #content .object-tools {
 
 #content a img.favicon {
     height: 20px;
-    width: 20px;
+    max-width: 28px;
     vertical-align: -5px;
     padding-right: 6px;
 }
@@ -177,7 +233,7 @@ body.model-snapshot.change-list #content .object-tools {
 
 #content th.field-added, #content td.field-updated {
     word-break: break-word;
-    min-width: 128px;
+    min-width: 135px;
     white-space: normal;
 }
 
@@ -237,3 +293,40 @@ body.model-snapshot.change-list #content .object-tools {
     opacity: 0.1;
     filter: grayscale(100%);
 }
+
+
+#result_list tbody td.field-cmd_str pre,
+#result_list tbody td.field-output_str pre {
+    max-width: 22vw;
+    word-wrap: anywhere;
+    white-space: break-spaces;
+    max-height: 40px;
+    overflow: hidden;
+    margin: 2px;
+    background-color: rgba(0,0,0,0.05);
+    padding: 1px 4px 16px 8px;
+    border-radius: 4px;
+}
+
+#result_list tbody td.field-extractor {
+    font-weight: 800;
+    font-variant: small-caps;
+}
+
+#result_list tbody td.field-status {
+    font-variant: small-caps;
+}
+
+.inline-group .tabular td.original p {
+    margin-top: -33px;
+}
+
+tbody .output-link {
+    float: right;
+    margin-bottom: -25px;
+    margin-right: -3px;
+    margin-top: -4px;
+    opacity: 0.4;
+    box-shadow:   4px 4px 4px rgba(0,0,0,0.1);
+}
+tbody .output-link:hover {opacity: 1;}

archivebox/templates/static/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

archivebox/util.py

@@ -11,7 +11,7 @@ from functools import wraps
 from hashlib import sha256
 from urllib.parse import urlparse, quote, unquote
 from html import escape, unescape
-from datetime import datetime
+from datetime import datetime, timezone
 from dateparser import parse as dateparser
 from requests.exceptions import RequestException, ReadTimeout
 
@@ -51,16 +51,18 @@ htmlencode = lambda s: s and escape(s, quote=True)
 htmldecode = lambda s: s and unescape(s)
 
 short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0]
-ts_to_date = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
+ts_to_date_str = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
 ts_to_iso = lambda ts: ts and parse_date(ts).isoformat()
 
 
 URL_REGEX = re.compile(
+    r'(?=('
     r'http[s]?://'                    # start matching from allowed schemes
     r'(?:[a-zA-Z]|[0-9]'              # followed by allowed alphanum characters
     r'|[$-_@.&+]|[!*\(\),]'           #    or allowed symbols
     r'|(?:%[0-9a-fA-F][0-9a-fA-F]))'  #    or allowed unicode bytes
-    r'[^\]\[\(\)<>"\'\s]+',         # stop parsing at these symbols
+    r'[^\]\[\(\)<>"\'\s]+'          # stop parsing at these symbols
+    r'))',
     re.IGNORECASE,
 )
 
@@ -142,13 +144,17 @@ def parse_date(date: Any) -> Optional[datetime]:
         return None
 
     if isinstance(date, datetime):
+        if date.tzinfo is None:
+            return date.replace(tzinfo=timezone.utc)
+
+        assert date.tzinfo.utcoffset(datetime.now()).seconds == 0, 'Refusing to load a non-UTC date!'
         return date
     
     if isinstance(date, (float, int)):
         date = str(date)
 
     if isinstance(date, str):
-        return dateparser(date)
+        return dateparser(date, settings={'TIMEZONE': 'UTC'}).replace(tzinfo=timezone.utc)
 
     raise ValueError('Tried to parse invalid date! {}'.format(date))
 
@@ -217,6 +223,9 @@ def chrome_args(**options) -> List[str]:
 
     options = {**CHROME_OPTIONS, **options}
 
+    if not options['CHROME_BINARY']:
+        raise Exception('Could not find any CHROME_BINARY installed on your system')
+
     cmd_args = [options['CHROME_BINARY']]
 
     if options['CHROME_HEADLESS']:
@@ -231,6 +240,8 @@ def chrome_args(**options) -> List[str]:
             '--disable-gpu',
             '--disable-dev-shm-usage',
             '--disable-software-rasterizer',
+            '--run-all-compositor-stages-before-draw',
+            '--hide-scrollbars',
         )
 
 
@@ -244,7 +255,7 @@ def chrome_args(**options) -> List[str]:
         cmd_args += ('--window-size={}'.format(options['RESOLUTION']),)
 
     if options['TIMEOUT']:
-        cmd_args += ('--timeout={}'.format((options['TIMEOUT']) * 1000),)
+        cmd_args += ('--timeout={}'.format(options['TIMEOUT'] * 1000),)
 
     if options['CHROME_USER_DATA_DIR']:
         cmd_args.append('--user-data-dir={}'.format(options['CHROME_USER_DATA_DIR']))

archivebox/vendor/atomicwrites.py

@@ -0,0 +1 @@
+python-atomicwrites/atomicwrites/__init__.py

bin/build_deb.sh

@@ -10,14 +10,6 @@ set -o nounset
 set -o pipefail
 IFS=$'\n'
 
-REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
-
-if [[ -f "$REPO_DIR/.venv/bin/activate" ]]; then
-    source "$REPO_DIR/.venv/bin/activate"
-else
-    echo "[!] Warning: No virtualenv presesnt in $REPO_DIR.venv"
-fi
-cd "$REPO_DIR"
 
 CURRENT_PLAFORM="$(uname)"
 REQUIRED_PLATFORM="Linux"
@@ -26,30 +18,27 @@ if [[ "$CURRENT_PLAFORM" != "$REQUIRED_PLATFORM" ]]; then
    exit 0
 fi
 
-VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
-DEBIAN_VERSION="1"
-PGP_KEY_ID="7D5695D3B618872647861D51C38137A7C1675988"
-# make sure you have this in ~/.dput.cf:
-#     [archivebox-ppa]
-#     fqdn: ppa.launchpad.net
-#     method: ftp
-#     incoming: ~archivebox/ubuntu/archivebox/
-#     login: anonymous
-#     allow_unsigned_uploads: 0
 
+REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
+VERSION="$(jq -r '.version' < "$REPO_DIR/package.json")"
+DEBIAN_VERSION="${DEBIAN_VERSION:-1}"
+cd "$REPO_DIR"
+
+
+if [[ -f "$REPO_DIR/.venv/bin/activate" ]]; then
+    source "$REPO_DIR/.venv/bin/activate"
+else
+    echo "[!] Warning: No virtualenv presesnt in $REPO_DIR.venv"
+fi
 
 # cleanup build artifacts
 rm -Rf build deb_dist dist archivebox-*.tar.gz
 
-# make sure the stdeb.cfg file is up-to-date with all the dependencies
 
 # build source and binary packages
+# make sure the stdeb.cfg file is up-to-date with all the dependencies
 python3 setup.py --command-packages=stdeb.command \
     sdist_dsc --debian-version=$DEBIAN_VERSION \
     bdist_deb
 
-# sign the build with your PGP key ID
-debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
-
-# push the build to launchpad ppa
-# dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+# should output deb_dist/archivebox_0.5.4-1.{deb,changes,buildinfo,tar.gz}

bin/build_docker.sh

@@ -17,7 +17,7 @@ cd "$REPO_DIR"
 
 which docker > /dev/null
 
-echo "[+] Building docker image in the background..."
+echo "[+] Building archivebox:$VERSION docker image..."
 docker build . -t archivebox \
                -t archivebox:latest \
                -t archivebox:$VERSION \
@@ -28,6 +28,6 @@ docker build . -t archivebox \
                -t docker.io/archivebox/archivebox:latest \
                -t docker.io/archivebox/archivebox:$VERSION \
                -t docker.io/archivebox/archivebox:$SHORT_VERSION \
-               -t docker.pkg.github.com/pirate/archivebox/archivebox:latest \
-               -t docker.pkg.github.com/pirate/archivebox/archivebox:$VERSION \
-               -t docker.pkg.github.com/pirate/archivebox/archivebox:$SHORT_VERSION
+               -t docker.pkg.github.com/archivebox/archivebox/archivebox:latest \
+               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$VERSION \
+               -t docker.pkg.github.com/archivebox/archivebox/archivebox:$SHORT_VERSION

bin/docker_entrypoint.sh

@@ -3,6 +3,7 @@
 DATA_DIR="${DATA_DIR:-/data}"
 ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
 
+
 # Set the archivebox user UID & GID
 if [[ -n "$PUID" && "$PUID" != 0 ]]; then
     usermod -u "$PUID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
@@ -11,6 +12,7 @@ if [[ -n "$PGID" && "$PGID" != 0 ]]; then
     groupmod -g "$PGID" "$ARCHIVEBOX_USER" > /dev/null 2>&1
 fi
 
+
 # Set the permissions of the data dir to match the archivebox user
 if [[ -d "$DATA_DIR/archive" ]]; then
     # check data directory permissions
@@ -21,7 +23,7 @@ if [[ -d "$DATA_DIR/archive" ]]; then
     fi
 else
     # create data directory
-    mkdir -p "$DATA_DIR"
+    mkdir -p "$DATA_DIR/logs"
     chown -R $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
 fi
 chown $ARCHIVEBOX_USER:$ARCHIVEBOX_USER "$DATA_DIR"
@@ -33,11 +35,11 @@ if [[ "$1" == /* || "$1" == "echo" || "$1" == "archivebox" ]]; then
     # e.g. "archivebox init"
     #      "/bin/bash"
     #      "echo"
-    gosu "$ARCHIVEBOX_USER" bash -c "$*"
+    exec gosu "$ARCHIVEBOX_USER" bash -c "$*"
 else
     # no command given, assume args were meant to be passed to archivebox cmd
     # e.g. "add https://example.com"
     #      "manage createsupseruser"
     #      "server 0.0.0.0:8000"
-    gosu "$ARCHIVEBOX_USER" bash -c "archivebox $*"
+    exec gosu "$ARCHIVEBOX_USER" bash -c "archivebox $*"
 fi