Update README.md

Don't run ls if using non-default directory

Dockerfile

@@ -34,7 +34,8 @@ RUN git clone https://github.com/pirate/ArchiveBox /home/pptruser/app \
     && mkdir -p /data \
     && chown -R pptruser:pptruser /data \
     && ln -s /data /home/pptruser/app/archivebox/output \
-    && ln -s /home/pptruser/app/bin/archivebox /bin/archive \
+    && ln -s /home/pptruser/app/bin/* /bin/ \
+    && ln -s /home/pptruser/app/bin/archive /bin/archivebox \
     && chown -R pptruser:pptruser /home/pptruser/app/archivebox
     # && pip3 install -r /home/pptruser/app/archivebox/requirements.txt
 

README.md

@@ -66,7 +66,7 @@ Using multiple methods and the market-dominant browser to execute JS ensures we
 ### Can import links from:
 
  - <img src="https://getpocket.com/favicon.ico" height="22px"/> Pocket, Pinboard, Instapaper
- - <img src="https://nicksweeting.com/images/rss.svg" height="22px"/> RSS, XML, JSON, or plain text lists
+ - <img src="https://nicksweeting.com/images/rss.svg" height="22px"/> RSS, XML, JSON, HTML, Markdown, or plain text lists
  - <img src="https://nicksweeting.com/images/bookmarks.png" height="22px"/> Browser history or bookmarks (Chrome, Firefox, Safari, IE, Opera, and more)
  - *Shaarli, Delicious, Reddit Saved Posts, Wallabag, Unmark.it, and any other text with links in it!*
 
@@ -83,14 +83,14 @@ Using multiple methods and the market-dominant browser to execute JS ensures we
  - `git/` clone of any repository for github, bitbucket, or gitlab links
  - `index.html` & `index.json` HTML and JSON index files containing metadata and details
 
-By default it does everything, but can disable or tweak individual options via environment variables or config file.
+By default it does everything, but can disable or tweak [individual options](https://github.com/pirate/ArchiveBox/wiki/Configuration) via environment variables or config file.
 
-The archiving is additive, so you can schedule `./archive` to run regularly and pull new links into the index.
+The archiving is additive, so you can schedule `./archive` to [run regularly](https://github.com/pirate/ArchiveBox/wiki/Scheduled-Archiving) and pull new links into the index.
 All the saved content is static and indexed with JSON files, so it lives forever & is easily parseable, it requires no always-running backend.
 
 ### Related Projects
 
-There are tons of great web archiving tools out there.  In particular https://Webrecorder.io (pywb) and https://getpolarized.io/ are robust, stable pieces of software that have lots of overlap with ArchiveBox.  ArchiveBox differentiates itself by being primarily a one-shot CLI tool that specializing in importing streams of links from RSS, JSON (good for automatically archiving from a stream of browser history or bookmarks on a schedule), as opposed to a desktop application or web service that requires human interaction to add links.
+There are tons of great web archiving tools out there.  In particular https://webrecorder.io ([pywb](https://github.com/webrecorder/pywb)) and https://getpolarized.io/ are robust, stable pieces of software that have lots of overlap with ArchiveBox.  ArchiveBox differentiates itself by being primarily a one-shot CLI tool that specializing in importing streams of links from RSS, JSON (good for automatically archiving from a stream of browser history or bookmarks on a schedule), as opposed to a desktop application or web service that requires human interaction to add links.
 
 To learn more about the motivation for this project and how it fits into the broader community, see our [Web Archiving Community](https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community) wiki page.
 

archivebox/archive.py

@@ -7,35 +7,31 @@ import os
 import sys
 
 from datetime import datetime
-from subprocess import run
+from peekable import Peekable
+
 
 from parse import parse_links
-from links import (
-    new_links,
-    validate_links
-)
-from archive_methods import archive_links, _RESULTS_TOTALS
+from links import validate_links, links_after_timestamp
+from archive_methods import archive_link, _RESULTS_TOTALS
 from index import (
     write_links_index,
-    write_link_index,
     parse_json_links_index,
-    parse_json_link_index,
 )
 from config import (
+    ARCHIVE_DIR,
     ONLY_NEW,
-    OUTPUT_PERMISSIONS,
     OUTPUT_DIR,
+    REPO_DIR,
     ANSI,
-    TIMEOUT,
     GIT_SHA,
 )
 from util import (
-    download_url,
-    save_source,
-    progress,
-    cleanup_archive,
+    check_dependencies,
+    save_remote_source,
+    save_stdin_source,
     pretty_path,
     migrate_data,
+    check_links_structure,
 )
 
 __AUTHOR__ = 'Nick Sweeting <git@nicksweeting.com>'
@@ -43,6 +39,7 @@ __VERSION__ = GIT_SHA
 __DESCRIPTION__ = 'ArchiveBox Usage:  Create a browsable html archive of a list of links.'
 __DOCUMENTATION__ = 'https://github.com/pirate/ArchiveBox/wiki'
 
+
 def print_help():
     print(__DESCRIPTION__)
     print("Documentation:     {}\n".format(__DOCUMENTATION__))
@@ -54,41 +51,34 @@ def print_help():
     print("    echo 'https://examplecom' | ./bin/archivebox\n")
 
 
-def merge_links(archive_path=OUTPUT_DIR, import_path=None, only_new=False):
+def load_links(archive_path=OUTPUT_DIR, import_path=None):
     """get new links from file and optionally append them to links in existing archive"""
-    all_links = []
-    if import_path:
-        # parse and validate the import file
-        raw_links, parser_name = parse_links(import_path)
-        all_links = validate_links(raw_links)
 
-    # merge existing links in archive_path and new links
     existing_links = []
     if archive_path:
         existing_links = parse_json_links_index(archive_path)
-        all_links = validate_links(existing_links + all_links)
+        check_links_structure(existing_links)
 
+    new_links = []
+    if import_path:
+        # parse and validate the import file
+        raw_links, parser_name = parse_links(import_path)
+        new_links = validate_links(raw_links)
+        check_links_structure(new_links)
+
+    # merge existing links in archive_path and new links
+    all_links = validate_links(existing_links + new_links)
+    check_links_structure(all_links)
     num_new_links = len(all_links) - len(existing_links)
-    if num_new_links and not only_new:
-        print('{green}[+] [{}] Adding {} new links to index from {} ({} format){reset}'.format(
-            datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+
+    if import_path and parser_name:
+        print('    > Adding {} new links to index (parsed import as {})'.format(
             num_new_links,
-            pretty_path(import_path),
             parser_name,
-            **ANSI,
         ))
-    # else:
-    #     print('[*] [{}] No new links added to {}/index.json{}'.format(
-    #         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-    #         archive_path,
-    #         ' from {}'.format(import_path) if import_path else '',
-    #         **ANSI,
-    #     ))
 
-    if only_new:
-        return new_links(all_links, existing_links)
+    return all_links, new_links
 
-    return all_links
 
 def update_archive(archive_path, links, source=None, resume=None, append=True):
     """update or create index.html+json given a path to an export file containing new links"""
@@ -102,14 +92,44 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
              **ANSI,
         ))
     else:
-        print('{green}[▶] [{}] Downloading content for {} pages in archive...{reset}'.format(
+        print('{green}[▶] [{}] Updating content for {} pages in archive...{reset}'.format(
              datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
              len(links),
              **ANSI,
         ))
 
+    check_links_structure(links)
+
+    # prefetch the first link off the generator so that if we pause or fail
+    # immediately we can show that we paused on the first link and not just None
+    to_archive = Peekable(links_after_timestamp(links, resume))
+    idx, link = 0, to_archive.peek(0)
+
     # loop over links and archive them
-    archive_links(archive_path, links, source=source, resume=resume)
+    try:
+        check_dependencies()
+        for idx, link in enumerate(to_archive):
+            link_dir = os.path.join(ARCHIVE_DIR, link['timestamp'])
+            archive_link(link_dir, link)
+
+    except (KeyboardInterrupt, SystemExit, Exception) as e:
+        print('\n{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
+            **ANSI,
+            now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+            idx=idx+1,
+            timestamp=link['timestamp'],
+            total=len(links),
+        ))
+        print('    To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', '')))
+        print('    Continue where you left off by running:')
+        print('        {} {}'.format(
+            pretty_path(sys.argv[0]),
+            link['timestamp'],
+        ))
+        if not isinstance(e, KeyboardInterrupt):
+            print()
+            raise e
+        raise SystemExit(1)
 
     # print timing information & summary
     end_ts = datetime.now().timestamp()
@@ -119,7 +139,7 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
     else:
         duration = '{0:.2f} sec'.format(seconds, 2)
 
-    print('{}[√] [{}] Update of {} links complete ({}){}'.format(
+    print('{}[√] [{}] Update of {} pages complete ({}){}'.format(
         ANSI['green'],
         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
         len(links),
@@ -129,6 +149,7 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
     print('    - {} entries skipped'.format(_RESULTS_TOTALS['skipped']))
     print('    - {} entries updated'.format(_RESULTS_TOTALS['succeded']))
     print('    - {} errors'.format(_RESULTS_TOTALS['failed']))
+    print('    To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', '')))
 
 
 if __name__ == '__main__':
@@ -143,7 +164,7 @@ if __name__ == '__main__':
     source = sys.argv[1] if argc > 1 else None  # path of links file to import
     resume = sys.argv[2] if argc > 2 else None  # timestamp to resume dowloading from
    
-    stdin_raw_text = []
+    stdin_raw_text = ''
 
     if not sys.stdin.isatty():
         stdin_raw_text = sys.stdin.read()
@@ -181,18 +202,22 @@ if __name__ == '__main__':
 
     # Step 0: Download url to local file (only happens if a URL is specified instead of local path) 
     if source and any(source.startswith(s) for s in ('http://', 'https://', 'ftp://')):
-        source = download_url(source)
+        source = save_remote_source(source)
     elif stdin_raw_text:
-        source = save_source(stdin_raw_text)
+        source = save_stdin_source(stdin_raw_text)
 
     # Step 1: Parse the links and dedupe them with existing archive
-    links = merge_links(archive_path=out_dir, import_path=source, only_new=ONLY_NEW)
+    all_links, new_links = load_links(archive_path=out_dir, import_path=source)
 
     # Step 2: Write new index
-    write_links_index(out_dir=out_dir, links=links)
+    write_links_index(out_dir=out_dir, links=all_links)
 
-    # Step 3: Verify folder structure is 1:1 with index
-    # cleanup_archive(out_dir, links)
+    # Step 3: Run the archive methods for each link
+    if ONLY_NEW:
+        update_archive(out_dir, new_links, source=source, resume=resume, append=True)
+    else:
+        update_archive(out_dir, all_links, source=source, resume=resume, append=True)
 
-    # Step 4: Run the archive methods for each link
-    update_archive(out_dir, links, source=source, resume=resume, append=True)
+    # Step 4: Re-write links index with updated titles, icons, and resources
+    all_links, _ = load_links(archive_path=out_dir)
+    write_links_index(out_dir=out_dir, links=all_links)

archivebox/archive_methods.py

@@ -1,17 +1,24 @@
 import os
-import re
-import sys
 
 from functools import wraps
 from collections import defaultdict
 from datetime import datetime
 
-from peekable import Peekable
-
-from index import wget_output_path, parse_json_link_index, write_link_index
-from links import links_after_timestamp
+from index import (
+    wget_output_path,
+    parse_json_link_index,
+    write_link_index,
+    patch_index_title_hack,
+)
 from config import (
+    OUTPUT_DIR,
+    CURL_BINARY,
+    GIT_BINARY,
+    WGET_BINARY,
+    YOUTUBEDL_BINARY,
     CHROME_BINARY,
+    FETCH_FAVICON,
+    FETCH_TITLE,
     FETCH_WGET,
     FETCH_WGET_REQUISITES,
     FETCH_PDF,
@@ -23,7 +30,7 @@ from config import (
     RESOLUTION,
     CHECK_SSL_VALIDITY,
     SUBMIT_ARCHIVE_DOT_ORG,
-    FETCH_FAVICON,
+    COOKIES_FILE,
     WGET_USER_AGENT,
     CHROME_USER_DATA_DIR,
     CHROME_SANDBOX,
@@ -35,11 +42,13 @@ from config import (
     GIT_SHA,
 )
 from util import (
-    check_dependencies,
+    without_fragment,
+    fetch_page_title,
     progress,
     chmod_file,
     pretty_path,
-    run, PIPE, DEVNULL
+    check_link_structure,
+    run, PIPE, DEVNULL,
 )
 
 
@@ -49,38 +58,12 @@ _RESULTS_TOTALS = {   # globals are bad, mmkay
     'failed': 0,
 }
 
-def archive_links(archive_path, links, source=None, resume=None):
-    check_dependencies()
-
-    to_archive = Peekable(links_after_timestamp(links, resume))
-    idx, link = 0, to_archive.peek(0)
-
-    try:
-        for idx, link in enumerate(to_archive):
-            link_dir = os.path.join(ARCHIVE_DIR, link['timestamp'])
-            archive_link(link_dir, link)
-    
-    except (KeyboardInterrupt, SystemExit, Exception) as e:
-        print('{lightyellow}[X] [{now}] Downloading paused on link {timestamp} ({idx}/{total}){reset}'.format(
-            **ANSI,
-            now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-            idx=idx+1,
-            timestamp=link['timestamp'],
-            total=len(links),
-        ))
-        print('    Continue where you left off by running:')
-        print('        {} {}'.format(
-            pretty_path(sys.argv[0]),
-            link['timestamp'],
-        ))
-        if not isinstance(e, KeyboardInterrupt):
-            raise e
-        raise SystemExit(1)
-
 
 def archive_link(link_dir, link, overwrite=True):
     """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
 
+    check_link_structure(link)
+
     try:
         update_existing = os.path.exists(link_dir)
         if update_existing:
@@ -91,11 +74,14 @@ def archive_link(link_dir, link, overwrite=True):
         else:
             os.makedirs(link_dir)
         
-        log_link_archive(link_dir, link, update_existing)
+        print_link_status_line(link_dir, link, update_existing)
 
         if FETCH_FAVICON:
             link = fetch_favicon(link_dir, link, overwrite=overwrite)
 
+        if FETCH_TITLE:
+            link = fetch_title(link_dir, link, overwrite=overwrite)
+
         if FETCH_WGET:
             link = fetch_wget(link_dir, link, overwrite=overwrite)
 
@@ -124,18 +110,18 @@ def archive_link(link_dir, link, overwrite=True):
     
     return link
 
-def log_link_archive(link_dir, link, update_existing):
+def print_link_status_line(link_dir, link, update_existing):
     print('[{symbol_color}{symbol}{reset}] [{now}] "{title}"\n    {blue}{url}{reset}'.format(
         symbol='*' if update_existing else '+',
         symbol_color=ANSI['black' if update_existing else 'green'],
         now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-        **link,
+        **{**link, 'title': link['title'] or link['url']},
         **ANSI,
     ))
 
     print('    > {}{}'.format(pretty_path(link_dir), '' if update_existing else ' (new)'))
-    if link['type']:
-        print('      i {}'.format(link['type']))
+    # if link['type']:
+    #     print('      i {}'.format(link['type']))
 
 
 
@@ -208,7 +194,7 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
 
     # WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
     CMD = [
-        'wget',
+        WGET_BINARY,
         # '--server-response',  # print headers for better error parsing
         '--no-verbose',
         '--adjust-extension',
@@ -217,12 +203,14 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
         '--backup-converted',
         '--span-hosts',
         '--no-parent',
+        '-e', 'robots=off',
         '--restrict-file-names=unix',
         '--timeout={}'.format(timeout),
         *(() if warc else ('--timestamping',)),
         *(('--warc-file={}'.format(warc_path),) if warc else ()),
         *(('--page-requisites',) if FETCH_WGET_REQUISITES else ()),
         *(('--user-agent={}'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()),
+        *(('--load-cookies', COOKIES_FILE) if COOKIES_FILE else ()),
         *((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', '--no-hsts'))),
         link['url'],
     ]
@@ -254,12 +242,19 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
             raise Exception('Got an error from the server')
     except Exception as e:
         end()
+
+        # to let the user copy-paste the command and run it safely we have
+        # to quote some of the arguments that could have spaces in them
+        quoted_cmd = ' '.join(CMD)
+        quoted_cmd = quoted_cmd.replace(WGET_USER_AGENT, '"{}"'.format(WGET_USER_AGENT))
+        if COOKIES_FILE:
+            quoted_cmd = quoted_cmd.replace(COOKIES_FILE, '"{}"'.format(COOKIES_FILE))
+
         print('        {}Some resources were skipped: {}{}'.format(ANSI['lightyellow'], e, ANSI['reset']))
         print('        Run to see full output:')
         print('            cd {};'.format(link_dir))
-        print('            {}'.format(' '.join(CMD).replace(WGET_USER_AGENT, '"{}"'.format(WGET_USER_AGENT))))
+        print('            {}'.format(quoted_cmd))
         output = e
-
     return {
         'cmd': CMD,
         'output': output,
@@ -402,7 +397,7 @@ def archive_dot_org(link_dir, link, timeout=TIMEOUT):
 
     success = False
     CMD = [
-        'curl',
+        CURL_BINARY,
         '--location',
         '--head',
         '--user-agent', 'ArchiveBox/{} (+https://github.com/pirate/ArchiveBox/)'.format(GIT_SHA),
@@ -466,8 +461,9 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
         return {'output': 'favicon.ico', 'status': 'skipped'}
 
     CMD = [
-        'curl',
+        CURL_BINARY,
         '--max-time', str(timeout),
+        *(() if CHECK_SSL_VALIDITY else ('--insecure',)),
         'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
     ]
     fout = open('{}/favicon.ico'.format(link_dir), 'w')
@@ -491,6 +487,36 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
         'output': output,
     }
 
+@attach_result_to_link('title')
+def fetch_title(link_dir, link, timeout=TIMEOUT):
+    """try to guess the page's title from its content"""
+
+    # if link already has valid title, skip it
+    if link['title'] and not link['title'].lower().startswith('http'):
+        return {'output': link['title'], 'status': 'skipped'}
+
+    end = progress(timeout, prefix='      ')
+    try:
+        title = fetch_page_title(link['url'], timeout=timeout, progress=False)
+        end()
+        output = title
+    except Exception as e:
+        end()
+        print('        {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
+        output = e
+
+    # titles should show up in the global index immediatley for better UX,
+    # do a hacky immediate replacement to add them in as we're archiving
+    # TODO: figure out how to do this without gnarly string replacement
+    if title:
+        link['title'] = title
+        patch_index_title_hack(link['url'], title)
+
+    return {
+        'cmd': 'fetch_page_title("{}")'.format(link['url']),
+        'output': output,
+    }
+
 @attach_result_to_link('media')
 def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
     """Download playlists or individual video, audio, and subtitles using youtube-dl"""
@@ -504,7 +530,7 @@ def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
 
     os.makedirs(output, exist_ok=True)
     CMD = [
-        'youtube-dl',
+        YOUTUBEDL_BINARY,
         '--write-description',
         '--write-info-json',
         '--write-annotations',
@@ -514,12 +540,15 @@ def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
         '--no-check-certificate',
         '--user-agent',
         '--all-subs',
-        '-x',
-        '-k',
+        '--extract-audio',
+        '--keep-video',
+        '--ignore-errors',
+        '--geo-bypass',
         '--audio-format', 'mp3',
         '--audio-quality', '320K',
         '--embed-thumbnail',
         '--add-metadata',
+        *(() if CHECK_SSL_VALIDITY else ('--no-check-certificate',)),
         link['url'],
     ]
 
@@ -567,7 +596,14 @@ def fetch_git(link_dir, link, timeout=TIMEOUT):
     if os.path.exists(os.path.join(link_dir, 'git')):
         return {'output': 'git', 'status': 'skipped'}
 
-    CMD = ['git', 'clone', '--mirror', '--recursive', link['url'].split('#')[0], 'git']
+    CMD = [
+        GIT_BINARY,
+        *(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')),
+        'clone',
+        '--mirror',
+        '--recursive',
+        without_fragment(link['url']),
+    ]
     output = 'git'
 
     end = progress(timeout, prefix='      ')

archivebox/config.py

@@ -11,12 +11,13 @@ from subprocess import run, PIPE
 # ******************************************************************************
 
 IS_TTY = sys.stdout.isatty()
-ONLY_NEW =               os.getenv('ONLY_NEW',               'False'            ).lower() == 'true'
 USE_COLOR =              os.getenv('USE_COLOR',              str(IS_TTY)        ).lower() == 'true'
 SHOW_PROGRESS =          os.getenv('SHOW_PROGRESS',          str(IS_TTY)        ).lower() == 'true'
-OUTPUT_PERMISSIONS =     os.getenv('OUTPUT_PERMISSIONS',     '755'              )
+ONLY_NEW =               os.getenv('ONLY_NEW',               'False'            ).lower() == 'true'
 MEDIA_TIMEOUT =          int(os.getenv('MEDIA_TIMEOUT',      '3600'))
 TIMEOUT =                int(os.getenv('TIMEOUT',            '60'))
+OUTPUT_PERMISSIONS =     os.getenv('OUTPUT_PERMISSIONS',     '755'              )
+FOOTER_INFO =            os.getenv('FOOTER_INFO',            'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.',)
 
 FETCH_WGET =             os.getenv('FETCH_WGET',             'True'             ).lower() == 'true'
 FETCH_WGET_REQUISITES =  os.getenv('FETCH_WGET_REQUISITES',  'True'             ).lower() == 'true'
@@ -27,17 +28,21 @@ FETCH_WARC =             os.getenv('FETCH_WARC',             'True'
 FETCH_GIT =              os.getenv('FETCH_GIT',              'True'             ).lower() == 'true'
 FETCH_MEDIA =            os.getenv('FETCH_MEDIA',            'True'             ).lower() == 'true'
 FETCH_FAVICON =          os.getenv('FETCH_FAVICON',          'True'             ).lower() == 'true'
+FETCH_TITLE =            os.getenv('FETCH_TITLE',            'True'             ).lower() == 'true'
 SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True'             ).lower() == 'true'
 
 CHECK_SSL_VALIDITY =     os.getenv('CHECK_SSL_VALIDITY',     'True'             ).lower() == 'true'
 RESOLUTION =             os.getenv('RESOLUTION',             '1440,2000'        )
 GIT_DOMAINS =            os.getenv('GIT_DOMAINS',            'github.com,bitbucket.org,gitlab.com').split(',')
 WGET_USER_AGENT =        os.getenv('WGET_USER_AGENT',        'ArchiveBox/{GIT_SHA} (+https://github.com/pirate/ArchiveBox/) wget/{WGET_VERSION}')
-CHROME_USER_DATA_DIR =   os.getenv('CHROME_USER_DATA_DIR',    None)
+COOKIES_FILE =           os.getenv('COOKIES_FILE',           None)
+CHROME_USER_DATA_DIR =   os.getenv('CHROME_USER_DATA_DIR',   None)
 
-CHROME_BINARY =          os.getenv('CHROME_BINARY',          None)  # change to google-chrome browser if using google-chrome
-WGET_BINARY =            os.getenv('WGET_BINARY',            'wget'             )
-FOOTER_INFO =            os.getenv('FOOTER_INFO',            'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.',)
+CURL_BINARY =            os.getenv('CURL_BINARY',            'curl')
+GIT_BINARY =             os.getenv('GIT_BINARY',             'git')
+WGET_BINARY =            os.getenv('WGET_BINARY',            'wget')
+YOUTUBEDL_BINARY =       os.getenv('YOUTUBEDL_BINARY',       'youtube-dl')
+CHROME_BINARY =          os.getenv('CHROME_BINARY',          None)
 
 ### Paths
 REPO_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
@@ -99,7 +104,7 @@ if not USE_COLOR:
 ### Confirm Environment Setup
 GIT_SHA = 'unknown'
 try:
-    GIT_SHA = run(["git", "rev-list", "-1", "HEAD", "./"], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode()
+    GIT_SHA = run([GIT_BINARY, 'rev-list', '-1', 'HEAD', './'], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode()
 except Exception:
     print('[!] Warning: unable to determine git version, is git installed and in your $PATH?')
 
@@ -113,7 +118,7 @@ except Exception:
 
 WGET_VERSION = 'unknown'
 try:
-    wget_vers_str = run(["wget", "--version"], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode()
+    wget_vers_str = run([WGET_BINARY, "--version"], stdout=PIPE, cwd=REPO_DIR).stdout.strip().decode()
     WGET_VERSION = wget_vers_str.split('\n')[0].split(' ')[2]
 except Exception:
     if USE_WGET:
@@ -121,6 +126,12 @@ except Exception:
 
 WGET_USER_AGENT = WGET_USER_AGENT.format(GIT_SHA=GIT_SHA[:9], WGET_VERSION=WGET_VERSION)
 
+try:
+    COOKIES_FILE = os.path.abspath(COOKIES_FILE) if COOKIES_FILE else None
+except Exception:
+    print('[!] Warning: unable to get full path to COOKIES_FILE, are you sure you specified it correctly?')
+    raise
+
 if sys.stdout.encoding.upper() not in ('UTF-8', 'UTF8'):
     print('[X] Your system is running python3 scripts with a bad locale setting: {} (it should be UTF-8).'.format(sys.stdout.encoding))
     print('    To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)')

archivebox/index.py

@@ -6,6 +6,7 @@ from string import Template
 from distutils.dir_util import copy_tree
 
 from config import (
+    OUTPUT_DIR,
     TEMPLATES_DIR,
     OUTPUT_PERMISSIONS,
     ANSI,
@@ -17,6 +18,8 @@ from util import (
     wget_output_path,
     derived_link_info,
     pretty_path,
+    check_link_structure,
+    check_links_structure,
 )
 
 
@@ -25,21 +28,27 @@ from util import (
 def write_links_index(out_dir, links):
     """create index.html file for a given list of links"""
 
+    check_links_structure(links)
+
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
 
-    write_json_links_index(out_dir, links)
-    write_html_links_index(out_dir, links)
-    
-    print('{green}[√] [{}] Updated main index files:{reset}'.format(
+    print('{green}[*] [{}] Updating main index files...{reset}'.format(
         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-        **ANSI))
+        **ANSI,
+    ))
+    write_json_links_index(out_dir, links)
     print('    > {}/index.json'.format(pretty_path(out_dir)))
+    
+    write_html_links_index(out_dir, links)
     print('    > {}/index.html'.format(pretty_path(out_dir)))
+    
 
 def write_json_links_index(out_dir, links):
     """write the json link index to a given path"""
 
+    check_links_structure(links)
+
     path = os.path.join(out_dir, 'index.json')
 
     index_json = {
@@ -61,13 +70,17 @@ def parse_json_links_index(out_dir):
     index_path = os.path.join(out_dir, 'index.json')
     if os.path.exists(index_path):
         with open(index_path, 'r', encoding='utf-8') as f:
-            return json.load(f)['links']
+            links = json.load(f)['links']
+            check_links_structure(links)
+            return links
 
     return []
 
 def write_html_links_index(out_dir, links):
     """write the html link index to a given path"""
 
+    check_links_structure(links)
+
     path = os.path.join(out_dir, 'index.html')
 
     copy_tree(os.path.join(TEMPLATES_DIR, 'static'), os.path.join(out_dir, 'static'))
@@ -102,6 +115,25 @@ def write_html_links_index(out_dir, links):
     chmod_file(path)
 
 
+def patch_index_title_hack(link_url, new_title):
+    """hack to update just one link's title in the link index json"""
+
+    json_path = os.path.join(OUTPUT_DIR, 'index.json')
+
+    links = parse_json_links_index(OUTPUT_DIR)
+
+    changed = False
+    for link in links:
+        if link['url'] == link_url:
+            link['title'] = new_title
+            changed = True
+            break
+
+    if changed:
+        write_json_links_index(OUTPUT_DIR, links)
+
+
+
 ### Individual link index
 
 def write_link_index(out_dir, link):
@@ -112,6 +144,7 @@ def write_link_index(out_dir, link):
 def write_json_link_index(out_dir, link):
     """write a json file with some info about the link"""
     
+    check_link_structure(link)
     path = os.path.join(out_dir, 'index.json')
 
     print('      √ index.json')
@@ -126,10 +159,13 @@ def parse_json_link_index(out_dir):
     existing_index = os.path.join(out_dir, 'index.json')
     if os.path.exists(existing_index):
         with open(existing_index, 'r', encoding='utf-8') as f:
-            return json.load(f)
+            link_json = json.load(f)
+            check_link_structure(link_json)
+            return link_json
     return {}
 
 def write_html_link_index(out_dir, link):
+    check_link_structure(link)
     with open(os.path.join(TEMPLATES_DIR, 'link_index_fancy.html'), 'r', encoding='utf-8') as f:
         link_html = f.read()
 
@@ -141,6 +177,7 @@ def write_html_link_index(out_dir, link):
         f.write(Template(link_html).substitute({
             **link,
             **link['latest'],
+            'title': link['title'] or link['url'],
             'type': link['type'] or 'website',
             'tags': link['tags'] or 'untagged',
             'bookmarked': datetime.fromtimestamp(float(link['timestamp'])).strftime('%Y-%m-%d %H:%M'),

archivebox/links.py

@@ -32,34 +32,33 @@ Link {
 
 """
 
-import datetime
 from html import unescape
 from collections import OrderedDict
 
 from util import (
-    domain,
-    base_url,
-    str_between,
-    get_link_type,
     merge_links,
     wget_output_path,
+    check_link_structure,
+    check_links_structure,
 )
-from config import ANSI
 
 
 def validate_links(links):
+    check_links_structure(links)
     links = archivable_links(links)  # remove chrome://, about:, mailto: etc.
     links = uniquefied_links(links)  # merge/dedupe duplicate timestamps & urls
     links = sorted_links(links)      # deterministically sort the links based on timstamp, url
-    
+
     if not links:
         print('[X] No links found :(')
         raise SystemExit(1)
 
     for link in links:
-        link['title'] = unescape(link['title'])
+        check_link_structure(link)
+
+        link['title'] = unescape(link['title']) if link['title'] else None
         link['latest'] = link.get('latest') or {}
-        
+
         latest = link['latest']
         if not link['latest'].get('wget'):
             link['latest']['wget'] = wget_output_path(link)
@@ -76,25 +75,21 @@ def validate_links(links):
         if not latest.get('favicon'):
             latest['favicon'] = None
 
+        if not link['latest'].get('title'):
+            link['latest']['title'] = link['title']
+
     return list(links)
 
-def new_links(all_links, existing_links):
-    """
-    Return all links which are in the all_links but not in the existing_links.
-    This is used to determine which links are new and not indexed jet. Set the
-    ONLY_NEW environment variable to activate this filter mechanism.
-    """
-    existing_urls = {link['url'] for link in existing_links}
-    return [link for link in all_links if link['url'] not in existing_urls]
 
 def archivable_links(links):
     """remove chrome://, about:// or other schemed links that cant be archived"""
     return (
         link
         for link in links
-        if any(link['url'].startswith(s) for s in ('http://', 'https://', 'ftp://'))
+        if any(link['url'].lower().startswith(s) for s in ('http://', 'https://', 'ftp://'))
     )
 
+
 def uniquefied_links(sorted_links):
     """
     ensures that all non-duplicate links have monotonically increasing timestamps
@@ -120,10 +115,12 @@ def uniquefied_links(sorted_links):
 
     return unique_timestamps.values()
 
+
 def sorted_links(links):
     sort_func = lambda link: (link['timestamp'].split('.', 1)[0], link['url'])
     return sorted(links, key=sort_func, reverse=True)
 
+
 def links_after_timestamp(links, timestamp=None):
     if not timestamp:
         yield from links
@@ -136,6 +133,7 @@ def links_after_timestamp(links, timestamp=None):
         except (ValueError, TypeError):
             print('Resume value and all timestamp values must be valid numbers.')
 
+
 def lowest_uniq_timestamp(used_timestamps, timestamp):
     """resolve duplicate timestamps by appending a decimal 1234, 1234 -> 1234.1, 1234.2"""
 

archivebox/parse.py

@@ -8,71 +8,54 @@ For examples of supported files see examples/.
 
 Parsed link schema: {
     'url': 'https://example.com/example/?abc=123&xyc=345#lmnop',
-    'domain': 'example.com',
-    'base_url': 'example.com/example/',
     'timestamp': '15442123124234',
-    'tags': 'abc,def',
     'title': 'Example.com Page Title',
+    'tags': 'abc,def',
     'sources': ['ril_export.html', 'downloads/getpocket.com.txt'],
 }
 """
 
 import re
 import json
-import urllib
+
+from datetime import datetime
 from collections import OrderedDict
 import xml.etree.ElementTree as etree
 
-from datetime import datetime
-
 from config import ANSI
 from util import (
-    domain,
-    base_url,
     str_between,
     get_link_type,
-    fetch_page_title,
     URL_REGEX,
+    check_url_parsing,
 )
 
 
-def get_parsers(file):
-    """return all parsers that work on a given file, defaults to all of them"""
-
-    return OrderedDict([
-        ('Pocket HTML', parse_pocket_html_export),
-        ('Pinboard JSON', parse_pinboard_json_export),
-        ('Netscape HTML', parse_netscape_html_export),
-        ('RSS', parse_rss_export),
-        ('Pinboard RSS', parse_pinboard_rss_export),
-        ('Shaarli RSS', parse_shaarli_rss_export),
-        ('Medium RSS', parse_medium_rss_export),
-        ('Plain Text', parse_plain_text_export),
-    ])
-
 def parse_links(path):
     """parse a list of links dictionaries from a bookmark export file"""
     
+    check_url_parsing()
+
     links = []
     with open(path, 'r', encoding='utf-8') as file:
-        print('{green}[*] [{}] Parsing new links from output/sources/{} and fetching titles...{reset}'.format(
+        print('{green}[*] [{}] Parsing new links from output/sources/{}...{reset}'.format(
             datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
             path.rsplit('/', 1)[-1],
             **ANSI,
         ))
 
-        for parser_name, parser_func in get_parsers(file).items():
-            # otherwise try all parsers until one works
+        for parser_name, parser_func in PARSERS.items():
             try:
                 links += list(parser_func(file))
                 if links:
                     break
-            except (ValueError, TypeError, IndexError, AttributeError, etree.ParseError):
-                # parser not supported on this file
+            except Exception as err:
+                # we try each parser one by one, wong parsers will throw exeptions
+                # if unsupported and we accept the first one that passes
+                # uncomment the following line to see why the parser was unsupported for each attempted format
+                # print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err))
                 pass
 
-    print()
-
     return links, parser_name
 
 
@@ -90,11 +73,9 @@ def parse_pocket_html_export(html_file):
             time = datetime.fromtimestamp(float(match.group(2)))
             info = {
                 'url': fixed_url,
-                'domain': domain(fixed_url),
-                'base_url': base_url(fixed_url),
                 'timestamp': str(time.timestamp()),
                 'tags': match.group(3),
-                'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or fetch_page_title(fixed_url),
+                'title': match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') or None,
                 'sources': [html_file.name],
             }
             info['type'] = get_link_type(info)
@@ -127,16 +108,15 @@ def parse_pinboard_json_export(json_file):
                 title = erg['title'].strip()
             info = {
                 'url': url,
-                'domain': domain(url),
-                'base_url': base_url(url),
                 'timestamp': timestamp,
                 'tags': erg.get('tags') or '',
-                'title': title or fetch_page_title(url),
+                'title': title or None,
                 'sources': [json_file.name],
             }
             info['type'] = get_link_type(info)
             yield info
 
+
 def parse_rss_export(rss_file):
     """Parse RSS XML-format files into links"""
 
@@ -157,7 +137,7 @@ def parse_rss_export(rss_file):
         rows = leading_removed.split('\n')
 
         def get_row(key):
-            return [r for r in rows if r.startswith('<{}>'.format(key))][0]
+            return [r for r in rows if r.strip().startswith('<{}>'.format(key))][0]
 
         title = str_between(get_row('title'), '<![CDATA[', ']]').strip()
         url = str_between(get_row('link'), '<link>', '</link>')
@@ -166,17 +146,16 @@ def parse_rss_export(rss_file):
 
         info = {
             'url': url,
-            'domain': domain(url),
-            'base_url': base_url(url),
-            'timestamp': str(datetime.now().timestamp()),
+            'timestamp': str(time.timestamp()),
             'tags': '',
-            'title': title or fetch_page_title(url),
+            'title': title or None,
             'sources': [rss_file.name],
         }
         info['type'] = get_link_type(info)
 
         yield info
 
+
 def parse_shaarli_rss_export(rss_file):
     """Parse Shaarli-specific RSS XML-format files into links"""
 
@@ -205,14 +184,11 @@ def parse_shaarli_rss_export(rss_file):
         ts_str = str_between(get_row('published'), '<published>', '</published>')
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
 
-
         info = {
             'url': url,
-            'domain': domain(url),
-            'base_url': base_url(url),
             'timestamp': str(time.timestamp()),
             'tags': '',
-            'title': title or fetch_page_title(url),
+            'title': title or None,
             'sources': [rss_file.name],
         }
         info['type'] = get_link_type(info)
@@ -235,11 +211,9 @@ def parse_netscape_html_export(html_file):
 
             info = {
                 'url': url,
-                'domain': domain(url),
-                'base_url': base_url(url),
                 'timestamp': str(time.timestamp()),
                 'tags': "",
-                'title': match.group(3).strip() or fetch_page_title(url),
+                'title': match.group(3).strip() or None,
                 'sources': [html_file.name],
             }
             info['type'] = get_link_type(info)
@@ -268,11 +242,9 @@ def parse_pinboard_rss_export(rss_file):
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
         info = {
             'url': url,
-            'domain': domain(url),
-            'base_url': base_url(url),
             'timestamp': str(time.timestamp()),
             'tags': tags,
-            'title': title or fetch_page_title(url),
+            'title': title or None,
             'sources': [rss_file.name],
         }
         info['type'] = get_link_type(info)
@@ -293,11 +265,9 @@ def parse_medium_rss_export(rss_file):
         time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z")
         info = {
             'url': url,
-            'domain': domain(url),
-            'base_url': base_url(url),
             'timestamp': str(time.timestamp()),
             'tags': '',
-            'title': title or fetch_page_title(url),
+            'title': title or None,
             'sources': [rss_file.name],
         }
         info['type'] = get_link_type(info)
@@ -317,13 +287,22 @@ def parse_plain_text_export(text_file):
                 url = url.strip()
                 info = {
                     'url': url,
-                    'domain': domain(url),
-                    'base_url': base_url(url),
                     'timestamp': str(datetime.now().timestamp()),
                     'tags': '',
-                    'title': fetch_page_title(url),
+                    'title': None,
                     'sources': [text_file.name],
                 }
                 info['type'] = get_link_type(info)
                 yield info
 
+
+PARSERS = OrderedDict([
+    ('Pocket HTML', parse_pocket_html_export),
+    ('Pinboard JSON', parse_pinboard_json_export),
+    ('Netscape HTML', parse_netscape_html_export),
+    ('RSS', parse_rss_export),
+    ('Pinboard RSS', parse_pinboard_rss_export),
+    ('Shaarli RSS', parse_shaarli_rss_export),
+    ('Medium RSS', parse_medium_rss_export),
+    ('Plain Text', parse_plain_text_export),
+])

archivebox/util.py

@@ -3,8 +3,8 @@ import re
 import sys
 import time
 import json
-import signal
-import urllib.request
+from urllib.request import Request, urlopen
+from urllib.parse import urlparse
 
 from decimal import Decimal
 from urllib.parse import quote
@@ -13,37 +13,64 @@ from subprocess import TimeoutExpired, Popen, PIPE, DEVNULL, CompletedProcess, C
 from multiprocessing import Process
 
 from config import (
-    IS_TTY,
-    OUTPUT_PERMISSIONS,
-    REPO_DIR,
-    SOURCES_DIR,
-    OUTPUT_DIR,
-    ARCHIVE_DIR,
-    TIMEOUT,
-    TERM_WIDTH,
-    SHOW_PROGRESS,
     ANSI,
+    IS_TTY,
+    TERM_WIDTH,
+    REPO_DIR,
+    OUTPUT_DIR,
+    SOURCES_DIR,
+    ARCHIVE_DIR,
+    OUTPUT_PERMISSIONS,
+    TIMEOUT,
+    SHOW_PROGRESS,
+    CHECK_SSL_VALIDITY,
+    WGET_USER_AGENT,
+    CURL_BINARY,
+    WGET_BINARY,
     CHROME_BINARY,
+    GIT_BINARY,
+    YOUTUBEDL_BINARY,
+    FETCH_TITLE,
+    FETCH_FAVICON,
     FETCH_WGET,
+    FETCH_WARC,
     FETCH_PDF,
     FETCH_SCREENSHOT,
     FETCH_DOM,
-    FETCH_FAVICON,
+    FETCH_GIT,
     FETCH_MEDIA,
     SUBMIT_ARCHIVE_DOT_ORG,
 )
 
-# URL helpers
-without_scheme = lambda url: url.replace('http://', '').replace('https://', '').replace('ftp://', '')
-without_query = lambda url: url.split('?', 1)[0]
-without_hash = lambda url: url.split('#', 1)[0]
-without_path = lambda url: url.split('/', 1)[0]
-domain = lambda url: without_hash(without_query(without_path(without_scheme(url))))
+# URL helpers: https://docs.python.org/3/library/urllib.parse.html#url-parsing
+scheme = lambda url: urlparse(url).scheme
+without_scheme = lambda url: urlparse(url)._replace(scheme='').geturl().strip('//')
+without_query = lambda url: urlparse(url)._replace(query='').geturl().strip('//')
+without_fragment = lambda url: urlparse(url)._replace(fragment='').geturl().strip('//')
+without_path = lambda url: urlparse(url)._replace(path='', fragment='', query='').geturl().strip('//')
+path = lambda url: urlparse(url).path
+basename = lambda url: urlparse(url).path.rsplit('/', 1)[-1]
+domain = lambda url: urlparse(url).netloc
+query = lambda url: urlparse(url).query
+fragment = lambda url: urlparse(url).fragment
+extension = lambda url: basename(url).rsplit('.', 1)[-1].lower() if '.' in basename(url) else ''
 base_url = lambda url: without_scheme(url)  # uniq base url used to dedupe links
 
 short_ts = lambda ts: ts.split('.')[0]
 
-URL_REGEX = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))[^<\""]+'
+URL_REGEX = re.compile(
+    r'http[s]?://'                    # start matching from allowed schemes
+    r'(?:[a-zA-Z]|[0-9]'              # followed by allowed alphanum characters
+    r'|[$-_@.&+]|[!*\(\),]'           #    or allowed symbols
+    r'|(?:%[0-9a-fA-F][0-9a-fA-F]))'  #    or allowed unicode bytes
+    r'[^\]\[\(\)<>\""\'\s]+',         # stop parsing at these symbols
+    re.IGNORECASE,
+)
+HTML_TITLE_REGEX = re.compile(
+    r'<title>'                         # start matching text after <title> tag
+    r'(.[^<>]+)',                      # get everything up to these symbols
+    re.IGNORECASE,
+)
 
 
 def check_dependencies():
@@ -55,6 +82,20 @@ def check_dependencies():
         print('    See https://github.com/pirate/ArchiveBox#troubleshooting for help upgrading your Python installation.')
         raise SystemExit(1)
 
+    if FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG:
+        if run(['which', CURL_BINARY], stdout=DEVNULL).returncode or run([CURL_BINARY, '--version'], stdout=DEVNULL).returncode:
+            print('{red}[X] Missing dependency: curl{reset}'.format(**ANSI))
+            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format(CURL_BINARY))
+            print('    See https://github.com/pirate/ArchiveBox for help.')
+            raise SystemExit(1)
+
+    if FETCH_WGET or FETCH_WARC:
+        if run(['which', WGET_BINARY], stdout=DEVNULL).returncode or run([WGET_BINARY, '--version'], stdout=DEVNULL).returncode:
+            print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI))
+            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format(WGET_BINARY))
+            print('    See https://github.com/pirate/ArchiveBox for help.')
+            raise SystemExit(1)
+
     if FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM:
         if run(['which', CHROME_BINARY], stdout=DEVNULL).returncode:
             print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset']))
@@ -79,28 +120,45 @@ def check_dependencies():
             print('    See https://github.com/pirate/ArchiveBox for help.')
             raise SystemExit(1)
 
-    if FETCH_WGET:
-        if run(['which', 'wget'], stdout=DEVNULL).returncode or run(['wget', '--version'], stdout=DEVNULL).returncode:
-            print('{red}[X] Missing dependency: wget{reset}'.format(**ANSI))
-            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format('wget'))
-            print('    See https://github.com/pirate/ArchiveBox for help.')
-            raise SystemExit(1)
-
-    if FETCH_FAVICON or SUBMIT_ARCHIVE_DOT_ORG:
-        if run(['which', 'curl'], stdout=DEVNULL).returncode or run(['curl', '--version'], stdout=DEVNULL).returncode:
-            print('{red}[X] Missing dependency: curl{reset}'.format(**ANSI))
-            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format('curl'))
+    if FETCH_GIT:
+        if run(['which', GIT_BINARY], stdout=DEVNULL).returncode or run([GIT_BINARY, '--version'], stdout=DEVNULL).returncode:
+            print('{red}[X] Missing dependency: git{reset}'.format(**ANSI))
+            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format(GIT_BINARY))
             print('    See https://github.com/pirate/ArchiveBox for help.')
             raise SystemExit(1)
 
     if FETCH_MEDIA:
-        if run(['which', 'youtube-dl'], stdout=DEVNULL).returncode or run(['youtube-dl', '--version'], stdout=DEVNULL).returncode:
+        if run(['which', YOUTUBEDL_BINARY], stdout=DEVNULL).returncode or run([YOUTUBEDL_BINARY, '--version'], stdout=DEVNULL).returncode:
             print('{red}[X] Missing dependency: youtube-dl{reset}'.format(**ANSI))
-            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format('youtube-dl'))
+            print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format(YOUTUBEDL_BINARY))
             print('    See https://github.com/pirate/ArchiveBox for help.')
             raise SystemExit(1)
 
 
+def check_url_parsing():
+    """Check that plain text regex URL parsing works as expected"""
+    test_urls = '''
+    https://example1.com/what/is/happening.html?what=1#how-about-this=1
+    https://example2.com/what/is/happening/?what=1#how-about-this=1
+    HTtpS://example3.com/what/is/happening/?what=1#how-about-this=1f
+    https://example4.com/what/is/happening.html
+    https://example5.com/
+    https://example6.com
+
+    <test>http://example7.com</test>
+    [https://example8.com/what/is/this.php?what=1]
+    [and http://example9.com?what=1&other=3#and-thing=2]
+    <what>https://example10.com#and-thing=2 "</about>
+    abc<this["https://example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def
+    sdflkf[what](https://example12.com/who/what.php?whoami=1#whatami=2)?am=hi
+    example13.bada
+    and example14.badb
+    <or>htt://example15.badc</that>
+    '''
+    # print('\n'.join(re.findall(URL_REGEX, test_urls)))
+    assert len(re.findall(URL_REGEX, test_urls)) == 12
+
+
 def chmod_file(path, cwd='.', permissions=OUTPUT_PERMISSIONS, timeout=30):
     """chmod -R <permissions> <cwd>/<path>"""
 
@@ -182,7 +240,7 @@ def pretty_path(path):
     return path.replace(REPO_DIR + '/', '')
 
 
-def save_source(raw_text):
+def save_stdin_source(raw_text):
     if not os.path.exists(SOURCES_DIR):
         os.makedirs(SOURCES_DIR)
 
@@ -196,7 +254,21 @@ def save_source(raw_text):
     return source_path
 
 
-def download_url(url):
+def fetch_page_content(url, timeout=TIMEOUT):
+    req = Request(url, headers={'User-Agent': WGET_USER_AGENT})
+
+    if CHECK_SSL_VALIDITY:
+        resp = urlopen(req, timeout=timeout)
+    else:
+        import ssl
+        insecure = ssl._create_unverified_context()
+        resp = urlopen(req, timeout=timeout, context=insecure)
+
+    encoding = resp.headers.get_content_charset() or 'utf-8'
+    return resp.read().decode(encoding)
+
+
+def save_remote_source(url, timeout=TIMEOUT):
     """download a given url's content into downloads/domain.txt"""
 
     if not os.path.exists(SOURCES_DIR):
@@ -206,42 +278,54 @@ def download_url(url):
 
     source_path = os.path.join(SOURCES_DIR, '{}-{}.txt'.format(domain(url), ts))
 
-    print('[*] [{}] Downloading {} > {}'.format(
+    print('{}[*] [{}] Downloading {}{}'.format(
+        ANSI['green'],
         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
         url,
-        pretty_path(source_path),
+        ANSI['reset'],
     ))
     end = progress(TIMEOUT, prefix='      ')
     try:
-        downloaded_xml = urllib.request.urlopen(url).read().decode('utf-8')
+        downloaded_xml = fetch_page_content(url, timeout=timeout)
         end()
     except Exception as e:
         end()
-        print('[!] Failed to download {}\n'.format(url))
+        print('{}[!] Failed to download {}{}\n'.format(
+            ANSI['red'],
+            url,
+            ANSI['reset'],
+        ))
         print('    ', e)
         raise SystemExit(1)
 
     with open(source_path, 'w', encoding='utf-8') as f:
         f.write(downloaded_xml)
 
+    print('    > {}'.format(pretty_path(source_path)))
+
     return source_path
 
 
-def fetch_page_title(url, default=True):
+def fetch_page_title(url, timeout=10, progress=SHOW_PROGRESS):
     """Attempt to guess a page's title by downloading the html"""
-    if default is True:
-        default = url
+    if not FETCH_TITLE:
+        return None
 
     try:
-        sys.stdout.write('.')
-        sys.stdout.flush()
-        html_content = urllib.request.urlopen(url, timeout=10).read().decode('utf-8')
-        match = re.search('<title>(.*?)</title>', html_content)
-        return match.group(1) if match else default or None
-    except Exception:
-        if default is False:
-            raise
-        return default
+        if progress:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+
+        html = fetch_page_content(url, timeout=timeout)
+
+        match = re.search(HTML_TITLE_REGEX, html)
+        return match.group(1).strip() if match else None
+    except Exception as err:
+        # print('[!] Failed to fetch title because of {}: {}'.format(
+        #     err.__class__.__name__,
+        #     err,
+        # ))
+        return None
 
 
 def str_between(string, start, end=None):
@@ -256,19 +340,19 @@ def str_between(string, start, end=None):
 def get_link_type(link):
     """Certain types of links need to be handled specially, this figures out when that's the case"""
 
-    if link['base_url'].endswith('.pdf'):
+    if extension(link['url']) == 'pdf':
         return 'PDF'
-    elif link['base_url'].rsplit('.', 1) in ('pdf', 'png', 'jpg', 'jpeg', 'svg', 'bmp', 'gif', 'tiff', 'webp'):
+    elif extension(link['url']) in ('pdf', 'png', 'jpg', 'jpeg', 'svg', 'bmp', 'gif', 'tiff', 'webp'):
         return 'image'
-    elif 'wikipedia.org' in link['domain']:
+    elif 'wikipedia.org' in domain(link['url']).lower():
         return 'wiki'
-    elif 'youtube.com' in link['domain']:
+    elif 'youtube.com' in domain(link['url']).lower():
         return 'youtube'
-    elif 'soundcloud.com' in link['domain']:
+    elif 'soundcloud.com' in domain(link['url']).lower():
         return 'soundcloud'
-    elif 'youku.com' in link['domain']:
+    elif 'youku.com' in domain(link['url']).lower():
         return 'youku'
-    elif 'vimeo.com' in link['domain']:
+    elif 'vimeo.com' in domain(link['url']).lower():
         return 'vimeo'
     return None
 
@@ -276,19 +360,19 @@ def merge_links(a, b):
     """deterministially merge two links, favoring longer field values over shorter,
     and "cleaner" values over worse ones.
     """
-    longer = lambda key: a[key] if len(a[key]) > len(b[key]) else b[key]
+    longer = lambda key: (a[key] if len(a[key]) > len(b[key]) else b[key]) if (a[key] and b[key]) else (a[key] or b[key])
     earlier = lambda key: a[key] if a[key] < b[key] else b[key]
     
     url = longer('url')
     longest_title = longer('title')
-    cleanest_title = a['title'] if '://' not in a['title'] else b['title']
+    cleanest_title = a['title'] if '://' not in (a['title'] or '') else b['title']
     link = {
         'timestamp': earlier('timestamp'),
         'url': url,
         'domain': domain(url),
         'base_url': base_url(url),
         'tags': longer('tags'),
-        'title': longest_title if '://' not in longest_title else cleanest_title,
+        'title': longest_title if '://' not in (longest_title or '') else cleanest_title,
         'sources': list(set(a.get('sources', []) + b.get('sources', []))),
     }
     link['type'] = get_link_type(link)
@@ -299,15 +383,15 @@ def find_link(folder, links):
     url = parse_url(folder)
     if url:
         for link in links:
-            if (link['base_url'] in url) or (url in link['url']):
+            if (base_url(link['url']) in url) or (url in link['url']):
                 return link
 
     timestamp = folder.split('.')[0]
     for link in links:
         if link['timestamp'].startswith(timestamp):
-            if link['domain'] in os.listdir(os.path.join(ARCHIVE_DIR, folder)):
+            if domain(link['url']) in os.listdir(os.path.join(ARCHIVE_DIR, folder)):
                 return link      # careful now, this isn't safe for most ppl
-            if link['domain'] in parse_url(folder):
+            if domain(link['url']) in parse_url(folder):
                 return link
     return None
 
@@ -321,7 +405,7 @@ def parse_url(folder):
                 link_json = f.read().strip()
                 if link_json:
                     link = json.loads(link_json)
-                    return link['base_url']
+                    return base_url(link['url'])
             except ValueError:
                 print('File contains invalid JSON: {}!'.format(link_json))
 
@@ -377,8 +461,8 @@ def fix_folder_path(archive_path, link_folder, link):
     target = os.path.join(archive_path, link['timestamp'])
 
     url_in_folder = parse_url(source)
-    if not (url_in_folder in link['base_url']
-            or link['base_url'] in url_in_folder):
+    if not (url_in_folder in base_url(link['url'])
+            or base_url(link['url']) in url_in_folder):
         raise ValueError('The link does not match the url for this folder.')
 
     if not os.path.exists(target):
@@ -466,12 +550,12 @@ def wget_output_path(link, look_in=None):
     urlencode = lambda s: quote(s, encoding='utf-8', errors='replace')
 
     if link['type'] in ('PDF', 'image'):
-        return urlencode(link['base_url'])
+        return urlencode(base_url(link['url']))
 
     # Since the wget algorithm to for -E (appending .html) is incredibly complex
     # instead of trying to emulate it here, we just look in the output folder
     # to see what html file wget actually created as the output
-    wget_folder = link['base_url'].rsplit('/', 1)[0].split('/')
+    wget_folder = base_url(link['url']).rsplit('/', 1)[0].split('/')
     look_in = os.path.join(ARCHIVE_DIR, link['timestamp'], *wget_folder)
 
     if look_in and os.path.exists(look_in):
@@ -491,7 +575,7 @@ def wget_output_path(link, look_in=None):
 
     # if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M):
     #     # already ends in .html
-    #     return urlencode(link['base_url'])
+    #     return urlencode(base_url(link['url']))
     # else:
     #     # .html needs to be appended
     #     without_scheme = split_url[0].split('://', 1)[-1].split('?', 1)[0]
@@ -504,36 +588,53 @@ def wget_output_path(link, look_in=None):
     #             return urlencode('#'.join([without_scheme + '/index.html' + query + '.html', *split_url[1:]]))
     #         elif '/' in without_scheme:
     #             return urlencode('#'.join([without_scheme + '.html', *split_url[1:]]))
-    #         return urlencode(link['base_url'] + '/index.html')
+    #         return urlencode(base_url(link['url']) + '/index.html')
 
 
 def derived_link_info(link):
     """extend link info with the archive urls and other derived data"""
 
-    link_info = {
+    url = link['url']
+
+    extended_info = {
         **link,
+        'title': link['title'] or base_url(url),
         'date': datetime.fromtimestamp(Decimal(link['timestamp'])).strftime('%Y-%m-%d %H:%M'),
-        'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**link),
-        'favicon_url': 'archive/{timestamp}/favicon.ico'.format(**link),
-        'files_url': 'archive/{timestamp}/index.html'.format(**link),
+        'base_url': base_url(url),
+        'domain': domain(url),
+        'basename': basename(url),
+        'path': path(url),
+    }
+
+    # Archive Method Output URLs
+    extended_info = {
+        **extended_info,
+        'favicon_url': 'archive/{timestamp}/favicon.ico'.format(**extended_info),
+        'google_favicon_url': 'https://www.google.com/s2/favicons?domain={domain}'.format(**extended_info),
+        'files_url': 'archive/{timestamp}/index.html'.format(**extended_info),
         'archive_url': 'archive/{}/{}'.format(link['timestamp'], wget_output_path(link) or 'index.html'),
-        'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
-        'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
-        'dom_link': 'archive/{timestamp}/output.html'.format(**link),
-        'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
+        'warc_url': 'archive/{timestamp}/warc'.format(**extended_info),
+        'pdf_link': 'archive/{timestamp}/output.pdf'.format(**extended_info),
+        'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**extended_info),
+        'dom_link': 'archive/{timestamp}/output.html'.format(**extended_info),
+        'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**extended_info),
+        'git_url': 'archive/{timestamp}/git'.format(**extended_info),
+        'media_url': 'archive/{timestamp}/media'.format(**extended_info),
+        
     }
 
     # PDF and images are handled slightly differently
     # wget, screenshot, & pdf urls all point to the same file
     if link['type'] in ('PDF', 'image'):
-        link_info.update({
-            'archive_url': 'archive/{timestamp}/{base_url}'.format(**link),
-            'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
-            'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
-            'dom_link': 'archive/{timestamp}/{base_url}'.format(**link),
-            'title': '{title} ({type})'.format(**link),
+        extended_info.update({
+            'archive_url': 'archive/{timestamp}/{base_url}'.format(**extended_info),
+            'pdf_link': 'archive/{timestamp}/{base_url}'.format(**extended_info),
+            'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**extended_info),
+            'dom_link': 'archive/{timestamp}/{base_url}'.format(**extended_info),
+            'title': link['title'] or basename(link['url']),
         })
-    return link_info
+
+    return extended_info
 
 
 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs):
@@ -570,3 +671,16 @@ def run(*popenargs, input=None, capture_output=False, timeout=None, check=False,
             raise CalledProcessError(retcode, process.args,
                                      output=stdout, stderr=stderr)
     return CompletedProcess(process.args, retcode, stdout, stderr)
+
+
+def check_link_structure(link):
+    assert isinstance(link, dict)
+    assert isinstance(link.get('url'), str)
+    assert len(link['url']) > 2
+    assert len(re.findall(URL_REGEX, link['url'])) == 1
+
+
+def check_links_structure(links):
+    assert isinstance(links, list)
+    if links:
+        check_link_structure(links[0])

bin/archivebox-export-browser-history

@@ -6,10 +6,10 @@ mkdir -p "$REPO_DIR/output/sources"
 
 if [[ "$1" == "--chrome" ]]; then
     # Google Chrome / Chromium
-    default=$(ls ~/Library/Application\ Support/Google/Chrome/Default/History)
     if [[ -e "$2" ]]; then
         cp "$2" "$REPO_DIR/output/sources/chrome_history.db.tmp"
     else
+        default=$(ls ~/Library/Application\ Support/Google/Chrome/Default/History)
         echo "Defaulting to history db: $default"
         echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
         cp "$default" "$REPO_DIR/output/sources/chrome_history.db.tmp"
@@ -22,10 +22,10 @@ fi
 
 if [[ "$1" == "--firefox" ]]; then
     # Firefox
-    default=$(ls ~/Library/Application\ Support/Firefox/Profiles/*.default/places.sqlite)
     if [[ -e "$2" ]]; then
         cp "$2" "$REPO_DIR/output/sources/firefox_history.db.tmp"
     else
+        default=$(ls ~/Library/Application\ Support/Firefox/Profiles/*.default/places.sqlite)
         echo "Defaulting to history db: $default"
         echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
         cp "$default" "$REPO_DIR/output/sources/firefox_history.db.tmp"

etc/ArchiveBox.conf.default

@@ -1,49 +1,65 @@
+# Example config file for ArchiveBox
+# Copy this file to ~/.ArchiveBox.conf and then edit it
 # Config file uses python syntax but can also be sourced as a .env
+# For documentation, see:
+#    https://github.com/pirate/ArchiveBox/wiki/Configuration
 
-### To disable specific archive methods, uncomment the line and change it to False
-# FETCH_WGET=True
-# FETCH_WGET_REQUISITES=True
-# FETCH_PDF=True
-# FETCH_SCREENSHOT=True
-# FETCH_DOM=True
-# FETCH_WARC=False
-# FETCH_MEDIA=False
-# FETCH_GIT=True
-# FETCH_FAVICON=True
-# SUBMIT_ARCHIVE_DOT_ORG=True
+################################################################################
+## General Settings
+################################################################################
 
-### To only download new links, and never attempt to update old ones, uncomment this line:
-# ONLY_NEW=True
+#OUTPUT_DIR=output
+#OUTPUT_PERMISSIONS=755
+#ONLY_NEW=False
+#TIMEOUT=60
+#MEDIA_TIMEOUT=3600
+#TEMPLATES_DIR=archivebox/templates
+#FOOTER_INFO="Run by ACME co."
 
-### To change your ouput folder permissions, uncomment and edit this line:
-# OUTPUT_PERMISSIONS=755
 
-### To change how much time is allowed before marking a link as failed, uncomment and edit these lines:
-# MEDIA_TIMEOUT=3600
-# TIMEOUT=60
+################################################################################
+## Archive Method Toggles
+################################################################################
 
-### To ignore SSL certificate errors, uncomment this line:
-# CHECK_SSL_VALIDITY=False
+#FETCH_TITLE=True
+#FETCH_FAVICON=True
+#FETCH_WGET=True
+#FETCH_WARC=True
+#FETCH_PDF=True
+#FETCH_SCREENSHOT=True
+#FETCH_DOM=True
+#FETCH_GIT=True
+#FETCH_MEDIA=False
+#SUBMIT_ARCHIVE_DOT_ORG=True
 
-### To customize the screenshot resolution, uncomment and change this line:
-# RESOLUTION="1440,2000"
 
-### To customize which domains we attempted to `git clone`, uncomment and change this line:
-# GIT_DOMAINS="github.com,bitbucket.org,gitlab.com"
+################################################################################
+## Archive Method Options
+################################################################################
 
-### To lie and pretend to be Google Chrome when archiving, uncomment this line:
-# WGET_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
+#CHECK_SSL_VALIDITY=True
+#FETCH_WGET_REQUISITES=True
+#RESOLUTION="1440,900"
+#WGET_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
+#GIT_DOMAINS="github.com,bitbucket.org,gitlab.com"
+#COOKIES_FILE="path/to/cookies.txt"
+#CHROME_USER_DATA_DIR="~/.config/google-chrome/Default"
 
-### To share a chrome data directory with your main browser, uncomment one of these lines:
-# CHROME_USER_DATA_DIR="~/Library/Application Support/Google/Chrome/Default"
-# CHROME_USER_DATA_DIR="~/.config/google-chrome/Default"
 
-### To use a custom chrome binary, uncomment and set to binary name or full path:
-# CHROME_BINARY="chromium-browser"
+################################################################################
+## Shell Options
+################################################################################
 
-### To use a custom wget binary, uncomment and set to binary name or full path:
-# WGET_BINARY="wget"
+#USE_COLOR=True
+#SHOW_PROGRESS=True
 
-### To customize Text displayed in html output footer, uncomment and edit this line:
-# FOOTER_INFO="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests."
 
+################################################################################
+## Dependency Options
+################################################################################
+
+#CURL_BINARY="curl"
+#GIT_BINARY="git"
+#WGET_BINARY="wget"
+#YOUTUBEDL_BINARY="youtube-dl"
+#CHROME_BINARY="chromium-browser"