Merge pull request #147 from nicolaschan/patch-1

Don't run ls if using non-default directory

README.md

@@ -90,7 +90,7 @@ All the saved content is static and indexed with JSON files, so it lives forever
 
 ### Related Projects
 
-There are tons of great web archiving tools out there.  In particular https://Webrecorder.io (pywb) and https://getpolarized.io/ are robust, stable pieces of software that have lots of overlap with ArchiveBox.  ArchiveBox differentiates itself by being primarily a one-shot CLI tool that specializing in importing streams of links from RSS, JSON (good for automatically archiving from a stream of browser history or bookmarks on a schedule), as opposed to a desktop application or web service that requires human interaction to add links.
+There are tons of great web archiving tools out there.  In particular https://webrecorder.io ([pywb](https://github.com/webrecorder/pywb)) and https://getpolarized.io/ are robust, stable pieces of software that have lots of overlap with ArchiveBox.  ArchiveBox differentiates itself by being primarily a one-shot CLI tool that specializing in importing streams of links from RSS, JSON (good for automatically archiving from a stream of browser history or bookmarks on a schedule), as opposed to a desktop application or web service that requires human interaction to add links.
 
 To learn more about the motivation for this project and how it fits into the broader community, see our [Web Archiving Community](https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community) wiki page.
 

archivebox/archive.py

@@ -10,10 +10,7 @@ from datetime import datetime
 from subprocess import run
 
 from parse import parse_links
-from links import (
-    new_links,
-    validate_links
-)
+from links import validate_links
 from archive_methods import archive_links, _RESULTS_TOTALS
 from index import (
     write_links_index,
@@ -25,8 +22,10 @@ from config import (
     ONLY_NEW,
     OUTPUT_PERMISSIONS,
     OUTPUT_DIR,
+    REPO_DIR,
     ANSI,
     TIMEOUT,
+    SHOW_PROGRESS,
     GIT_SHA,
 )
 from util import (
@@ -54,41 +53,33 @@ def print_help():
     print("    echo 'https://examplecom' | ./bin/archivebox\n")
 
 
-def merge_links(archive_path=OUTPUT_DIR, import_path=None, only_new=False):
+def load_links(archive_path=OUTPUT_DIR, import_path=None):
     """get new links from file and optionally append them to links in existing archive"""
-    all_links = []
-    if import_path:
-        # parse and validate the import file
-        raw_links, parser_name = parse_links(import_path)
-        all_links = validate_links(raw_links)
-
-    # merge existing links in archive_path and new links
+    
     existing_links = []
     if archive_path:
         existing_links = parse_json_links_index(archive_path)
-        all_links = validate_links(existing_links + all_links)
 
+    new_links = []
+    if import_path:
+        # parse and validate the import file
+        raw_links, parser_name = parse_links(import_path)
+        new_links = validate_links(raw_links)
+        if SHOW_PROGRESS:
+            print()
+
+    # merge existing links in archive_path and new links
+    all_links = validate_links(existing_links + new_links)
     num_new_links = len(all_links) - len(existing_links)
-    if num_new_links and not only_new:
-        print('{green}[+] [{}] Adding {} new links to index from {} ({} format){reset}'.format(
-            datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
+
+    if import_path and parser_name:
+        print('    > Adding {} new links to index from {} (parsed as {} format)'.format(
             num_new_links,
             pretty_path(import_path),
             parser_name,
-            **ANSI,
         ))
-    # else:
-    #     print('[*] [{}] No new links added to {}/index.json{}'.format(
-    #         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-    #         archive_path,
-    #         ' from {}'.format(import_path) if import_path else '',
-    #         **ANSI,
-    #     ))
 
-    if only_new:
-        return new_links(all_links, existing_links)
-
-    return all_links
+    return all_links, new_links
 
 def update_archive(archive_path, links, source=None, resume=None, append=True):
     """update or create index.html+json given a path to an export file containing new links"""
@@ -102,7 +93,7 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
              **ANSI,
         ))
     else:
-        print('{green}[▶] [{}] Downloading content for {} pages in archive...{reset}'.format(
+        print('{green}[▶] [{}] Updating content for {} pages in archive...{reset}'.format(
              datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
              len(links),
              **ANSI,
@@ -119,7 +110,7 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
     else:
         duration = '{0:.2f} sec'.format(seconds, 2)
 
-    print('{}[√] [{}] Update of {} links complete ({}){}'.format(
+    print('{}[√] [{}] Update of {} pages complete ({}){}'.format(
         ANSI['green'],
         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
         len(links),
@@ -129,6 +120,7 @@ def update_archive(archive_path, links, source=None, resume=None, append=True):
     print('    - {} entries skipped'.format(_RESULTS_TOTALS['skipped']))
     print('    - {} entries updated'.format(_RESULTS_TOTALS['succeded']))
     print('    - {} errors'.format(_RESULTS_TOTALS['failed']))
+    print('    To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', '')))
 
 
 if __name__ == '__main__':
@@ -185,14 +177,18 @@ if __name__ == '__main__':
     elif stdin_raw_text:
         source = save_source(stdin_raw_text)
 
+
     # Step 1: Parse the links and dedupe them with existing archive
-    links = merge_links(archive_path=out_dir, import_path=source, only_new=ONLY_NEW)
+    all_links, new_links = load_links(archive_path=out_dir, import_path=source)
 
     # Step 2: Write new index
-    write_links_index(out_dir=out_dir, links=links)
+    write_links_index(out_dir=out_dir, links=all_links)
 
     # Step 3: Verify folder structure is 1:1 with index
     # cleanup_archive(out_dir, links)
 
     # Step 4: Run the archive methods for each link
-    update_archive(out_dir, links, source=source, resume=resume, append=True)
+    if ONLY_NEW:
+        update_archive(out_dir, new_links, source=source, resume=resume, append=True)
+    else:
+        update_archive(out_dir, all_links, source=source, resume=resume, append=True)

archivebox/archive_methods.py

@@ -12,6 +12,8 @@ from index import wget_output_path, parse_json_link_index, write_link_index
 from links import links_after_timestamp
 from config import (
     CHROME_BINARY,
+    FETCH_FAVICON,
+    FETCH_TITLE,
     FETCH_WGET,
     FETCH_WGET_REQUISITES,
     FETCH_PDF,
@@ -23,7 +25,6 @@ from config import (
     RESOLUTION,
     CHECK_SSL_VALIDITY,
     SUBMIT_ARCHIVE_DOT_ORG,
-    FETCH_FAVICON,
     WGET_USER_AGENT,
     CHROME_USER_DATA_DIR,
     CHROME_SANDBOX,
@@ -36,6 +37,7 @@ from config import (
 )
 from util import (
     check_dependencies,
+    fetch_page_title,
     progress,
     chmod_file,
     pretty_path,
@@ -96,6 +98,9 @@ def archive_link(link_dir, link, overwrite=True):
         if FETCH_FAVICON:
             link = fetch_favicon(link_dir, link, overwrite=overwrite)
 
+        if FETCH_TITLE:
+            link = fetch_title(link_dir, link, overwrite=overwrite)
+
         if FETCH_WGET:
             link = fetch_wget(link_dir, link, overwrite=overwrite)
 
@@ -129,13 +134,13 @@ def log_link_archive(link_dir, link, update_existing):
         symbol='*' if update_existing else '+',
         symbol_color=ANSI['black' if update_existing else 'green'],
         now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-        **link,
+        **{**link, 'title': link['title'] or link['url']},
         **ANSI,
     ))
 
     print('    > {}{}'.format(pretty_path(link_dir), '' if update_existing else ' (new)'))
-    if link['type']:
-        print('      i {}'.format(link['type']))
+    # if link['type']:
+    #     print('      i {}'.format(link['type']))
 
 
 
@@ -217,6 +222,7 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
         '--backup-converted',
         '--span-hosts',
         '--no-parent',
+        '-e', 'robots=off',
         '--restrict-file-names=unix',
         '--timeout={}'.format(timeout),
         *(() if warc else ('--timestamping',)),
@@ -491,6 +497,29 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
         'output': output,
     }
 
+@attach_result_to_link('title')
+def fetch_title(link_dir, link, timeout=TIMEOUT):
+    """try to guess the page's title from its content"""
+
+    # if link already has valid title, skip it
+    if link['title'] and not link['title'].lower().startswith('http'):
+        return {'output': link['title'], 'cmd': 'fetch_page_title("{}")'.format(link['url'])}
+
+    end = progress(timeout, prefix='      ')
+    try:
+        title = fetch_page_title(link['url'], timeout=timeout, progress=False)
+        end()
+        output = title
+    except Exception as e:
+        end()
+        print('        {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
+        output = e
+
+    return {
+        'cmd': 'fetch_page_title("{}")'.format(link['url']),
+        'output': output,
+    }
+
 @attach_result_to_link('media')
 def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
     """Download playlists or individual video, audio, and subtitles using youtube-dl"""

archivebox/config.py

@@ -27,6 +27,7 @@ FETCH_WARC =             os.getenv('FETCH_WARC',             'True'
 FETCH_GIT =              os.getenv('FETCH_GIT',              'True'             ).lower() == 'true'
 FETCH_MEDIA =            os.getenv('FETCH_MEDIA',            'True'             ).lower() == 'true'
 FETCH_FAVICON =          os.getenv('FETCH_FAVICON',          'True'             ).lower() == 'true'
+FETCH_TITLE =            os.getenv('FETCH_TITLE',            'True'             ).lower() == 'true'
 SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True'             ).lower() == 'true'
 
 CHECK_SSL_VALIDITY =     os.getenv('CHECK_SSL_VALIDITY',     'True'             ).lower() == 'true'

archivebox/index.py

@@ -28,14 +28,16 @@ def write_links_index(out_dir, links):
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
 
-    write_json_links_index(out_dir, links)
-    write_html_links_index(out_dir, links)
-    
-    print('{green}[√] [{}] Updated main index files:{reset}'.format(
+    print('{green}[*] [{}] Updating main index files...{reset}'.format(
         datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-        **ANSI))
+        **ANSI,
+    ))
+    write_json_links_index(out_dir, links)
     print('    > {}/index.json'.format(pretty_path(out_dir)))
+    
+    write_html_links_index(out_dir, links)
     print('    > {}/index.html'.format(pretty_path(out_dir)))
+    
 
 def write_json_links_index(out_dir, links):
     """write the json link index to a given path"""
@@ -141,6 +143,7 @@ def write_html_link_index(out_dir, link):
         f.write(Template(link_html).substitute({
             **link,
             **link['latest'],
+            'title': link['title'] or link['url'],
             'type': link['type'] or 'website',
             'tags': link['tags'] or 'untagged',
             'bookmarked': datetime.fromtimestamp(float(link['timestamp'])).strftime('%Y-%m-%d %H:%M'),

archivebox/links.py

@@ -57,7 +57,7 @@ def validate_links(links):
         raise SystemExit(1)
 
     for link in links:
-        link['title'] = unescape(link['title'])
+        link['title'] = unescape(link['title']) if link['title'] else None
         link['latest'] = link.get('latest') or {}
         
         latest = link['latest']
@@ -76,16 +76,10 @@ def validate_links(links):
         if not latest.get('favicon'):
             latest['favicon'] = None
 
-    return list(links)
+        if not link['latest'].get('title'):
+            link['latest']['title'] = link['title']
 
-def new_links(all_links, existing_links):
-    """
-    Return all links which are in the all_links but not in the existing_links.
-    This is used to determine which links are new and not indexed jet. Set the
-    ONLY_NEW environment variable to activate this filter mechanism.
-    """
-    existing_urls = {link['url'] for link in existing_links}
-    return [link for link in all_links if link['url'] not in existing_urls]
+    return list(links)
 
 def archivable_links(links):
     """remove chrome://, about:// or other schemed links that cant be archived"""

archivebox/parse.py

@@ -18,6 +18,7 @@ Parsed link schema: {
 """
 
 import re
+import sys
 import json
 import urllib
 from collections import OrderedDict
@@ -25,7 +26,7 @@ import xml.etree.ElementTree as etree
 
 from datetime import datetime
 
-from config import ANSI
+from config import ANSI, SHOW_PROGRESS
 from util import (
     domain,
     base_url,
@@ -60,6 +61,8 @@ def parse_links(path):
             path.rsplit('/', 1)[-1],
             **ANSI,
         ))
+        if SHOW_PROGRESS:
+            sys.stdout.write('    ')
 
         for parser_name, parser_func in get_parsers(file).items():
             # otherwise try all parsers until one works
@@ -67,12 +70,13 @@ def parse_links(path):
                 links += list(parser_func(file))
                 if links:
                     break
-            except (ValueError, TypeError, IndexError, AttributeError, etree.ParseError):
-                # parser not supported on this file
+            except Exception as err:
+                # we try each parser one by one, each parser will throw exeption an exception if unsupported
+                # so we accept the first one that
+                # uncomment the following line to see why the parser was unsupported for each attempted format
+                # print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err))
                 pass
 
-    print()
-
     return links, parser_name
 
 
@@ -157,7 +161,7 @@ def parse_rss_export(rss_file):
         rows = leading_removed.split('\n')
 
         def get_row(key):
-            return [r for r in rows if r.startswith('<{}>'.format(key))][0]
+            return [r for r in rows if r.strip().startswith('<{}>'.format(key))][0]
 
         title = str_between(get_row('title'), '<![CDATA[', ']]').strip()
         url = str_between(get_row('link'), '<link>', '</link>')
@@ -168,7 +172,7 @@ def parse_rss_export(rss_file):
             'url': url,
             'domain': domain(url),
             'base_url': base_url(url),
-            'timestamp': str(datetime.now().timestamp()),
+            'timestamp': str(time.timestamp()),
             'tags': '',
             'title': title or fetch_page_title(url),
             'sources': [rss_file.name],
@@ -205,7 +209,6 @@ def parse_shaarli_rss_export(rss_file):
         ts_str = str_between(get_row('published'), '<published>', '</published>')
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
 
-
         info = {
             'url': url,
             'domain': domain(url),

archivebox/util.py

@@ -4,7 +4,8 @@ import sys
 import time
 import json
 import signal
-import urllib.request
+from urllib.request import urlopen
+from urllib.parse import urlparse
 
 from decimal import Decimal
 from urllib.parse import quote
@@ -33,17 +34,24 @@ from config import (
     SUBMIT_ARCHIVE_DOT_ORG,
 )
 
-# URL helpers
-without_scheme = lambda url: url.replace('http://', '').replace('https://', '').replace('ftp://', '')
-without_query = lambda url: url.split('?', 1)[0]
-without_hash = lambda url: url.split('#', 1)[0]
-without_path = lambda url: url.split('/', 1)[0]
-domain = lambda url: without_hash(without_query(without_path(without_scheme(url))))
+# URL helpers: https://docs.python.org/3/library/urllib.parse.html#url-parsing
+scheme = lambda url: urlparse(url).scheme
+without_scheme = lambda url: urlparse(url)._replace(scheme='').geturl().strip('//')
+without_query = lambda url: urlparse(url)._replace(query='').geturl().strip('//')
+without_fragment = lambda url: urlparse(url)._replace(fragment='').geturl().strip('//')
+without_path = lambda url: urlparse(url)._replace(path='', fragment='', query='').geturl().strip('//')
+path = lambda url: urlparse(url).path
+basename = lambda url: urlparse(url).path.rsplit('/', 1)[-1]
+domain = lambda url: urlparse(url).netloc
+query = lambda url: urlparse(url).query
+fragment = lambda url: urlparse(url).fragment
+extension = lambda url: basename(url).rsplit('.', 1)[-1].lower() if '.' in basename(url) else ''
 base_url = lambda url: without_scheme(url)  # uniq base url used to dedupe links
 
 short_ts = lambda ts: ts.split('.')[0]
 
 URL_REGEX = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))[^<\""]+'
+HTML_TITLE_REGEX = '<title>(.[^<>]+)'
 
 
 def check_dependencies():
@@ -213,7 +221,7 @@ def download_url(url):
     ))
     end = progress(TIMEOUT, prefix='      ')
     try:
-        downloaded_xml = urllib.request.urlopen(url).read().decode('utf-8')
+        downloaded_xml = urlopen(url).read().decode('utf-8')
         end()
     except Exception as e:
         end()
@@ -227,21 +235,18 @@ def download_url(url):
     return source_path
 
 
-def fetch_page_title(url, default=True):
+def fetch_page_title(url, timeout=10, progress=SHOW_PROGRESS):
     """Attempt to guess a page's title by downloading the html"""
-    if default is True:
-        default = url
-
     try:
-        sys.stdout.write('.')
-        sys.stdout.flush()
-        html_content = urllib.request.urlopen(url, timeout=10).read().decode('utf-8')
-        match = re.search('<title>(.*?)</title>', html_content)
-        return match.group(1) if match else default or None
+        if progress:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+
+        html_content = urlopen(url, timeout=timeout).read().decode('utf-8')
+        match = re.search(HTML_TITLE_REGEX, html_content)
+        return match.group(1).strip() if match else None
     except Exception:
-        if default is False:
-            raise
-        return default
+        return None
 
 
 def str_between(string, start, end=None):
@@ -276,19 +281,19 @@ def merge_links(a, b):
     """deterministially merge two links, favoring longer field values over shorter,
     and "cleaner" values over worse ones.
     """
-    longer = lambda key: a[key] if len(a[key]) > len(b[key]) else b[key]
+    longer = lambda key: (a[key] if len(a[key]) > len(b[key]) else b[key]) if (a[key] and b[key]) else (a[key] or b[key])
     earlier = lambda key: a[key] if a[key] < b[key] else b[key]
     
     url = longer('url')
     longest_title = longer('title')
-    cleanest_title = a['title'] if '://' not in a['title'] else b['title']
+    cleanest_title = a['title'] if '://' not in (a['title'] or '') else b['title']
     link = {
         'timestamp': earlier('timestamp'),
         'url': url,
         'domain': domain(url),
         'base_url': base_url(url),
         'tags': longer('tags'),
-        'title': longest_title if '://' not in longest_title else cleanest_title,
+        'title': longest_title if '://' not in (longest_title or '') else cleanest_title,
         'sources': list(set(a.get('sources', []) + b.get('sources', []))),
     }
     link['type'] = get_link_type(link)
@@ -521,6 +526,7 @@ def derived_link_info(link):
         'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
         'dom_link': 'archive/{timestamp}/output.html'.format(**link),
         'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
+        'title': link['title'] or link['url'],
     }
 
     # PDF and images are handled slightly differently
@@ -531,7 +537,7 @@ def derived_link_info(link):
             'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
             'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
             'dom_link': 'archive/{timestamp}/{base_url}'.format(**link),
-            'title': '{title} ({type})'.format(**link),
+            'title': link['title'] or basename(link['url']),
         })
     return link_info
 

bin/archivebox-export-browser-history

@@ -6,10 +6,10 @@ mkdir -p "$REPO_DIR/output/sources"
 
 if [[ "$1" == "--chrome" ]]; then
     # Google Chrome / Chromium
-    default=$(ls ~/Library/Application\ Support/Google/Chrome/Default/History)
     if [[ -e "$2" ]]; then
         cp "$2" "$REPO_DIR/output/sources/chrome_history.db.tmp"
     else
+        default=$(ls ~/Library/Application\ Support/Google/Chrome/Default/History)
         echo "Defaulting to history db: $default"
         echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
         cp "$default" "$REPO_DIR/output/sources/chrome_history.db.tmp"
@@ -22,10 +22,10 @@ fi
 
 if [[ "$1" == "--firefox" ]]; then
     # Firefox
-    default=$(ls ~/Library/Application\ Support/Firefox/Profiles/*.default/places.sqlite)
     if [[ -e "$2" ]]; then
         cp "$2" "$REPO_DIR/output/sources/firefox_history.db.tmp"
     else
+        default=$(ls ~/Library/Application\ Support/Firefox/Profiles/*.default/places.sqlite)
         echo "Defaulting to history db: $default"
         echo "Optionally specify the path to a different sqlite history database as the 2nd argument."
         cp "$default" "$REPO_DIR/output/sources/firefox_history.db.tmp"

etc/ArchiveBox.conf.default

@@ -10,6 +10,7 @@
 # FETCH_MEDIA=False
 # FETCH_GIT=True
 # FETCH_FAVICON=True
+# FETCH_TITLE=True
 # SUBMIT_ARCHIVE_DOT_ORG=True
 
 ### To only download new links, and never attempt to update old ones, uncomment this line: