Merge branch 'dev' into link-removal2

archivebox/core/admin.py

@@ -22,7 +22,7 @@ from index.html import snapshot_icons
 from logging_util import printable_filesize
 from main import add, remove
 from config import OUTPUT_DIR
-from extractors import archive_links
+from extractors import archive_snapshots
 
 # Admin URLs
 # /admin/

archivebox/core/models.py

@@ -1,6 +1,10 @@
 __package__ = 'archivebox.core'
 
 import uuid
+from pathlib import Path
+from typing import Dict, Optional, List
+from datetime import datetime, timedelta
+from collections import defaultdict
 
 from django.db import models, transaction
 from django.utils.functional import cached_property
@@ -9,9 +13,11 @@ from django.db.models import Case, When, Value, IntegerField
 
 from ..util import parse_date
 from ..index.schema import Link
-from ..extractors import get_default_archive_methods, ARCHIVE_METHODS_INDEXING_PRECEDENCE
+from ..config import CONFIG
+from ..system import get_dir_size
 
-EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
+#EXTRACTORS = [(extractor[0], extractor[0]) for extractor in get_default_archive_methods()]
+EXTRACTORS = [("title", "title"), ("wget", "wget")]
 STATUS_CHOICES = [
     ("succeeded", "succeeded"),
     ("failed", "failed"),
@@ -24,7 +30,6 @@ except AttributeError:
     import jsonfield
     JSONField = jsonfield.JSONField
 
-
 class Tag(models.Model):
     """
     Based on django-taggit model
@@ -79,7 +84,7 @@ class Snapshot(models.Model):
     updated = models.DateTimeField(null=True, blank=True, db_index=True)
     tags = models.ManyToManyField(Tag)
 
-    keys = ('url', 'timestamp', 'title', 'tags', 'updated')
+    keys = ('id', 'url', 'timestamp', 'title', 'tags', 'updated', 'base_url')
 
     def __repr__(self) -> str:
         title = self.title or '-'
@@ -89,18 +94,63 @@ class Snapshot(models.Model):
         title = self.title or '-'
         return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
 
+    def field_names():
+        fields = self._meta.get_field_names()
+        exclude = ["tags", "archiveresult"] # Exclude relationships for now
+        return [field.name for field in fields if field.name not in exclude]
+
+
     @classmethod
     def from_json(cls, info: dict):
         info = {k: v for k, v in info.items() if k in cls.keys}
+        if "tags" in info:
+            # TODO: Handle tags
+            info.pop("tags")
+        info.pop("base_url", None) 
         return cls(**info)
 
+    def get_history(self) -> dict:
+        """
+        Generates the history dictionary out of the stored ArchiveResults
+        """
+        history_list = self.archiveresult_set.all()
+        history = defaultdict(list)
+        for history_item in history_list:
+            history[history_item.extractor].append(
+                {
+                    "cmd": history_item.cmd,
+                    "cmd_version": history_item.cmd_version,
+                    "end_ts": history_item.end_ts.isoformat(),
+                    "start_ts": history_item.start_ts.isoformat(),
+                    "pwd": history_item.pwd,
+                    "output": history_item.output,
+                    "schema": "ArchiveResult",
+                    "status": history_item.status
+                }
+            )
+        return dict(history)
+
     def as_json(self, *args) -> dict:
+        """
+        Returns the snapshot in json format.
+        id is converted to str
+        history is extracted from ArchiveResult
+        """
         args = args or self.keys
-        return {
+        output = {
             key: getattr(self, key)
             if key != 'tags' else self.tags_str()
             for key in args
         }
+        if "id" in output.keys():
+            output["id"] = str(output["id"])
+
+        output["history"] = self.get_history()
+        return output
+
+    def as_csv(self, cols: Optional[List[str]]=None, separator: str=',', ljust: int=0) -> str:
+        from ..index.csv import to_csv
+        return to_csv(self, cols=cols or self.field_names(), separator=separator, ljust=ljust)
 
     def as_link(self) -> Link:
         return Link.from_json(self.as_json())
@@ -117,8 +167,49 @@ class Snapshot(models.Model):
         return parse_date(self.timestamp)
 
     @cached_property
-    def is_archived(self):
-        return self.as_link().is_archived
+    def bookmarked_date(self) -> Optional[str]:
+        from ..util import ts_to_date
+
+        max_ts = (datetime.now() + timedelta(days=30)).timestamp()
+
+        if self.timestamp and self.timestamp.replace('.', '').isdigit():
+            if 0 < float(self.timestamp) < max_ts:
+                return ts_to_date(datetime.fromtimestamp(float(self.timestamp)))
+            else:
+                return str(self.timestamp)
+        return None
+
+    @cached_property
+    def is_archived(self) -> bool:
+        from ..config import ARCHIVE_DIR
+        from ..util import domain
+
+        output_paths = (
+            domain(self.url),
+            'output.pdf',
+            'screenshot.png',
+            'output.html',
+            'media',
+            'singlefile.html'
+        )
+
+        return any(
+            (Path(ARCHIVE_DIR) / self.timestamp / path).exists()
+            for path in output_paths
+        )
+
+    @cached_property
+    def archive_dates(self) -> List[datetime]:
+        return [
+            result.start_ts
+            for result in self.archiveresult_set.all()
+        ]
+
+    @cached_property
+    def oldest_archive_date(self) -> Optional[datetime]:
+        oldest = self.archiveresult_set.all().order_by("-start_ts")[:1]
+        if len(oldest) > 0:
+            return oldest[0].start_ts
 
     @cached_property
     def num_outputs(self):
@@ -129,20 +220,26 @@ class Snapshot(models.Model):
         return self.as_link().url_hash
 
     @cached_property
-    def base_url(self):
-        return self.as_link().base_url
+    def base_url(self) -> str:
+        from ..util import base_url
+        return base_url(self.url)
 
     @cached_property
-    def link_dir(self):
-        return self.as_link().link_dir
+    def snapshot_dir(self):
+        from ..config import CONFIG
+        return Path(CONFIG['ARCHIVE_DIR']) / self.timestamp
 
     @cached_property
     def archive_path(self):
-        return self.as_link().archive_path
+        from ..config import ARCHIVE_DIR_NAME
+        return '{}/{}'.format(ARCHIVE_DIR_NAME, self.timestamp)
 
     @cached_property
-    def archive_size(self):
-        return self.as_link().archive_size
+    def archive_size(self) -> float:
+        try:
+            return get_dir_size(self.archive_path)[0]
+        except Exception:
+            return 0
 
     @cached_property
     def history(self):
@@ -158,6 +255,64 @@ class Snapshot(models.Model):
             return self.history['title'][-1].output.strip()
         return None
 
+    @cached_property
+    def domain(self) -> str:
+        from ..util import domain
+        return domain(self.url)
+
+    @cached_property
+    def is_static(self) -> bool:
+        from ..util import is_static_file
+        return is_static_file(self.url)
+
+    @cached_property
+    def details(self) -> Dict:
+        # TODO: Define what details are, and return them accordingly
+        return {"history": {}}
+
+    @property
+    def extension(self) -> str:
+        from ..util import extension
+        return extension(self.url)
+
+    def canonical_outputs(self) -> Dict[str, Optional[str]]:
+        """predict the expected output paths that should be present after archiving"""
+
+        from ..extractors.wget import wget_output_path
+        canonical = {
+            'index_path': 'index.html',
+            'favicon_path': 'favicon.ico',
+            'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
+            'wget_path': wget_output_path(self),
+            'warc_path': 'warc',
+            'singlefile_path': 'singlefile.html',
+            'readability_path': 'readability/content.html',
+            'mercury_path': 'mercury/content.html',
+            'pdf_path': 'output.pdf',
+            'screenshot_path': 'screenshot.png',
+            'dom_path': 'output.html',
+            'archive_org_path': 'https://web.archive.org/web/{}'.format(self.base_url),
+            'git_path': 'git',
+            'media_path': 'media',
+        }
+        if self.is_static:
+            # static binary files like PDF and images are handled slightly differently.
+            # they're just downloaded once and aren't archived separately multiple times, 
+            # so the wget, screenshot, & pdf urls should all point to the same file
+
+            static_path = wget_output_path(self)
+            canonical.update({
+                'title': self.basename,
+                'wget_path': static_path,
+                'pdf_path': static_path,
+                'screenshot_path': static_path,
+                'dom_path': static_path,
+                'singlefile_path': static_path,
+                'readability_path': static_path,
+                'mercury_path': static_path,
+            })
+        return canonical
+
     def save_tags(self, tags=()):
         tags_id = []
         for tag in tags:
@@ -168,6 +323,7 @@ class Snapshot(models.Model):
 
 class ArchiveResultManager(models.Manager):
     def indexable(self, sorted: bool = True):
+        from ..extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
         INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
         qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS,status='succeeded')
 
@@ -192,3 +348,6 @@ class ArchiveResult(models.Model):
 
     def __str__(self):
         return self.extractor
+
+    class Meta:
+        ordering = ["-start_ts"]

archivebox/core/views.py

@@ -61,7 +61,7 @@ class SnapshotView(View):
         by_ts = {page.timestamp: page for page in all_pages}
         try:
             # print('SERVING STATICFILE', by_ts[slug].link_dir, request.path, path)
-            response = static.serve(request, archivefile, document_root=by_ts[slug].link_dir, show_indexes=True)
+            response = static.serve(request, archivefile, document_root=by_ts[slug].snapshot_dir, show_indexes=True)
             response["Link"] = f'<{by_ts[slug].url}>; rel="canonical"'
             return response
         except KeyError:

archivebox/extractors/__init__.py

@@ -4,22 +4,20 @@ import os
 from pathlib import Path
 
 from typing import Optional, List, Iterable, Union
-from datetime import datetime
-from django.db.models import QuerySet
 
-from ..index.schema import Link
-from ..index.sql import write_link_to_sql_index
+from django.db.models import QuerySet, Model
+
 from ..index import (
-    load_link_details,
-    write_link_details,
+    load_snapshot_details,
+    write_snapshot_details,
 )
 from ..util import enforce_types
 from ..logging_util import (
     log_archiving_started,
     log_archiving_paused,
     log_archiving_finished,
-    log_link_archiving_started,
-    log_link_archiving_finished,
+    log_snapshot_archiving_started,
+    log_snapshot_archiving_finished,
     log_archive_method_started,
     log_archive_method_finished,
 )
@@ -67,15 +65,9 @@ def ignore_methods(to_ignore: List[str]):
     return list(methods)
 
 @enforce_types
-def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Link:
+def archive_snapshot(snapshot: Model, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> Model:
     """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
-
-    # TODO: Remove when the input is changed to be a snapshot. Suboptimal approach.
-    from core.models import Snapshot, ArchiveResult
-    try:
-        snapshot = Snapshot.objects.get(url=link.url) # TODO: This will be unnecessary once everything is a snapshot
-    except Snapshot.DoesNotExist:
-        snapshot = write_link_to_sql_index(link)
+    from core.models import ArchiveResult
 
     ARCHIVE_METHODS = get_default_archive_methods()
     
@@ -85,33 +77,33 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
             if method[0] in methods
         ]
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     try:
         is_new = not Path(out_dir).exists()
         if is_new:
             os.makedirs(out_dir)
+            details = {"history": {}}
+            write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=False)
+        else:
+            details = snapshot.details #TODO: This can be retrieved from the sqlite database too.
+                                       # If that makes more sense, it can be easily changed.
 
-        link = load_link_details(link, out_dir=out_dir)
-        write_link_details(link, out_dir=out_dir, skip_sql_index=False)
-        log_link_archiving_started(link, out_dir, is_new)
-        link = link.overwrite(updated=datetime.now())
+        log_snapshot_archiving_started(snapshot, out_dir, is_new)
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
 
         for method_name, should_run, method_function in ARCHIVE_METHODS:
             try:
-                if method_name not in link.history:
-                    link.history[method_name] = []
+                if method_name not in details["history"]:
+                    details["history"][method_name] = []
 
-                if should_run(link, out_dir, overwrite):
+                if should_run(snapshot, out_dir, overwrite):
                     log_archive_method_started(method_name)
 
-                    result = method_function(link=link, out_dir=out_dir)
-
-                    link.history[method_name].append(result)
+                    result = method_function(snapshot=snapshot, out_dir=out_dir)
 
                     stats[result.status] += 1
                     log_archive_method_finished(result)
-                    write_search_index(link=link, texts=result.index_texts)
+                    write_search_index(snapshot=snapshot, texts=result.index_texts)
                     ArchiveResult.objects.create(snapshot=snapshot, extractor=method_name, cmd=result.cmd, cmd_version=result.cmd_version,
                                                  output=result.output, pwd=result.pwd, start_ts=result.start_ts, end_ts=result.end_ts, status=result.status)
 
@@ -119,27 +111,29 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
                     stats['skipped'] += 1
             except Exception as e:
-                raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
+                raise Exception('Exception in archive_methods.save_{}(Snapshot(url={}))'.format(
                     method_name,
-                    link.url,
+                    snapshot.url,
                 )) from e
 
         # print('    ', stats)
 
         try:
-            latest_title = link.history['title'][-1].output.strip()
-            if latest_title and len(latest_title) >= len(link.title or ''):
-                link = link.overwrite(title=latest_title)
+            latest_title_archive_result = snapshot.archiveresult_set.filter(extractor="title")
+            if latest_title_archive_result.count() > 0:
+                latest_title = latest_title_archive_result.output.strip()
+                if len(latest_title) >= len(snapshot.title or ''):
+                    snapshot.title = latest_title
         except Exception:
             pass
 
-        write_link_details(link, out_dir=out_dir, skip_sql_index=False)
+        write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=False)
 
-        log_link_archiving_finished(link, link.link_dir, is_new, stats)
+        log_snapshot_archiving_finished(snapshot, snapshot.snapshot_dir, is_new, stats)
 
     except KeyboardInterrupt:
         try:
-            write_link_details(link, out_dir=link.link_dir)
+            write_snapshot_details(snapshot, out_dir=snapshot.snapshot_dir)
         except:
             pass
         raise
@@ -148,35 +142,29 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         print('    ! Failed to archive link: {}: {}'.format(err.__class__.__name__, err))
         raise
 
-    return link
+    return snapshot
 
 @enforce_types
-def archive_links(all_links: Union[Iterable[Link], QuerySet], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> List[Link]:
+def archive_snapshots(all_snapshots: Union[QuerySet, List[Model]], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[Path]=None) -> QuerySet:
 
-    if type(all_links) is QuerySet:
-        num_links: int = all_links.count()
-        get_link = lambda x: x.as_link()
-        all_links = all_links.iterator()
-    else:
-        num_links: int = len(all_links)
-        get_link = lambda x: x
+    all_snapshots = list(all_snapshots)
+    num_snapshots: int = len(all_snapshots)
 
-    if num_links == 0:
+    if num_snapshots == 0:
         return []
 
-    log_archiving_started(num_links)
+    log_archiving_started(num_snapshots)
     idx: int = 0
     try:
-        for link in all_links:
+        for snapshot in all_snapshots:
             idx += 1
-            to_archive = get_link(link)
-            archive_link(to_archive, overwrite=overwrite, methods=methods, out_dir=Path(link.link_dir))
+            archive_snapshot(snapshot, overwrite=overwrite, methods=methods, out_dir=Path(snapshot.snapshot_dir))
     except KeyboardInterrupt:
-        log_archiving_paused(num_links, idx, link.timestamp)
+        log_archiving_paused(num_snapshots, idx, snapshot.timestamp)
         raise SystemExit(0)
     except BaseException:
         print()
         raise
 
-    log_archiving_finished(num_links)
-    return all_links
+    log_archiving_finished(num_snapshots)
+    return all_snapshots

archivebox/extractors/archive_org.py

@@ -5,7 +5,9 @@ from pathlib import Path
 from typing import Optional, List, Dict, Tuple
 from collections import defaultdict
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -23,10 +25,13 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = '{domain}/'
+
 
 @enforce_types
-def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_archive_dot_org(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -37,13 +42,13 @@ def should_save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, overwr
     return SAVE_ARCHIVE_DOT_ORG
 
 @enforce_types
-def save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_archive_dot_org(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """submit site to archive.org for archiving via their service, save returned archive url"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'archive.org.txt'
     archive_org_url = None
-    submit_url = 'https://web.archive.org/save/{}'.format(link.url)
+    submit_url = 'https://web.archive.org/save/{}'.format(snapshot.url)
     cmd = [
         CURL_BINARY,
         *CURL_ARGS,

archivebox/extractors/dom.py

@@ -3,7 +3,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file, atomic_write
 from ..util import (
     enforce_types,
@@ -18,10 +20,13 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'output.html'
+
 
 @enforce_types
-def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_dom(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -31,16 +36,16 @@ def should_save_dom(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
     return SAVE_DOM
 
 @enforce_types
-def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_dom(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print HTML of site to file using chrome --dump-html"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'output.html'
     output_path = out_dir / output
     cmd = [
         *chrome_args(TIMEOUT=timeout),
         '--dump-dom',
-        link.url
+        snapshot.url
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/favicon.py

@@ -4,7 +4,9 @@ from pathlib import Path
 
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput
 from ..system import chmod_file, run
 from ..util import enforce_types, domain
 from ..config import (
@@ -19,19 +21,22 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'favicon.ico'
+
+
 @enforce_types
-def should_save_favicon(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'favicon.ico').exists():
+def should_save_favicon(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+    if not overwrite and (Path(out_dir) / 'favicon.ico').exists():
         return False
 
     return SAVE_FAVICON
 
 @enforce_types
-def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_favicon(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download site favicon from google's favicon api"""
 
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or snapshot.snapshot_dir
     output: ArchiveOutput = 'favicon.ico'
     cmd = [
         CURL_BINARY,
@@ -40,7 +45,7 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         '--output', str(output),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
+        'https://www.google.com/s2/favicons?domain={}'.format(domain(snapshot.url)),
     ]
     status = 'failed'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/git.py

@@ -4,7 +4,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -26,10 +28,16 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'git/'
+# @contents = output.glob('*.*')
+# @exists = self.contents.exists()
+# @size => get_size(self.contents)
+# @num_files => len(self.contents)
 
 @enforce_types
-def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_git(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -37,8 +45,8 @@ def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
         return False
 
     is_clonable_url = (
-        (domain(link.url) in GIT_DOMAINS)
-        or (extension(link.url) == 'git')
+        (domain(snapshot.url) in GIT_DOMAINS)
+        or (extension(snapshot.url) == 'git')
     )
     if not is_clonable_url:
         return False
@@ -47,10 +55,10 @@ def should_save_git(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
 
 
 @enforce_types
-def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_git(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using git"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'git'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
@@ -59,7 +67,7 @@ def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
         'clone',
         *GIT_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']),
-        without_query(without_fragment(link.url)),
+        without_query(without_fragment(snapshot.url)),
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/headers.py

@@ -4,7 +4,9 @@ from pathlib import Path
 
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput
 from ..system import atomic_write
 from ..util import (
     enforce_types,
@@ -21,20 +23,28 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
+
+# output = 'headers.json'
+
 @enforce_types
-def should_save_headers(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'headers.json').exists():
+def should_save_headers(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+
+    if not SAVE_HEADERS:
         return False
+    
+    if overwrite:
+        return True
 
-    return SAVE_HEADERS
+    output = Path(out_dir or snapshot.snapshot_dir) / 'headers.json'
+    return not output.exists()
 
 
 @enforce_types
-def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_headers(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """Download site headers"""
 
-    out_dir = Path(out_dir or link.link_dir)
+    out_dir = Path(out_dir or snapshot.snapshot_dir)
     output_folder = out_dir.absolute()
     output: ArchiveOutput = 'headers.json'
 
@@ -48,10 +58,10 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        link.url,
+        snapshot.url,
     ]
     try:
-        json_headers = get_headers(link.url, timeout=timeout)
+        json_headers = get_headers(snapshot.url, timeout=timeout)
         output_folder.mkdir(exist_ok=True)
         atomic_write(str(output_folder / "headers.json"), json_headers)
     except (Exception, OSError) as err:

archivebox/extractors/media.py

@@ -3,7 +3,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -20,9 +22,13 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'media/'
+
 @enforce_types
-def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_media(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -32,10 +38,10 @@ def should_save_media(link: Link, out_dir: Optional[Path]=None, overwrite: Optio
     return SAVE_MEDIA
 
 @enforce_types
-def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
+def save_media(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIMEOUT) -> ArchiveResult:
     """Download playlists or individual video, audio, and subtitles using youtube-dl"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'media'
     output_path = out_dir / output
     output_path.mkdir(exist_ok=True)
@@ -43,7 +49,7 @@ def save_media(link: Link, out_dir: Optional[Path]=None, timeout: int=MEDIA_TIME
         YOUTUBEDL_BINARY,
         *YOUTUBEDL_ARGS,
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate']),
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/mercury.py

@@ -6,7 +6,9 @@ from subprocess import CompletedProcess
 from typing import Optional, List
 import json
 
-from ..index.schema import Link, ArchiveResult, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
@@ -37,22 +39,23 @@ def ShellError(cmd: List[str], result: CompletedProcess, lines: int=20) -> Archi
 
 
 @enforce_types
-def should_save_mercury(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_mercury(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.snapshot_dir
+    if is_static_file(snapshot.url):
         return False
 
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'mercury').exists():
+    output = Path(out_dir or snapshot.snapshot_dir) / 'mercury'
+    if not overwrite and output.exists():
         return False
 
-    return SAVE_MERCURY
+    return SAVE_MERCURY and MERCURY_VERSION
 
 
 @enforce_types
-def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_mercury(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download reader friendly version using @postlight/mercury-parser"""
 
-    out_dir = Path(out_dir or link.link_dir)
+    out_dir = Path(out_dir or snapshot.snapshot_dir)
     output_folder = out_dir.absolute() / "mercury"
     output = str(output_folder)
 
@@ -62,7 +65,7 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         # Get plain text version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
-            link.url,
+            snapshot.url,
             "--format=text"
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
@@ -74,7 +77,7 @@ def save_mercury(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
         # Get HTML version of article
         cmd = [
             DEPENDENCIES['MERCURY_BINARY']['path'],
-            link.url
+            snapshot.url
         ]
         result = run(cmd, cwd=out_dir, timeout=timeout)
         try:

archivebox/extractors/pdf.py

@@ -3,7 +3,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -17,10 +19,12 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
+# output = 'output.pdf'
 
 @enforce_types
-def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_pdf(snapshot: Model, verwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -31,15 +35,15 @@ def should_save_pdf(link: Link, out_dir: Optional[Path]=None, overwrite: Optiona
 
 
 @enforce_types
-def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_pdf(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """print PDF of site to file using chrome --headless"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'output.pdf'
     cmd = [
         *chrome_args(TIMEOUT=timeout),
         '--print-to-pdf',
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/readability.py

@@ -6,7 +6,9 @@ from tempfile import NamedTemporaryFile
 from typing import Optional
 import json
 
-from ..index.schema import Link, ArchiveResult, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveError
 from ..system import run, atomic_write
 from ..util import (
     enforce_types,
@@ -23,13 +25,14 @@ from ..config import (
 )
 from ..logging_util import TimedProgress
 
+
 @enforce_types
-def get_html(link: Link, path: Path) -> str:
+def get_html(snapshot: Model, path: Path) -> str:
     """
     Try to find wget, singlefile and then dom files.
     If none is found, download the url again.
     """
-    canonical = link.canonical_outputs()
+    canonical = snapshot.canonical_outputs()
     abs_path = path.absolute()
     sources = [canonical["singlefile_path"], canonical["wget_path"], canonical["dom_path"]]
     document = None
@@ -41,27 +44,31 @@ def get_html(link: Link, path: Path) -> str:
         except (FileNotFoundError, TypeError):
             continue
     if document is None:
-        return download_url(link.url)
+        return download_url(snapshot.url)
     else:
         return document
 
+
+# output = 'readability/'
+
 @enforce_types
-def should_save_readability(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_readability(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[str]=None) -> bool:
+    out_dir = out_dir or snapshot.link_dir
+    if is_static_file(snapshot.url):
         return False
 
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and (out_dir / 'readability').exists():
+    output = Path(out_dir or snapshot.snapshot_dir) / 'readability'
+    if not overwrite and output.exists():
         return False
 
-    return SAVE_READABILITY
+    return SAVE_READABILITY and READABILITY_VERSION
 
 
 @enforce_types
-def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_readability(snapshot: Model, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download reader friendly version using @mozilla/readability"""
 
-    out_dir = Path(out_dir or link.link_dir)
+    out_dir = Path(out_dir or snapshot.snapshot_dir)
     output_folder = out_dir.absolute() / "readability"
     output = str(output_folder)
 
@@ -71,12 +78,12 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
     # fake command to show the user so they have something to try debugging if get_html fails
     cmd = [
         CURL_BINARY,
-        link.url
+        snapshot.url
     ]
     readability_content = None
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        document = get_html(link, out_dir)
+        document = get_html(snapshot, out_dir)
         temp_doc = NamedTemporaryFile(delete=False)
         temp_doc.write(document.encode("utf-8"))
         temp_doc.close()

archivebox/extractors/screenshot.py

@@ -3,7 +3,9 @@ __package__ = 'archivebox.extractors'
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -18,10 +20,12 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
+# output = 'screenshot.png'
 
 @enforce_types
-def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_screenshot(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -31,15 +35,15 @@ def should_save_screenshot(link: Link, out_dir: Optional[Path]=None, overwrite:
     return SAVE_SCREENSHOT
 
 @enforce_types
-def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_screenshot(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """take screenshot of site using chrome --headless"""
     
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output: ArchiveOutput = 'screenshot.png'
     cmd = [
         *chrome_args(TIMEOUT=timeout),
         '--screenshot',
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')

archivebox/extractors/singlefile.py

@@ -5,7 +5,9 @@ from pathlib import Path
 from typing import Optional
 import json
 
-from ..index.schema import Link, ArchiveResult, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -23,8 +25,9 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    if is_static_file(link.url):
+def should_save_singlefile(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if is_static_file(snapshot.url):
         return False
 
     out_dir = out_dir or Path(link.link_dir)
@@ -35,10 +38,10 @@ def should_save_singlefile(link: Link, out_dir: Optional[Path]=None, overwrite:
 
 
 @enforce_types
-def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_singlefile(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using single-file"""
 
-    out_dir = out_dir or Path(link.link_dir)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
     output = "singlefile.html"
 
     browser_args = chrome_args(TIMEOUT=0)
@@ -49,7 +52,8 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
         DEPENDENCIES['SINGLEFILE_BINARY']['path'],
         '--browser-executable-path={}'.format(CHROME_BINARY),
         browser_args,
-        link.url,
+        snapshot.url,
+        output
         output,
     ]
 

archivebox/extractors/title.py

@@ -5,7 +5,9 @@ from html.parser import HTMLParser
 from pathlib import Path
 from typing import Optional
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..util import (
     enforce_types,
     download_url,
@@ -59,10 +61,12 @@ class TitleParser(HTMLParser):
             self.inside_title_tag = False
 
 
+# output = '{title}'
+
 @enforce_types
-def should_save_title(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
+def should_save_title(snapshot: Model, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
     # if link already has valid title, skip it
-    if not overwrite and link.title and not link.title.lower().startswith('http'):
+    if not overwrite and snapshot.title and not snapshot.title.lower().startswith('http'):
         return False
 
     return SAVE_TITLE
@@ -73,7 +77,7 @@ def extract_title_with_regex(html):
     return output
 
 @enforce_types
-def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_title(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """try to guess the page's title from its content"""
 
     from core.models import Snapshot
@@ -85,12 +89,12 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
         '--max-time', str(timeout),
         *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
-        link.url,
+        snapshot.url,
     ]
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
-        html = download_url(link.url, timeout=timeout)
+        html = download_url(snapshot.url, timeout=timeout)
         try:
             # try using relatively strict html parser first
             parser = TitleParser()
@@ -104,10 +108,11 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
         
         # if title is better than the one in the db, update db with new title
         if isinstance(output, str) and output:
-            if not link.title or len(output) >= len(link.title):
-                Snapshot.objects.filter(url=link.url,
-                                        timestamp=link.timestamp)\
+            if not snapshot.title or len(output) >= len(snapshot.title):
+                Snapshot.objects.filter(url=snapshot.url,
+                                        timestamp=snapshot.timestamp)\
                                 .update(title=output)
+                snapshot.title = output
         else:
             # if no content was returned, dont save a title (because it might be a temporary error)
             if not html:

archivebox/extractors/wget.py

@@ -6,7 +6,9 @@ from pathlib import Path
 from typing import Optional
 from datetime import datetime
 
-from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
+from django.db.models import Model
+
+from ..index.schema import ArchiveResult, ArchiveOutput, ArchiveError
 from ..system import run, chmod_file
 from ..util import (
     enforce_types,
@@ -34,20 +36,20 @@ from ..logging_util import TimedProgress
 
 
 @enforce_types
-def should_save_wget(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
-    output_path = wget_output_path(link)
-    out_dir = out_dir or Path(link.link_dir)
-    if not overwrite and output_path and (out_dir / output_path).exists():
+def should_save_wget(snapshot: Model, overwrite: Optional[bool]=False, out_dir: Optional[Path]=None) -> bool:
+    output_path = wget_output_path(snapshot)
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
+    if not overwrite output_path and (out_dir / output_path).exists():
         return False
 
     return SAVE_WGET
 
 
 @enforce_types
-def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
+def save_wget(snapshot: Model, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
     """download full site using wget"""
 
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or snapshot.snapshot_dir
     if SAVE_WARC:
         warc_dir = out_dir / "warc"
         warc_dir.mkdir(exist_ok=True)
@@ -68,14 +70,14 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
         *(['--compression=auto'] if WGET_AUTO_COMPRESSION else []),
         *([] if SAVE_WARC else ['--timestamping']),
         *([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),
-        link.url,
+        snapshot.url,
     ]
 
     status = 'succeeded'
     timer = TimedProgress(timeout, prefix='      ')
     try:
         result = run(cmd, cwd=str(out_dir), timeout=timeout)
-        output = wget_output_path(link)
+        output = wget_output_path(snapshot)
 
         # parse out number of files downloaded from last line of stderr:
         #  "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
@@ -126,13 +128,13 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
 
 
 @enforce_types
-def wget_output_path(link: Link) -> Optional[str]:
+def wget_output_path(snapshot: Model) -> Optional[str]:
     """calculate the path to the wgetted .html file, since wget may
     adjust some paths to be different than the base_url path.
 
     See docs on wget --adjust-extension (-E)
     """
-    
+
     # Wget downloads can save in a number of different ways depending on the url:
     #    https://example.com
     #       > example.com/index.html
@@ -164,8 +166,8 @@ def wget_output_path(link: Link) -> Optional[str]:
     # and there's no way to get the computed output path from wget
     # in order to avoid having to reverse-engineer how they calculate it,
     # we just look in the output folder read the filename wget used from the filesystem
-    full_path = without_fragment(without_query(path(link.url))).strip('/')
-    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+") / urldecode(full_path)
+    full_path = without_fragment(without_query(path(snapshot.url))).strip('/')
+    search_dir = Path(snapshot.snapshot_dir) / domain(snapshot.url).replace(":", "+") / urldecode(full_path)
     for _ in range(4):
         if search_dir.exists():
             if search_dir.is_dir():
@@ -174,7 +176,7 @@ def wget_output_path(link: Link) -> Optional[str]:
                     if re.search(".+\\.[Ss]?[Hh][Tt][Mm][Ll]?$", str(f), re.I | re.M)
                 ]
                 if html_files:
-                    return str(html_files[0].relative_to(link.link_dir))
+                    return str(html_files[0].relative_to(snapshot.snapshot_dir))
 
                 # sometimes wget'd URLs have no ext and return non-html
                 # e.g. /some/example/rss/all -> some RSS XML content)
@@ -183,23 +185,23 @@ def wget_output_path(link: Link) -> Optional[str]:
                 last_part_of_url = urldecode(full_path.rsplit('/', 1)[-1])
                 for file_present in search_dir.iterdir():
                     if file_present == last_part_of_url:
-                        return str((search_dir / file_present).relative_to(link.link_dir))
+                        return str((search_dir / file_present).relative_to(snapshot.snapshot_dir))
 
         # Move up one directory level
         search_dir = search_dir.parent
 
-        if str(search_dir) == link.link_dir:
+        if search_dir == snapshot.snapshot_dir:
             break
 
     # check for literally any file present that isnt an empty folder
-    domain_dir = Path(domain(link.url).replace(":", "+"))
-    files_within = list((Path(link.link_dir) / domain_dir).glob('**/*.*'))
+    domain_dir = Path(domain(snapshot.url).replace(":", "+"))
+    files_within = list((Path(snapshot.snapshot_dir) / domain_dir).glob('**/*.*'))
     if files_within:
-        return str((domain_dir / files_within[-1]).relative_to(link.link_dir))
+        return str((domain_dir / files_within[-1]).relative_to(snapshot.snapshot_dir))
     
-    # fallback to just the domain dir
-    search_dir = Path(link.link_dir) / domain(link.url).replace(":", "+")
+    # fallback to just the domain dir, dont try to introspect further inside it
+    search_dir = Path(snapshot.snapshot_dir) / domain(snapshot.url).replace(":", "+")
     if search_dir.is_dir():
-        return domain(link.url).replace(":", "+")
+        return domain(snapshot.url).replace(":", "+")
 
     return None

archivebox/index/__init__.py

@@ -9,7 +9,7 @@ from typing import List, Tuple, Dict, Optional, Iterable
 from collections import OrderedDict
 from contextlib import contextmanager
 from urllib.parse import urlparse
-from django.db.models import QuerySet, Q
+from django.db.models import QuerySet, Q, Model
 
 from ..util import (
     scheme,
@@ -38,16 +38,16 @@ from ..logging_util import (
 
 from .schema import Link, ArchiveResult
 from .html import (
-    write_html_link_details,
+    write_html_snapshot_details,
 )
 from .json import (
     pyjson,
-    parse_json_link_details, 
-    write_json_link_details,
+    load_json_snapshot,
+    write_json_snapshot_details,
 )
 from .sql import (
     write_sql_main_index,
-    write_sql_link_details,
+    write_sql_snapshot_details,
 )
 
 from ..search import search_backend_enabled, query_search_index
@@ -55,10 +55,12 @@ from ..search import search_backend_enabled, query_search_index
 ### Link filtering and checking
 
 @enforce_types
-def merge_links(a: Link, b: Link) -> Link:
-    """deterministially merge two links, favoring longer field values over shorter,
+def merge_snapshots(a: Model, b: Model) -> Model:
+    """deterministially merge two snapshots, favoring longer field values over shorter,
     and "cleaner" values over worse ones.
+    TODO: Check if this makes sense with the new setup
     """
+    return a
     assert a.base_url == b.base_url, f'Cannot merge two links with different URLs ({a.base_url} != {b.base_url})'
 
     # longest url wins (because a fuzzy url will always be shorter)
@@ -109,55 +111,55 @@ def merge_links(a: Link, b: Link) -> Link:
             key=lambda result: result.start_ts,
         )))
 
-    return Link(
+    return Snapshot(
         url=url,
         timestamp=timestamp,
         title=title,
         tags=tags,
-        sources=sources,
-        history=history,
+        #sources=sources,
+        #history=history,
     )
 
 
 @enforce_types
-def validate_links(links: Iterable[Link]) -> List[Link]:
+def validate_snapshots(snapshots: List[Model]) -> List[Model]:
     timer = TimedProgress(TIMEOUT * 4)
     try:
-        links = archivable_links(links)  # remove chrome://, about:, mailto: etc.
-        links = sorted_links(links)      # deterministically sort the links based on timestamp, url
-        links = fix_duplicate_links(links)  # merge/dedupe duplicate timestamps & urls
+        snapshots = archivable_snapshots(snapshots)  # remove chrome://, about:, mailto: etc.
+        snapshots = sorted_snapshots(snapshots)      # deterministically sort the links based on timestamp, url
+        snapshots = fix_duplicate_snapshots(snapshots)  # merge/dedupe duplicate timestamps & urls
     finally:
         timer.end()
 
-    return list(links)
+    return list(snapshots)
 
 @enforce_types
-def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
+def archivable_snapshots(snapshots: Iterable[Model]) -> Iterable[Model]:
     """remove chrome://, about:// or other schemed links that cant be archived"""
-    for link in links:
+    for snapshot in snapshots:
         try:
-            urlparse(link.url)
+            urlparse(snapshot.url)
         except ValueError:
             continue
-        if scheme(link.url) not in ('http', 'https', 'ftp'):
+        if scheme(snapshot.url) not in ('http', 'https', 'ftp'):
             continue
-        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(link.url):
+        if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(snapshot.url):
             continue
 
-        yield link
+        yield snapshot
 
 
 @enforce_types
-def fix_duplicate_links(sorted_links: Iterable[Link]) -> Iterable[Link]:
+def fix_duplicate_snapshots(sorted_snapshots: Iterable[Model]) -> Iterable[Model]:
     """
     ensures that all non-duplicate links have monotonically increasing timestamps
+    TODO: Review how to do this with the new snapshots refactor
     """
-    # from core.models import Snapshot
-
+    return sorted_snapshots
     unique_urls: OrderedDict[str, Link] = OrderedDict()
 
-    for link in sorted_links:
-        if link.url in unique_urls:
+    for snapshot in sorted_snapshots:
+        if snapshot.url in unique_urls:
             # merge with any other links that share the same url
             link = merge_links(unique_urls[link.url], link)
         unique_urls[link.url] = link
@@ -166,9 +168,9 @@ def fix_duplicate_links(sorted_links: Iterable[Link]) -> Iterable[Link]:
 
 
 @enforce_types
-def sorted_links(links: Iterable[Link]) -> Iterable[Link]:
-    sort_func = lambda link: (link.timestamp.split('.', 1)[0], link.url)
-    return sorted(links, key=sort_func, reverse=True)
+def sorted_snapshots(snapshots: Iterable[Model]) -> Iterable[Model]:
+    sort_func = lambda snapshot: (snapshot.timestamp.split('.', 1)[0], snapshot.url)
+    return sorted(snapshots, key=sort_func, reverse=True)
 
 
 @enforce_types
@@ -222,14 +224,14 @@ def timed_index_update(out_path: Path):
 
 
 @enforce_types
-def write_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
+def write_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
     """Writes links to sqlite3 file for a given list of links"""
 
-    log_indexing_process_started(len(links))
+    log_indexing_process_started(len(snapshots))
 
     try:
         with timed_index_update(out_dir / SQL_INDEX_FILENAME):
-            write_sql_main_index(links, out_dir=out_dir)
+            write_sql_main_index(snapshots, out_dir=out_dir)
             os.chmod(out_dir / SQL_INDEX_FILENAME, int(OUTPUT_PERMISSIONS, base=8)) # set here because we don't write it with atomic writes
 
     except (KeyboardInterrupt, SystemExit):
@@ -244,7 +246,9 @@ def write_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
 
 @enforce_types
 def load_main_index(out_dir: Path=OUTPUT_DIR, warn: bool=True) -> List[Link]:
-    """parse and load existing index with any new links from import_path merged in"""
+    """
+    Returns all of the snapshots currently in index
+    """
     from core.models import Snapshot
     try:
         return Snapshot.objects.all()
@@ -265,88 +269,62 @@ def load_main_index_meta(out_dir: Path=OUTPUT_DIR) -> Optional[dict]:
 
 
 @enforce_types
-def parse_links_from_source(source_path: str, root_url: Optional[str]=None) -> Tuple[List[Link], List[Link]]:
+def parse_snapshots_from_source(source_path: str, root_url: Optional[str]=None) -> List[Model]:
 
-    from ..parsers import parse_links
+    from ..parsers import parse_snapshots
 
-    new_links: List[Link] = []
+    new_links: List[Model] = []
 
     # parse and validate the import file
-    raw_links, parser_name = parse_links(source_path, root_url=root_url)
-    new_links = validate_links(raw_links)
+    raw_snapshots, parser_name = parse_snapshots(source_path, root_url=root_url)
+    new_snapshots = validate_snapshots(raw_snapshots)
 
     if parser_name:
-        num_parsed = len(raw_links)
+        num_parsed = len(raw_snapshots)
         log_parsing_finished(num_parsed, parser_name)
 
-    return new_links
+    return new_snapshots
 
 @enforce_types
-def fix_duplicate_links_in_index(snapshots: QuerySet, links: Iterable[Link]) -> Iterable[Link]:
+def filter_new_urls(snapshots: QuerySet,
+                 new_snapshots: List) -> List:
     """
-    Given a list of in-memory Links, dedupe and merge them with any conflicting Snapshots in the DB.
+    Returns a list of Snapshots corresponding to the urls that were not present in the index
     """
-    unique_urls: OrderedDict[str, Link] = OrderedDict()
+    urls = {snapshot.url: snapshot for snapshot in new_snapshots}
+    filtered_snapshots = snapshots.filter(url__in=urls.keys())
 
-    for link in links:
-        index_link = snapshots.filter(url=link.url)
-        if index_link:
-            link = merge_links(index_link[0].as_link(), link)
-
-        unique_urls[link.url] = link
-
-    return unique_urls.values()
-
-@enforce_types
-def dedupe_links(snapshots: QuerySet,
-                 new_links: List[Link]) -> List[Link]:
-    """
-    The validation of links happened at a different stage. This method will
-    focus on actual deduplication and timestamp fixing.
-    """
+    for found_snapshot in filtered_snapshots:
+        urls.pop(found_snapshot.url)
     
-    # merge existing links in out_dir and new links
-    dedup_links = fix_duplicate_links_in_index(snapshots, new_links)
+    log_deduping_finished(len(urls.keys()))
 
-    new_links = [
-        link for link in new_links
-        if not snapshots.filter(url=link.url).exists()
-    ]
-
-    dedup_links_dict = {link.url: link for link in dedup_links}
-
-    # Replace links in new_links with the dedup version
-    for i in range(len(new_links)):
-        if new_links[i].url in dedup_links_dict.keys():
-            new_links[i] = dedup_links_dict[new_links[i].url]
-    log_deduping_finished(len(new_links))
-
-    return new_links
+    return list(urls.values())
 
 ### Link Details Index
 
 @enforce_types
-def write_link_details(link: Link, out_dir: Optional[str]=None, skip_sql_index: bool=False) -> None:
-    out_dir = out_dir or link.link_dir
+def write_snapshot_details(snapshot: List[Model], out_dir: Optional[str]=None, skip_sql_index: bool=False) -> None:
+    out_dir = out_dir or snapshot.snapshot_dir
 
-    write_json_link_details(link, out_dir=out_dir)
-    write_html_link_details(link, out_dir=out_dir)
+    write_json_snapshot_details(snapshot, out_dir=out_dir)
+    write_html_snapshot_details(snapshot, out_dir=out_dir)
     if not skip_sql_index:
-        write_sql_link_details(link)
+        write_sql_snapshot_details(snapshot)
 
 
 @enforce_types
-def load_link_details(link: Link, out_dir: Optional[str]=None) -> Link:
+def load_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> Model:
     """check for an existing link archive in the given directory, 
        and load+merge it into the given link dict
     """
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or Path(snapshot.snapshot_dir)
 
-    existing_link = parse_json_link_details(out_dir)
-    if existing_link:
-        return merge_links(existing_link, link)
+    existing_snapshot = load_json_snapshot(Path(out_dir))
+    if existing_snapshot:
+        return merge_snapshots(existing_snapshot, snapshot)
 
-    return link
+    return snapshot
 
 
 
@@ -401,56 +379,41 @@ def snapshot_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type
         return search_filter(snapshots, filter_patterns, filter_type)
 
 
-def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_indexed_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """indexed links without checking archive status or data directory validity"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in links
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in snapshots}
 
-def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_archived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """indexed links that are archived with a valid data directory"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in filter(is_archived, links)
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_archived, snapshots)}
 
-def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_unarchived_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """indexed links that are unarchived with no data directory or an empty data directory"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in filter(is_unarchived, links)
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_unarchived, snapshots)}
 
-def get_present_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_present_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that actually exist in the archive/ folder"""
+    from core.models import Snapshot
 
     all_folders = {}
 
     for entry in (out_dir / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
-            link = None
+            snapshot = None
             try:
-                link = parse_json_link_details(entry.path)
+                snapshot = load_json_snapshot(Path(entry.path))
             except Exception:
                 pass
 
-            all_folders[entry.name] = link
+            all_folders[entry.name] = snapshot
 
     return all_folders
 
-def get_valid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_valid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs with a valid index matched to the main index and archived content"""
-    links = [snapshot.as_link_with_details() for snapshot in snapshots.iterator()]
-    return {
-        link.link_dir: link
-        for link in filter(is_valid, links)
-    }
+    return {snapshot.snapshot_dir: snapshot for snapshot in filter(is_valid, snapshots)}
 
-def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that are invalid for any reason: corrupted/duplicate/orphaned/unrecognized"""
     duplicate = get_duplicate_folders(snapshots, out_dir=OUTPUT_DIR)
     orphaned = get_orphaned_folders(snapshots, out_dir=OUTPUT_DIR)
@@ -459,7 +422,7 @@ def get_invalid_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Option
     return {**duplicate, **orphaned, **corrupted, **unrecognized}
 
 
-def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that conflict with other directories that have the same link URL or timestamp"""
     by_url = {}
     by_timestamp = {}
@@ -472,91 +435,92 @@ def get_duplicate_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Opti
     )
 
     for path in chain(snapshots.iterator(), data_folders):
-        link = None
+        snapshot = None
         if type(path) is not str:
-            path = path.as_link().link_dir
+            path = path.snapshot_dir
 
         try:
-            link = parse_json_link_details(path)
+            snapshot = load_json_snapshot(Path(path))
         except Exception:
             pass
 
-        if link:
-            # link folder has same timestamp as different link folder
-            by_timestamp[link.timestamp] = by_timestamp.get(link.timestamp, 0) + 1
-            if by_timestamp[link.timestamp] > 1:
-                duplicate_folders[path] = link
+        if snapshot:
+            # snapshot folder has same timestamp as different link folder
+            by_timestamp[snapshot.timestamp] = by_timestamp.get(snapshot.timestamp, 0) + 1
+            if by_timestamp[snapshot.timestamp] > 1:
+                duplicate_folders[path] = snapshot
 
             # link folder has same url as different link folder
-            by_url[link.url] = by_url.get(link.url, 0) + 1
-            if by_url[link.url] > 1:
-                duplicate_folders[path] = link
+            by_url[snapshot.url] = by_url.get(snapshot.url, 0) + 1
+            if by_url[snapshot.url] > 1:
+                duplicate_folders[path] = snapshot
     return duplicate_folders
 
-def get_orphaned_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_orphaned_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that contain a valid index but aren't listed in the main index"""
     orphaned_folders = {}
 
     for entry in (Path(out_dir) / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
-            link = None
+            snapshot = None
             try:
-                link = parse_json_link_details(str(entry))
+                snapshot = load_json_snapshot(entry)
             except Exception:
                 pass
 
-            if link and not snapshots.filter(timestamp=entry.name).exists():
+            if snapshot and not snapshots.filter(timestamp=entry.name).exists():
                 # folder is a valid link data dir with index details, but it's not in the main index
-                orphaned_folders[str(entry)] = link
+                orphaned_folders[str(entry)] = snapshot
 
     return orphaned_folders
 
-def get_corrupted_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_corrupted_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that don't contain a valid index and aren't listed in the main index"""
     corrupted = {}
     for snapshot in snapshots.iterator():
-        link = snapshot.as_link()
-        if is_corrupt(link):
-            corrupted[link.link_dir] = link
+        if is_corrupt(snapshot):
+            corrupted[snapshot.snapshot_dir] = snapshot
     return corrupted
 
-def get_unrecognized_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+def get_unrecognized_folders(snapshots, out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     """dirs that don't contain recognizable archive data and aren't listed in the main index"""
-    unrecognized_folders: Dict[str, Optional[Link]] = {}
+    unrecognized_folders: Dict[str, Optional[Model]] = {}
 
     for entry in (Path(out_dir) / ARCHIVE_DIR_NAME).iterdir():
         if entry.is_dir():
             index_exists = (entry / "index.json").exists()
-            link = None
+            snapshot = None
             try:
-                link = parse_json_link_details(str(entry))
+                snapshot = load_json_snapshot(entry)
             except KeyError:
                 # Try to fix index
                 if index_exists:
-                    try:
+                    pass
+                    # TODO: Implement the `guess` bit for snapshots
+                    # try:
                         # Last attempt to repair the detail index
-                        link_guessed = parse_json_link_details(str(entry), guess=True)
-                        write_json_link_details(link_guessed, out_dir=str(entry))
-                        link = parse_json_link_details(str(entry))
-                    except Exception:
-                        pass
+                        # link_guessed = parse_json_snapshot_details(str(entry), guess=True)
+                        # write_json_snapshot_details(link_guessed, out_dir=str(entry))
+                        # link = parse_json_link_details(str(entry))
+                    # except Exception:
+                    #     pass
 
-            if index_exists and link is None:
+            if index_exists and snapshot is None:
                 # index exists but it's corrupted or unparseable
-                unrecognized_folders[str(entry)] = link
+                unrecognized_folders[str(entry)] = snapshot
             
             elif not index_exists:
                 # link details index doesn't exist and the folder isn't in the main index
                 timestamp = entry.name
                 if not snapshots.filter(timestamp=timestamp).exists():
-                    unrecognized_folders[str(entry)] = link
+                    unrecognized_folders[str(entry)] = snapshot
 
     return unrecognized_folders
 
 
-def is_valid(link: Link) -> bool:
-    dir_exists = Path(link.link_dir).exists()
-    index_exists = (Path(link.link_dir) / "index.json").exists()
+def is_valid(snapshot: Model) -> bool:
+    dir_exists = Path(snapshot.snapshot_dir).exists()
+    index_exists = (Path(snapshot.snapshot_dir) / "index.json").exists()
     if not dir_exists:
         # unarchived links are not included in the valid list
         return False
@@ -564,29 +528,30 @@ def is_valid(link: Link) -> bool:
         return False
     if dir_exists and index_exists:
         try:
-            parsed_link = parse_json_link_details(link.link_dir, guess=True)
-            return link.url == parsed_link.url
+            # TODO: review if the `guess` was necessary here
+            parsed_snapshot = load_json_snapshot(snapshot.snapshot_dir)
+            return snapshot.url == parsed_snapshot.url
         except Exception:
             pass
     return False
 
-def is_corrupt(link: Link) -> bool:
-    if not Path(link.link_dir).exists():
+def is_corrupt(snapshot: Model) -> bool:
+    if not Path(snapshot.snapshot_dir).exists():
         # unarchived links are not considered corrupt
         return False
 
-    if is_valid(link):
+    if is_valid(snapshot):
         return False
 
     return True
 
-def is_archived(link: Link) -> bool:
-    return is_valid(link) and link.is_archived
+def is_archived(snapshot: Model) -> bool:
+    return is_valid(snapshot) and snapshot.is_archived
     
-def is_unarchived(link: Link) -> bool:
-    if not Path(link.link_dir).exists():
+def is_unarchived(snapshot: Model) -> bool:
+    if not Path(snapshot.snapshot_dir).exists():
         return True
-    return not link.is_archived
+    return not snapshot.is_archived
 
 
 def fix_invalid_folder_locations(out_dir: Path=OUTPUT_DIR) -> Tuple[List[str], List[str]]:
@@ -596,22 +561,22 @@ def fix_invalid_folder_locations(out_dir: Path=OUTPUT_DIR) -> Tuple[List[str], L
         if entry.is_dir(follow_symlinks=True):
             if (Path(entry.path) / 'index.json').exists():
                 try:
-                    link = parse_json_link_details(entry.path)
+                    snapshot = load_json_snapshot(Path(entry.path))
                 except KeyError:
-                    link = None
-                if not link:
+                    snapshot = None
+                if not snapshot:
                     continue
 
-                if not entry.path.endswith(f'/{link.timestamp}'):
-                    dest = out_dir / ARCHIVE_DIR_NAME / link.timestamp
+                if not entry.path.endswith(f'/{snapshot.timestamp}'):
+                    dest = out_dir / ARCHIVE_DIR_NAME / snapshot.timestamp
                     if dest.exists():
                         cant_fix.append(entry.path)
                     else:
                         shutil.move(entry.path, dest)
                         fixed.append(dest)
                         timestamp = entry.path.rsplit('/', 1)[-1]
-                        assert link.link_dir == entry.path
-                        assert link.timestamp == timestamp
-                        write_json_link_details(link, out_dir=entry.path)
+                        assert snapshot.snapshot_dir == entry.path
+                        assert snapshot.timestamp == timestamp
+                        write_json_snapshot_details(snapshot, out_dir=entry.path)
 
     return fixed, cant_fix

archivebox/index/csv.py

@@ -2,12 +2,14 @@ __package__ = 'archivebox.index'
 
 from typing import List, Optional, Any
 
+from django.db.models import Model
+
 from ..util import enforce_types
 from .schema import Link
 
 
 @enforce_types
-def links_to_csv(links: List[Link],
+def snapshots_to_csv(snapshots: List[Model],
                  cols: Optional[List[str]]=None,
                  header: bool=True,
                  separator: str=',',
@@ -20,8 +22,8 @@ def links_to_csv(links: List[Link],
         header_str = separator.join(col.ljust(ljust) for col in cols)
 
     row_strs = (
-        link.to_csv(cols=cols, ljust=ljust, separator=separator)
-        for link in links
+        snapshot.as_csv(cols=cols, ljust=ljust, separator=separator)
+        for snapshot in snapshots
     )
 
     return '\n'.join((header_str, *row_strs))

archivebox/index/html.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from typing import List, Optional, Iterator, Mapping
 from pathlib import Path
 
+from django.db.models import Model
 from django.utils.html import format_html, mark_safe
 from collections import defaultdict
 
@@ -47,24 +48,24 @@ def parse_html_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[str]:
     return ()
 
 @enforce_types
-def generate_index_from_links(links: List[Link], with_headers: bool):
+def generate_index_from_snapshots(snapshots: List[Model], with_headers: bool):
     if with_headers:
-        output = main_index_template(links)
+        output = main_index_template(snapshots)
     else:
-        output = main_index_template(links, template=MINIMAL_INDEX_TEMPLATE)
+        output = main_index_template(snapshots, template=MINIMAL_INDEX_TEMPLATE)
     return output
 
 @enforce_types
-def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) -> str:
+def main_index_template(snapshots: List[Model], template: str=MAIN_INDEX_TEMPLATE) -> str:
     """render the template for the entire main index"""
 
     return render_django_template(template, {
         'version': VERSION,
         'git_sha': GIT_SHA,
-        'num_links': str(len(links)),
+        'num_snapshots': str(len(snapshots)),
         'date_updated': datetime.now().strftime('%Y-%m-%d'),
         'time_updated': datetime.now().strftime('%Y-%m-%d %H:%M'),
-        'links': [link._asdict(extended=True) for link in links],
+        'snapshots': snapshots,
         'FOOTER_INFO': FOOTER_INFO,
     })
 
@@ -72,38 +73,36 @@ def main_index_template(links: List[Link], template: str=MAIN_INDEX_TEMPLATE) ->
 ### Link Details Index
 
 @enforce_types
-def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
-    out_dir = out_dir or link.link_dir
+def write_html_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> None:
+    out_dir = out_dir or snapshot.snapshot_dir
 
-    rendered_html = link_details_template(link)
+    rendered_html = snapshot_details_template(snapshot)
     atomic_write(str(Path(out_dir) / HTML_INDEX_FILENAME), rendered_html)
 
 
 @enforce_types
-def link_details_template(link: Link) -> str:
+def snapshot_details_template(snapshot: Model) -> str:
 
     from ..extractors.wget import wget_output_path
 
-    link_info = link._asdict(extended=True)
-
     return render_django_template(LINK_DETAILS_TEMPLATE, {
-        **link_info,
-        **link_info['canonical'],
+        **snapshot.as_json(),
+        **snapshot.canonical_outputs(),
         'title': htmlencode(
-            link.title
-            or (link.base_url if link.is_archived else TITLE_LOADING_MSG)
+            snapshot.title
+            or (snapshot.base_url if snapshot.is_archived else TITLE_LOADING_MSG)
         ),
-        'url_str': htmlencode(urldecode(link.base_url)),
+        'url_str': htmlencode(urldecode(snapshot.base_url)),
         'archive_url': urlencode(
-            wget_output_path(link)
-            or (link.domain if link.is_archived else '')
+            wget_output_path(snapshot)
+            or (snapshot.domain if snapshot.is_archived else '')
         ) or 'about:blank',
-        'extension': link.extension or 'html',
-        'tags': link.tags or 'untagged',
-        'size': printable_filesize(link.archive_size) if link.archive_size else 'pending',
-        'status': 'archived' if link.is_archived else 'not yet archived',
-        'status_color': 'success' if link.is_archived else 'danger',
-        'oldest_archive_date': ts_to_date(link.oldest_archive_date),
+        'extension': snapshot.extension or 'html',
+        'tags': snapshot.tags_str() or 'untagged',
+        'size': printable_filesize(snapshot.archive_size) if snapshot.archive_size else 'pending',
+        'status': 'archived' if snapshot.is_archived else 'not yet archived',
+        'status_color': 'success' if snapshot.is_archived else 'danger',
+        'oldest_archive_date': ts_to_date(snapshot.oldest_archive_date),
         'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
     })
 
@@ -121,9 +120,8 @@ def snapshot_icons(snapshot) -> str:
     # start = datetime.now()
 
     archive_results = snapshot.archiveresult_set.filter(status="succeeded")
-    link = snapshot.as_link()
-    path = link.archive_path
-    canon = link.canonical_outputs()
+    path = snapshot.archive_path
+    canon = snapshot.canonical_outputs()
     output = ""
     output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> &nbsp;'
     icons = {

archivebox/index/json.py

@@ -7,6 +7,7 @@ from pathlib import Path
 
 from datetime import datetime
 from typing import List, Optional, Iterator, Any, Union
+from django.db.models import Model
 
 from .schema import Link
 from ..system import atomic_write
@@ -40,17 +41,18 @@ MAIN_INDEX_HEADER = {
 }
 
 @enforce_types
-def generate_json_index_from_links(links: List[Link], with_headers: bool):
+def generate_json_index_from_snapshots(snapshots: List[Model], with_headers: bool):
+    snapshots_json = [snapshot.as_json() for snapshot in snapshots]
     if with_headers:
         output = {
             **MAIN_INDEX_HEADER,
-            'num_links': len(links),
+            'num_links': len(snapshots),
             'updated': datetime.now(),
             'last_run_cmd': sys.argv,
-            'links': links,
+            'links': snapshots_json,
         }
     else:
-        output = links
+        output = snapshots_json
     return to_json(output, indent=4, sort_keys=True)
 
 
@@ -81,41 +83,46 @@ def parse_json_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]:
 ### Link Details Index
 
 @enforce_types
-def write_json_link_details(link: Link, out_dir: Optional[str]=None) -> None:
-    """write a json file with some info about the link"""
+def write_json_snapshot_details(snapshot: Model, out_dir: Optional[str]=None) -> None:
+    """write a json file with some info about the snapshot"""
     
-    out_dir = out_dir or link.link_dir
+    out_dir = out_dir or snapshot.snapshot_dir
     path = Path(out_dir) / JSON_INDEX_FILENAME
-    atomic_write(str(path), link._asdict(extended=True))
+    atomic_write(str(path), snapshot.as_json())
 
 
 @enforce_types
-def parse_json_link_details(out_dir: Union[Path, str], guess: Optional[bool]=False) -> Optional[Link]:
-    """load the json link index from a given directory"""
+def load_json_snapshot(out_dir: Path) -> Optional[Model]:
+    """
+    Loads the detail from the local json index
+    """
+    from core.models import Snapshot
+
     existing_index = Path(out_dir) / JSON_INDEX_FILENAME
     if existing_index.exists():
         with open(existing_index, 'r', encoding='utf-8') as f:
             try:
-                link_json = pyjson.load(f)
-                return Link.from_json(link_json, guess)
+                output = pyjson.load(f)
+                output = Snapshot.from_json(output)
+                return output
             except pyjson.JSONDecodeError:
                 pass
     return None
 
 
 @enforce_types
-def parse_json_links_details(out_dir: Union[Path, str]) -> Iterator[Link]:
-    """read through all the archive data folders and return the parsed links"""
+def parse_json_snapshot_details(out_dir: Union[Path, str]) -> Iterator[dict]:
+    """read through all the archive data folders and return the parsed snapshots"""
 
     for entry in os.scandir(Path(out_dir) / ARCHIVE_DIR_NAME):
         if entry.is_dir(follow_symlinks=True):
             if (Path(entry.path) / 'index.json').exists():
                 try:
-                    link = parse_json_link_details(entry.path)
+                    snapshot_details = load_json_snapshot(Path(entry.path))
                 except KeyError:
-                    link = None
-                if link:
-                    yield link
+                    snapshot_details = None
+                if snapshot_details:
+                    yield snapshot_details
 
 
 

archivebox/index/schema.py

@@ -408,6 +408,7 @@ class Link:
         return latest
 
 
+
     def canonical_outputs(self) -> Dict[str, Optional[str]]:
         """predict the expected output paths that should be present after archiving"""
 

archivebox/index/sql.py

@@ -3,8 +3,9 @@ __package__ = 'archivebox.index'
 from io import StringIO
 from pathlib import Path
 from typing import List, Tuple, Iterator
-from django.db.models import QuerySet
+from django.db.models import QuerySet, Model
 from django.db import transaction
+from datetime import datetime
 
 from .schema import Link
 from ..util import enforce_types
@@ -28,49 +29,50 @@ def remove_from_sql_main_index(snapshots: QuerySet, out_dir: Path=OUTPUT_DIR) ->
         snapshots.delete()
 
 @enforce_types
-def write_link_to_sql_index(link: Link):
+def write_snapshot_to_index(snapshot: Model):
     from core.models import Snapshot
-    info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}
-    tags = info.pop("tags")
-    if tags is None:
-        tags = []
-
     try:
-        info["timestamp"] = Snapshot.objects.get(url=link.url).timestamp
+        timestamp = Snapshot.objects.get(url=snapshot.url).timestamp
     except Snapshot.DoesNotExist:
-        while Snapshot.objects.filter(timestamp=info["timestamp"]).exists():
-            info["timestamp"] = str(float(info["timestamp"]) + 1.0)
+        timestamp = snapshot.timestamp
+        if not timestamp:
+            timestamp = str(datetime.now().timestamp())
+        while Snapshot.objects.filter(timestamp=timestamp).exists():
+            print("the timestamp is: ", timestamp)
+            timestamp = str(float(timestamp) + 1.0)
 
-    snapshot, _ = Snapshot.objects.update_or_create(url=link.url, defaults=info)
-    snapshot.save_tags(tags)
+    snapshot.timestamp = timestamp
+    snapshot.save()
     return snapshot
 
 
 @enforce_types
-def write_sql_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None:
+def write_sql_main_index(snapshots: List[Model], out_dir: Path=OUTPUT_DIR) -> None:
     with transaction.atomic():
-        for link in links:
-            write_link_to_sql_index(link)
+        for snapshot in snapshots:
+            write_snapshot_to_index(snapshot)
             
 
 @enforce_types
-def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None:
+def write_sql_snapshot_details(snapshot: Model, out_dir: Path=OUTPUT_DIR) -> None:
     from core.models import Snapshot
 
     with transaction.atomic():
         try:
-            snap = Snapshot.objects.get(url=link.url)
+            snap = Snapshot.objects.get(url=snapshot.url)
         except Snapshot.DoesNotExist:
-            snap = write_link_to_sql_index(link)
-        snap.title = link.title
+            snap = write_snapshot_to_index(snapshot)
+        snap.title = snapshot.title
 
-        tag_set = (
-            set(tag.strip() for tag in (link.tags or '').split(','))
-        )
-        tag_list = list(tag_set) or []
+        # TODO: If there are actual tags, this will break
+        #tag_set = (
+        #    set(tag.strip() for tag in (snapshot.tags.all() or '').split(','))
+        #)
+        #tag_list = list(tag_set) or []
 
         snap.save()
-        snap.save_tags(tag_list)
+        #snap.save_tags(tag_list)
+        return snap
 
 
 

archivebox/logging_util.py

@@ -303,7 +303,7 @@ def log_archiving_finished(num_links: int):
     print('        archivebox server 0.0.0.0:8000')
 
 
-def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
+def log_snapshot_archiving_started(snapshot: Model, snapshot_dir: str, is_new: bool):
     # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
     #     http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
     #     > output/archive/1478739709
@@ -312,16 +312,16 @@ def log_link_archiving_started(link: "Link", link_dir: str, is_new: bool):
         symbol_color=ANSI['green' if is_new else 'black'],
         symbol='+' if is_new else '√',
         now=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-        title=link.title or link.base_url,
+        title=snapshot.title or snapshot.base_url,
         **ANSI,
     ))
-    print('    {blue}{url}{reset}'.format(url=link.url, **ANSI))
+    print('    {blue}{url}{reset}'.format(url=snapshot.url, **ANSI))
     print('    {} {}'.format(
         '>' if is_new else '√',
-        pretty_path(link_dir),
+        pretty_path(snapshot_dir),
     ))
 
-def log_link_archiving_finished(link: "Link", link_dir: str, is_new: bool, stats: dict):
+def log_snapshot_archiving_finished(snapshot: Model, snapshot_dir: str, is_new: bool, stats: dict):
     total = sum(stats.values())
 
     if stats['failed'] > 0 :
@@ -395,49 +395,49 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str):
     ))
     print('    {}'.format(' '.join(filter_patterns or ())))
 
-def log_list_finished(links):
-    from .index.csv import links_to_csv
+def log_list_finished(snapshots):
+    from .index.csv import snapshots_to_csv
     print()
     print('---------------------------------------------------------------------------------------------------')
-    print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
+    print(snapshots_to_csv(snapshots, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
     print('---------------------------------------------------------------------------------------------------')
     print()
 
 
-def log_removal_started(links: List["Link"], yes: bool, delete: bool):
-    print('{lightyellow}[i] Found {} matching URLs to remove.{reset}'.format(len(links), **ANSI))
+def log_removal_started(snapshots: List["Snapshot"], yes: bool, delete: bool):
+    print('{lightyellow}[i] Found {} matching URLs to remove.{reset}'.format(len(snapshots), **ANSI))
     if delete:
-        file_counts = [link.num_outputs for link in links if Path(link.link_dir).exists()]
+        file_counts = [snapshot.num_outputs for snapshot in snapshots if Path(snapshot.snapshot_dir).exists()]
         print(
-            f'    {len(links)} Links will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
+            f'    {len(snapshots)} Snapshots will be de-listed from the main index, and their archived content folders will be deleted from disk.\n'
             f'    ({len(file_counts)} data folders with {sum(file_counts)} archived files will be deleted!)'
         )
     else:
         print(
-            '    Matching links will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
+            '    Matching snapshots will be de-listed from the main index, but their archived content folders will remain in place on disk.\n'
             '    (Pass --delete if you also want to permanently delete the data folders)'
         )
 
     if not yes:
         print()
-        print('{lightyellow}[?] Do you want to proceed with removing these {} links?{reset}'.format(len(links), **ANSI))
+        print('{lightyellow}[?] Do you want to proceed with removing these {} snapshots?{reset}'.format(len(snapshots), **ANSI))
         try:
             assert input('    y/[n]: ').lower() == 'y'
         except (KeyboardInterrupt, EOFError, AssertionError):
             raise SystemExit(0)
 
-def log_removal_finished(all_links: int, to_remove: int):
-    if all_links == 0:
+def log_removal_finished(all_snapshots: int, to_remove: int):
+    if to_remove == 0:
         print()
-        print('{red}[X] No matching links found.{reset}'.format(**ANSI))
+        print('{red}[X] No matching snapshots found.{reset}'.format(**ANSI))
     else:
         print()
-        print('{red}[√] Removed {} out of {} links from the archive index.{reset}'.format(
+        print('{red}[√] Removed {} out of {} snapshots from the archive index.{reset}'.format(
             to_remove,
-            all_links,
+            all_snapshots,
             **ANSI,
         ))
-        print('    Index now contains {} links.'.format(all_links - to_remove))
+        print('    Index now contains {} snapshots.'.format(all_snapshots - to_remove))
 
 
 def log_shell_welcome_msg():

archivebox/main.py

@@ -9,7 +9,7 @@ from datetime import date
 
 from typing import Dict, List, Optional, Iterable, IO, Union
 from crontab import CronTab, CronSlices
-from django.db.models import QuerySet
+from django.db.models import QuerySet, Model
 
 from .cli import (
     list_subcommands,
@@ -22,15 +22,15 @@ from .cli import (
 from .parsers import (
     save_text_as_source,
     save_file_as_source,
-    parse_links_memory,
+    parse_snapshots_memory,
 )
 from .index.schema import Link
 from .util import enforce_types                         # type: ignore
 from .system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
 from .index import (
     load_main_index,
-    parse_links_from_source,
-    dedupe_links,
+    parse_snapshots_from_source,
+    filter_new_urls,
     write_main_index,
     snapshot_filter,
     get_indexed_folders,
@@ -44,12 +44,12 @@ from .index import (
     get_corrupted_folders,
     get_unrecognized_folders,
     fix_invalid_folder_locations,
-    write_link_details,
+    write_snapshot_details,
 )
 from .index.json import (
     parse_json_main_index,
-    parse_json_links_details,
-    generate_json_index_from_links,
+    parse_json_snapshot_details,
+    generate_json_index_from_snapshots,
 )
 from .index.sql import (
     get_admins,
@@ -57,10 +57,10 @@ from .index.sql import (
     remove_from_sql_main_index,
 )
 from .index.html import (
-    generate_index_from_links,
+    generate_index_from_snapshots,
 )
-from .index.csv import links_to_csv
-from .extractors import archive_links, archive_link, ignore_methods
+from .index.csv import snapshots_to_csv
+from .extractors import archive_snapshots, archive_snapshot, ignore_methods
 from .config import (
     stderr,
     hint,
@@ -113,7 +113,7 @@ from .logging_util import (
     printable_dependency_version,
 )
 
-from .search import flush_search_index, index_links
+from .search import flush_search_index, index_snapshots
 
 ALLOWED_IN_OUTPUT_DIR = {
     'lost+found',
@@ -334,12 +334,12 @@ def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
     print()
     print('{green}[*] Collecting links from any existing indexes and archive folders...{reset}'.format(**ANSI))
 
-    all_links = Snapshot.objects.none()
-    pending_links: Dict[str, Link] = {}
+    all_snapshots = Snapshot.objects.none()
+    pending_snapshots: Dict[str, Snapshot] = {}
 
     if existing_index:
-        all_links = load_main_index(out_dir=out_dir, warn=False)
-        print('    √ Loaded {} links from existing main index.'.format(all_links.count()))
+        all_snapshots = load_main_index(out_dir=out_dir, warn=False)
+        print('    √ Loaded {} snapshots from existing main index.'.format(all_snapshots.count()))
 
     # Links in data folders that dont match their timestamp
     fixed, cant_fix = fix_invalid_folder_locations(out_dir=out_dir)
@@ -349,46 +349,51 @@ def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None:
         print('    {lightyellow}! Could not fix {} data directory locations due to conflicts with existing folders.{reset}'.format(len(cant_fix), **ANSI))
 
     # Links in JSON index but not in main index
-    orphaned_json_links = {
-        link.url: link
-        for link in parse_json_main_index(out_dir)
-        if not all_links.filter(url=link.url).exists()
+    orphaned_json_snapshots = {
+        snapshot.url: snapshot
+        for snapshot in parse_json_main_index(out_dir)
+        if not all_snapshots.filter(url=snapshot.url).exists()
     }
-    if orphaned_json_links:
-        pending_links.update(orphaned_json_links)
-        print('    {lightyellow}√ Added {} orphaned links from existing JSON index...{reset}'.format(len(orphaned_json_links), **ANSI))
+    if orphaned_json_snapshots:
+        pending_snapshots.update(orphaned_json_snapshots)
+        print('    {lightyellow}√ Added {} orphaned snapshots from deprecated JSON index...{reset}'.format(len(orphaned_json_snapshots), **ANSI))
 
     # Links in data dir indexes but not in main index
-    orphaned_data_dir_links = {
-        link.url: link
-        for link in parse_json_links_details(out_dir)
-        if not all_links.filter(url=link.url).exists()
+    orphaned_data_dir_snapshots = {
+        snapshot.url: snapshot
+        for snapshot in parse_json_snapshot_details(out_dir)
+        if not all_snapshots.filter(url=snapshot.url).exists()
     }
-    if orphaned_data_dir_links:
-        pending_links.update(orphaned_data_dir_links)
-        print('    {lightyellow}√ Added {} orphaned links from existing archive directories.{reset}'.format(len(orphaned_data_dir_links), **ANSI))
+    if orphaned_data_dir_snapshots:
+        pending_snapshots.update(orphaned_data_dir_snapshots)
+        print('    {lightyellow}√ Added {} orphaned snapshots from existing archive directories.{reset}'.format(len(orphaned_data_dir_snapshots), **ANSI))
+
+
+    # TODO: Should we remove orphaned folders from the invalid list? With init they are being imported, but the same links that were
+    # listed as just imported are listed as skipped because they are invalid. At the very least I think we should improve this message,
+    # because it makes this command a little more confusing.
 
     # Links in invalid/duplicate data dirs
     invalid_folders = {
-        folder: link
-        for folder, link in get_invalid_folders(all_links, out_dir=out_dir).items()
+        folder: snapshot
+        for folder, snapshot in get_invalid_folders(all_snapshots, out_dir=out_dir).items()
     }
     if invalid_folders:
-        print('    {lightyellow}! Skipped adding {} invalid link data directories.{reset}'.format(len(invalid_folders), **ANSI))
-        print('        X ' + '\n        X '.join(f'{folder} {link}' for folder, link in invalid_folders.items()))
+        print('    {lightyellow}! Skipped adding {} invalid snapshot data directories.{reset}'.format(len(invalid_folders), **ANSI))
+        print('        X ' + '\n        X '.join(f'{folder} {snapshot}' for folder, snapshot in invalid_folders.items()))
         print()
         print('    {lightred}Hint:{reset} For more information about the link data directories that were skipped, run:'.format(**ANSI))
         print('        archivebox status')
         print('        archivebox list --status=invalid')
 
 
-    write_main_index(list(pending_links.values()), out_dir=out_dir)
+    write_main_index(list(pending_snapshots.values()), out_dir=out_dir)
 
     print('\n{green}------------------------------------------------------------------{reset}'.format(**ANSI))
     if existing_index:
         print('{green}[√] Done. Verified and updated the existing ArchiveBox collection.{reset}'.format(**ANSI))
     else:
-        print('{green}[√] Done. A new ArchiveBox collection was initialized ({} links).{reset}'.format(len(all_links), **ANSI))
+        print('{green}[√] Done. A new ArchiveBox collection was initialized ({} snapshots).{reset}'.format(len(all_snapshots), **ANSI))
     print()
     print('    {lightred}Hint:{reset} To view your archive index, run:'.format(**ANSI))
     print('        archivebox server  # then visit http://127.0.0.1:8000')
@@ -426,11 +431,11 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
     print(f'    Index size: {size} across {num_files} files')
     print()
 
-    links = load_main_index(out_dir=out_dir)
-    num_sql_links = links.count()
-    num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir))
-    print(f'    > SQL Main Index: {num_sql_links} links'.ljust(36), f'(found in {SQL_INDEX_FILENAME})')
-    print(f'    > JSON Link Details: {num_link_details} links'.ljust(36), f'(found in {ARCHIVE_DIR_NAME}/*/index.json)')
+    snapshots = load_main_index(out_dir=out_dir)
+    num_sql_snapshots = snapshots.count()
+    num_snapshot_details = sum(1 for snapshot in parse_json_snapshot_details(out_dir=out_dir))
+    print(f'    > SQL Main Index: {num_sql_snapshots} snapshots'.ljust(36), f'(found in {SQL_INDEX_FILENAME})')
+    print(f'    > JSON Link Details: {num_snapshot_details} snapshots'.ljust(36), f'(found in {ARCHIVE_DIR_NAME}/*/index.json)')
     print()
     print('{green}[*] Scanning archive data directories...{reset}'.format(**ANSI))
     print(ANSI['lightyellow'], f'   {ARCHIVE_DIR}/*', ANSI['reset'])
@@ -438,23 +443,23 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
     size = printable_filesize(num_bytes)
     print(f'    Size: {size} across {num_files} files in {num_dirs} directories')
     print(ANSI['black'])
-    num_indexed = len(get_indexed_folders(links, out_dir=out_dir))
-    num_archived = len(get_archived_folders(links, out_dir=out_dir))
-    num_unarchived = len(get_unarchived_folders(links, out_dir=out_dir))
+    num_indexed = len(get_indexed_folders(snapshots, out_dir=out_dir))
+    num_archived = len(get_archived_folders(snapshots, out_dir=out_dir))
+    num_unarchived = len(get_unarchived_folders(snapshots, out_dir=out_dir))
     print(f'    > indexed: {num_indexed}'.ljust(36), f'({get_indexed_folders.__doc__})')
     print(f'      > archived: {num_archived}'.ljust(36), f'({get_archived_folders.__doc__})')
     print(f'      > unarchived: {num_unarchived}'.ljust(36), f'({get_unarchived_folders.__doc__})')
     
-    num_present = len(get_present_folders(links, out_dir=out_dir))
-    num_valid = len(get_valid_folders(links, out_dir=out_dir))
+    num_present = len(get_present_folders(snapshots, out_dir=out_dir))
+    num_valid = len(get_valid_folders(snapshots, out_dir=out_dir))
     print()
     print(f'    > present: {num_present}'.ljust(36), f'({get_present_folders.__doc__})')
     print(f'      > valid: {num_valid}'.ljust(36), f'({get_valid_folders.__doc__})')
     
-    duplicate = get_duplicate_folders(links, out_dir=out_dir)
-    orphaned = get_orphaned_folders(links, out_dir=out_dir)
-    corrupted = get_corrupted_folders(links, out_dir=out_dir)
-    unrecognized = get_unrecognized_folders(links, out_dir=out_dir)
+    duplicate = get_duplicate_folders(snapshots, out_dir=out_dir)
+    orphaned = get_orphaned_folders(snapshots, out_dir=out_dir)
+    corrupted = get_corrupted_folders(snapshots, out_dir=out_dir)
+    unrecognized = get_unrecognized_folders(snapshots, out_dir=out_dir)
     num_invalid = len({**duplicate, **orphaned, **corrupted, **unrecognized})
     print(f'      > invalid: {num_invalid}'.ljust(36), f'({get_invalid_folders.__doc__})')
     print(f'        > duplicate: {len(duplicate)}'.ljust(36), f'({get_duplicate_folders.__doc__})')
@@ -465,7 +470,7 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
     print(ANSI['reset'])
 
     if num_indexed:
-        print('    {lightred}Hint:{reset} You can list link data directories by status like so:'.format(**ANSI))
+        print('    {lightred}Hint:{reset} You can list snapshot data directories by status like so:'.format(**ANSI))
         print('        archivebox list --status=<status>  (e.g. indexed, corrupted, archived, etc.)')
 
     if orphaned:
@@ -494,7 +499,7 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
         print('        archivebox manage createsuperuser')
 
     print()
-    for snapshot in links.order_by('-updated')[:10]:
+    for snapshot in snapshots.order_by('-updated')[:10]:
         if not snapshot.updated:
             continue
         print(
@@ -515,8 +520,8 @@ def oneshot(url: str, extractors: str="", out_dir: Path=OUTPUT_DIR):
     Create a single URL archive folder with an index.json and index.html, and all the archive method outputs.
     You can run this to archive single pages without needing to create a whole collection with archivebox init.
     """
-    oneshot_link, _ = parse_links_memory([url])
-    if len(oneshot_link) > 1:
+    oneshot_snapshots, _ = parse_snapshots_memory([url])
+    if len(oneshot_snapshots) > 1:
         stderr(
                 '[X] You should pass a single url to the oneshot command',
                 color='red'
@@ -524,8 +529,10 @@ def oneshot(url: str, extractors: str="", out_dir: Path=OUTPUT_DIR):
         raise SystemExit(2)
 
     methods = extractors.split(",") if extractors else ignore_methods(['title'])
-    archive_link(oneshot_link[0], out_dir=out_dir, methods=methods)
-    return oneshot_link
+    snapshot = oneshot_snapshots[0]
+    snapshot.save() # Oneshot uses an in-memory database, so this is safe
+    archive_snapshot(snapshot, out_dir=out_dir, methods=methods)
+    return snapshot
 
 @enforce_types
 def add(urls: Union[str, List[str]],
@@ -537,6 +544,7 @@ def add(urls: Union[str, List[str]],
         extractors: str="",
         out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Add a new URL or list of URLs to your archive"""
+    from core.models import Snapshot
 
     assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)'
 
@@ -548,8 +556,8 @@ def add(urls: Union[str, List[str]],
     # Load list of links from the existing index
     check_data_folder(out_dir=out_dir)
     check_dependencies()
-    new_links: List[Link] = []
-    all_links = load_main_index(out_dir=out_dir)
+    new_snapshots: List[Snapshot] = []
+    all_snapshots = load_main_index(out_dir=out_dir)
 
     log_importing_started(urls=urls, depth=depth, index_only=index_only)
     if isinstance(urls, str):
@@ -559,24 +567,25 @@ def add(urls: Union[str, List[str]],
         # save verbatim args to sources
         write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir)
     
-    new_links += parse_links_from_source(write_ahead_log, root_url=None)
+    new_snapshots += parse_snapshots_from_source(write_ahead_log, root_url=None)
 
     # If we're going one level deeper, download each link and look for more links
-    new_links_depth = []
-    if new_links and depth == 1:
-        log_crawl_started(new_links)
-        for new_link in new_links:
-            downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir)
-            new_links_depth += parse_links_from_source(downloaded_file, root_url=new_link.url)
+    new_snapshots_depth = []
+    if new_snapshots and depth == 1:
+        log_crawl_started(new_snapshots)
+        for new_snapshot in new_snapshots:
+            # TODO: Check if we need to add domain to the Snapshot model
+            downloaded_file = save_file_as_source(new_snapshot.url, filename=f'{new_snapshot.timestamp}-crawl-{new_snapshot.domain}.txt', out_dir=out_dir)
+            new_snapshots_depth += parse_snapshots_from_source(downloaded_file, root_url=new_snapshot.url)
 
-    imported_links = list({link.url: link for link in (new_links + new_links_depth)}.values())
-    new_links = dedupe_links(all_links, imported_links)
+    imported_snapshots = [Snapshot(url=snapshot.url) for snapshot in new_snapshots + new_snapshots_depth]
+    new_snapshots = filter_new_urls(all_snapshots, imported_snapshots)
 
-    write_main_index(links=new_links, out_dir=out_dir)
-    all_links = load_main_index(out_dir=out_dir)
+    write_main_index(snapshots=new_snapshots, out_dir=out_dir)
+    all_snapshots = load_main_index(out_dir=out_dir)
 
     if index_only:
-        return all_links
+        return all_snapshots
 
     # Run the archive methods for each link
     archive_kwargs = {
@@ -585,13 +594,14 @@ def add(urls: Union[str, List[str]],
     if extractors:
         archive_kwargs["methods"] = extractors
     if update_all:
-        archive_links(all_links, overwrite=overwrite, **archive_kwargs)
+        archive_snapshots(all_snapshots, overwrite=overwrite, **archive_kwargs)
     elif overwrite:
-        archive_links(imported_links, overwrite=True, **archive_kwargs)
-    elif new_links:
-        archive_links(new_links, overwrite=False, **archive_kwargs)
+        imported_snapshots = Snapshot.objects.filter(url__in=[imported_snapshot.url for imported_snapshot in imported_snapshots])
+        archive_snapshots(imported_snapshots, overwrite=True, **archive_kwargs)
+    elif new_snapshots:
+        archive_snapshots(new_snapshots, overwrite=False, **archive_kwargs)
 
-    return all_links
+    return all_snapshots
 
 @enforce_types
 def remove(filter_str: Optional[str]=None,
@@ -641,7 +651,7 @@ def remove(filter_str: Optional[str]=None,
     log_list_started(filter_patterns, filter_type)
     timer = TimedProgress(360, prefix='      ')
     try:
-        snapshots = list_links(**list_kwargs)
+        snapshots = list_snapshots(**list_kwargs)
     finally:
         timer.end()
 
@@ -651,24 +661,23 @@ def remove(filter_str: Optional[str]=None,
         raise SystemExit(1)
 
 
-    log_links = [link.as_link() for link in snapshots]
-    log_list_finished(log_links)
-    log_removal_started(log_links, yes=yes, delete=delete)
+    log_list_finished(snapshots)
+    log_removal_started(snapshots, yes=yes, delete=delete)
 
     timer = TimedProgress(360, prefix='      ')
     try:
         for snapshot in snapshots:
             if delete:
-                shutil.rmtree(snapshot.as_link().link_dir, ignore_errors=True)
+                shutil.rmtree(snapshot.snapshot_dir, ignore_errors=True)
     finally:
         timer.end()
 
     to_remove = snapshots.count()
+    all_snapshots = load_main_index(out_dir=out_dir).count()
 
     flush_search_index(snapshots=snapshots)
     remove_from_sql_main_index(snapshots=snapshots, out_dir=out_dir)
-    all_snapshots = load_main_index(out_dir=out_dir)
-    log_removal_finished(all_snapshots.count(), to_remove)
+    log_removal_finished(all_snapshots, to_remove)
     
     return all_snapshots
 
@@ -686,15 +695,16 @@ def update(resume: Optional[float]=None,
            extractors: str="",
            out_dir: Path=OUTPUT_DIR) -> List[Link]:
     """Import any new links from subscriptions and retry any previously failed/skipped links"""
+    from core.models import Snapshot
 
     check_data_folder(out_dir=out_dir)
     check_dependencies()
-    new_links: List[Link] = [] # TODO: Remove input argument: only_new
+    new_snapshots: List[Snapshot] = [] # TODO: Remove input argument: only_new
 
     extractors = extractors.split(",") if extractors else []
 
     # Step 1: Filter for selected_links
-    matching_snapshots = list_links(
+    matching_snapshots = list_snapshots(
         filter_patterns=filter_patterns,
         filter_type=filter_type,
         before=before,
@@ -702,29 +712,29 @@ def update(resume: Optional[float]=None,
     )
 
     matching_folders = list_folders(
-        links=matching_snapshots,
+        snapshots=matching_snapshots,
         status=status,
         out_dir=out_dir,
     )
-    all_links = [link for link in matching_folders.values() if link]
+    all_snapshots = [snapshot for snapshot in matching_folders.values()]
 
     if index_only:
-        for link in all_links:
-            write_link_details(link, out_dir=out_dir, skip_sql_index=True)
-        index_links(all_links, out_dir=out_dir)
-        return all_links
+        for snapshot in all_snapshots:
+            write_snapshot_details(snapshot, out_dir=out_dir, skip_sql_index=True)
+        index_snapshots(all_snapshots, out_dir=out_dir)
+        return all_snapshots
         
     # Step 2: Run the archive methods for each link
-    to_archive = new_links if only_new else all_links
+    to_archive = new_snapshots if only_new else all_snapshots
     if resume:
         to_archive = [
-            link for link in to_archive
-            if link.timestamp >= str(resume)
+            snapshot for snapshot in to_archive
+            if snapshot.timestamp >= str(resume)
         ]
         if not to_archive:
             stderr('')
             stderr(f'[√] Nothing found to resume after {resume}', color='green')
-            return all_links
+            return all_snapshots
 
     archive_kwargs = {
         "out_dir": out_dir,
@@ -732,11 +742,11 @@ def update(resume: Optional[float]=None,
     if extractors:
         archive_kwargs["methods"] = extractors
 
-    archive_links(to_archive, overwrite=overwrite, **archive_kwargs)
+    archive_snapshots(to_archive, overwrite=overwrite, **archive_kwargs)
 
     # Step 4: Re-write links index with updated titles, icons, and resources
-    all_links = load_main_index(out_dir=out_dir)
-    return all_links
+    all_snapshots = load_main_index(out_dir=out_dir)
+    return all_snapshots
 
 @enforce_types
 def list_all(filter_patterns_str: Optional[str]=None,
@@ -765,7 +775,7 @@ def list_all(filter_patterns_str: Optional[str]=None,
     elif filter_patterns_str:
         filter_patterns = filter_patterns_str.split('\n')
 
-    snapshots = list_links(
+    snapshots = list_snapshots(
         filter_patterns=filter_patterns,
         filter_type=filter_type,
         before=before,
@@ -776,17 +786,17 @@ def list_all(filter_patterns_str: Optional[str]=None,
         snapshots = snapshots.order_by(sort)
 
     folders = list_folders(
-        links=snapshots,
+        snapshots=snapshots,
         status=status,
         out_dir=out_dir,
     )
 
     if json: 
-        output = generate_json_index_from_links(folders.values(), with_headers)
+        output = generate_json_index_from_snapshots(folders.values(), with_headers)
     elif html:
-        output = generate_index_from_links(folders.values(), with_headers)
+        output = generate_index_from_snapshots(folders.values(), with_headers)
     elif csv:
-        output = links_to_csv(folders.values(), cols=csv.split(','), header=with_headers)
+        output = snapshots_to_csv(folders.values(), cols=csv.split(','), header=with_headers)
     else:
         output = printable_folders(folders, with_headers=with_headers)
     print(output)
@@ -794,7 +804,7 @@ def list_all(filter_patterns_str: Optional[str]=None,
 
 
 @enforce_types
-def list_links(snapshots: Optional[QuerySet]=None,
+def list_snapshots(snapshots: Optional[QuerySet]=None,
                filter_patterns: Optional[List[str]]=None,
                filter_type: str='exact',
                after: Optional[float]=None,
@@ -817,9 +827,9 @@ def list_links(snapshots: Optional[QuerySet]=None,
     return all_snapshots
 
 @enforce_types
-def list_folders(links: List[Link],
+def list_folders(snapshots: List[Model],
                  status: str,
-                 out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Link]]:
+                 out_dir: Path=OUTPUT_DIR) -> Dict[str, Optional[Model]]:
     
     check_data_folder(out_dir=out_dir)
 
@@ -837,7 +847,7 @@ def list_folders(links: List[Link],
     }
 
     try:
-        return STATUS_FUNCTIONS[status](links, out_dir=out_dir)
+        return STATUS_FUNCTIONS[status](snapshots, out_dir=out_dir)
     except KeyError:
         raise ValueError('Status not recognized.')
 

archivebox/parsers/__init__.py

@@ -14,6 +14,8 @@ from typing import IO, Tuple, List, Optional
 from datetime import datetime
 from pathlib import Path 
 
+from django.db.models import Model
+
 from ..system import atomic_write
 from ..config import (
     ANSI,
@@ -64,7 +66,7 @@ PARSERS = (
 
 
 @enforce_types
-def parse_links_memory(urls: List[str], root_url: Optional[str]=None):
+def parse_snapshots_memory(urls: List[str], root_url: Optional[str]=None):
     """
     parse a list of URLS without touching the filesystem
     """
@@ -75,16 +77,16 @@ def parse_links_memory(urls: List[str], root_url: Optional[str]=None):
     file = StringIO()
     file.writelines(urls)
     file.name = "io_string"
-    links, parser = run_parser_functions(file, timer, root_url=root_url)
+    snapshots, parser = run_parser_functions(file, timer, root_url=root_url)
     timer.end()
 
     if parser is None:
         return [], 'Failed to parse'
-    return links, parser
+    return snapshots, parser
     
 
 @enforce_types
-def parse_links(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Link], str]:
+def parse_snapshots(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Model], str]:
     """parse a list of URLs with their metadata from an 
        RSS feed, bookmarks export, or text file
     """
@@ -93,27 +95,27 @@ def parse_links(source_file: str, root_url: Optional[str]=None) -> Tuple[List[Li
 
     timer = TimedProgress(TIMEOUT * 4)
     with open(source_file, 'r', encoding='utf-8') as file:
-        links, parser = run_parser_functions(file, timer, root_url=root_url)
+        snapshots, parser = run_parser_functions(file, timer, root_url=root_url)
 
     timer.end()
     if parser is None:
         return [], 'Failed to parse'
-    return links, parser
+    return snapshots, parser
 
 
-def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None) -> Tuple[List[Link], Optional[str]]:
-    most_links: List[Link] = []
+def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None) -> Tuple[List[Model], Optional[str]]:
+    most_snapshots: List[Model] = []
     best_parser_name = None
 
     for parser_name, parser_func in PARSERS:
         try:
-            parsed_links = list(parser_func(to_parse, root_url=root_url))
-            if not parsed_links:
+            parsed_snapshots = list(parser_func(to_parse, root_url=root_url))
+            if not parsed_snapshots:
                 raise Exception('no links found')
 
             # print(f'[√] Parser {parser_name} succeeded: {len(parsed_links)} links parsed')
-            if len(parsed_links) > len(most_links):
-                most_links = parsed_links
+            if len(parsed_snapshots) > len(most_snapshots):
+                most_snapshots = parsed_snapshots
                 best_parser_name = parser_name
                 
         except Exception as err:                                                # noqa
@@ -125,7 +127,7 @@ def run_parser_functions(to_parse: IO[str], timer, root_url: Optional[str]=None)
             # raise
             pass
     timer.end()
-    return most_links, best_parser_name
+    return most_snapshots, best_parser_name
 
 
 @enforce_types

archivebox/parsers/generic_html.py

@@ -6,7 +6,8 @@ import re
 from typing import IO, Iterable, Optional
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -29,8 +30,9 @@ class HrefParser(HTMLParser):
 
 
 @enforce_types
-def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None, **_kwargs) -> Iterable[Link]:
+def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None, **_kwargs) -> Iterable[Model]:
     """Parse Generic HTML for href tags and use only the url (support for title coming later)"""
+    from core.models import Snapshot
 
     html_file.seek(0)
     for line in html_file:
@@ -44,10 +46,10 @@ def parse_generic_html_export(html_file: IO[str], root_url: Optional[str]=None,
                 url = urljoin(root_url, url)
             
             for archivable_url in re.findall(URL_REGEX, url):
-                yield Link(
+                yield Snapshot(
                     url=htmldecode(archivable_url),
                     timestamp=str(datetime.now().timestamp()),
                     title=None,
-                    tags=None,
-                    sources=[html_file.name],
+                    #tags=None,
+                    #sources=[html_file.name],
                 )

archivebox/parsers/generic_json.py

@@ -5,7 +5,8 @@ import json
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -13,8 +14,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)"""
+    from core.models import Snapshot
 
     json_file.seek(0)
     links = json.load(json_file)
@@ -56,10 +58,10 @@ def parse_generic_json_export(json_file: IO[str], **_kwargs) -> Iterable[Link]:
             elif link.get('name'):
                 title = link['name'].strip()
 
-            yield Link(
+            yield Snapshot(
                 url=htmldecode(url),
                 timestamp=ts_str,
                 title=htmldecode(title) or None,
-                tags=htmldecode(link.get('tags')) or '',
-                sources=[json_file.name],
+                #tags=htmldecode(link.get('tags')) or '',
+                #sources=[json_file.name],
             )

archivebox/parsers/generic_rss.py

@@ -4,7 +4,8 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -12,8 +13,9 @@ from ..util import (
 )
 
 @enforce_types
-def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse RSS XML-format files into links"""
+    from core.models import Snapshot
 
     rss_file.seek(0)
     items = rss_file.read().split('<item>')
@@ -40,10 +42,10 @@ def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z")
         title = str_between(get_row('title'), '<![CDATA[', ']]').strip()
 
-        yield Link(
+        yield Snapshot(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            tags=None,
-            sources=[rss_file.name],
+            #tags=None,
+            #sources=[rss_file.name],
         )

archivebox/parsers/generic_txt.py

@@ -7,7 +7,8 @@ from typing import IO, Iterable
 from datetime import datetime
 from pathlib import Path
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -16,8 +17,10 @@ from ..util import (
 
 
 @enforce_types
-def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse raw links from each line in a text file"""
+    # TODO: Check if we should add sources list to the database
+    from core.models import Snapshot
 
     text_file.seek(0)
     for line in text_file.readlines():
@@ -27,12 +30,12 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
         # if the line is a local file path that resolves, then we can archive it
         try:
             if Path(line).exists():
-                yield Link(
+                yield Snapshot(
                     url=line,
                     timestamp=str(datetime.now().timestamp()),
                     title=None,
-                    tags=None,
-                    sources=[text_file.name],
+                    #tags=None,
+                    #sources=[text_file.name],
                 )
         except (OSError, PermissionError):
             # nvm, not a valid path...
@@ -40,22 +43,22 @@ def parse_generic_txt_export(text_file: IO[str], **_kwargs) -> Iterable[Link]:
 
         # otherwise look for anything that looks like a URL in the line
         for url in re.findall(URL_REGEX, line):
-            yield Link(
+            yield Snapshot(
                 url=htmldecode(url),
                 timestamp=str(datetime.now().timestamp()),
                 title=None,
-                tags=None,
-                sources=[text_file.name],
+                #tags=None,
+                #sources=[text_file.name],
             )
 
             # look inside the URL for any sub-urls, e.g. for archive.org links
             # https://web.archive.org/web/20200531203453/https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             # -> https://www.reddit.com/r/socialism/comments/gu24ke/nypd_officers_claim_they_are_protecting_the_rule/fsfq0sw/
             for sub_url in re.findall(URL_REGEX, line[1:]):
-                yield Link(
+                yield Snapshot(
                     url=htmldecode(sub_url),
                     timestamp=str(datetime.now().timestamp()),
                     title=None,
-                    tags=None,
-                    sources=[text_file.name],
+                    #tags=None,
+                    #sources=[text_file.name],
                 )

archivebox/parsers/medium_rss.py

@@ -4,9 +4,10 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
+from django.db.models import Model
+
 from xml.etree import ElementTree
 
-from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,8 +15,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse Medium RSS feed files into links"""
+    from core.models import Snapshot
 
     rss_file.seek(0)
     root = ElementTree.parse(rss_file).getroot()
@@ -26,10 +28,10 @@ def parse_medium_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         ts_str = item.find("pubDate").text                              # type: ignore
         time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z")    # type: ignore
         
-        yield Link(
+        yield Snapshot(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            tags=None,
-            sources=[rss_file.name],
+            #tags=None,
+            #sources=[rss_file.name],
         )

archivebox/parsers/netscape_html.py

@@ -6,7 +6,8 @@ import re
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,8 +15,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse netscape-format bookmarks export files (produced by all browsers)"""
+    from core.models import Snapshot
 
     html_file.seek(0)
     pattern = re.compile("<a href=\"(.+?)\" add_date=\"(\\d+)\"[^>]*>(.+)</a>", re.UNICODE | re.IGNORECASE)
@@ -29,11 +31,11 @@ def parse_netscape_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
             time = datetime.fromtimestamp(float(match.group(2)))
             title = match.group(3).strip()
 
-            yield Link(
+            yield Snapshot(
                 url=htmldecode(url),
                 timestamp=str(time.timestamp()),
                 title=htmldecode(title) or None,
-                tags=None,
-                sources=[html_file.name],
+                #tags=None,
+                #sources=[html_file.name],
             )
 

archivebox/parsers/pinboard_rss.py

@@ -4,9 +4,10 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
+from django.db.models import Model
+
 from xml.etree import ElementTree
 
-from ..index.schema import Link
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,8 +15,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse Pinboard RSS feed files into links"""
+    from core.models import Snapshot
 
     rss_file.seek(0)
     root = ElementTree.parse(rss_file).getroot()
@@ -38,10 +40,10 @@ def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         else:
             time = datetime.now()
 
-        yield Link(
+        yield Snapshot(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            tags=htmldecode(tags) or None,
-            sources=[rss_file.name],
+            #tags=htmldecode(tags) or None,
+            #sources=[rss_file.name],
         )

archivebox/parsers/pocket_api.py

@@ -7,9 +7,11 @@ from typing import IO, Iterable, Optional
 from configparser import ConfigParser
 
 from pathlib import Path
+
+from django.db.models import Model
+
 from ..vendor.pocket import Pocket
 
-from ..index.schema import Link
 from ..util import enforce_types
 from ..system import atomic_write
 from ..config import (
@@ -46,19 +48,21 @@ def get_pocket_articles(api: Pocket, since=None, page=0):
         api.last_since = body['since']
 
 
-def link_from_article(article: dict, sources: list):
+def snapshot_from_article(article: dict, sources: list):
+    from core.models import Snapshot
+
     url: str = article['resolved_url'] or article['given_url']
     broken_protocol = _BROKEN_PROTOCOL_RE.match(url)
     if broken_protocol:
         url = url.replace(f'{broken_protocol.group(1)}:/', f'{broken_protocol.group(1)}://')
     title = article['resolved_title'] or article['given_title'] or url
 
-    return Link(
+    return Snapshot(
         url=url,
         timestamp=article['time_read'],
         title=title,
-        tags=article.get('tags'),
-        sources=sources
+        #tags=article.get('tags'),
+        #sources=sources
     )
 
 
@@ -95,7 +99,7 @@ def should_parse_as_pocket_api(text: str) -> bool:
 
 
 @enforce_types
-def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse bookmarks from the Pocket API"""
 
     input_buffer.seek(0)
@@ -108,6 +112,6 @@ def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
             api.last_since = None
     
             for article in get_pocket_articles(api, since=read_since(username)):
-                yield link_from_article(article, sources=[line])
+                yield snapshot_from_article(article, sources=[line])
     
             write_since(username, api.last_since)

archivebox/parsers/pocket_html.py

@@ -6,7 +6,8 @@ import re
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -14,8 +15,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse Pocket-format bookmarks export files (produced by getpocket.com/export/)"""
+    from core.models import Snapshot
 
     html_file.seek(0)
     pattern = re.compile("^\\s*<li><a href=\"(.+)\" time_added=\"(\\d+)\" tags=\"(.*)\">(.+)</a></li>", re.UNICODE)
@@ -29,10 +31,10 @@ def parse_pocket_html_export(html_file: IO[str], **_kwargs) -> Iterable[Link]:
             tags = match.group(3)
             title = match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '')
             
-            yield Link(
+            yield Snapshot(
                 url=htmldecode(url),
                 timestamp=str(time.timestamp()),
                 title=htmldecode(title) or None,
-                tags=tags or '',
-                sources=[html_file.name],
+                #tags=tags or '',
+                #sources=[html_file.name],
             )

archivebox/parsers/shaarli_rss.py

@@ -4,7 +4,8 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -13,8 +14,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse Shaarli-specific RSS XML-format files into links"""
+    from core.models import Snapshot
 
     rss_file.seek(0)
     entries = rss_file.read().split('<entry>')[1:]
@@ -41,10 +43,10 @@ def parse_shaarli_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         ts_str = str_between(get_row('published'), '<published>', '</published>')
         time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
 
-        yield Link(
+        yield Snapshot(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            tags=None,
-            sources=[rss_file.name],
+            #tags=None,
+            #sources=[rss_file.name],
         )

archivebox/parsers/wallabag_atom.py

@@ -4,7 +4,8 @@ __package__ = 'archivebox.parsers'
 from typing import IO, Iterable
 from datetime import datetime
 
-from ..index.schema import Link
+from django.db.models import Model
+
 from ..util import (
     htmldecode,
     enforce_types,
@@ -13,8 +14,9 @@ from ..util import (
 
 
 @enforce_types
-def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
+def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Model]:
     """Parse Wallabag Atom files into links"""
+    from core.models import Snapshot
 
     rss_file.seek(0)
     entries = rss_file.read().split('<entry>')[1:]
@@ -48,10 +50,10 @@ def parse_wallabag_atom_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
         except Exception:
             tags = None
 
-        yield Link(
+        yield Snapshot(
             url=htmldecode(url),
             timestamp=str(time.timestamp()),
             title=htmldecode(title) or None,
-            tags=tags or '',
-            sources=[rss_file.name],
+            #tags=tags or '',
+            #sources=[rss_file.name],
         )

archivebox/search/__init__.py

@@ -2,7 +2,7 @@ from typing import List, Union
 from pathlib import Path
 from importlib import import_module
 
-from django.db.models import QuerySet
+from django.db.models import QuerySet, Model
 
 from archivebox.index.schema import Link
 from archivebox.util import enforce_types
@@ -28,24 +28,22 @@ def import_backend():
     return backend
 
 @enforce_types
-def write_search_index(link: Link, texts: Union[List[str], None]=None, out_dir: Path=OUTPUT_DIR, skip_text_index: bool=False) -> None:
+def write_search_index(snapshot: Model, texts: Union[List[str], None]=None, out_dir: Path=OUTPUT_DIR, skip_text_index: bool=False) -> None:
     if not indexing_enabled():
         return
 
     if not skip_text_index and texts:
         from core.models import Snapshot
 
-        snap = Snapshot.objects.filter(url=link.url).first()
         backend = import_backend()
-        if snap:
-            try:
-                backend.index(snapshot_id=str(snap.id), texts=texts)
-            except Exception as err:
-                stderr()
-                stderr(
-                    f'[X] The search backend threw an exception={err}:',
+        try:
+            backend.index(snapshot_id=str(snapshot.id), texts=texts)
+        except Exception as err:
+            stderr()
+            stderr(
+                f'[X] The search backend threw an exception={err}:',
                 color='red',
-                )
+            )
 
 @enforce_types
 def query_search_index(query: str, out_dir: Path=OUTPUT_DIR) -> QuerySet:
@@ -85,17 +83,17 @@ def flush_search_index(snapshots: QuerySet):
         )
 
 @enforce_types
-def index_links(links: Union[List[Link],None], out_dir: Path=OUTPUT_DIR):
-    if not links:
+def index_snapshots(snapshots: Union[List[Model],None], out_dir: Path=OUTPUT_DIR):
+    if not snapshots:
         return
 
     from core.models import Snapshot, ArchiveResult
 
-    for link in links:
-        snap = Snapshot.objects.filter(url=link.url).first()
+    for snapshot in snapshots:
+        snap = Snapshot.objects.filter(url=snapshot.url).first()
         if snap: 
             results = ArchiveResult.objects.indexable().filter(snapshot=snap)
-            log_index_started(link.url)
+            log_index_started(snapshot.url)
             try:
                 texts = get_indexable_content(results)
             except Exception as err:
@@ -105,4 +103,4 @@ def index_links(links: Union[List[Link],None], out_dir: Path=OUTPUT_DIR):
                     color='red',
                     ) 
             else:
-                write_search_index(link, texts, out_dir=out_dir)
+                write_search_index(snapshot, texts, out_dir=out_dir)

archivebox/templates/core/index_row.html

@@ -1,37 +1,59 @@
 {% load static %}
 
 <tr>
-    <td title="{{link.timestamp}}"> {% if link.bookmarked_date  %} {{ link.bookmarked_date }} {% else %} {{ link.added }} {% endif %} </td>
-    <td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}">
-        {% if link.is_archived %}
-            <a href="archive/{{link.timestamp}}/index.html"><img src="archive/{{link.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
+{% comment %}
+    <!-- from upstream, may need to merge these two -->
+    <td title="{{snapshot.timestamp}}"> {% if snapshot.bookmarked_date  %} {{ snapshot.bookmarked_date }} {% else %} {{ snapshot.added }} {% endif %} </td>
+    <td class="title-col">
+        {% if snapshot.is_archived %}
+            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="archive/{{snapshot.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async"></a>
         {% else %}
-            <a href="archive/{{link.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
+            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async"></a>
+        {% endif %}
+            <a href="archive/{{snapshot.timestamp}}/{{snapshot.canonical_outputs.wget_path}}" title="{{snapshot.title}}">
+                <span data-title-for="{{snapshot.url}}" data-archived="{{snapshot.is_archived}}">{{snapshot.title|default:'Loading...'}}</span>
+                <small style="float:right">{% if snapshot.tags_str != None %} {{snapshot.tags_str|default:''}} {% else %} untagged {% endif %}</small>
+            </a>
+    </td>
+    <td>
+        <a href="archive/{{snapshot.timestamp}}/index.html">📄 
+            <span data-number-for="{{snapshot.url}}" title="Fetching any missing files...">{% if snapshot.icons  %} {{snapshot.icons}} {% else %} {{ snapshot.num_outputs}} {% endif %}<img src="{% static 'spinner.gif' %}" class="files-spinner" decoding="async"/></span>
+        </a>
+    </td>
+   <td style="text-align:left"><a href="{{snapshot.url}}">{{snapshot.url}}</a></td>
+{% endcomment %}
+
+    <td title="{{snapshot.timestamp}}"> {% if snapshot.bookmarked_date  %} {{ snapshot.bookmarked_date }} {% else %} {{ snapshot.added }} {% endif %} </td>
+    <td class="title-col" style="opacity: {% if snapshot.title %}1{% else %}0.3{% endif %}">
+        {% if snapshot.is_archived %}
+            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="archive/{{snapshot.timestamp}}/favicon.ico" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async"></a>
+        {% else %}
+            <a href="archive/{{snapshot.timestamp}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="snapshot-favicon" decoding="async" style="height: 15px"></a>
         {% endif %}
 
-        <a href="archive/{{link.timestamp}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
-            <span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">{{link.title|default:'Loading...'}}</span>
-            {% if link.tags_str %}
+        <a href="archive/{{snapshot.timestamp}}/index.html" title="{{snapshot.title|default:'Not yet archived...'}}">
+            <span data-title-for="{{snapshot.url}}" data-archived="{{snapshot.is_archived}}">{{snapshot.title|default:'Loading...'}}</span>
+            {% if snapshot.tags_str %}
                 <span class="tags" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
-                    {% if link.tags_str != None %}
-                        {{link.tags_str|default:''}}
+                    {% if snapshot.tags_str != None %}
+                        {{snapshot.tags_str|default:''}}
                     {% else %}
-                        {{ link.tags|default:'' }}
+                        {{ snapshot.tags|default:'' }}
                     {% endif %}
                 </span>
             {% endif %}
         </a>
     </td>
     <td>
-        <span data-number-for="{{link.url}}" title="Fetching any missing files...">
-            {% if link.icons  %}
-                {{link.icons}} <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
+        <span data-number-for="{{snapshot.url}}" title="Fetching any missing files...">
+            {% if snapshot.icons  %}
+                {{snapshot.icons}} <small style="float:right; opacity: 0.5">{{snapshot.num_outputs}}</small>
             {% else %}
-                <a href="archive/{{link.timestamp}}/index.html">📄 
-                    {{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
+                <a href="archive/{{snapshot.timestamp}}/index.html">📄 
+                    {{snapshot.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
                 </a>
             {% endif %}
         </span>
     </td>
-   <td style="text-align:left"><a href="{{link.url}}">{{link.url}}</a></td>
+   <td style="text-align:left"><a href="{{snapshot.url}}">{{snapshot.url}}</a></td>
 </tr>

archivebox/templates/core/minimal_index.html

@@ -1,24 +1,30 @@
 <!DOCTYPE html>
 <html lang="en">
-    <head>
-        <title>Archived Sites</title>
-        <meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
-    </head>
-    <body data-status="{{status}}">
-        <table id="table-bookmarks">
-            <thead>
-                <tr class="thead-tr">
-                    <th style="width: 100px;">Bookmarked</th>
-                    <th style="width: 26vw;">Saved Link ({{num_links}})</th>
-                    <th style="width: 50px">Files</th>
-                    <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for link in links %}
-                    {% include "index_row.html" with link=link %} 
-                {% endfor %}
-            </tbody>
-        </table>
-    </body>
+  <head>
+    <title>Archived Sites</title>
+    <meta
+      charset="utf-8"
+      name="viewport"
+      content="width=device-width, initial-scale=1"
+    />
+  </head>
+  <body data-status="{{status}}">
+    <table id="table-bookmarks">
+      <thead>
+        <tr class="thead-tr">
+          <th style="width: 100px">Bookmarked</th>
+          <th style="width: 26vw">Saved Link ({{num_links}})</th>
+          <th style="width: 50px">Files</th>
+          <th style="width: 16vw; whitespace: nowrap; overflow-x: hidden">
+            Original URL
+          </th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for snapshot in snapshots %}
+          {% include "index_row.html" with snapshot=snapshot %}
+        {% endfor %}
+      </tbody>
+    </table>
+  </body>
 </html>

archivebox/templates/core/snapshot.html

@@ -263,7 +263,7 @@
                 }
             }
         </style>
-        <link rel="stylesheet" href="../../static/bootstrap.min.css">
+        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
     </head>
     <body>
         <header>

archivebox/templates/core/static_index.html

@@ -227,14 +227,14 @@
             <thead>
                 <tr>
                     <th style="width: 100px;">Bookmarked</th>
-                    <th style="width: 26vw;">Saved Link ({{num_links}})</th>
+                    <th style="width: 26vw;">Saved Link ({{num_snapshots}})</th>
                     <th style="width: 50px">Files</th>
                     <th style="width: 16vw;whitespace:nowrap;overflow-x:hidden;">Original URL</th>
                 </tr>
             </thead>
             <tbody>
-                {% for link in links %}
-                   {% include 'index_row.html' with link=link %}
+                {% for snapshot in snapshots %}
+                   {% include 'index_row.html' with snapshot=snapshot %}
                 {% endfor %}
             </tbody>
         </table>

package-lock.json

@@ -1,6 +1,6 @@
 {
 	"name": "archivebox",
-	"version": "0.4.21",
+	"version": "0.5.1",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {

tests/test_add.py

@@ -35,6 +35,7 @@ def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process, disable_extrac
     archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
     with open(archived_item_path / "index.json", "r") as f:
         output_json = json.load(f)
+
     assert output_json["base_url"] == "127.0.0.1:8080/static/example.com.html"
 
 
@@ -90,4 +91,5 @@ def test_extract_input_uses_only_passed_extractors(tmp_path, process):
     archived_item_path = list(tmp_path.glob('archive/**/*'))[0]
 
     assert (archived_item_path / "warc").exists()
-    assert not (archived_item_path / "singlefile.html").exists()
+    assert not (archived_item_path / "singlefile.html").exists()
+

tests/test_init.py

@@ -84,7 +84,7 @@ def test_collision_urls_different_timestamps(tmp_path, process, disable_extracto
 
     init_process = subprocess.run(['archivebox', 'init'], capture_output=True, env=disable_extractors_dict)
     # 1 from duplicated url, 1 from corrupted index
-    assert "Skipped adding 2 invalid link data directories" in init_process.stdout.decode("utf-8")
+    assert "Skipped adding 2 invalid snapshot data directories" in init_process.stdout.decode("utf-8")
     assert init_process.returncode == 0
 
 def test_collision_timestamps_different_urls(tmp_path, process, disable_extractors_dict):
@@ -107,7 +107,7 @@ def test_collision_timestamps_different_urls(tmp_path, process, disable_extracto
         json.dump(link_details, f)
 
     init_process = subprocess.run(['archivebox', 'init'], capture_output=True, env=disable_extractors_dict)
-    assert "Skipped adding 1 invalid link data directories" in init_process.stdout.decode("utf-8")
+    assert "Skipped adding 1 invalid snapshot data directories" in init_process.stdout.decode("utf-8")
     assert init_process.returncode == 0
 
 def test_orphaned_folders(tmp_path, process, disable_extractors_dict):
@@ -124,7 +124,7 @@ def test_orphaned_folders(tmp_path, process, disable_extractors_dict):
     conn.close()
 
     init_process = subprocess.run(['archivebox', 'init'], capture_output=True, env=disable_extractors_dict)
-    assert "Added 1 orphaned links from existing JSON index" in init_process.stdout.decode("utf-8")
+    assert "Added 1 orphaned snapshots from deprecated JSON index" in init_process.stdout.decode("utf-8")
     assert init_process.returncode == 0
 
 def test_unrecognized_folders(tmp_path, process, disable_extractors_dict):
@@ -134,7 +134,7 @@ def test_unrecognized_folders(tmp_path, process, disable_extractors_dict):
     (tmp_path / "archive" / "some_random_folder").mkdir()
 
     init_process = subprocess.run(['archivebox', 'init'], capture_output=True, env=disable_extractors_dict)
-    assert "Skipped adding 1 invalid link data directories" in init_process.stdout.decode("utf-8")
+    assert "Skipped adding 1 invalid snapshot data directories" in init_process.stdout.decode("utf-8")
     assert init_process.returncode == 0
 
 def test_tags_migration(tmp_path, disable_extractors_dict):