Update README.md

.gitignore

@@ -11,6 +11,7 @@ venv/
 .venv/
 .docker-venv/
 node_modules/
+typings/
 
 # Ignore dev lockfiles (should always be built fresh)
 pdm.dev.lock
@@ -25,6 +26,7 @@ build/
 dist/
 
 # Data folders
+tmp/
 data/
 data*/
 output/

Dockerfile

@@ -253,7 +253,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
     && apt-get update -qq \
     && apt-get install -qq -y -t bookworm-backports \
-        build-essential \
+        build-essential gcc \
         libssl-dev libldap2-dev libsasl2-dev \
         python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
     # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
@@ -264,7 +264,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     # && source $GLOBAL_VENV/bin/activate \
     && pip install -r requirements.txt \
     && apt-get purge -y \
-        build-essential \
+        build-essential gcc \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 

README.md

@@ -790,7 +790,19 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
 </blockquote>
 
-These optional dependencies used for archiving sites include:
+<ul>
+<li>Language: Python <code>&gt;=3.10</code></li>
+<li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
+<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
+<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
+<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
+<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
+<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+</ul>
+
+
+These optional subdependencies used for archiving sites include:
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right" style="max-width: 100%;">
 

archivebox/abid_utils/abid.py

@@ -1,4 +1,6 @@
-from typing import NamedTuple, Any, Union, Optional
+__package__ = 'archivebox.abid_utils'
+
+from typing import NamedTuple, Any, Union, Dict
 
 import ulid
 import uuid6
@@ -9,6 +11,7 @@ from uuid import UUID
 from typeid import TypeID            # type: ignore[import-untyped]
 from datetime import datetime
 
+from ..util import enforce_types
 
 
 ABID_PREFIX_LEN = 4
@@ -108,9 +111,10 @@ class ABID(NamedTuple):
 ####################################################
 
 
+@enforce_types
 def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
     """
-    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
+    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
     """
     if isinstance(uri, bytes):
         uri_str: str = uri.decode()
@@ -126,35 +130,45 @@ def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
         except AttributeError:
             pass
     
+    # the uri hash is the sha256 of the domain + salt
     uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
 
     return hashlib.sha256(uri_bytes).hexdigest().upper()
 
-def abid_part_from_prefix(prefix: Optional[str]) -> str:
+@enforce_types
+def abid_part_from_prefix(prefix: str) -> str:
     """
     'snp_'
     """
-    if prefix is None:
-        return 'obj_'
+    # if prefix is None:
+    #     return 'obj_'
 
     prefix = prefix.strip('_').lower()
     assert len(prefix) == 3
     return prefix + '_'
 
-def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+@enforce_types
+def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
     """
     'E4A5CCD9'     # takes first 8 characters of sha256(url)
     """
-    uri = str(uri)
+    uri = str(uri).strip()
+    assert uri not in ('None', '')
     return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
 
-def abid_part_from_ts(ts: Optional[datetime]) -> str:
+@enforce_types
+def abid_part_from_ts(ts: datetime) -> str:
     """
     '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
     """
-    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
+    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
 
-def abid_part_from_subtype(subtype: str) -> str:
+@enforce_types
+def ts_from_abid(abid: str) -> datetime:
+    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
+
+@enforce_types
+def abid_part_from_subtype(subtype: str | int) -> str:
     """
     Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
     Also allows us to change the ulid spec later by putting special sigil values here.
@@ -165,6 +179,7 @@ def abid_part_from_subtype(subtype: str) -> str:
 
     return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
 
+@enforce_types
 def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
     """
     'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
@@ -186,17 +201,23 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
     return str(rand)[-ABID_RAND_LEN:].upper()
 
 
-def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
+@enforce_types
+def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
+    return {
+        'prefix': abid_part_from_prefix(prefix),
+        'ts': abid_part_from_ts(ts),
+        'uri': abid_part_from_uri(uri, salt=salt),
+        'subtype': abid_part_from_subtype(subtype),
+        'rand': abid_part_from_rand(rand),
+        # 'salt': don't add this, salt combined with uri above to form a single hash
+    }
+
+@enforce_types
+def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
     """
     Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
     """
 
-    abid = ABID(
-        prefix=abid_part_from_prefix(prefix),
-        ts=abid_part_from_ts(ts),
-        uri=abid_part_from_uri(uri, salt=salt),
-        subtype=abid_part_from_subtype(subtype),
-        rand=abid_part_from_rand(rand),
-    )
+    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
     assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
     return abid

archivebox/abid_utils/admin.py

@@ -1,18 +1,173 @@
-from django.contrib import admin
+__package__ = 'archivebox.abid_utils'
+
+from typing import Any
+
+from django.contrib import admin, messages
+from django.core.exceptions import ValidationError
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+from django.shortcuts import redirect
+
+from django_object_actions import DjangoObjectActions, action
 
 
-class ABIDModelAdmin(admin.ModelAdmin):
-    list_display = ('created', 'created_by', 'abid', '__str__')
-    sort_fields = ('created', 'created_by', 'abid', '__str__')
-    readonly_fields = ('abid', 'created', '__str__')
+from api.auth import get_or_create_api_token
+
+from ..util import parse_date
+from .abid import ABID
+
+def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
+    """highlight each character in red that differs with the char at the same index in compare_val"""
+
+    display_val = str(display_val)
+    compare_val = str(compare_val)
+
+    if len(compare_val) < len(display_val):
+        compare_val += ' ' * (len(display_val) - len(compare_val))
+
+    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
+    if invert:
+        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
+
+    return mark_safe(''.join(
+        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
+        if display_val[i] != compare_val[i] else
+        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
+        for i in range(len(display_val))
+    ))
+
+def get_abid_info(self, obj, request=None):
+    try:
+        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
+
+        fresh_values = obj.ABID_FRESH_VALUES
+        fresh_hashes = obj.ABID_FRESH_HASHES
+        fresh_diffs = obj.ABID_FRESH_DIFFS
+        fresh_abid = ABID(**fresh_hashes)
+        
+        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
+        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
+
+        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
+
+        fresh_ts = parse_date(fresh_values['ts']) or None
+        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
+
+        derived_uri = fresh_hashes['uri']
+        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
+
+        derived_subtype = fresh_hashes['subtype']
+        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
+
+        derived_rand = fresh_hashes['rand']
+        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
+
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
+            <br/><hr/>
+            <div style="opacity: 0.8">
+            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
+            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
+            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
+            <hr/>
+            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
+            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
+            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
+            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
+            <br/><hr/>
+            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
+            </div>
+            ''',
+            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
+            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
+            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
+            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
+            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
+            # str(fresh_abid), mark_safe(fresh_abid_diff),
+            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
+            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
+            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
+            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
+            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
+            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
+            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
+        )
+    except Exception as e:
+        # import ipdb; ipdb.set_trace()
+        return str(e)
+
+
+class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
+    list_display = ('created_at', 'created_by', 'abid')
+    sort_fields = ('created_at', 'created_by', 'abid')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
+    # fields = [*readonly_fields]
+    
+    change_actions = ("regenerate_abid",)
+    # changelist_actions = ("regenerate_abid",)
+
+    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
+        try:
+            object_pk = self.model.id_from_abid(object_id)
+            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
+        except (self.model.DoesNotExist, ValidationError):
+            pass
+        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
+    
+    def queryset(self, request):
+        self.request = request
+        return super().queryset(request)                                                # type: ignore
+    
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        return super().change_view(request, object_id, form_url, extra_context)
 
     def get_form(self, request, obj=None, **kwargs):
+        self.request = request
         form = super().get_form(request, obj, **kwargs)
         if 'created_by' in form.base_fields:
             form.base_fields['created_by'].initial = request.user
+            
+        if obj:
+            if obj.ABID_FRESH_DIFFS:
+                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
+
         return form
 
-    # def save_model(self, request, obj, form, change):
-    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
-    #         obj.created_by = request.user
-    #     obj.save()
+    def get_formset(self, request, formset=None, obj=None, **kwargs):
+        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
+        formset.form.base_fields['created_at'].disabled = True
+        
+        return formset
+
+    def save_model(self, request, obj, form, change):
+        self.request = request
+
+        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
+
+        super().save_model(request, obj, form, change)
+        obj.refresh_from_db()
+
+        new_abid = obj.abid
+        if new_abid != old_abid:
+            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
+        # import ipdb; ipdb.set_trace()
+
+    @admin.display(description='API Identifiers')
+    def abid_info(self, obj):
+        return get_abid_info(self, obj, request=self.request)
+
+    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
+    def regenerate_abid(self, request, obj):
+        old_abid = str(obj.abid)
+        obj.abid = obj.issue_new_abid(overwrite=True)
+        obj.save()
+        obj.refresh_from_db()
+        new_abid = str(obj.abid)
+
+        if new_abid != old_abid:
+            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
+        else:
+            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")

archivebox/abid_utils/models.py

@@ -1,23 +1,21 @@
 """
 This file provides the Django ABIDField and ABIDModel base model to inherit from.
-
-It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
 """
 
-from typing import Any, Dict, Union, List, Set, NamedTuple, cast
 
-from ulid import ULID
-from uuid import uuid4, UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
+from typing import Any, Dict, Union, List, Set, cast
+
+from uuid import uuid4
 from functools import partial
 from charidfield import CharIDField  # type: ignore[import-untyped]
 
-from django.conf import settings
+from django.contrib import admin
+from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
 from django.db import models
 from django.utils import timezone
 from django.db.utils import OperationalError
 from django.contrib.auth import get_user_model
+from django.urls import reverse_lazy
 
 from django_stubs_ext.db.models import TypedModelMeta
 
@@ -29,7 +27,9 @@ from .abid import (
     DEFAULT_ABID_PREFIX,
     DEFAULT_ABID_URI_SALT,
     abid_part_from_prefix,
-    abid_from_values
+    abid_hashes_from_values,
+    ts_from_abid,
+    abid_part_from_ts,
 )
 
 ####################################################
@@ -57,144 +57,269 @@ def get_or_create_system_user_pk(username='system'):
         return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
 
     # otherwise, create a dedicated "system" user
-    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
     return user.pk
 
 
+class AutoDateTimeField(models.DateTimeField):
+    # def pre_save(self, model_instance, add):
+    #     return timezone.now()
+    pass
+
+class ABIDError(Exception):
+    pass
+
+
 class ABIDModel(models.Model):
     """
     Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
     """
-    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
-    abid_ts_src = 'None'                    # e.g. 'self.created'
-    abid_uri_src = 'None'                   # e.g. 'self.uri'
-    abid_subtype_src = 'None'               # e.g. 'self.extractor'
-    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
+    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
+    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
+    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
+    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
+    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
+    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
+    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
 
-    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
-    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
-    abid = ABIDField(prefix=abid_prefix)
+    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    # abid = ABIDField(prefix=abid_prefix)
 
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
+    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    # modified_at = models.DateTimeField(auto_now=True)
+
+    _prefetched_objects_cache: Dict[str, Any]
 
     class Meta(TypedModelMeta):
         abstract = True
 
-    def save(self, *args: Any, **kwargs: Any) -> None:
-        # when first creating a row, self.ABID is the source of truth
-        # overwrite default prefilled self.id & self.abid with generated self.ABID value
-        if self._state.adding or not self.id:
-            self.id = self.ABID.uuid
-        if self._state.adding or not self.abid:
-            self.abid = str(self.ABID)
+    @admin.display(description='Summary')
+    def __str__(self) -> str:
+        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
 
-        super().save(*args, **kwargs)
-        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
-        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
+        super().__init__(*args, **kwargs)
+        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
+        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
+        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
+        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
+        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
+
+    def clean(self, abid_drift_allowed: bool | None=None) -> None:
+        if self._state.adding:
+            # only runs once when a new object is first saved to the DB
+            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
+            self._previous_abid = None
+            self.abid = str(self.issue_new_abid())
+
+        else:
+            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
+            abid_diffs = self.ABID_FRESH_DIFFS
+            if abid_diffs:
+                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
+
+                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
+                full_summary = (
+                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
+                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
+                )
+
+                change_error = ValidationError({
+                    **{
+                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
+                        diff['abid_src'].replace('self.', '')
+                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
+                            else NON_FIELD_ERRORS
+                        : ValidationError(
+                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
+                            code='ABIDConflict',
+                            params=diff,
+                        )
+                        for diff in abid_diffs.values()
+                    },
+                    NON_FIELD_ERRORS: ValidationError(full_summary),
+                })
+
+                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
+                if allowed_to_invalidate_abid:
+                    print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
+                    print(change_error)
+                    print('--------------------------------------------------------------------------------------------------')
+                else:
+                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
+                    print(change_error)
+                    print('--------------------------------------------------------------------------------------------------')
+                    raise change_error
+
+    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
+        """Overriden save method ensures new ABID is generated while a new object is first saving."""
+
+        self.clean(abid_drift_allowed=abid_drift_allowed)
+
+        return super().save(*args, **kwargs)
+    
+    @classmethod
+    def id_from_abid(cls, abid: str) -> str:
+        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
 
     @property
-    def abid_values(self) -> Dict[str, Any]:
+    def ABID_SOURCES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        assert self.abid_prefix
         return {
-            'prefix': self.abid_prefix,
-            'ts': eval(self.abid_ts_src),
-            'uri': eval(self.abid_uri_src),
-            'subtype': eval(self.abid_subtype_src),
-            'rand': eval(self.abid_rand_src),
+            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
+            'ts': self.abid_ts_src,
+            'uri': self.abid_uri_src,
+            'subtype': self.abid_subtype_src,
+            'rand': self.abid_rand_src,
+            'salt': 'self.abid_salt',               # defined as static class vars at build time
         }
 
-    def generate_abid(self) -> ABID:
-        """
-        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-        """
-        prefix, ts, uri, subtype, rand = self.abid_values.values()
-
-        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
-            suggested_abid = self.__class__.__name__[:3].lower()
-            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
-
-        if not ts:
-            # default to unix epoch with 00:00:00 UTC
-            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
-            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
-
-        if not uri:
-            uri = str(self)
-            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
-
-        if not subtype:
-            subtype = self.__class__.__name__
-            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
-
-        if not rand:
-            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
-            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
-
-        abid = abid_from_values(
-            prefix=prefix,
-            ts=ts,
-            uri=uri,
-            subtype=subtype,
-            rand=rand,
-            salt=DEFAULT_ABID_URI_SALT,
+    @property
+    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        abid_sources = self.ABID_SOURCES
+        assert all(src != 'None' for src in abid_sources.values())
+        return {
+            'prefix': eval(abid_sources['prefix']),
+            'ts': eval(abid_sources['ts']),
+            'uri': eval(abid_sources['uri']),
+            'subtype': eval(abid_sources['subtype']),
+            'rand': eval(abid_sources['rand']),
+            'salt': eval(abid_sources['salt']),
+        }
+    
+    @property
+    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
+        abid_values = self.ABID_FRESH_VALUES
+        assert all(val for val in abid_values.values())
+        return abid_hashes_from_values(
+            prefix=abid_values['prefix'],
+            ts=abid_values['ts'],
+            uri=abid_values['uri'],
+            subtype=abid_values['subtype'],
+            rand=abid_values['rand'],
+            salt=abid_values['salt'],
         )
-        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
-        return abid
+    
+    @property
+    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
+        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
+        existing_abid = self.ABID
+        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
+        abid_sources = self.ABID_SOURCES
+        fresh_values = self.ABID_FRESH_VALUES
+        fresh_hashes = self.ABID_FRESH_HASHES
+        return {
+            key: {
+                'key': key,
+                'model': self.__class__.__name__,
+                'pk': self.pk,
+                'abid_src': abid_sources[key],
+                'old_val': existing_values.get(key, None),
+                'old_hash': getattr(existing_abid, key),
+                'new_val': fresh_values[key],
+                'new_hash': new_hash,
+                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
+            }
+            for key, new_hash in fresh_hashes.items()
+            if getattr(existing_abid, key) != new_hash
+        }
+
+    def issue_new_abid(self, overwrite=False) -> ABID:
+        """
+        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
+        """
+        if not overwrite:
+            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
+        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
+
+        # Setup Field defaults to be ready for ABID generation
+        self.abid = None
+        self.id = self.id or uuid4()
+        self.pk = self.id
+        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
+        self.modified_at = self.modified_at or self.created_at
+        self.created_by = self.created_by or get_or_create_system_user_pk()
+
+        # Compute fresh ABID values & hashes based on object's live properties
+        abid_fresh_values = self.ABID_FRESH_VALUES
+        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
+        abid_fresh_hashes = self.ABID_FRESH_HASHES
+        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
+        
+        new_abid = ABID(**abid_fresh_hashes)
+        
+        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
+
+        return new_abid
 
     @property
     def ABID(self) -> ABID:
         """
-        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
+        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
+        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
         """
+
+        if self.abid:
+            return ABID.parse(cast(str, self.abid))
         
-        # if object is not yet saved to DB, always generate fresh ABID from values
-        if self._state.adding:
-            return self.generate_abid()
-        
-        # otherwise DB is single source of truth, load ABID from existing db pk
-        abid: ABID | None = None
-        try:
-            abid = abid or ABID.parse(self.pk)
-        except Exception:
-            pass
+        return self.issue_new_abid()
 
-        try:
-            abid = abid or ABID.parse(self.id)
-        except Exception:
-            pass
+    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
+    # @property
+    # def UUID(self) -> UUID:
+    #     """
+    #     Get a uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return self.ABID.uuid
+    
+    # @property
+    # def uuid(self) -> str:
+    #     """
+    #     Get a str uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return str(self.ABID.uuid)
+    
+    # @property
+    # def ULID(self) -> ULID:
+    #     """
+    #     Get a ulid.ULID representation of the object's ABID.
+    #     """
+    #     return self.ABID.ulid
 
-        try:
-            abid = abid or ABID.parse(cast(str, self.abid))
-        except Exception:
-            pass
-
-        abid = abid or self.generate_abid()
-
-        return abid
+    # @property
+    # def TypeID(self) -> TypeID:
+    #     """
+    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+    #     """
+    #     return self.ABID.typeid
+    
+    @property
+    def api_url(self) -> str:
+        """
+        Compute the REST API URL to access this object.
+        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
+        """
+        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
 
     @property
-    def ULID(self) -> ULID:
+    def api_docs_url(self) -> str:
         """
-        Get a ulid.ULID representation of the object's ABID.
+        Compute the REST API Documentation URL to learn about accessing this object.
+        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
         """
-        return self.ABID.ulid
+        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
 
     @property
-    def UUID(self) -> UUID:
-        """
-        Get a uuid.UUID (v4) representation of the object's ABID.
-        """
-        return self.ABID.uuid
-
-    @property
-    def TypeID(self) -> TypeID:
-        """
-        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-        """
-        return self.ABID.typeid
-
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
 
+    def get_absolute_url(self):
+        return self.api_docs_url
 
 ####################################################
 
@@ -255,6 +380,7 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
     """
     Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
     e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    Honestly should only be used for debugging, no reason to expose this ability to users.
     """
 
     # convert str to ABID if necessary
@@ -304,7 +430,7 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
                 )
 
             for obj in qs:
-                if obj.generate_abid() == abid:
+                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
                     # found exact match, no need to keep iterating
                     return [obj]
                 partial_matches.append(obj)

archivebox/api/auth.py

@@ -1,13 +1,34 @@
 __package__ = 'archivebox.api'
 
-from typing import Optional, cast
+from typing import Any, Optional, cast
+from datetime import timedelta
 
 from django.http import HttpRequest
+from django.utils import timezone
 from django.contrib.auth import login
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import AbstractBaseUser
 
 from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
+from ninja.errors import HttpError
+
+
+def get_or_create_api_token(user):
+    from api.models import APIToken
+    
+    if user and user.is_superuser:
+        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
+        if api_tokens.exists():
+            # unexpired token exists, use it
+            api_token = api_tokens.last()
+        else:
+            # does not exist, create a new one
+            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
+        
+        assert api_token.is_valid(), f"API token is not valid {api_token}"
+
+        return api_token
+    return None
 
 
 def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
@@ -16,39 +37,35 @@ def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[Abs
     
     user = None
 
-    submitted_empty_form = token in ('string', '', None)
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
+    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
+    if not submitted_empty_form:
         try:
             token = APIToken.objects.get(token=token)
             if token.is_valid():
                 user = token.created_by
+                request._api_token = token
         except APIToken.DoesNotExist:
             pass
 
     if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
-
-    return None
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        return None
+    
+    return cast(AbstractBaseUser, user)
 
 def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user = None
     
     submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
+    if not submitted_empty_form:
         user = authenticate(
             username=username,
             password=password,
         )
 
     if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
+        # print('[❌] Failed to authenticate API user using API Key:', request)
         user = None
 
     return cast(AbstractBaseUser | None, user)
@@ -56,28 +73,41 @@ def auth_using_password(username, password, request: Optional[HttpRequest]=None)
 
 ### Base Auth Types
 
+
 class APITokenAuthCheck:
     """The base class for authentication methods that use an api.models.APIToken"""
     def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_token(
+        request.user = auth_using_token(
             token=key,
             request=request,
         )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+        return request.user
+
 
 class UserPassAuthCheck:
     """The base class for authentication methods that use a username & password"""
     def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_password(
+        request.user = auth_using_password(
             username=username,
             password=password,
             request=request,
         )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+
+        return request.user
 
 
 ### Django-Ninja-Provided Auth Methods
@@ -98,13 +128,12 @@ class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
     """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
     pass
 
-
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [
     HeaderTokenAuth(),
     BearerTokenAuth(),
     QueryParamTokenAuth(), 
-    django_auth_superuser,
+    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
     UsernameAndPasswordAuth(),
 ]

archivebox/api/migrations/0008_alter_apitoken_created_alter_apitoken_created_by_and_more.py

@@ -0,0 +1,47 @@
+# Generated by Django 5.1 on 2024-09-04 23:32
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0007_alter_apitoken_created_by'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+    ]

archivebox/api/migrations/0009_rename_created_apitoken_created_at_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 5.1 on 2024-09-05 00:26
+
+import abid_utils.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.AddField(
+            model_name='outboundwebhook',
+            name='created_at',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
+        ),
+    ]

archivebox/api/models.py

@@ -12,7 +12,7 @@ from signal_webhooks.models import WebhookBase
 
 from django_stubs_ext.db.models import TypedModelMeta
 
-from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
+from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
 
 
 
@@ -27,16 +27,18 @@ class APIToken(ABIDModel):
     """
     # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
     abid_prefix = 'apt_'
-    abid_ts_src = 'self.created'
-    abid_uri_src = 'self.token'
-    abid_subtype_src = 'self.created_by_id'
+    abid_ts_src = 'self.created_at'
+    abid_uri_src = 'self.created_by_id'
+    abid_subtype_src = '"01"'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
     expires = models.DateTimeField(null=True, blank=True)
@@ -58,7 +60,7 @@ class APIToken(ABIDModel):
             "abid":             str(self.ABID),
             "created_by_id":    str(self.created_by_id),
             "token":            self.token,
-            "created":          self.created.isoformat(),
+            "created_at":       self.created_at.isoformat(),
             "expires":          self.expires_as_iso8601,
         }
 
@@ -94,17 +96,18 @@ class OutboundWebhook(ABIDModel, WebhookBase):
         settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
     """
     abid_prefix = 'whk_'
-    abid_ts_src = 'self.created'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.endpoint'
     abid_subtype_src = 'self.ref'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     # More fields here: WebhookBase...
 
@@ -120,3 +123,6 @@ class OutboundWebhook(ABIDModel, WebhookBase):
     class Meta(WebhookBase.Meta):
         verbose_name = 'API Outbound Webhook'
 
+
+    def __str__(self) -> str:
+        return f'[{self.abid}] {self.ref} -> {self.endpoint}'

archivebox/api/v1_api.py

@@ -6,6 +6,7 @@ from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
 from django.http import HttpRequest, HttpResponse
+from django.views.decorators.csrf import csrf_exempt
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
 
 from ninja import NinjaAPI, Swagger
@@ -52,7 +53,26 @@ class NinjaAPIWithIOCapture(NinjaAPI):
 
                 response = super().create_temporal_response(request)
 
-        print('RESPONDING NOW', response)
+        # Diable caching of API responses entirely
+        response['Cache-Control'] = 'no-store'
+
+        # Add debug stdout and stderr headers to response
+        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
+        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
+        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
+
+        # Add Auth Headers to response
+        api_token = getattr(request, '_api_token', None)
+        token_expiry = api_token.expires.isoformat() if api_token else 'Never'
+
+        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
+        response['X-ArchiveBox-Auth-Expires'] = token_expiry
+        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
+        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
+        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
+
+        # import ipdb; ipdb.set_trace()
+        # print('RESPONDING NOW', response)
 
         return response
 

archivebox/api/v1_auth.py

@@ -3,12 +3,14 @@ __package__ = 'archivebox.api'
 from typing import Optional
 
 from ninja import Router, Schema
+from django.utils import timezone
+from datetime import timedelta
 
 from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password
+from api.auth import auth_using_token, auth_using_password, get_or_create_api_token
 
 
-router = Router(tags=['Authentication'])
+router = Router(tags=['Authentication'], auth=None)
 
 
 class PasswordAuthSchema(Schema):
@@ -25,10 +27,9 @@ def get_api_token(request, auth_data: PasswordAuthSchema):
         request=request,
     )
 
-    if user:
-        # TODO: support multiple tokens in the future, for now we just have one per user
-        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
-
+    if user and user.is_superuser:
+        api_token = get_or_create_api_token(user)
+        assert api_token is not None, "Failed to create API token"
         return api_token.__json__()
     
     return {"success": False, "errors": ["Invalid credentials"]}

archivebox/api/v1_cli.py

@@ -16,8 +16,10 @@ from ..util import ansi_to_html
 from ..config import ONLY_NEW
 
 
+from .auth import API_AUTH_METHODS
+
 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'])
+router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
 
 
 # Schemas
@@ -92,7 +94,7 @@ class ListCommandSchema(Schema):
     status: Optional[StatusChoices] = StatusChoices.indexed
     after: Optional[float] = 0
     before: Optional[float] = 999999999999999
-    sort: str = 'added'
+    sort: str = 'bookmarked_at'
     as_json: bool = True
     as_html: bool = False
     as_csv: str | bool = 'timestamp,url'

archivebox/api/v1_core.py

@@ -12,11 +12,15 @@ from django.contrib.auth import get_user_model
 
 from ninja import Router, Schema, FilterSchema, Field, Query
 from ninja.pagination import paginate, PaginationBase
+from ninja.errors import HttpError
 
 from core.models import Snapshot, ArchiveResult, Tag
+from api.models import APIToken, OutboundWebhook
 from abid_utils.abid import ABID
 
-router = Router(tags=['Core Models'])
+from .auth import API_AUTH_METHODS
+
+router = Router(tags=['Core Models'], auth=API_AUTH_METHODS)
 
 
 
@@ -56,23 +60,17 @@ class CustomPagination(PaginationBase):
 
 ### ArchiveResult #########################################################################
 
-class ArchiveResultSchema(Schema):
+class MinimalArchiveResultSchema(Schema):
     TYPE: str = 'core.models.ArchiveResult'
 
     id: UUID
-    old_id: int
     abid: str
 
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
 
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: str
-
     extractor: str
     cmd_version: Optional[str]
     cmd: List[str]
@@ -80,6 +78,9 @@ class ArchiveResultSchema(Schema):
     status: str
     output: str
 
+    start_ts: Optional[datetime]
+    end_ts: Optional[datetime]
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -89,20 +90,12 @@ class ArchiveResultSchema(Schema):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
 
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
     @staticmethod
     def resolve_abid(obj):
         return str(obj.ABID)
 
     @staticmethod
-    def resolve_created(obj):
+    def resolve_created_at(obj):
         return obj.start_ts
 
     @staticmethod
@@ -113,19 +106,34 @@ class ArchiveResultSchema(Schema):
     def resolve_snapshot_url(obj):
         return obj.snapshot.url
 
+    @staticmethod
+    def resolve_snapshot_id(obj):
+        return str(obj.snapshot_id)
+    
     @staticmethod
     def resolve_snapshot_abid(obj):
         return str(obj.snapshot.ABID)
 
     @staticmethod
     def resolve_snapshot_tags(obj):
-        return obj.snapshot.tags_str()
+        return sorted(tag.name for tag in obj.snapshot.tags.all())
+
+class ArchiveResultSchema(MinimalArchiveResultSchema):
+    TYPE: str = 'core.models.ArchiveResult'
+
+    # ... Extends MinimalArchiveResultSchema fields ...
+
+    snapshot_id: UUID
+    snapshot_abid: str
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: List[str]
 
 
 class ArchiveResultFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
 
-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
     snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
     snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
     snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
@@ -137,9 +145,9 @@ class ArchiveResultFilterSchema(FilterSchema):
     pwd: Optional[str] = Field(None, q='pwd__icontains')
     cmd_version: Optional[str] = Field(None, q='cmd_version')
 
-    created: Optional[datetime] = Field(None, q='updated')
-    created__gte: Optional[datetime] = Field(None, q='updated__gte')
-    created__lt: Optional[datetime] = Field(None, q='updated__lt')
+    created_at: Optional[datetime] = Field(None, q='created_at')
+    created_at__gte: Optional[datetime] = Field(None, q='created_at__gte')
+    created_at__lt: Optional[datetime] = Field(None, q='created_at__lt')
 
 
 @router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
@@ -153,8 +161,8 @@ def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...))
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by pk, abid, or old_id."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
+    """Get a specific ArchiveResult by id or abid."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id))
 
 
 # @router.post("/archiveresult", response=ArchiveResultSchema)
@@ -189,26 +197,27 @@ class SnapshotSchema(Schema):
     TYPE: str = 'core.models.Snapshot'
 
     id: UUID
-    old_id: UUID
     abid: str
 
-    modified: datetime
-    created: datetime
     created_by_id: str
     created_by_username: str
+    created_at: datetime
+    modified_at: datetime
+
+    bookmarked_at: datetime
+    downloaded_at: Optional[datetime]
 
     url: str
-    tags: str
+    tags: List[str]
     title: Optional[str]
     timestamp: str
     archive_path: str
 
-    bookmarked: datetime
-    added: datetime
-    updated: Optional[datetime]
+    # url_for_admin: str
+    # url_for_view: str
 
     num_archiveresults: int
-    archiveresults: List[ArchiveResultSchema]
+    archiveresults: List[MinimalArchiveResultSchema]
 
     @staticmethod
     def resolve_created_by_id(obj):
@@ -219,21 +228,21 @@ class SnapshotSchema(Schema):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
 
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
     @staticmethod
     def resolve_abid(obj):
         return str(obj.ABID)
 
     @staticmethod
     def resolve_tags(obj):
-        return obj.tags_str()
+        return sorted(tag.name for tag in obj.tags.all())
+
+    # @staticmethod
+    # def resolve_url_for_admin(obj):
+    #     return f"/admin/core/snapshot/{obj.id}/change/"
+    
+    # @staticmethod
+    # def resolve_url_for_view(obj):
+    #     return f"/{obj.archive_path}"
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
@@ -247,29 +256,27 @@ class SnapshotSchema(Schema):
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
-
-    old_id: Optional[str] = Field(None, q='old_id__icontains')
+    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'timestamp__startswith'])
     abid: Optional[str] = Field(None, q='abid__icontains')
 
     created_by_id: str = Field(None, q='created_by_id')
     created_by_username: str = Field(None, q='created_by__username__icontains')
 
-    created__gte: datetime = Field(None, q='created__gte')
-    created__lt: datetime = Field(None, q='created__lt')
-    created: datetime = Field(None, q='created')
-    modified: datetime = Field(None, q='modified')
-    modified__gte: datetime = Field(None, q='modified__gte')
-    modified__lt: datetime = Field(None, q='modified__lt')
+    created_at__gte: datetime = Field(None, q='created_at__gte')
+    created_at__lt: datetime = Field(None, q='created_at__lt')
+    created_at: datetime = Field(None, q='created_at')
+    modified_at: datetime = Field(None, q='modified_at')
+    modified_at__gte: datetime = Field(None, q='modified_at__gte')
+    modified_at__lt: datetime = Field(None, q='modified_at__lt')
 
-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'timestamp__startswith'])
     url: Optional[str] = Field(None, q='url')
     tag: Optional[str] = Field(None, q='tags__name')
     title: Optional[str] = Field(None, q='title__icontains')
     timestamp: Optional[str] = Field(None, q='timestamp__startswith')
     
-    added__gte: Optional[datetime] = Field(None, q='added__gte')
-    added__lt: Optional[datetime] = Field(None, q='added__lt')
+    bookmarked_at__gte: Optional[datetime] = Field(None, q='bookmarked_at__gte')
+    bookmarked_at__lt: Optional[datetime] = Field(None, q='bookmarked_at__lt')
 
 
 
@@ -285,16 +292,16 @@ def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_arch
 
 @router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
 def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid, uuid, or pk."""
+    """Get a specific Snapshot by abid or id."""
     request.with_archiveresults = with_archiveresults
     snapshot = None
     try:
-        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
         pass
 
     try:
-        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
+        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
     except Snapshot.DoesNotExist:
         pass
 
@@ -311,7 +318,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
 # def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
 #
 #     for attr, value in payload.dict().items():
 #         setattr(snapshot, attr, value)
@@ -321,7 +328,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.delete("/snapshot/{snapshot_id}")
 # def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
 #     snapshot.delete()
 #     return {"success": True}
 
@@ -334,11 +341,10 @@ class TagSchema(Schema):
     TYPE: str = 'core.models.Tag'
 
     id: UUID
-    old_id: str
     abid: str
 
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
 
@@ -347,10 +353,6 @@ class TagSchema(Schema):
     num_snapshots: int
     snapshots: List[SnapshotSchema]
 
-    @staticmethod
-    def resolve_old_id(obj):
-        return str(obj.old_id)
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -382,11 +384,6 @@ def get_tag(request, tag_id: str, with_snapshots: bool=True):
     request.with_snapshots = with_snapshots
     request.with_archiveresults = False
     tag = None
-    try:
-        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError, ValueError):
-        pass
-
     try:
         tag = Tag.objects.get(abid__icontains=tag_id)
     except (Tag.DoesNotExist, ValidationError):
@@ -421,4 +418,10 @@ def get_any(request, abid: str):
     except Exception:
         pass
 
-    return response
+    if abid.startswith(APIToken.abid_prefix):
+        raise HttpError(403, 'APIToken objects are not accessible via REST API')
+    
+    if abid.startswith(OutboundWebhook.abid_prefix):
+        raise HttpError(403, 'OutboundWebhook objects are not accessible via REST API')
+
+    raise HttpError(404, 'Object with given ABID not found')

archivebox/builtin_plugins/base/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

archivebox/builtin_plugins/base/apps.py

@@ -1,82 +0,0 @@
-import sys
-import inspect
-from typing import List, Dict, Any, Optional
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-from plugantic.replayers import Replayer
-
-
-class PythonBinary(Binary):
-    name: BinName = 'python'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides: Dict[str, Any] = {
-        'env': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(Binary):
-    name: BinName = 'sqlite'
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(sqlite3),
-            'version': \
-                lambda: sqlite3.version,
-        },
-    }
-
-class DjangoBinary(Binary):
-    name: BinName = 'django'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(Replayer):
-    name: str = 'basic'
-
-
-class BasePlugin(Plugin):
-    name: str = 'base'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = []
-    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
-
-
-PLUGINS = [BasePlugin()]
-
-
-class BaseConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'builtin_plugins.base'

archivebox/builtin_plugins/base/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/builtin_plugins/base/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/base/views.py

@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.

archivebox/builtin_plugins/npm/apps.py

@@ -0,0 +1,70 @@
+__package__ = 'archivebox.builtin_plugins.npm'
+
+from typing import List, Optional
+from pydantic import InstanceOf, Field
+
+from django.conf import settings
+
+from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
+from plugantic.base_hook import BaseHook
+
+from ...config import CONFIG
+
+###################### Config ##########################
+
+
+class NpmDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_NPM: bool = True
+    NPM_BINARY: str = Field(default='npm')
+    NPM_ARGS: Optional[List[str]] = Field(default=None)
+    NPM_EXTRA_ARGS: List[str] = []
+    NPM_DEFAULT_ARGS: List[str] = []
+
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+class CustomNpmProvider(NpmProvider, BaseBinProvider):
+    PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
+
+NPM_BINPROVIDER = CustomNpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
+npm = NPM_BINPROVIDER
+
+class NpmBinary(BaseBinary):
+    name: BinName = 'npm'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+
+NPM_BINARY = NpmBinary()
+
+class NodeBinary(BaseBinary):
+    name: BinName = 'node'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+
+NODE_BINARY = NodeBinary()
+
+
+
+class NpmPlugin(BasePlugin):
+    app_label: str = 'npm'
+    verbose_name: str = 'NPM'
+    
+    hooks: List[InstanceOf[BaseHook]] = [
+        NPM_CONFIG,
+        NPM_BINPROVIDER,
+        NODE_BINARY,
+        NPM_BINARY,
+    ]
+
+
+PLUGIN = NpmPlugin()
+PLUGIN.register(settings)
+DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/pip/apps.py

@@ -0,0 +1,144 @@
+import os
+import sys
+import inspect
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+import django
+
+from django.db.backends.sqlite3.base import Database as sqlite3     # type: ignore[import-type]
+from django.core.checks import Error, Tags
+from django.conf import settings
+
+from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
+from plugantic.base_plugin import BasePlugin
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_check import BaseCheck
+from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
+from plugantic.base_hook import BaseHook
+
+
+###################### Config ##########################
+
+
+class PipDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_PIP: bool = True
+    PIP_BINARY: str = Field(default='pip')
+    PIP_ARGS: Optional[List[str]] = Field(default=None)
+    PIP_EXTRA_ARGS: List[str] = []
+    PIP_DEFAULT_ARGS: List[str] = []
+
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+class CustomPipProvider(PipProvider, BaseBinProvider):
+    name: str = 'pip'
+    INSTALLER_BIN: str = 'pip'
+    PATH: PATHStr = str(Path(sys.executable).parent)
+
+
+PIP_BINPROVIDER = CustomPipProvider(PATH=str(Path(sys.executable).parent))
+pip = PIP_BINPROVIDER
+
+class PipBinary(BaseBinary):
+    name: BinName = 'pip'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+
+PIP_BINARY = PipBinary()
+
+
+
+
+
+class PythonBinary(BaseBinary):
+    name: BinName = 'python'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
+        'apt': {
+            'packages': \
+                lambda: 'python3 python3-minimal python3-pip python3-setuptools python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+PYTHON_BINARY = PythonBinary()
+
+class SqliteBinary(BaseBinary):
+    name: BinName = 'sqlite'
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: Path(inspect.getfile(sqlite3)),
+            'version': \
+                lambda: SemVer(sqlite3.version),
+        },
+    }
+
+SQLITE_BINARY = SqliteBinary()
+
+
+class DjangoBinary(BaseBinary):
+    name: BinName = 'django'
+
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+DJANGO_BINARY = DjangoBinary()
+
+
+class CheckUserIsNotRoot(BaseCheck):
+    label: str = 'CheckUserIsNotRoot'
+    tag: str = Tags.database
+
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        errors = []
+        if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
+            errors.append(
+                Error(
+                    "Cannot run as root!",
+                    id="core.S001",
+                    hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
+                )
+            )
+        logger.debug('[√] UID is not root')
+        return errors
+
+
+USER_IS_NOT_ROOT_CHECK = CheckUserIsNotRoot()
+
+
+class PipPlugin(BasePlugin):
+    app_label: str = 'pip'
+    verbose_name: str = 'PIP'
+
+    hooks: List[InstanceOf[BaseHook]] = [
+        PIP_CONFIG,
+        PIP_BINPROVIDER,
+        PIP_BINARY,
+        PYTHON_BINARY,
+        SQLITE_BINARY,
+        DJANGO_BINARY,
+        USER_IS_NOT_ROOT_CHECK,
+    ]
+
+PLUGIN = PipPlugin()
+PLUGIN.register(settings)
+DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,38 +1,33 @@
-from typing import List, Optional
 from pathlib import Path
+from typing import List, Dict, Optional
 
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
+from django.conf import settings
 
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
+# Depends on other PyPI/vendor packages:
+from pydantic import InstanceOf, Field
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
 
-from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+# Depends on other Django apps:
+from plugantic.base_plugin import BasePlugin
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_binary import BaseBinary, env
+from plugantic.base_extractor import BaseExtractor
+from plugantic.base_queue import BaseQueue
+from plugantic.base_hook import BaseHook
 
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
+# Depends on Other Plugins:
+from builtin_plugins.npm.apps import npm
 
 
 ###################### Config ##########################
 
-class SinglefileToggleConfig(ConfigSet):
+class SinglefileToggleConfigs(BaseConfigSet):
     section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
 
     SAVE_SINGLEFILE: bool = True
 
 
-class SinglefileDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    SINGLEFILE_BINARY: str = Field(default='wget')
-    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
-    SINGLEFILE_EXTRA_ARGS: List[str] = []
-    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class SinglefileOptionsConfig(ConfigSet):
+class SinglefileOptionsConfigs(BaseConfigSet):
     section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
 
     # loaded from shared config
@@ -43,52 +38,84 @@ class SinglefileOptionsConfig(ConfigSet):
     SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
 
 
+class SinglefileDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
 
-DEFAULT_CONFIG = {
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileConfigs(SinglefileToggleConfigs, SinglefileOptionsConfigs, SinglefileDependencyConfigs):
+    # section: ConfigSectionName = 'ALL_CONFIGS'
+    pass
+
+DEFAULT_GLOBAL_CONFIG = {
     'CHECK_SSL_VALIDITY': False,
     'SAVE_SINGLEFILE': True,
     'TIMEOUT': 120,
 }
 
-PLUGIN_CONFIG = [
-    SinglefileToggleConfig(**DEFAULT_CONFIG),
-    SinglefileDependencyConfig(**DEFAULT_CONFIG),
-    SinglefileOptionsConfig(**DEFAULT_CONFIG),
-]
+SINGLEFILE_CONFIG = SinglefileConfigs(**DEFAULT_GLOBAL_CONFIG)
 
-###################### Binaries ############################
 
-class SinglefileBinary(Binary):
+
+min_version: str = "1.1.54"
+max_version: str = "2.0.0"
+
+def get_singlefile_abspath() -> Optional[Path]:
+    return 
+
+
+class SinglefileBinary(BaseBinary):
     name: BinName = 'single-file'
-    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [npm, env]
 
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={
+        # 'env': {
+        #     'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
+        # },
+        # 'npm': {
+        #     'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
+        #     'packages': lambda: f'single-file-cli@>={min_version} <{max_version}',
+        # },
+    }
 
-###################### Extractors ##########################
+SINGLEFILE_BINARY = SinglefileBinary()
 
-class SinglefileExtractor(Extractor):
-    name: ExtractorName = 'singlefile'
-    binary: Binary = SinglefileBinary()
+PLUGIN_BINARIES = [SINGLEFILE_BINARY]
+
+class SinglefileExtractor(BaseExtractor):
+    name: str = 'singlefile'
+    binary: BinName = SINGLEFILE_BINARY.name
 
     def get_output_path(self, snapshot) -> Path:
         return Path(snapshot.link_dir) / 'singlefile.html'
 
 
-###################### Plugins #############################
+SINGLEFILE_BINARY = SinglefileBinary()
+SINGLEFILE_EXTRACTOR = SinglefileExtractor()
 
-
-class SinglefilePlugin(Plugin):
+class SinglefileQueue(BaseQueue):
     name: str = 'singlefile'
-    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
+    
+    binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
 
-PLUGINS = [SinglefilePlugin()]
+SINGLEFILE_QUEUE = SinglefileQueue()
 
-###################### Django Apps #########################
+class SinglefilePlugin(BasePlugin):
+    app_label: str ='singlefile'
+    verbose_name: str = 'SingleFile'
 
-class SinglefileConfig(AppConfig):
-    name = 'builtin_plugins.singlefile'
-    verbose_name = 'SingleFile'
+    hooks: List[InstanceOf[BaseHook]] = [
+        SINGLEFILE_CONFIG,
+        SINGLEFILE_BINARY,
+        SINGLEFILE_EXTRACTOR,
+        SINGLEFILE_QUEUE,
+    ]
 
-    def ready(self):
-        print('Loaded singlefile plugin')
+
+
+PLUGIN = SinglefilePlugin()
+PLUGIN.register(settings)
+DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/singlefile/config.yaml

@@ -1,66 +0,0 @@
-name: singlefile
-plugin_version: '0.0.1'
-plugin_spec: '0.0.1'
-
-binaries:
-    singlefile:
-        providers:
-            - env
-            - npm
-
-commands:
-    - singlefile.exec
-    - singlefile.extract
-    - singlefile.should_extract
-    - singlefile.get_output_path
-
-extractors:
-    singlefile:
-        binary: singlefile
-        test: singlefile.should_extract
-        extract: singlefile.extract
-        output_files:
-            - singlefile.html
-
-configs:
-    ARCHIVE_METHOD_TOGGLES:
-        SAVE_SINGLEFILE:
-            type: bool
-            default: true
-
-    DEPENDENCY_CONFIG:
-        SINGLEFILE_BINARY:
-            type: str
-            default: wget
-        SINGLEFILE_ARGS:
-            type: Optional[List[str]]
-            default: null
-        SINGLEFILE_EXTRA_ARGS:
-            type: List[str]
-            default: []
-        SINGLEFILE_DEFAULT_ARGS:
-            type: List[str]
-            default: 
-            - "--timeout={TIMEOUT-10}"
-
-    ARCHIVE_METHOD_OPTIONS:
-        SINGLEFILE_USER_AGENT:
-            type: str
-            default: ""
-            alias: USER_AGENT
-        SINGLEFILE_TIMEOUT:
-            type: int
-            default: 60
-            alias: TIMEOUT
-        SINGLEFILE_CHECK_SSL_VALIDITY:
-            type: bool
-            default: true
-            alias: CHECK_SSL_VALIDITY
-        SINGLEFILE_RESTRICT_FILE_NAMES:
-            type: str
-            default: windows
-            alias: RESTRICT_FILE_NAMES
-        SINGLEFILE_COOKIES_FILE:
-            type: Optional[Path]
-            default: null
-            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/migrations/0001_initial.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.1.1 on 2024-09-10 05:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('core', '0074_alter_snapshot_downloaded_at'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SinglefileResult',
+            fields=[
+            ],
+            options={
+                'proxy': True,
+                'indexes': [],
+                'constraints': [],
+            },
+            bases=('core.archiveresult',),
+        ),
+    ]

archivebox/builtin_plugins/singlefile/models.py

@@ -0,0 +1,14 @@
+from django.db import models
+
+from core.models import ArchiveResult
+
+class SinglefileResultManager(models.Manager):
+    def get_queryset(self):
+        return super().get_queryset().filter(extractor='singlefile')
+
+
+class SinglefileResult(ArchiveResult):
+    objects = SinglefileResultManager()
+
+    class Meta:
+        proxy = True

archivebox/builtin_plugins/singlefile/tasks.py

@@ -0,0 +1,40 @@
+__package__ = 'archivebox.queues'
+
+import time
+
+from django.core.cache import cache
+
+from huey import crontab
+from django_huey import db_task, on_startup, db_periodic_task
+from huey_monitor.models import TaskModel
+from huey_monitor.tqdm import ProcessInfo
+
+@db_task(queue="singlefile", context=True)
+def extract(url, out_dir, config, task=None, parent_task_id=None):
+    if task and parent_task_id:
+        TaskModel.objects.set_parent_task(main_task_id=parent_task_id, sub_task_id=task.id)
+
+    process_info = ProcessInfo(task, desc="extract_singlefile", parent_task_id=parent_task_id, total=1)
+
+    time.sleep(5)
+
+    process_info.update(n=1)
+    return {'output': 'singlefile.html', 'status': 'succeeded'}
+
+
+# @on_startup(queue='singlefile')
+# def start_singlefile_queue():
+#     print("[+] Starting singlefile worker...")
+#     update_version.call_local()
+
+
+# @db_periodic_task(crontab(minute='*/5'), queue='singlefile')
+# def update_version():
+#     print('[*] Updating singlefile version... 5 minute interval')
+#     from django.conf import settings
+    
+#     bin = settings.BINARIES.SinglefileBinary.load()
+#     if bin.version:
+#         cache.set(f"bin:abspath:{bin.name}", bin.abspath)
+#         cache.set(f"bin:version:{bin.name}:{bin.abspath}", bin.version)
+#         print('[√] Updated singlefile version:', bin.version, bin.abspath)

archivebox/builtin_plugins/singlefile/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/ytdlp/apps.py

@@ -0,0 +1,79 @@
+from typing import List, Dict
+from subprocess import run, PIPE
+from pydantic import InstanceOf, Field
+
+from django.conf import settings
+
+from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
+from plugantic.base_plugin import BasePlugin
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_binary import BaseBinary, env, apt, brew
+from plugantic.base_hook import BaseHook
+
+from builtin_plugins.pip.apps import pip
+
+###################### Config ##########################
+
+
+class YtdlpDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_YTDLP: bool = True
+
+    YTDLP_BINARY: str = Field(default='yt-dlp')
+
+DEFAULT_GLOBAL_CONFIG = {}
+YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+
+class YtdlpBinary(BaseBinary):
+    name: BinName = YTDLP_CONFIG.YTDLP_BINARY
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+
+class FfmpegBinary(BaseBinary):
+    name: BinName = 'ffmpeg'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            # 'abspath': lambda: shutil.which('ffmpeg', PATH=env.PATH),
+            # 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
+        },
+        'apt': {
+            # 'abspath': lambda: shutil.which('ffmpeg', PATH=apt.PATH),
+            'version': lambda: run(['apt', 'show', 'ffmpeg'], stdout=PIPE, stderr=PIPE, text=True).stdout,
+        },
+        'brew': {
+            # 'abspath': lambda: shutil.which('ffmpeg', PATH=brew.PATH),
+            'version': lambda: run(['brew', 'info', 'ffmpeg', '--quiet'], stdout=PIPE, stderr=PIPE, text=True).stdout,
+        },
+    }
+
+    # def get_ffmpeg_version(self) -> Optional[str]:
+    #     return self.exec(cmd=['-version']).stdout
+
+
+YTDLP_BINARY = YtdlpBinary()
+FFMPEG_BINARY = FfmpegBinary()
+
+# class YtdlpExtractor(BaseExtractor):
+#     name: str = 'ytdlp'
+#     binary: str = 'ytdlp'
+
+
+
+class YtdlpPlugin(BasePlugin):
+    app_label: str = 'ytdlp'
+    verbose_name: str = 'YTDLP'
+
+    hooks: List[InstanceOf[BaseHook]] = [
+        YTDLP_CONFIG,
+        YTDLP_BINARY,
+        FFMPEG_BINARY,
+    ]
+
+
+PLUGIN = YtdlpPlugin()
+PLUGIN.register(settings)
+DJANGO_APP = PLUGIN.AppConfig

archivebox/cli/__init__.py

@@ -37,7 +37,7 @@ is_valid_cli_module = lambda module, subcommand: (
 )
 
 
-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting
+IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
 
 
 def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
@@ -68,7 +68,7 @@ def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: It
         else:
             return tries
 
-    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')
+    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
 
 
 def list_subcommands() -> Dict[str, str]:

archivebox/config.py

@@ -103,7 +103,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'PUBLIC_SNAPSHOTS':          {'type': bool,  'default': True},
         'PUBLIC_ADD_VIEW':           {'type': bool,  'default': False},
         'FOOTER_INFO':               {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
-        'SNAPSHOTS_PER_PAGE':        {'type': int,   'default': 100},
+        'SNAPSHOTS_PER_PAGE':        {'type': int,   'default': 40},
         'CUSTOM_TEMPLATES_DIR':      {'type': str,   'default': None},
         'TIME_ZONE':                 {'type': str,   'default': 'UTC'},
         'TIMEZONE':                  {'type': str,   'default': 'UTC'},
@@ -152,10 +152,10 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
         'CHECK_SSL_VALIDITY':       {'type': bool,  'default': True},
         'MEDIA_MAX_SIZE':           {'type': str,   'default': '750m'},
 
-        'USER_AGENT':               {'type': str,   'default': None},
-        'CURL_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) curl/{CURL_VERSION}'},
-        'WGET_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/) wget/{WGET_VERSION}'},
-        'CHROME_USER_AGENT':        {'type': str,   'default': lambda c: c['USER_AGENT'] or 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'USER_AGENT':               {'type': str,   'default': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)'},
+        'CURL_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] + ' curl/{CURL_VERSION}'},
+        'WGET_USER_AGENT':          {'type': str,   'default': lambda c: c['USER_AGENT'] + ' wget/{WGET_VERSION}'},
+        'CHROME_USER_AGENT':        {'type': str,   'default': lambda c: c['USER_AGENT']},
 
         'COOKIES_FILE':             {'type': str,   'default': None},
         'CHROME_USER_DATA_DIR':     {'type': str,   'default': None},
@@ -301,7 +301,7 @@ ROBOTS_TXT_FILENAME = 'robots.txt'
 FAVICON_FILENAME = 'favicon.ico'
 CONFIG_FILENAME = 'ArchiveBox.conf'
 
-DEFAULT_CLI_COLORS = {
+DEFAULT_CLI_COLORS = AttrDict({
     'reset': '\033[00;00m',
     'lightblue': '\033[01;30m',
     'lightyellow': '\033[01;33m',
@@ -311,8 +311,8 @@ DEFAULT_CLI_COLORS = {
     'blue': '\033[01;34m',
     'white': '\033[01;37m',
     'black': '\033[01;30m',
-}
-ANSI = {k: '' for k in DEFAULT_CLI_COLORS.keys()}
+})
+ANSI = AttrDict({k: '' for k in DEFAULT_CLI_COLORS.keys()})
 
 COLOR_DICT = defaultdict(lambda: [(0, 0, 0), (0, 0, 0)], {
     '00': [(0, 0, 0), (0, 0, 0)],
@@ -353,20 +353,20 @@ STATICFILE_EXTENSIONS = {
 # These files are exceptions to the is_empty check when we're trying to init a new dir,
 # as they could be from a previous archivebox version, system artifacts, dependencies, etc.
 ALLOWED_IN_OUTPUT_DIR = {
-    '.gitignore',
-    'lost+found',
-    '.DS_Store',
-    '.venv',
-    'venv',
-    'virtualenv',
-    '.virtualenv',
-    'node_modules',
-    'package.json',
-    'package-lock.json',
-    'yarn.lock',
-    'static',
-    'sonic',
-    'search.sqlite3',
+    ".gitignore",
+    "lost+found",
+    ".DS_Store",
+    ".venv",
+    "venv",
+    "virtualenv",
+    ".virtualenv",
+    "node_modules",
+    "package.json",
+    "package-lock.json",
+    "yarn.lock",
+    "static",
+    "sonic",
+    "search.sqlite3",
     CRONTABS_DIR_NAME,
     ARCHIVE_DIR_NAME,
     SOURCES_DIR_NAME,
@@ -374,15 +374,18 @@ ALLOWED_IN_OUTPUT_DIR = {
     CACHE_DIR_NAME,
     PERSONAS_DIR_NAME,
     SQL_INDEX_FILENAME,
-    f'{SQL_INDEX_FILENAME}-wal',
-    f'{SQL_INDEX_FILENAME}-shm',
+    f"{SQL_INDEX_FILENAME}-wal",
+    f"{SQL_INDEX_FILENAME}-shm",
+    "queue.sqlite3",
+    "queue.sqlite3-wal",
+    "queue.sqlite3-shm",
     JSON_INDEX_FILENAME,
     HTML_INDEX_FILENAME,
     ROBOTS_TXT_FILENAME,
     FAVICON_FILENAME,
     CONFIG_FILENAME,
-    f'{CONFIG_FILENAME}.bak',
-    'static_index.json',
+    f"{CONFIG_FILENAME}.bak",
+    "static_index.json",
 }
 
 
@@ -540,7 +543,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
 
     'TERM_WIDTH':               {'default': lambda c: lambda: shutil.get_terminal_size((100, 10)).columns},
     'USER':                     {'default': lambda c: get_system_user()},
-    'ANSI':                     {'default': lambda c: DEFAULT_CLI_COLORS if c['USE_COLOR'] else {k: '' for k in DEFAULT_CLI_COLORS.keys()}},
+    'ANSI':                     {'default': lambda c: DEFAULT_CLI_COLORS if c['USE_COLOR'] else AttrDict({k: '' for k in DEFAULT_CLI_COLORS.keys()})},
 
     'PACKAGE_DIR':              {'default': lambda c: Path(__file__).resolve().parent},
     'TEMPLATES_DIR':            {'default': lambda c: c['PACKAGE_DIR'] / TEMPLATES_DIR_NAME},
@@ -560,6 +563,7 @@ DYNAMIC_CONFIG_SCHEMA: ConfigDefaultDict = {
     'DIR_OUTPUT_PERMISSIONS':   {'default': lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5')},  # exec is always needed to list directories
 
     'ARCHIVEBOX_BINARY':        {'default': lambda c: sys.argv[0] or bin_path('archivebox')},
+    'NODE_BIN_PATH':            {'default': lambda c: str((Path(c["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))},
 
     'VERSION':                  {'default': lambda c: get_version(c).split('+', 1)[0]},     # remove +editable from user-displayed version string
     'COMMIT_HASH':              {'default': lambda c: get_commit_hash(c)},                  # short git commit hash of codebase HEAD commit
@@ -1269,8 +1273,7 @@ os.environ["TZ"] = TIMEZONE                                                  # n
 os.umask(0o777 - int(DIR_OUTPUT_PERMISSIONS, base=8))                        # noqa: F821
 
 # add ./node_modules/.bin to $PATH so we can use node scripts in extractors
-NODE_BIN_PATH = str((Path(CONFIG["OUTPUT_DIR"]).absolute() / 'node_modules' / '.bin'))
-sys.path.append(NODE_BIN_PATH)
+sys.path.append(CONFIG.NODE_BIN_PATH)
 
 # OPTIONAL: also look around the host system for node modules to use
 #   avoid enabling this unless absolutely needed,

archivebox/core/admin.py

@@ -1,25 +1,21 @@
 __package__ = 'archivebox.core'
 
 import os
-import json
 
-from io import StringIO
+import threading
 from pathlib import Path
-from contextlib import redirect_stdout
-from datetime import datetime, timezone
-from typing import Dict, Any
 
-from django.contrib import admin
-from django.db.models import Count, Q, Prefetch
+from django.contrib import admin, messages
 from django.urls import path, reverse, resolve
 from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
-from django.shortcuts import render, redirect
-from django.contrib.auth import get_user_model
+from django.contrib.auth import get_user_model, get_permission_codename
+from django.contrib.auth.admin import UserAdmin
 from django.core.paginator import Paginator
 from django.core.exceptions import ValidationError
+from django.template import Template, RequestContext
 from django.conf import settings
 from django import forms
 
@@ -28,14 +24,13 @@ from signal_webhooks.admin import WebhookAdmin
 from signal_webhooks.utils import get_webhook_model
 # from plugantic.admin import CustomPlugin
 
-from ..util import htmldecode, urldecode, ansi_to_html
+from ..util import htmldecode, urldecode
 
-from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
-from core.forms import AddLinkForm
+from core.models import Snapshot, ArchiveResult, Tag
 from core.mixins import SearchResultsAdminMixin
 from api.models import APIToken
-from abid_utils.models import get_or_create_system_user_pk
 from abid_utils.admin import ABIDModelAdmin
+from queues.tasks import bg_archive_links, bg_add
 
 from index.html import snapshot_icons
 from logging_util import printable_filesize
@@ -66,53 +61,87 @@ class ArchiveBoxAdmin(admin.AdminSite):
     site_title = 'Index'
     namespace = 'admin'
 
-    def get_urls(self):
-        return [
-            path('core/snapshot/add/', self.add_view, name='Add'),
-        ] + super().get_urls()
 
-    def add_view(self, request):
-        if not request.user.is_authenticated:
-            return redirect(f'/admin/login/?next={request.path}')
+class CustomUserAdmin(UserAdmin):
+    sort_fields = ['id', 'email', 'username', 'is_superuser', 'last_login', 'date_joined']
+    list_display = ['username', 'id', 'email', 'is_superuser', 'last_login', 'date_joined']
+    readonly_fields = ('snapshot_set', 'archiveresult_set', 'tag_set', 'apitoken_set', 'outboundwebhook_set')
+    fieldsets = [*UserAdmin.fieldsets, ('Data', {'fields': readonly_fields})]
 
-        request.current_app = self.name
-        context: Dict[str, Any] = {
-            **self.each_context(request),
-            'title': 'Add URLs',
-        }
+    @admin.display(description='Snapshots')
+    def snapshot_set(self, obj):
+        total_count = obj.snapshot_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> {}',
+                snap.pk,
+                snap.abid,
+                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
+                snap.url[:64],
+            )
+            for snap in obj.snapshot_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
-        if request.method == 'GET':
-            context['form'] = AddLinkForm()
+    @admin.display(description='Archive Result Logs')
+    def archiveresult_set(self, obj):
+        total_count = obj.archiveresult_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/core/archiveresult/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> <b>📄 {}</b> {}',
+                result.pk,
+                result.abid,
+                result.snapshot.downloaded_at.strftime('%Y-%m-%d %H:%M') if result.snapshot.downloaded_at else 'pending...',
+                result.extractor,
+                result.snapshot.url[:64],
+            )
+            for result in obj.archiveresult_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/core/archiveresult/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
-        elif request.method == 'POST':
-            form = AddLinkForm(request.POST)
-            if form.is_valid():
-                url = form.cleaned_data["url"]
-                print(f'[+] Adding URL: {url}')
-                depth = 0 if form.cleaned_data["depth"] == "0" else 1
-                input_kwargs = {
-                    "urls": url,
-                    "depth": depth,
-                    "update_all": False,
-                    "out_dir": CONFIG.OUTPUT_DIR,
-                }
-                add_stdout = StringIO()
-                with redirect_stdout(add_stdout):
-                   add(**input_kwargs)
-                print(add_stdout.getvalue())
+    @admin.display(description='Tags')
+    def tag_set(self, obj):
+        total_count = obj.tag_set.count()
+        return mark_safe(', '.join(
+            format_html(
+                '<code><a href="/admin/core/tag/{}/change"><b>{}</b></a></code>',
+                tag.pk,
+                tag.name,
+            )
+            for tag in obj.tag_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/core/tag/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
+
+    @admin.display(description='API Tokens')
+    def apitoken_set(self, obj):
+        total_count = obj.apitoken_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/api/apitoken/{}/change"><b>[{}]</b></a></code> {} (expires {})',
+                apitoken.pk,
+                apitoken.abid,
+                apitoken.token_redacted[:64],
+                apitoken.expires,
+            )
+            for apitoken in obj.apitoken_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/api/apitoken/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
+
+    @admin.display(description='API Outbound Webhooks')
+    def outboundwebhook_set(self, obj):
+        total_count = obj.outboundwebhook_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/api/outboundwebhook/{}/change"><b>[{}]</b></a></code> {} -> {}',
+                outboundwebhook.pk,
+                outboundwebhook.abid,
+                outboundwebhook.referenced_model,
+                outboundwebhook.endpoint,
+            )
+            for outboundwebhook in obj.outboundwebhook_set.order_by('-modified_at')[:10]
+        ) + f'<br/><a href="/admin/api/outboundwebhook/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
-                context.update({
-                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
-                    "form": AddLinkForm(),
-                })
-            else:
-                context["form"] = form
 
-        return render(template_name='add.html', request=request, context=context)
 
 
 archivebox_admin = ArchiveBoxAdmin()
-archivebox_admin.register(get_user_model())
+archivebox_admin.register(get_user_model(), CustomUserAdmin)
 archivebox_admin.disable_action('delete_selected')
 
 # archivebox_admin.register(CustomPlugin)
@@ -129,6 +158,28 @@ archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_ad
 archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
 
 
+from huey_monitor.apps import HueyMonitorConfig
+HueyMonitorConfig.verbose_name = 'Background Workers'
+
+from huey_monitor.admin import TaskModel, TaskModelAdmin, SignalInfoModel, SignalInfoModelAdmin
+archivebox_admin.register(SignalInfoModel, SignalInfoModelAdmin)
+
+
+class CustomTaskModelAdmin(TaskModelAdmin):
+    actions = ["delete_selected"]
+
+    def has_delete_permission(self, request, obj=None):
+        codename = get_permission_codename("delete", self.opts)
+        return request.user.has_perm("%s.%s" % (self.opts.app_label, codename))
+
+
+archivebox_admin.register(TaskModel, CustomTaskModelAdmin)
+
+def result_url(result: TaskModel) -> str:
+    url = reverse("admin:huey_monitor_taskmodel_change", args=[str(result.id)])
+    return format_html('<a href="{url}" class="fade-in-progress-url">See progress...</a>'.format(url=url))
+
+
 class AccelleratedPaginator(Paginator):
     """
     Accellerated Pagniator ignores DISTINCT when counting total number of rows.
@@ -138,7 +189,7 @@ class AccelleratedPaginator(Paginator):
 
     @cached_property
     def count(self):
-        if self.object_list._has_filters():
+        if self.object_list._has_filters():                             # type: ignore
             # fallback to normal count method on filtered queryset
             return super().count
         else:
@@ -162,8 +213,8 @@ class ArchiveResultInline(admin.TabularInline):
     # fk_name = 'snapshot'
     extra = 0
     sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
-    readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
-    fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
+    readonly_fields = ('id', 'result_id', 'completed', 'command', 'version')
+    fields = ('start_ts', 'end_ts', *readonly_fields, 'extractor', 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
     # exclude = ('id',)
     ordering = ('end_ts',)
     show_change_link = True
@@ -172,7 +223,10 @@ class ArchiveResultInline(admin.TabularInline):
 
     def get_parent_object_from_request(self, request):
         resolved = resolve(request.path_info)
-        return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
+        try:
+            return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
+        except (self.parent_model.DoesNotExist, ValidationError):
+            return self.parent_model.objects.get(pk=self.parent_model.id_from_abid(resolved.kwargs['object_id']))
 
     @admin.display(
         description='Completed',
@@ -195,7 +249,7 @@ class ArchiveResultInline(admin.TabularInline):
         snapshot = self.get_parent_object_from_request(request)
 
         # import ipdb; ipdb.set_trace()
-        formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
+        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
         
         # default values for new entries
         formset.form.base_fields['status'].initial = 'succeeded'
@@ -251,6 +305,7 @@ from django.contrib.admin.widgets import FilteredSelectMultiple
 
 class SnapshotActionForm(ActionForm):
     tags = forms.ModelMultipleChoiceField(
+        label='Edit tags',
         queryset=Tag.objects.all(),
         required=False,
         widget=FilteredSelectMultiple(
@@ -267,48 +322,18 @@ class SnapshotActionForm(ActionForm):
     # )
 
 
-def get_abid_info(self, obj):
-    return format_html(
-        # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-        '''
-        <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-        <br/><hr/>
-        <div style="opacity: 0.8">
-        &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        <hr/>
-        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; {}: <code style="user-select: all">{}</code><br/>
-        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; "><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <span style="display:inline-block; vertical-align: -4px; width: 290px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span>
-        &nbsp; SALT: &nbsp; <code style="font-size: 10px;"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code><br/>
-        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}: <code style="user-select: all">{}</code><br/>
-        &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  {}: <code style="user-select: all">{}</code>
-        <br/><hr/>
-        &nbsp; &nbsp; <small style="opacity: 0.5">.old_id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        </div>
-        ''',
-        obj.api_url, obj.api_url, obj.api_docs_url,
-        str(obj.abid),
-        str(obj.ABID.uuid),
-        str(obj.id),
-        obj.ABID.ts, str(obj.ABID.uuid)[0:14], obj.abid_ts_src, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
-        obj.ABID.uri, str(obj.ABID.uuid)[14:26], obj.abid_uri_src, str(obj.abid_values['uri']),
-        obj.ABID.uri_salt,
-        obj.ABID.subtype, str(obj.ABID.uuid)[26:28], obj.abid_subtype_src, str(obj.abid_values['subtype']),
-        obj.ABID.rand, str(obj.ABID.uuid)[28:36], obj.abid_rand_src, str(obj.abid_values['rand'])[-7:],
-        str(getattr(obj, 'old_id', '')),
-    )
+
 
 
 @admin.register(Snapshot, site=archivebox_admin)
 class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
-    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
-    sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
-    search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
-    list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
-    fields = ('url', 'created_by', 'title', *readonly_fields)
-    ordering = ['-added']
+    list_display = ('created_at', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'created_at')
+    readonly_fields = ('admin_actions', 'status_info', 'tags_str', 'imported_timestamp', 'created_at', 'modified_at', 'downloaded_at', 'abid_info', 'link_dir')
+    search_fields = ('id', 'url', 'abid', 'timestamp', 'title', 'tags__name')
+    list_filter = ('created_at', 'downloaded_at', 'archiveresult__status', 'created_by', 'tags__name')
+    fields = ('url', 'title', 'created_by', 'bookmarked_at', *readonly_fields)
+    ordering = ['-created_at']
     actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
     inlines = [TagInline, ArchiveResultInline]
     list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
@@ -320,6 +345,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     show_full_result_count = False
 
     def changelist_view(self, request, extra_context=None):
+        self.request = request
         extra_context = extra_context or {}
         try:
             return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
@@ -327,34 +353,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
             self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
             return super().changelist_view(request, GLOBAL_CONTEXT)
 
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        snapshot = None
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-        
-        try:
-            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
-        except (Snapshot.DoesNotExist, ValidationError):
-            pass
-
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-
-        if snapshot:
-            object_id = str(snapshot.id)
-
-        return super().change_view(
-            request,
-            object_id,
-            form_url,
-            extra_context=extra_context,
-        )
 
     def get_urls(self):
         urls = super().get_urls()
@@ -370,8 +368,20 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     #     self.request = request
     #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
 
-    def tag_list(self, obj):
-        return ', '.join(tag.name for tag in obj.tags.all())
+    @admin.action(
+        description="Imported Timestamp"
+    )
+    def imported_timestamp(self, obj):
+        context = RequestContext(self.request, {
+            'bookmarked_date': obj.bookmarked,
+            'timestamp': obj.timestamp,
+        })
+
+        html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
+        return mark_safe(html.render(context))
+    
+        # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
+        # return f'{pretty_time} ({obj.timestamp})'
 
     # TODO: figure out a different way to do this, you cant nest forms so this doenst work
     # def action(self, obj):
@@ -429,12 +439,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
             obj.extension or '-',
         )
 
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            return str(e)
-
     @admin.display(
         description='Title',
         ordering='title',
@@ -464,6 +468,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         # ordering='archiveresult_count',
     )
     def files(self, obj):
+        # return '-'
         return snapshot_icons(obj)
 
 
@@ -526,46 +531,73 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     #     return super().changelist_view(request, extra_context=None)
 
     @admin.action(
-        description="Pull"
-    )
-    def update_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], out_dir=CONFIG.OUTPUT_DIR)
-
-    @admin.action(
-        description="⬇️ Title"
+        description="ℹ️ Get Title"
     )
     def update_titles(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, methods=('title','favicon'), out_dir=CONFIG.OUTPUT_DIR)
+        links = [snapshot.as_link() for snapshot in queryset]
+        if len(links) < 3:
+            # run syncronously if there are only 1 or 2 links
+            archive_links(links, overwrite=True, methods=('title','favicon'), out_dir=CONFIG.OUTPUT_DIR)
+            messages.success(request, f"Title and favicon have been fetched and saved for {len(links)} URLs.")
+        else:
+            # otherwise run in a background worker
+            result = bg_archive_links((links,), kwargs={"overwrite": True, "methods": ["title", "favicon"], "out_dir": CONFIG.OUTPUT_DIR})
+            messages.success(
+                request,
+                mark_safe(f"Title and favicon are updating in the background for {len(links)} URLs. {result_url(result)}"),
+            )
 
     @admin.action(
-        description="Re-Snapshot"
+        description="⬇️ Get Missing"
+    )
+    def update_snapshots(self, request, queryset):
+        links = [snapshot.as_link() for snapshot in queryset]
+
+        result = bg_archive_links((links,), kwargs={"overwrite": False, "out_dir": CONFIG.OUTPUT_DIR})
+
+        messages.success(
+            request,
+            mark_safe(f"Re-trying any previously failed methods for {len(links)} URLs in the background. {result_url(result)}"),
+        )
+
+
+    @admin.action(
+        description="🆕 Archive Again"
     )
     def resnapshot_snapshot(self, request, queryset):
         for snapshot in queryset:
-            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
+            timestamp = timezone.now().isoformat('T', 'seconds')
             new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
-            add(new_url, tag=snapshot.tags_str())
+
+            result = bg_add({'urls': new_url, 'tag': snapshot.tags_str()})
+
+        messages.success(
+            request,
+            mark_safe(f"Creating new fresh snapshots for {queryset.count()} URLs in the background. {result_url(result)}"),
+        )
 
     @admin.action(
-        description="Reset"
+        description="🔄 Redo"
     )
     def overwrite_snapshots(self, request, queryset):
-        archive_links([
-            snapshot.as_link()
-            for snapshot in queryset
-        ], overwrite=True, out_dir=CONFIG.OUTPUT_DIR)
+        links = [snapshot.as_link() for snapshot in queryset]
+
+        result = bg_archive_links((links,), kwargs={"overwrite": True, "out_dir": CONFIG.OUTPUT_DIR})
+
+        messages.success(
+            request,
+            mark_safe(f"Clearing all previous results and re-downloading {len(links)} URLs in the background. {result_url(result)}"),
+        )
 
     @admin.action(
-        description="Delete"
+        description="☠️ Delete"
     )
     def delete_snapshots(self, request, queryset):
         remove(snapshots=queryset, yes=True, delete=True, out_dir=CONFIG.OUTPUT_DIR)
+        messages.success(
+            request,
+            mark_safe(f"Succesfully deleted {queryset.count()} Snapshots. Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed."),
+        )
 
 
     @admin.action(
@@ -576,6 +608,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         print('[+] Adding tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.add(*tags)
+        messages.success(
+            request,
+            f"Added {len(tags)} tags to {queryset.count()} Snapshots.",
+        )
 
 
     @admin.action(
@@ -586,10 +622,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
         print('[-] Removing tags', tags, 'to Snapshots', queryset)
         for obj in queryset:
             obj.tags.remove(*tags)
-
-
-        
-
+        messages.success(
+            request,
+            f"Removed {len(tags)} tags from {queryset.count()} Snapshots.",
+        )
 
 
 # @admin.register(SnapshotTag, site=archivebox_admin)
@@ -601,27 +637,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
 #     actions = ['delete_selected']
 #     ordering = ['-id']
 
-#     def API(self, obj):
-#         return get_abid_info(self, obj)
 
 
 @admin.register(Tag, site=archivebox_admin)
 class TagAdmin(ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
-    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
-    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
+    list_display = ('created_at', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
+    list_filter = ('created_at', 'created_by')
+    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created_at')
+    readonly_fields = ('slug', 'abid', 'created_at', 'modified_at', 'abid_info', 'snapshots')
     search_fields = ('abid', 'name', 'slug')
     fields = ('name', 'created_by', *readonly_fields)
     actions = ['delete_selected']
-    ordering = ['-created']
+    ordering = ['-created_at']
 
     paginator = AccelleratedPaginator
 
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            return str(e)
 
     def num_snapshots(self, tag):
         return format_html(
@@ -636,19 +666,19 @@ class TagAdmin(ABIDModelAdmin):
             format_html(
                 '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
                 snap.pk,
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
                 snap.url[:64],
             )
-            for snap in tag.snapshot_set.order_by('-updated')[:10]
-        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
+            for snap in tag.snapshot_set.order_by('-downloaded_at')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">{total_count} total snapshots...<a>'))
 
 
 @admin.register(ArchiveResult, site=archivebox_admin)
 class ArchiveResultAdmin(ABIDModelAdmin):
     list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
-    search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created_at', 'modified_at', 'abid_info', 'output_summary')
+    search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
     fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
     autocomplete_fields = ['snapshot']
 
@@ -657,6 +687,17 @@ class ArchiveResultAdmin(ABIDModelAdmin):
     list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
     
     paginator = AccelleratedPaginator
+    save_on_top = True
+    
+    actions = ['delete_selected']
+    
+    class Meta:
+        verbose_name = 'Archive Result'
+        verbose_name_plural = 'Archive Results'
+
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        return super().change_view(request, object_id, form_url, extra_context)
 
     @admin.display(
         description='Snapshot Info'
@@ -666,16 +707,10 @@ class ArchiveResultAdmin(ABIDModelAdmin):
             '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
             result.snapshot.timestamp,
             result.snapshot.abid,
-            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
+            result.snapshot.bookmarked_at.strftime('%Y-%m-%d %H:%M'),
             result.snapshot.url[:128],
         )
 
-    def API(self, obj):
-        try:
-            return get_abid_info(self, obj)
-        except Exception as e:
-            raise e
-            return str(e)
 
     @admin.display(
         description='Snapshot Tags'
@@ -700,20 +735,13 @@ class ArchiveResultAdmin(ABIDModelAdmin):
     def output_summary(self, result):
         snapshot_dir = Path(CONFIG.OUTPUT_DIR) / str(result.pwd).split('data/', 1)[-1]
         output_str = format_html(
-            '<pre style="display: block; max-width: 75vw; white-space: pre-line;">{}</pre><br/>',
+            '<pre style="display: inline-block">{}</pre><br/>',
             result.output,
         )
         output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
         path_from_output_str = (snapshot_dir / result.output)
         output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
-        
-        path_exists = False
-        try:
-            path_exists = path_from_output_str.exists()
-        except OSError:
-            pass
-
-        if path_exists:
+        if path_from_output_str.exists():
             root_dir = str(path_from_output_str)
         else:
             root_dir = str(snapshot_dir)
@@ -737,18 +765,18 @@ class ArchiveResultAdmin(ABIDModelAdmin):
 
 @admin.register(APIToken, site=archivebox_admin)
 class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created', 'created_by', 'expires')
-    readonly_fields = ('abid', 'created')
+    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
     search_fields = ('id', 'abid', 'created_by__username', 'token')
     fields = ('created_by', 'token', 'expires', *readonly_fields)
 
     list_filter = ('created_by',)
-    ordering = ['-created']
+    ordering = ['-created_at']
     list_per_page = 100
 
 @admin.register(get_webhook_model(), site=archivebox_admin)
 class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('abid', 'created', *WebhookAdmin.readonly_fields)
+    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
+    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)

archivebox/core/asgi.py

@@ -0,0 +1,28 @@
+"""
+WSGI config for archivebox project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from archivebox.config import setup_django
+
+setup_django(in_memory_db=False, check_db=True)
+
+
+from django.core.asgi import get_asgi_application
+from channels.routing import ProtocolTypeRouter
+
+
+django_asgi_app = get_asgi_application()
+
+application = ProtocolTypeRouter(
+    {
+        "http": django_asgi_app,
+        # Just HTTP for now. (We can add other protocols later.)
+    }
+)

archivebox/core/migrations/0027_update_snapshot_ids.py

@@ -2,7 +2,6 @@
 
 from django.db import migrations
 
-from django.db import migrations
 from datetime import datetime
 from abid_utils.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
 
@@ -52,7 +51,7 @@ def update_snapshot_ids(apps, schema_editor):
     Snapshot = apps.get_model("core", "Snapshot")
     num_total = Snapshot.objects.all().count()
     print(f'   Updating {num_total} Snapshot.id, Snapshot.uuid values in place...')
-    for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator()):
+    for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator(chunk_size=500)):
         assert snapshot.abid
         snapshot.abid_prefix = 'snp_'
         snapshot.abid_ts_src = 'self.added'
@@ -72,7 +71,7 @@ def update_archiveresult_ids(apps, schema_editor):
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     num_total = ArchiveResult.objects.all().count()
     print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
-    for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator()):
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator(chunk_size=500)):
         assert result.abid
         result.abid_prefix = 'res_'
         result.snapshot = Snapshot.objects.get(pk=result.snapshot_id)

archivebox/core/migrations/0032_alter_archiveresult_id.py

@@ -1,9 +1,13 @@
 # Generated by Django 5.0.6 on 2024-08-18 05:20
 
 import core.models
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -14,6 +18,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
+            field=models.BigIntegerField(default=rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
         ),
     ]

archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py

@@ -2,16 +2,21 @@
 
 import core.models
 import uuid
+import random
 from django.db import migrations, models
 
 from abid_utils.abid import ABID
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
+
 def update_archiveresult_ids(apps, schema_editor):
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     num_total = ArchiveResult.objects.all().count()
     print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
-    for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()):
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator(chunk_size=500)):
         assert result.abid
         result.uuid = ABID.parse(result.abid).uuid
         result.save(update_fields=["uuid"])
@@ -30,7 +35,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='ID'),
         ),
         migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
         migrations.AlterField(

archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py

@@ -2,9 +2,13 @@
 
 import core.models
 import uuid
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -20,6 +24,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0051_snapshottag_snapshot_alter_snapshottag_snapshot_old.py

@@ -9,7 +9,7 @@ def update_snapshottag_ids(apps, schema_editor):
     SnapshotTag = apps.get_model("core", "SnapshotTag")
     num_total = SnapshotTag.objects.all().count()
     print(f'   Updating {num_total} SnapshotTag.snapshot_id values in place... (may take an hour or longer for large collections...)')
-    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator()):
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator(chunk_size=500)):
         assert snapshottag.snapshot_old_id
         snapshot = Snapshot.objects.get(old_id=snapshottag.snapshot_old_id)
         snapshottag.snapshot_id = snapshot.id

archivebox/core/migrations/0058_alter_tag_old_id.py

@@ -1,9 +1,12 @@
 # Generated by Django 5.0.6 on 2024-08-20 03:30
 
-import core.models
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -14,6 +17,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='tag',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0059_tag_id.py

@@ -1,8 +1,9 @@
 # Generated by Django 5.0.6 on 2024-08-20 03:33
 
+from datetime import datetime
 from django.db import migrations, models
-from abid_utils.models import ABID, abid_from_values
-
+from abid_utils.abid import abid_from_values
+from abid_utils.models import ABID
 
 def calculate_abid(self):
     """
@@ -49,7 +50,7 @@ def update_archiveresult_ids(apps, schema_editor):
     Tag = apps.get_model("core", "Tag")
     num_total = Tag.objects.all().count()
     print(f'   Updating {num_total} Tag.id, ArchiveResult.uuid values in place...')
-    for idx, tag in enumerate(Tag.objects.all().iterator()):
+    for idx, tag in enumerate(Tag.objects.all().iterator(chunk_size=500)):
         if not tag.slug:
             tag.slug = tag.name.lower().replace(' ', '_')
         if not tag.name:

archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py

@@ -9,7 +9,7 @@ def update_snapshottag_ids(apps, schema_editor):
     SnapshotTag = apps.get_model("core", "SnapshotTag")
     num_total = SnapshotTag.objects.all().count()
     print(f'   Updating {num_total} SnapshotTag.tag_id values in place... (may take an hour or longer for large collections...)')
-    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator()):
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator(chunk_size=500)):
         assert snapshottag.old_tag_id
         tag = Tag.objects.get(old_id=snapshottag.old_tag_id)
         snapshottag.tag_id = tag.id

archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py

@@ -3,8 +3,11 @@
 import core.models
 import django.db.models.deletion
 import uuid
+import random
 from django.db import migrations, models
 
+def rand_int_id():
+    return random.getrandbits(32)
 
 class Migration(migrations.Migration):
 
@@ -26,6 +29,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='tag',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0069_alter_archiveresult_created_alter_snapshot_added_and_more.py

@@ -0,0 +1,35 @@
+# Generated by Django 5.1 on 2024-08-28 09:40
+
+import abid_utils.models
+import django.utils.timezone
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0068_alter_archiveresult_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='added',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+    ]

archivebox/core/migrations/0070_alter_archiveresult_created_by_alter_snapshot_added_and_more.py

@@ -0,0 +1,52 @@
+# Generated by Django 5.1 on 2024-09-04 09:00
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0069_alter_archiveresult_created_alter_snapshot_added_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='added',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='id',
+            field=models.UUIDField(default=None, primary_key=True, serialize=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='old_id',
+            field=models.UUIDField(default=None, editable=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+    ]

archivebox/core/migrations/0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more.py

@@ -0,0 +1,65 @@
+# Generated by Django 5.1 on 2024-09-04 23:23
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0070_alter_archiveresult_created_by_alter_snapshot_added_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='archiveresult',
+            name='old_id',
+        ),
+        migrations.RemoveField(
+            model_name='snapshot',
+            name='old_id',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='old_id',
+        ),
+        
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='tag_set', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+    ]

archivebox/core/migrations/0072_rename_added_snapshot_bookmarked_at_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1 on 2024-09-05 00:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='added',
+            new_name='bookmarked_at',
+        ),
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='updated',
+            new_name='downloaded_at',
+        ),
+    ]

archivebox/core/migrations/0073_rename_created_archiveresult_created_at_and_more.py

@@ -0,0 +1,43 @@
+# Generated by Django 5.1 on 2024-09-05 00:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0072_rename_added_snapshot_bookmarked_at_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='tag',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='tag',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+    ]

archivebox/core/migrations/0074_alter_snapshot_downloaded_at.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1 on 2024-09-05 01:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0073_rename_created_archiveresult_created_at_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='downloaded_at',
+            field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
+        ),
+    ]

archivebox/core/models.py

@@ -1,14 +1,11 @@
 __package__ = 'archivebox.core'
 
 
-from typing import Optional, List, Dict, Iterable
+from typing import Optional, Dict, Iterable
 from django_stubs_ext.db.models import TypedModelMeta
 
 import json
-import random
 
-import uuid
-from uuid import uuid4
 from pathlib import Path
 
 from django.db import models
@@ -17,9 +14,10 @@ from django.utils.text import slugify
 from django.core.cache import cache
 from django.urls import reverse, reverse_lazy
 from django.db.models import Case, When, Value, IntegerField
+from django.contrib import admin
 from django.conf import settings
 
-from abid_utils.models import ABIDModel, ABIDField
+from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
 
 from ..system import get_dir_size
 from ..util import parse_date, base_url
@@ -28,13 +26,10 @@ from ..index.html import snapshot_icons
 from ..extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE, EXTRACTORS
 
 
-def rand_int_id():
-    return random.getrandbits(32)
-
 
 # class BaseModel(models.Model):
 #     # TODO: migrate all models to a shared base class with all our standard fields and helpers:
-#     #       ulid/created/modified/owner/is_deleted/as_json/from_json/etc.
+#     #       ulid/created_at/modified_at/created_by/is_deleted/as_json/from_json/etc.
 #     #
 #     # id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
 #     # ulid = models.CharField(max_length=26, null=True, blank=True, db_index=True, unique=True)
@@ -50,16 +45,18 @@ class Tag(ABIDModel):
     Based on django-taggit model + ABID base.
     """
     abid_prefix = 'tag_'
-    abid_ts_src = 'self.created'          # TODO: add created/modified time
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.slug'
     abid_subtype_src = '"03"'
-    abid_rand_src = 'self.old_id'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    old_id = models.BigIntegerField(unique=True, default=rand_int_id, serialize=False, verbose_name='Old ID')  # legacy PK
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='tag_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     name = models.CharField(unique=True, blank=False, max_length=100)
     slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
@@ -74,10 +71,6 @@ class Tag(ABIDModel):
     def __str__(self):
         return self.name
 
-    # @property
-    # def old_id(self):
-    #     return self.id
-
     def slugify(self, tag, i=None):
         slug = slugify(tag)
         if i is not None:
@@ -109,11 +102,11 @@ class Tag(ABIDModel):
     @property
     def api_url(self) -> str:
         # /api/v1/core/snapshot/{uulid}
-        return reverse_lazy('api-1:get_tag', args=[self.abid])
+        return reverse_lazy('api-1:get_tag', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
 
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
 
 class SnapshotTag(models.Model):
     id = models.AutoField(primary_key=True)
@@ -133,34 +126,40 @@ class SnapshotManager(models.Manager):
 
 class Snapshot(ABIDModel):
     abid_prefix = 'snp_'
-    abid_ts_src = 'self.added'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.url'
     abid_subtype_src = '"01"'
-    abid_rand_src = 'self.old_id'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
-    old_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)  # legacy pk
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)  # loaded from self._init_timestamp
+    modified_at = models.DateTimeField(auto_now=True)
+
+    # legacy ts fields
+    bookmarked_at = AutoDateTimeField(default=None, null=False, editable=True, db_index=True)
+    downloaded_at = models.DateTimeField(default=None, null=True, editable=False, db_index=True, blank=True)
+
     url = models.URLField(unique=True, db_index=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
-
-    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
-    
     tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
+    title = models.CharField(max_length=512, null=True, blank=True, db_index=True)    
 
-    added = models.DateTimeField(auto_now_add=True, db_index=True)
-    updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
-
-    keys = ('url', 'timestamp', 'title', 'tags', 'updated')
+    keys = ('url', 'timestamp', 'title', 'tags', 'downloaded_at')
 
     archiveresult_set: models.Manager['ArchiveResult']
 
     objects = SnapshotManager()
 
-    @property
-    def uuid(self):
-        return self.id
+    def save(self, *args, **kwargs):
+        if not self.bookmarked_at:
+            self.bookmarked_at = self.created_at or self._init_timestamp
+        
+        super().save(*args, **kwargs)
+
 
     def __repr__(self) -> str:
         title = (self.title_stripped or '-')[:64]
@@ -170,13 +169,6 @@ class Snapshot(ABIDModel):
         title = (self.title_stripped or '-')[:64]
         return f'[{self.timestamp}] {self.url[:64]} ({title})'
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        try:
-            assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'Snapshot.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
-        except AssertionError as e:
-            print(e)
-
     @classmethod
     def from_json(cls, info: dict):
         info = {k: v for k, v in info.items() if k in cls.keys}
@@ -196,9 +188,10 @@ class Snapshot(ABIDModel):
         from ..index import load_link_details
         return load_link_details(self.as_link())
 
+    @admin.display(description='Tags')
     def tags_str(self, nocache=True) -> str | None:
         calc_tags_str = lambda: ','.join(sorted(tag.name for tag in self.tags.all()))
-        cache_key = f'{self.pk}-{(self.updated or self.added).timestamp()}-tags'
+        cache_key = f'{self.pk}-{(self.downloaded_at or self.bookmarked_at).timestamp()}-tags'
         
         if hasattr(self, '_prefetched_objects_cache') and 'tags' in self._prefetched_objects_cache:
             # tags are pre-fetched already, use them directly (best because db is always freshest)
@@ -217,11 +210,14 @@ class Snapshot(ABIDModel):
     @property
     def api_url(self) -> str:
         # /api/v1/core/snapshot/{uulid}
-        return reverse_lazy('api-1:get_snapshot', args=[self.abid])
+        return reverse_lazy('api-1:get_snapshot', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
     
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
+    
+    def get_absolute_url(self):
+        return f'/{self.archive_path}'
     
     @cached_property
     def title_stripped(self) -> str:
@@ -266,7 +262,7 @@ class Snapshot(ABIDModel):
 
     @cached_property
     def archive_size(self):
-        cache_key = f'{str(self.pk)[:12]}-{(self.updated or self.added).timestamp()}-size'
+        cache_key = f'{str(self.pk)[:12]}-{(self.downloaded_at or self.bookmarked_at).timestamp()}-size'
 
         def calc_dir_size():
             try:
@@ -285,7 +281,7 @@ class Snapshot(ABIDModel):
                     for result in self.archiveresult_set.all()
                     if result.extractor == 'screenshot' and result.status =='succeeded' and result.output
                 ),
-                key=lambda result: result.created,
+                key=lambda result: result.created_at,
             ) or [None])[-1]
         else:
             result = self.archiveresult_set.filter(
@@ -318,7 +314,7 @@ class Snapshot(ABIDModel):
     def latest_title(self) -> Optional[str]:
         if self.title:
             return self.title   # whoopdedoo that was easy
-        
+
         # check if ArchiveResult set has already been prefetched, if so use it instead of fetching it from db again
         if hasattr(self, '_prefetched_objects_cache') and 'archiveresult_set' in self._prefetched_objects_cache:
             try:
@@ -332,7 +328,7 @@ class Snapshot(ABIDModel):
                 ) or [None])[-1]
             except IndexError:
                 pass
-        
+
 
         try:
             # take longest successful title from ArchiveResult db history
@@ -370,7 +366,7 @@ class Snapshot(ABIDModel):
 
 
     # def get_storage_dir(self, create=True, symlink=True) -> Path:
-    #     date_str = self.added.strftime('%Y%m%d')
+    #     date_str = self.bookmarked_at.strftime('%Y%m%d')
     #     domain_str = domain(self.url)
     #     abs_storage_dir = Path(settings.CONFIG.ARCHIVE_DIR) / 'snapshots' / date_str / domain_str / str(self.ulid)
 
@@ -398,7 +394,7 @@ class Snapshot(ABIDModel):
 class ArchiveResultManager(models.Manager):
     def indexable(self, sorted: bool = True):
         """Return only ArchiveResults containing text suitable for full-text search (sorted in order of typical result quality)"""
-        
+
         INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
         qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS, status='succeeded')
 
@@ -418,10 +414,11 @@ class ArchiveResultManager(models.Manager):
 
 class ArchiveResult(ABIDModel):
     abid_prefix = 'res_'
-    abid_ts_src = 'self.snapshot.added'
+    abid_ts_src = 'self.snapshot.created_at'
     abid_uri_src = 'self.snapshot.url'
     abid_subtype_src = 'self.extractor'
-    abid_rand_src = 'self.old_id'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
     EXTRACTOR_CHOICES = (
         ('htmltotext', 'htmltotext'),
@@ -445,11 +442,13 @@ class ArchiveResult(ABIDModel):
         ("skipped", "skipped")
     ]
 
-    old_id = models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID')
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True, verbose_name='ID')
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='archiveresult_set')
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')
 
     extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32)
@@ -466,22 +465,12 @@ class ArchiveResult(ABIDModel):
     class Meta(TypedModelMeta):
         verbose_name = 'Archive Result'
         verbose_name_plural = 'Archive Results Log'
-        
+
 
     def __str__(self):
+        # return f'[{self.abid}] 📅 {self.start_ts.strftime("%Y-%m-%d %H:%M")} 📄 {self.extractor} {self.snapshot.url}'
         return self.extractor
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        try:
-            assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'ArchiveResult.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
-        except AssertionError as e:
-            print(e)
-
-    @property
-    def uuid(self):
-        return self.id
-
     @cached_property
     def snapshot_dir(self):
         return Path(self.snapshot.link_dir)
@@ -489,11 +478,14 @@ class ArchiveResult(ABIDModel):
     @property
     def api_url(self) -> str:
         # /api/v1/core/archiveresult/{uulid}
-        return reverse_lazy('api-1:get_archiveresult', args=[self.abid])
-    
+        return reverse_lazy('api-1:get_archiveresult', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
+
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
+
+    def get_absolute_url(self):
+        return f'/{self.snapshot.archive_path}/{self.output_path()}'
 
     @property
     def extractor_module(self):
@@ -523,7 +515,7 @@ class ArchiveResult(ABIDModel):
 
 
     # def get_storage_dir(self, create=True, symlink=True):
-    #     date_str = self.snapshot.added.strftime('%Y%m%d')
+    #     date_str = self.snapshot.bookmarked_at.strftime('%Y%m%d')
     #     domain_str = domain(self.snapshot.url)
     #     abs_storage_dir = Path(settings.CONFIG.ARCHIVE_DIR) / 'results' / date_str / domain_str / self.extractor / str(self.ulid)
 

archivebox/core/serve_static.py

@@ -0,0 +1,169 @@
+import os
+import stat
+import posixpath
+import mimetypes
+from pathlib import Path
+
+from django.contrib.staticfiles import finders
+from django.views import static
+from django.http import StreamingHttpResponse, Http404, HttpResponse, HttpResponseNotModified
+from django.utils._os import safe_join
+from django.utils.http import http_date
+from django.utils.translation import gettext as _
+
+
+def serve_static_with_byterange_support(request, path, document_root=None, show_indexes=False):
+    """
+    Overrides Django's built-in django.views.static.serve function to support byte range requests.
+    This allows you to do things like seek into the middle of a huge mp4 or WACZ without downloading the whole file.
+    https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
+    """
+    assert document_root
+    path = posixpath.normpath(path).lstrip("/")
+    fullpath = Path(safe_join(document_root, path))
+    if fullpath.is_dir():
+        if show_indexes:
+            return static.directory_index(path, fullpath)
+        raise Http404(_("Directory indexes are not allowed here."))
+    if not fullpath.exists():
+        raise Http404(_("“%(path)s” does not exist") % {"path": fullpath})
+    
+    # Respect the If-Modified-Since header.
+    statobj = fullpath.stat()
+    if not static.was_modified_since(request.META.get("HTTP_IF_MODIFIED_SINCE"), statobj.st_mtime):
+        return HttpResponseNotModified()
+    
+    content_type, encoding = mimetypes.guess_type(str(fullpath))
+    content_type = content_type or "application/octet-stream"
+    
+    # setup resposne object
+    ranged_file = RangedFileReader(open(fullpath, "rb"))
+    response = StreamingHttpResponse(ranged_file, content_type=content_type)
+    response.headers["Last-Modified"] = http_date(statobj.st_mtime)
+
+    # handle byte-range requests by serving chunk of file    
+    if stat.S_ISREG(statobj.st_mode):
+        size = statobj.st_size
+        response["Content-Length"] = size
+        response["Accept-Ranges"] = "bytes"
+        response["X-Django-Ranges-Supported"] = "1"
+        # Respect the Range header.
+        if "HTTP_RANGE" in request.META:
+            try:
+                ranges = parse_range_header(request.META['HTTP_RANGE'], size)
+            except ValueError:
+                ranges = None
+            # only handle syntactically valid headers, that are simple (no
+            # multipart byteranges)
+            if ranges is not None and len(ranges) == 1:
+                start, stop = ranges[0]
+                if stop > size:
+                    # requested range not satisfiable
+                    return HttpResponse(status=416)
+                ranged_file.start = start
+                ranged_file.stop = stop
+                response["Content-Range"] = "bytes %d-%d/%d" % (start, stop - 1, size)
+                response["Content-Length"] = stop - start
+                response.status_code = 206
+    if encoding:
+        response.headers["Content-Encoding"] = encoding
+    return response
+
+
+def serve_static(request, path, **kwargs):
+    """
+    Serve static files below a given point in the directory structure or
+    from locations inferred from the staticfiles finders.
+
+    To use, put a URL pattern such as::
+
+        from django.contrib.staticfiles import views
+
+        path('<path:path>', views.serve)
+
+    in your URLconf.
+
+    It uses the django.views.static.serve() view to serve the found files.
+    """
+
+    normalized_path = posixpath.normpath(path).lstrip("/")
+    absolute_path = finders.find(normalized_path)
+    if not absolute_path:
+        if path.endswith("/") or path == "":
+            raise Http404("Directory indexes are not allowed here.")
+        raise Http404("'%s' could not be found" % path)
+    document_root, path = os.path.split(absolute_path)
+    return serve_static_with_byterange_support(request, path, document_root=document_root, **kwargs)
+
+
+def parse_range_header(header, resource_size):
+    """
+    Parses a range header into a list of two-tuples (start, stop) where `start`
+    is the starting byte of the range (inclusive) and `stop` is the ending byte
+    position of the range (exclusive).
+    Returns None if the value of the header is not syntatically valid.
+    https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
+    """
+    if not header or "=" not in header:
+        return None
+
+    ranges = []
+    units, range_ = header.split("=", 1)
+    units = units.strip().lower()
+
+    if units != "bytes":
+        return None
+
+    for val in range_.split(","):
+        val = val.strip()
+        if "-" not in val:
+            return None
+
+        if val.startswith("-"):
+            # suffix-byte-range-spec: this form specifies the last N bytes of an
+            # entity-body
+            start = resource_size + int(val)
+            if start < 0:
+                start = 0
+            stop = resource_size
+        else:
+            # byte-range-spec: first-byte-pos "-" [last-byte-pos]
+            start, stop = val.split("-", 1)
+            start = int(start)
+            # the +1 is here since we want the stopping point to be exclusive, whereas in
+            # the HTTP spec, the last-byte-pos is inclusive
+            stop = int(stop) + 1 if stop else resource_size
+            if start >= stop:
+                return None
+
+        ranges.append((start, stop))
+
+    return ranges
+
+
+class RangedFileReader:
+    """
+    Wraps a file like object with an iterator that runs over part (or all) of
+    the file defined by start and stop. Blocks of block_size will be returned
+    from the starting position, up to, but not including the stop point.
+    https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
+    """
+
+    block_size = 8192
+
+    def __init__(self, file_like, start=0, stop=float("inf"), block_size=None):
+        self.f = file_like
+        self.block_size = block_size or RangedFileReader.block_size
+        self.start = start
+        self.stop = stop
+
+    def __iter__(self):
+        self.f.seek(self.start)
+        position = self.start
+        while position < self.stop:
+            data = self.f.read(min(self.block_size, self.stop - position))
+            if not data:
+                break
+
+            yield data
+            position += self.block_size

archivebox/core/settings.py

@@ -6,9 +6,11 @@ import re
 import logging
 import inspect
 import tempfile
-from typing import Any, Dict
 
+from typing import Dict
 from pathlib import Path
+
+import django
 from django.utils.crypto import get_random_string
 
 from ..config import CONFIG
@@ -19,11 +21,44 @@ IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
 
+
+################################################################################
+### ArchiveBox Plugin Settings
+################################################################################
+
+BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins'  # /app/archivebox/builtin_plugins
+USERDATA_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins'     # /data/user_plugins
+
+def find_plugins_in_dir(plugins_dir, prefix: str) -> Dict[str, Path]:
+    return {
+        f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent
+        for plugin_entrypoint in sorted(plugins_dir.glob('*/apps.py'))
+    }
+
+INSTALLED_PLUGINS = {
+    **find_plugins_in_dir(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'),
+    **find_plugins_in_dir(USERDATA_PLUGINS_DIR, prefix='user_plugins'),
+}
+
+### Plugins Globals (filled by builtin_plugins.npm.apps.NpmPlugin.register() after Django startup)
+PLUGINS = AttrDict({})
+HOOKS = AttrDict({})
+
+# CONFIGS = AttrDict({})
+# BINPROVIDERS = AttrDict({})
+# BINARIES = AttrDict({})
+# EXTRACTORS = AttrDict({})
+# REPLAYERS = AttrDict({})
+# CHECKS = AttrDict({})
+# ADMINDATAVIEWS = AttrDict({})
+
+
 ################################################################################
 ### Django Core Settings
 ################################################################################
 
 WSGI_APPLICATION = 'core.wsgi.application'
+ASGI_APPLICATION = "core.asgi.application"
 ROOT_URLCONF = 'core.urls'
 
 LOGIN_URL = '/accounts/login/'
@@ -35,52 +70,41 @@ APPEND_SLASH = True
 DEBUG = CONFIG.DEBUG or ('--debug' in sys.argv)
 
 
-BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins'
-USER_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins'
-
-def find_plugins(plugins_dir, prefix: str) -> Dict[str, Any]:
-    plugins = {
-        f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent
-        for plugin_entrypoint in plugins_dir.glob('*/apps.py')
-    }
-    # print(f'Found {prefix} plugins:\n', '\n    '.join(plugins.keys()))
-    return plugins
-
-INSTALLED_PLUGINS = {
-    **find_plugins(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'),
-    **find_plugins(USER_PLUGINS_DIR, prefix='user_plugins'),
-}
-
-
 INSTALLED_APPS = [
+    'daphne',
+    
+    # Django default apps
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
-    'django_jsonform',
+
+    # 3rd-party apps from PyPI
+    'django_jsonform',           # handles rendering Pydantic models to Django HTML widgets/forms  https://github.com/bhch/django-jsonform
+    'signal_webhooks',           # handles REST API outbound webhooks                              https://github.com/MrThearMan/django-signal-webhooks
+    'django_object_actions',     # provides easy Django Admin action buttons on change views       https://github.com/crccheck/django-object-actions
     
-    'signal_webhooks',
-    'abid_utils',
-    'plugantic',
-    'core',
-    'api',
-    'pkgs',
+    # Our ArchiveBox-provided apps
+    'queues',                    # handles starting and managing background workers and processes
+    'abid_utils',                # handles ABID ID creation, handling, and models
+    'plugantic',                 # ArchiveBox plugin API definition + finding/registering/calling interface
+    'core',                      # core django model with Snapshot, ArchiveResult, etc.
+    'api',                       # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
 
-    *INSTALLED_PLUGINS.keys(),
+    # ArchiveBox plugins
+    *INSTALLED_PLUGINS.keys(),   # all plugin django-apps found in archivebox/builtin_plugins and data/user_plugins
 
-    'admin_data_views',
-    'django_extensions',
+    # 3rd-party apps from PyPI that need to be loaded last
+    'admin_data_views',          # handles rendering some convenient automatic read-only views of data in Django admin
+    'django_extensions',         # provides Django Debug Toolbar (and other non-debug helpers)
+    'django_huey',               # provides multi-queue support for django huey https://github.com/gaiacoop/django-huey
+    'bx_django_utils',           # needed for huey_monitor https://github.com/boxine/bx_django_utils
+    'huey_monitor',              # adds an admin UI for monitoring background huey tasks https://github.com/boxine/django-huey-monitor
 ]
 
 
-# For usage with https://www.jetadmin.io/integrations/django
-# INSTALLED_APPS += ['jet_django']
-# JET_PROJECT = 'archivebox'
-# JET_TOKEN = 'some-api-token-here'
-
-
 MIDDLEWARE = [
     'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
@@ -192,17 +216,28 @@ CACHE_DB_TABLE = 'django_cache'
 DATABASE_FILE = Path(CONFIG.OUTPUT_DIR) / CONFIG.SQL_INDEX_FILENAME
 DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(DATABASE_FILE))
 
+QUEUE_DATABASE_NAME = DATABASE_NAME.replace('index.sqlite3', 'queue.sqlite3')
+
 DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': DATABASE_NAME,
-        'OPTIONS': {
-            'timeout': 60,
-            'check_same_thread': False,
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": DATABASE_NAME,
+        "OPTIONS": {
+            "timeout": 60,
+            "check_same_thread": False,
         },
-        'TIME_ZONE': CONFIG.TIMEZONE,
+        "TIME_ZONE": CONFIG.TIMEZONE,
         # DB setup is sometimes modified at runtime by setup_django() in config.py
     },
+    "queue": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": QUEUE_DATABASE_NAME,
+        "OPTIONS": {
+            "timeout": 60,
+            "check_same_thread": False,
+        },
+        "TIME_ZONE": CONFIG.TIMEZONE,
+    },
     # 'cache': {
     #     'ENGINE': 'django.db.backends.sqlite3',
     #     'NAME': CACHE_DB_PATH,
@@ -219,6 +254,64 @@ MIGRATION_MODULES = {'signal_webhooks': None}
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 
+HUEY = {
+    "huey_class": "huey.SqliteHuey",
+    "filename": QUEUE_DATABASE_NAME,
+    "name": "system_tasks",
+    "results": True,
+    "store_none": True,
+    "immediate": False,
+    "utc": True,
+    "consumer": {
+        "workers": 1,
+        "worker_type": "thread",
+        "initial_delay": 0.1,  # Smallest polling interval, same as -d.
+        "backoff": 1.15,  # Exponential backoff using this rate, -b.
+        "max_delay": 10.0,  # Max possible polling interval, -m.
+        "scheduler_interval": 1,  # Check schedule every second, -s.
+        "periodic": True,  # Enable crontab feature.
+        "check_worker_health": True,  # Enable worker health checks.
+        "health_check_interval": 1,  # Check worker health every second.
+    },
+}
+
+# https://huey.readthedocs.io/en/latest/contrib.html#setting-things-up
+# https://github.com/gaiacoop/django-huey
+DJANGO_HUEY = {
+    "default": "system_tasks",
+    "queues": {
+        HUEY["name"]: HUEY.copy(),
+        # more registered here at plugin import-time by BaseQueue.register()
+    },
+}
+
+class HueyDBRouter:
+    """A router to store all the Huey Monitor models in the queue.sqlite3 database."""
+
+    route_app_labels = {"huey_monitor", "django_huey", "djhuey"}
+
+    def db_for_read(self, model, **hints):
+        if model._meta.app_label in self.route_app_labels:
+            return "queue"
+        return 'default'
+
+    def db_for_write(self, model, **hints):
+        if model._meta.app_label in self.route_app_labels:
+            return "queue"
+        return 'default'
+
+    def allow_relation(self, obj1, obj2, **hints):
+        if obj1._meta.app_label in self.route_app_labels or obj2._meta.app_label in self.route_app_labels:
+            return obj1._meta.app_label == obj2._meta.app_label
+        return None
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        if app_label in self.route_app_labels:
+            return db == "queue"
+        return db == "default"
+
+DATABASE_ROUTERS = ['core.settings.HueyDBRouter']
+
 CACHES = {
     'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'},
     # 'sqlite': {'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache'},
@@ -275,10 +368,11 @@ SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
 
 CSRF_COOKIE_SECURE = False
 SESSION_COOKIE_SECURE = False
+SESSION_COOKIE_HTTPONLY = True
 SESSION_COOKIE_DOMAIN = None
 SESSION_COOKIE_AGE = 1209600  # 2 weeks
 SESSION_EXPIRE_AT_BROWSER_CLOSE = False
-SESSION_SAVE_EVERY_REQUEST = True
+SESSION_SAVE_EVERY_REQUEST = False
 
 SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
@@ -290,6 +384,7 @@ AUTH_PASSWORD_VALIDATORS = [
 ]
 
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
+DATA_UPLOAD_MAX_MEMORY_SIZE = 26_214_400  # 25MB
 
 ################################################################################
 ### Shell Settings
@@ -310,8 +405,8 @@ if IS_SHELL:
 LANGUAGE_CODE = 'en-us'
 USE_I18N = True
 USE_TZ = True
-DATETIME_FORMAT = 'Y-m-d g:iA'
-SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'
+DATETIME_FORMAT = 'Y-m-d h:i:s A'
+SHORT_DATETIME_FORMAT = 'Y-m-d h:i:s A'
 TIME_ZONE = CONFIG.TIMEZONE        # django convention is TIME_ZONE, archivebox config uses TIMEZONE, they are equivalent
 
 
@@ -325,39 +420,44 @@ en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
 ### Logging Settings
 ################################################################################
 
-IGNORABLE_404_URLS = [
-    re.compile(r'apple-touch-icon.*\.png$'),
-    re.compile(r'favicon\.ico$'),
-    re.compile(r'robots\.txt$'),
-    re.compile(r'.*\.(css|js)\.map$'),
-]
-IGNORABLE_200_URLS = [
-    re.compile(r'^"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M),
-    re.compile(r'^"GET /admin/jsi18n/ HTTP/.*" (200|30.) .+', re.I | re.M),
+IGNORABLE_URL_PATTERNS = [
+    re.compile(r"/.*/?apple-touch-icon.*\.png"),
+    re.compile(r"/.*/?favicon\.ico"),
+    re.compile(r"/.*/?robots\.txt"),
+    re.compile(r"/.*/?.*\.(css|js)\.map"),
+    re.compile(r"/.*/?.*\.(css|js)\.map"),
+    re.compile(r"/static/.*"),
+    re.compile(r"/admin/jsi18n/"),
 ]
 
 class NoisyRequestsFilter(logging.Filter):
     def filter(self, record) -> bool:
         logline = record.getMessage()
+        # '"GET /api/v1/docs HTTP/1.1" 200 1023'
+        # '"GET /static/admin/js/SelectFilter2.js HTTP/1.1" 200 15502'
+        # '"GET /static/admin/js/SelectBox.js HTTP/1.1" 304 0'
+        # '"GET /admin/jsi18n/ HTTP/1.1" 200 3352'
+        # '"GET /admin/api/apitoken/0191bbf8-fd5e-0b8c-83a8-0f32f048a0af/change/ HTTP/1.1" 200 28778'
 
-        # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
-        for ignorable_url_pattern in IGNORABLE_404_URLS:
-            ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
-            if ignorable_log_pattern.match(logline):
+        # ignore harmless 404s for the patterns in IGNORABLE_URL_PATTERNS
+        for pattern in IGNORABLE_URL_PATTERNS:
+            ignorable_GET_request = re.compile(f'"GET {pattern.pattern} HTTP/.*" (2..|30.|404) .+$', re.I | re.M)
+            if ignorable_GET_request.match(logline):
                 return False
 
-            ignorable_log_pattern = re.compile(f'^Not Found: /.*/?{ignorable_url_pattern.pattern}', re.I | re.M)
-            if ignorable_log_pattern.match(logline):
+            ignorable_404_pattern = re.compile(f'Not Found: {pattern.pattern}', re.I | re.M)
+            if ignorable_404_pattern.match(logline):
                 return False
 
-        # ignore staticfile requests that 200 or 30*
-        for ignorable_url_pattern in IGNORABLE_200_URLS:
-            if ignorable_log_pattern.match(logline):
-                return False
-            
         return True
 
 
+class CustomOutboundWebhookLogFormatter(logging.Formatter):
+    def format(self, record):
+        result = super().format(record)
+        return result.replace('HTTP Request: ', 'OutboundWebhook: ')
+
+
 ERROR_LOG = tempfile.NamedTemporaryFile().name
 
 if CONFIG.LOGS_DIR.exists():
@@ -367,37 +467,143 @@ else:
     # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr
     print(f'[!] WARNING: data/logs dir does not exist. Logging to temp file: {ERROR_LOG}')
 
+
+LOG_LEVEL_DATABASE = 'DEBUG' if DEBUG else 'WARNING'
+LOG_LEVEL_REQUEST = 'DEBUG' if DEBUG else 'WARNING'
+
+import pydantic
+import django.template
+
 LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "rich": {
+            "datefmt": "[%Y-%m-%d %H:%M:%S]",
+            # "format": "{asctime} {levelname} {module} {name} {message} {username}",
+            "format": "%(name)s %(message)s",
         },
-        'logfile': {
-            'level': 'ERROR',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'filename': ERROR_LOG,
-            'maxBytes': 1024 * 1024 * 25,  # 25 MB
-            'backupCount': 10,
+        "outbound_webhooks": {
+            "()": CustomOutboundWebhookLogFormatter,
+            "datefmt": "[%Y-%m-%d %H:%M:%S]",
         },
     },
-    'filters': {
-        'noisyrequestsfilter': {
-            '()': NoisyRequestsFilter,
-        }
+    "filters": {
+        "noisyrequestsfilter": {
+            "()": NoisyRequestsFilter,
+        },
+        "require_debug_false": {
+            "()": "django.utils.log.RequireDebugFalse",
+        },
+        "require_debug_true": {
+            "()": "django.utils.log.RequireDebugTrue",
+        },
     },
-    'loggers': {
-        'django': {
-            'handlers': ['console', 'logfile'],
-            'level': 'INFO',
-            'filters': ['noisyrequestsfilter'],
+    "handlers": {
+        # "console": {
+        #     "level": "DEBUG",
+        #     'formatter': 'simple',
+        #     "class": "logging.StreamHandler",
+        #     'filters': ['noisyrequestsfilter', 'add_extra_logging_attrs'],
+        # },
+        "default": {
+            "class": "rich.logging.RichHandler",
+            "formatter": "rich",
+            "level": "DEBUG",
+            "markup": False,
+            "rich_tracebacks": True,
+            "filters": ["noisyrequestsfilter"],
+            "tracebacks_suppress": [
+                django,
+                pydantic,
+            ],
+        },
+        "logfile": {
+            "level": "INFO",
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": ERROR_LOG,
+            "maxBytes": 1024 * 1024 * 25,  # 25 MB
+            "backupCount": 10,
+            "formatter": "rich",
+            "filters": ["noisyrequestsfilter"],
+        },
+        "outbound_webhooks": {
+            "class": "rich.logging.RichHandler",
+            "markup": False,
+            "rich_tracebacks": True,
+            "formatter": "outbound_webhooks",
+        },
+        # "mail_admins": {
+        #     "level": "ERROR",
+        #     "filters": ["require_debug_false"],
+        #     "class": "django.utils.log.AdminEmailHandler",
+        # },
+        "null": {
+            "class": "logging.NullHandler",
+        },
+    },
+    "root": {
+        "handlers": ["default", "logfile"],
+        "level": "INFO",
+        "formatter": "rich",
+    },
+    "loggers": {
+        "api": {
+            "handlers": ["default", "logfile"],
+            "level": "DEBUG",
+        },
+        "checks": {
+            "handlers": ["default", "logfile"],
+            "level": "DEBUG",
+        },
+        "core": {
+            "handlers": ["default", "logfile"],
+            "level": "DEBUG",
+        },
+        "builtin_plugins": {
+            "handlers": ["default", "logfile"],
+            "level": "DEBUG",
+        },
+        "httpx": {
+            "handlers": ["outbound_webhooks"],
+            "level": "INFO",
+            "formatter": "outbound_webhooks",
+            "propagate": False,
+        },
+        "django": {
+            "handlers": ["default", "logfile"],
+            "level": "INFO",
+            "filters": ["noisyrequestsfilter"],
+        },
+        "django.utils.autoreload": {
+            "propagate": False,
+            "handlers": [],
+            "level": "ERROR",
+        },
+        "django.channels.server": {
+            # see archivebox.monkey_patches.ModifiedAccessLogGenerator for dedicated daphne server logging settings
+            "propagate": False,
+            "handlers": ["default", "logfile"],
+            "level": "INFO",
+            "filters": ["noisyrequestsfilter"],
+        },
+        "django.server": {  # logs all requests (2xx, 3xx, 4xx)
+            "propagate": False,
+            "handlers": ["default", "logfile"],
+            "level": "INFO",
+            "filters": ["noisyrequestsfilter"],
+        },
+        "django.request": {  # only logs 4xx and 5xx errors
+            "propagate": False,
+            "handlers": ["default", "logfile"],
+            "level": "ERROR",
+            "filters": ["noisyrequestsfilter"],
+        },
+        "django.db.backends": {
+            "propagate": False,
+            "handlers": ["default"],
+            "level": LOG_LEVEL_DATABASE,
         },
-        'django.server': {
-            'handlers': ['console', 'logfile'],
-            'level': 'INFO',
-            'filters': ['noisyrequestsfilter'],
-        }
     },
 }
 
@@ -456,6 +662,26 @@ ADMIN_DATA_VIEWS = {
                 "name": "plugin",
             },
         },
+        {
+            "route": "workers/",
+            "view": "plugantic.views.worker_list_view",
+            "name": "Workers",
+            "items": {
+                "route": "<str:key>/",
+                "view": "plugantic.views.worker_detail_view",
+                "name": "worker",
+            },
+        },
+        {
+            "route": "logs/",
+            "view": "plugantic.views.log_list_view",
+            "name": "Logs",
+            "items": {
+                "route": "<str:key>/",
+                "view": "plugantic.views.log_detail_view",
+                "name": "log",
+            },
+        },
     ],
 }
 
@@ -541,3 +767,9 @@ if DEBUG_REQUESTS_TRACKER:
 # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
 DEBUG_LOGFIRE = False
 DEBUG_LOGFIRE = DEBUG_LOGFIRE and (Path(CONFIG.OUTPUT_DIR) / '.logfire').is_dir()
+
+
+# For usage with https://www.jetadmin.io/integrations/django
+# INSTALLED_APPS += ['jet_django']
+# JET_PROJECT = 'archivebox'
+# JET_TOKEN = 'some-api-token-here'

archivebox/core/urls.py

@@ -1,14 +1,13 @@
 __package__ = 'archivebox.core'
 
-from django.urls import path, include
+from django.urls import path, re_path, include
 from django.views import static
-from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.conf import settings
 from django.views.generic.base import RedirectView
 
 from .admin import archivebox_admin
 from .views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
-
+from .serve_static import serve_static
 
 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
 # from config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
@@ -18,13 +17,16 @@ from .views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthC
 # print('DEBUG', settings.DEBUG)
 
 urlpatterns = [
-    path('public/', PublicIndexView.as_view(), name='public-index'),
+    re_path(r"^static/(?P<path>.*)$", serve_static),
+    # re_path(r"^media/(?P<path>.*)$", static.serve, {"document_root": settings.MEDIA_ROOT}),
 
     path('robots.txt', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'robots.txt'}),
     path('favicon.ico', static.serve, {'document_root': settings.STATICFILES_DIRS[0], 'path': 'favicon.ico'}),
 
     path('docs/', RedirectView.as_view(url='https://github.com/ArchiveBox/ArchiveBox/wiki'), name='Docs'),
 
+    path('public/', PublicIndexView.as_view(), name='public-index'),
+    
     path('archive/', RedirectView.as_view(url='/')),
     path('archive/<path:path>', SnapshotView.as_view(), name='Snapshot'),
 
@@ -41,7 +43,7 @@ urlpatterns = [
     path("api/",      include('api.urls'), name='api'),
 
     path('health/', HealthCheckView.as_view(), name='healthcheck'),
-    path('error/', lambda *_: 1/0),
+    path('error/', lambda *_: 1/0),                                             # type: ignore
 
     # path('jet_api/', include('jet_django.urls')),  Enable to use https://www.jetadmin.io/integrations/django
 
@@ -49,7 +51,6 @@ urlpatterns = [
     path('index.json', static.serve, {'document_root': settings.CONFIG.OUTPUT_DIR, 'path': 'index.json'}),
     path('', HomepageView.as_view(), name='Home'),
 ]
-urlpatterns += staticfiles_urlpatterns()
 
 if settings.DEBUG_TOOLBAR:
     urlpatterns += [path('__debug__/', include("debug_toolbar.urls"))]

archivebox/core/views.py

@@ -2,17 +2,17 @@ __package__ = 'archivebox.core'
 
 from typing import Callable
 
-from io import StringIO
+import threading
 from pathlib import Path
-from contextlib import redirect_stdout
 
 from django.shortcuts import render, redirect
 from django.http import HttpRequest, HttpResponse, Http404
 from django.utils.html import format_html, mark_safe
-from django.views import View, static
+from django.views import View
 from django.views.generic.list import ListView
 from django.views.generic import FormView
 from django.db.models import Q
+from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.decorators import method_decorator
@@ -23,6 +23,9 @@ from admin_data_views.utils import render_with_table_view, render_with_item_view
 
 from core.models import Snapshot
 from core.forms import AddLinkForm
+from core.admin import result_url
+
+from queues.tasks import bg_add
 
 from ..config import (
     OUTPUT_DIR,
@@ -46,6 +49,7 @@ from ..main import add
 from ..util import base_url, ansi_to_html, htmlencode, urldecode, urlencode, ts_to_date_str
 from ..search import query_search_index
 from ..extractors.wget import wget_output_path
+from .serve_static import serve_static_with_byterange_support
 
 
 class HomepageView(View):
@@ -197,7 +201,9 @@ class SnapshotView(View):
                         # if they requested snapshot index, serve live rendered template instead of static html
                         response = self.render_live_index(request, snapshot)
                     else:
-                        response = static.serve(request, archivefile, document_root=snapshot.link_dir, show_indexes=True)
+                        response = serve_static_with_byterange_support(
+                            request, archivefile, document_root=snapshot.link_dir, show_indexes=True,
+                        )
                     response["Link"] = f'<{snapshot.url}>; rel="canonical"'
                     return response
                 except Snapshot.DoesNotExist:
@@ -211,7 +217,7 @@ class SnapshotView(View):
                     format_html(
                         (
                             '<center><br/><br/><br/>'
-                            'No Snapshot directories match the given timestamp or UUID: <code>{}</code><br/><br/>'
+                            'No Snapshot directories match the given timestamp/ID/ABID: <code>{}</code><br/><br/>'
                             'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
                             '</center>'
                         ),
@@ -225,18 +231,18 @@ class SnapshotView(View):
                 snapshot_hrefs = mark_safe('<br/>').join(
                     format_html(
                         '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                        snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                        snap.bookmarked_at.strftime('%Y-%m-%d %H:%M:%S'),
                         snap.timestamp,
                         snap.timestamp,
                         snap.url,
                         snap.title_stripped[:64] or '',
                     )
-                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
+                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'bookmarked_at').order_by('-bookmarked_at')
                 )
                 return HttpResponse(
                     format_html(
                         (
-                            'Multiple Snapshots match the given timestamp/UUID <code>{}</code><br/><pre>'
+                            'Multiple Snapshots match the given timestamp/ID/ABID <code>{}</code><br/><pre>'
                         ),
                         slug,
                     ) + snapshot_hrefs + format_html(
@@ -257,12 +263,12 @@ class SnapshotView(View):
                         (
                             '<center><br/><br/><br/>'
                             f'Snapshot <a href="/archive/{snapshot.timestamp}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a>: <a href="{snapshot.url}" target="_blank" rel="noreferrer">{snapshot.url}</a><br/>'
-                            f'was queued on {str(snapshot.added).split(".")[0]}, '
+                            f'was queued on {str(snapshot.bookmarked_at).split(".")[0]}, '
                             f'but no files have been saved yet in:<br/><b><a href="/archive/{snapshot.timestamp}/" target="_top"><code>{snapshot.timestamp}</code></a><code>/'
                             '{}'
                             f'</code></b><br/><br/>'
                             'It\'s possible {} '
-                            f'during the last capture on {str(snapshot.added).split(".")[0]},<br/>or that the archiving process has not completed yet.<br/>'
+                            f'during the last capture on {str(snapshot.bookmarked_at).split(".")[0]},<br/>or that the archiving process has not completed yet.<br/>'
                             f'<pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>'
                             f'<code style="user-select: all; color: #333">archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
                             '<div class="text-align: left; width: 100%; max-width: 400px">'
@@ -270,7 +276,7 @@ class SnapshotView(View):
                             f'- list all the <a href="/archive/{snapshot.timestamp}/" target="_top">Snapshot files <code>.*</code></a><br/>'
                             f'- view the <a href="/archive/{snapshot.timestamp}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
                             f'- go to the <a href="/admin/core/snapshot/{snapshot.pk}/change/" target="_top">Snapshot admin</a> to edit<br/>'
-                            f'- go to the <a href="/admin/core/snapshot/?uuid__startswith={snapshot.uuid}" target="_top">Snapshot actions</a> to re-archive<br/>'
+                            f'- go to the <a href="/admin/core/snapshot/?id__exact={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
                             '- or return to <a href="/" target="_top">the main index...</a></div>'
                             '</center>'
                         ),
@@ -285,17 +291,17 @@ class SnapshotView(View):
         # ulid = slug.split('_', 1)[-1]
         # try:
         #     try:
-        #         snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid) | Q(old_id=ulid))
+        #         snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid))
         #     except Snapshot.DoesNotExist:
         #         pass
 
         #     try:
-        #         snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug) | Q(old_id__startswith=slug))
+        #         snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug))
         #     except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned):
         #         pass
 
         #     try:
-        #         snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
+        #         snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
         #     except Snapshot.DoesNotExist:
         #         pass
         #     return redirect(f'/archive/{snapshot.timestamp}/index.html')
@@ -308,7 +314,7 @@ class SnapshotView(View):
                 # try exact match on full url / ABID first
                 snapshot = Snapshot.objects.get(
                     Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
-                    | Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
+                    | Q(abid__icontains=path) | Q(id__icontains=path)
                 )
             except Snapshot.DoesNotExist:
                 # fall back to match on exact base_url
@@ -343,7 +349,7 @@ class SnapshotView(View):
             snapshot_hrefs = mark_safe('<br/>').join(
                 format_html(
                     '{} <code style="font-size: 0.8em">{}</code> <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                    snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                    snap.bookmarked_at.strftime('%Y-%m-%d %H:%M:%S'),
                     snap.abid,
                     snap.timestamp,
                     snap.timestamp,
@@ -352,8 +358,8 @@ class SnapshotView(View):
                 )
                 for snap in Snapshot.objects.filter(
                     Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
-                    | Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
-                ).only('url', 'timestamp', 'title', 'added').order_by('-added')
+                    | Q(abid__icontains=path) | Q(id__icontains=path)
+                ).only('url', 'timestamp', 'title', 'bookmarked_at').order_by('-bookmarked_at')
             )
             return HttpResponse(
                 format_html(
@@ -376,7 +382,7 @@ class PublicIndexView(ListView):
     template_name = 'public_index.html'
     model = Snapshot
     paginate_by = SNAPSHOTS_PER_PAGE
-    ordering = ['-added']
+    ordering = ['-bookmarked_at', '-created_at']
 
     def get_context_data(self, **kwargs):
         return {
@@ -474,18 +480,18 @@ class AddView(UserPassesTestMixin, FormView):
         }
         if extractors:
             input_kwargs.update({"extractors": extractors})
-        add_stdout = StringIO()
-        with redirect_stdout(add_stdout):
-            add(**input_kwargs)
-            print(add_stdout.getvalue())
 
-        context = self.get_context_data()
+        result = bg_add(input_kwargs, parent_task_id=None)
+        print('Started background add job:', result)
 
-        context.update({
-            "stdout": ansi_to_html(add_stdout.getvalue().strip()),
-            "form": AddLinkForm()
-        })
-        return render(template_name=self.template_name, request=self.request, context=context)
+        rough_url_count = url.count('://')
+
+        messages.success(
+            self.request,
+            mark_safe(f"Adding {rough_url_count} URLs in the background. (refresh in a few minutes to see results) {result_url(result)}"),
+        )
+
+        return redirect("/admin/core/snapshot/")
 
 
 class HealthCheckView(View):

archivebox/extractors/__init__.py

@@ -134,7 +134,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         link = load_link_details(link, out_dir=out_dir)
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
         log_link_archiving_started(link, str(out_dir), is_new)
-        link = link.overwrite(updated=datetime.now(timezone.utc))
+        link = link.overwrite(downloaded_at=datetime.now(timezone.utc))
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
         start_ts = datetime.now(timezone.utc)
 
@@ -157,11 +157,11 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                                                  output=result.output, pwd=result.pwd, start_ts=result.start_ts, end_ts=result.end_ts, status=result.status, created_by_id=snapshot.created_by_id)
 
 
-                    # bump the updated time on the main Snapshot here, this is critical
+                    # bump the downloaded_at time on the main Snapshot here, this is critical
                     # to be able to cache summaries of the ArchiveResults for a given
                     # snapshot without having to load all the results from the DB each time.
-                    # (we use {Snapshot.pk}-{Snapshot.updated} as the cache key and assume
-                    # ArchiveResults are unchanged as long as the updated timestamp is unchanged)
+                    # (we use {Snapshot.pk}-{Snapshot.downloaded_at} as the cache key and assume
+                    # ArchiveResults are unchanged as long as the downloaded_at timestamp is unchanged)
                     snapshot.save()
                 else:
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))
@@ -178,7 +178,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                         ts
                     ) + "\n" + str(e) + "\n"))
                     #f.write(f"\n> {command}; ts={ts} version={config['VERSION']} docker={config['IN_DOCKER']} is_tty={config['IS_TTY']}\n")
-               
+
                 # print(f'        ERROR: {method_name} {e.__class__.__name__}: {e} {getattr(e, "hints", "")}', ts, link.url, command)
                 raise Exception('Exception in archive_methods.save_{}(Link(url={}))'.format(
                     method_name,
@@ -218,7 +218,7 @@ def archive_links(all_links: Union[Iterable[Link], QuerySet], overwrite: bool=Fa
     if type(all_links) is QuerySet:
         num_links: int = all_links.count()
         get_link = lambda x: x.as_link_with_details()
-        all_links = all_links.iterator()
+        all_links = all_links.iterator(chunk_size=500)
     else:
         num_links: int = len(all_links)
         get_link = lambda x: x

archivebox/extractors/wget.py

@@ -245,7 +245,7 @@ def wget_output_path(link: Link, nocache: bool=False) -> Optional[str]:
     #    https://example.com/abc/test/?v=zzVa_tX1OiI
     #       > example.com/abc/test/index.html@v=zzVa_tX1OiI.html
 
-    cache_key = f'{link.url_hash}:{link.timestamp}-{link.updated and link.updated.timestamp()}-wget-output-path'
+    cache_key = f'{link.url_hash}:{link.timestamp}-{link.downloaded_at and link.downloaded_at.timestamp()}-wget-output-path'
     
     if not nocache:
         from django.core.cache import cache

archivebox/index/html.py

@@ -118,7 +118,7 @@ def render_django_template(template: str, context: Mapping[str, str]) -> str:
 
 
 def snapshot_icons(snapshot) -> str:
-    cache_key = f'{snapshot.pk}-{(snapshot.updated or snapshot.added).timestamp()}-snapshot-icons'
+    cache_key = f'result_icons:{snapshot.pk}:{(snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at or snapshot.bookmarked_at).timestamp()}'
     
     def calc_snapshot_icons():
         from core.models import ArchiveResult
@@ -133,6 +133,7 @@ def snapshot_icons(snapshot) -> str:
         else:
             archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
 
+        # import ipdb; ipdb.set_trace()
         link = snapshot.as_link()
         path = link.archive_path
         canon = link.canonical_outputs()
@@ -197,7 +198,12 @@ def snapshot_icons(snapshot) -> str:
         # print(((end - start).total_seconds()*1000) // 1, 'ms')
         return result
 
-    return cache.get_or_set(cache_key, calc_snapshot_icons)
-    # return calc_snapshot_icons()
+    cache_result = cache.get(cache_key)
+    if cache_result:
+        return cache_result
+    
+    fresh_result = calc_snapshot_icons()
+    cache.set(cache_key, fresh_result, timeout=60 * 60 * 24)
+    return fresh_result
 
    

archivebox/index/schema.py

@@ -132,7 +132,7 @@ class Link:
     tags: Optional[str]
     sources: List[str]
     history: Dict[str, List[ArchiveResult]] = field(default_factory=lambda: {})
-    updated: Optional[datetime] = None
+    downloaded_at: Optional[datetime] = None
     schema: str = 'Link'
 
     def __str__(self) -> str:
@@ -164,7 +164,7 @@ class Link:
             assert isinstance(self.timestamp, str) and self.timestamp
             assert self.timestamp.replace('.', '').isdigit()
             assert isinstance(self.url, str) and '://' in self.url
-            assert self.updated is None or isinstance(self.updated, datetime)
+            assert self.downloaded_at is None or isinstance(self.downloaded_at, datetime)
             assert self.title is None or (isinstance(self.title, str) and self.title)
             assert self.tags is None or isinstance(self.tags, str)
             assert isinstance(self.sources, list)
@@ -184,7 +184,7 @@ class Link:
             'url': self.url,
             'title': self.title or None,
             'timestamp': self.timestamp,
-            'updated': self.updated or None,
+            'downloaded_at': self.downloaded_at or None,
             'tags': self.tags or None,
             'sources': self.sources or [],
             'history': self.history or {},
@@ -192,7 +192,6 @@ class Link:
         if extended:
             info.update({
                 'snapshot_id': self.snapshot_id,
-                'snapshot_old_id': self.snapshot_old_id,
                 'snapshot_abid': self.snapshot_abid,
 
                 'link_dir': self.link_dir,
@@ -211,7 +210,7 @@ class Link:
                 'icons': None,           # only used to render static index in index/html.py, remove if no longer needed there
 
                 'bookmarked_date': self.bookmarked_date,
-                'updated_date': self.updated_date,
+                'downloaded_datestr': self.downloaded_datestr,
                 'oldest_archive_date': self.oldest_archive_date,
                 'newest_archive_date': self.newest_archive_date,
         
@@ -237,7 +236,7 @@ class Link:
             for key, val in json_info.items()
             if key in cls.field_names()
         }
-        info['updated'] = parse_date(info.get('updated'))
+        info['downloaded_at'] = parse_date(info.get('updated') or info.get('downloaded_at'))
         info['sources'] = info.get('sources') or []
 
         json_history = info.get('history') or {}
@@ -266,16 +265,12 @@ class Link:
     @cached_property
     def snapshot(self):
         from core.models import Snapshot
-        return Snapshot.objects.only('id', 'old_id', 'abid').get(url=self.url)
+        return Snapshot.objects.only('id', 'abid').get(url=self.url)
 
     @cached_property
     def snapshot_id(self):
         return str(self.snapshot.pk)
 
-    @cached_property
-    def snapshot_old_id(self):
-        return str(self.snapshot.old_id)
-
     @cached_property
     def snapshot_abid(self):
         return str(self.snapshot.ABID)
@@ -352,8 +347,8 @@ class Link:
 
 
     @property
-    def updated_date(self) -> Optional[str]:
-        return ts_to_date_str(self.updated) if self.updated else None
+    def downloaded_datestr(self) -> Optional[str]:
+        return ts_to_date_str(self.downloaded_at) if self.downloaded_at else None
 
     @property
     def archive_dates(self) -> List[datetime]:

archivebox/index/sql.py

@@ -37,9 +37,11 @@ def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir:
 @enforce_types
 def write_link_to_sql_index(link: Link, created_by_id: int | None=None):
     from core.models import Snapshot, ArchiveResult
+    from abid_utils.models import get_or_create_system_user_pk
+
     info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}
 
-    info['created_by_id'] = created_by_id
+    info['created_by_id'] = created_by_id or get_or_create_system_user_pk()
 
     tag_list = list(dict.fromkeys(
         tag.strip() for tag in re.split(TAG_SEPARATOR_PATTERN, link.tags or '')
@@ -146,17 +148,16 @@ def list_migrations(out_dir: Path=OUTPUT_DIR) -> List[Tuple[bool, str]]:
 @enforce_types
 def apply_migrations(out_dir: Path=OUTPUT_DIR) -> List[str]:
     from django.core.management import call_command
-    null, out = StringIO(), StringIO()
-    try:
-        call_command("makemigrations", interactive=False, stdout=null)
-    except Exception as e:
-        print('[!] Failed to create some migrations. Please open an issue and copy paste this output for help: {}'.format(e))
-        print()
+    out1, out2 = StringIO(), StringIO()
     
-    call_command("migrate", interactive=False, stdout=out)
-    out.seek(0)
+    call_command("migrate", interactive=False, database='default', stdout=out1)
+    out1.seek(0)
+    call_command("migrate", "huey_monitor", interactive=False, database='queue', stdout=out2)
+    out2.seek(0)
 
-    return [line.strip() for line in out.readlines() if line.strip()]
+    return [
+        line.strip() for line in out1.readlines() + out2.readlines() if line.strip()
+    ]
 
 @enforce_types
 def get_admins(out_dir: Path=OUTPUT_DIR) -> List[str]:

archivebox/main.py

@@ -1,16 +1,21 @@
 __package__ = 'archivebox'
 
 import os
+import time
 import sys
 import shutil
+import signal
 import platform
-from django.utils import timezone
+import subprocess
+
+from typing import Dict, List, Optional, Iterable, IO, Union
 from pathlib import Path
 from datetime import date, datetime
 
-from typing import Dict, List, Optional, Iterable, IO, Union
 from crontab import CronTab, CronSlices
+
 from django.db.models import QuerySet
+from django.utils import timezone
 
 from .cli import (
     list_subcommands,
@@ -318,7 +323,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
         print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     elif existing_index:
         # TODO: properly detect and print the existing version in current index as well
-        print('{green}[^] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
+        print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
         print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     else:
         if force:
@@ -540,9 +545,9 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
     last_login = User.objects.order_by('last_login').last()
     if last_login:
         print(f'    Last UI login: {last_login.username} @ {str(last_login.last_login)[:16]}')
-    last_updated = Snapshot.objects.order_by('updated').last()
-    if last_updated:
-        print(f'    Last changes: {str(last_updated.updated)[:16]}')
+    last_downloaded = Snapshot.objects.order_by('downloaded_at').last()
+    if last_downloaded:
+        print(f'    Last changes: {str(last_downloaded.downloaded_at)[:16]}')
 
     if not users:
         print()
@@ -550,13 +555,13 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
         print('        archivebox manage createsuperuser')
 
     print()
-    for snapshot in links.order_by('-updated')[:10]:
-        if not snapshot.updated:
+    for snapshot in links.order_by('-downloaded_at')[:10]:
+        if not snapshot.downloaded_at:
             continue
         print(
             ANSI['black'],
             (
-                f'   > {str(snapshot.updated)[:16]} '
+                f'   > {str(snapshot.downloaded_at)[:16]} '
                 f'[{snapshot.num_outputs} {("X", "√")[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] '
                 f'"{snapshot.title}": {snapshot.url}'
             )[:TERM_WIDTH()],
@@ -960,7 +965,8 @@ def setup(out_dir: Path=OUTPUT_DIR) -> None:
         run_subcommand('init', stdin=None, pwd=out_dir)
 
     setup_django(out_dir=out_dir, check_db=True)
-    from core.models import User
+    from django.contrib.auth import get_user_model
+    User = get_user_model()
 
     if not User.objects.filter(is_superuser=True).exists():
         stderr('\n[+] Creating new admin user for the Web UI...', color='green')
@@ -979,16 +985,16 @@ def setup(out_dir: Path=OUTPUT_DIR) -> None:
                 '--upgrade',
                 '--no-cache-dir',
                 '--no-warn-script-location',
-                'youtube_dl',
+                'yt-dlp',
             ], capture_output=False, cwd=out_dir)
             pkg_path = run_shell([
                 PYTHON_BINARY, '-m', 'pip',
                 'show',
-                'youtube_dl',
+                'yt-dlp',
             ], capture_output=True, text=True, cwd=out_dir).stdout.decode().split('Location: ')[-1].split('\n', 1)[0]
-            NEW_YOUTUBEDL_BINARY = Path(pkg_path) / 'youtube_dl' / '__main__.py'
+            NEW_YOUTUBEDL_BINARY = Path(pkg_path) / 'yt-dlp' / '__main__.py'
             os.chmod(NEW_YOUTUBEDL_BINARY, 0o777)
-            assert NEW_YOUTUBEDL_BINARY.exists(), f'youtube_dl must exist inside {pkg_path}'
+            assert NEW_YOUTUBEDL_BINARY.exists(), f'yt-dlp must exist inside {pkg_path}'
             config(f'YOUTUBEDL_BINARY={NEW_YOUTUBEDL_BINARY}', set=True, out_dir=out_dir)
         except BaseException as e:                                              # lgtm [py/catch-base-exception]
             stderr(f'[X] Failed to install python packages: {e}', color='red')
@@ -1334,9 +1340,9 @@ def server(runserver_args: Optional[List[str]]=None,
         print('        archivebox manage createsuperuser')
         print()
 
-    # fallback to serving staticfiles insecurely with django when DEBUG=False
-    if not config.DEBUG:
-        runserver_args.append('--insecure')  # TODO: serve statics w/ nginx instead
+    # fallback to serving staticfiles insecurely with django when DEBUG=False (not compatible with daphne)
+    # if not config.DEBUG:
+    #     runserver_args.append('--insecure')  # TODO: serve statics w/ nginx instead
     
     # toggle autoreloading when archivebox code changes (it's on by default)
     if not reload:
@@ -1345,7 +1351,71 @@ def server(runserver_args: Optional[List[str]]=None,
     config.SHOW_PROGRESS = False
     config.DEBUG = config.DEBUG or debug
 
-    call_command("runserver", *runserver_args)
+    if reload or debug:
+        call_command("runserver", *runserver_args)
+    else:
+        
+        host = '127.0.0.1'
+        port = '8000'
+        
+        try:
+            host_and_port = [arg for arg in runserver_args if arg.replace('.', '').replace(':', '').isdigit()][0]
+            if ':' in host_and_port:
+                host, port = host_and_port.split(':')
+            else:
+                if '.' in host_and_port:
+                    host = host_and_port
+                else:
+                    port = host_and_port
+        except IndexError:
+            pass
+
+        from queues.supervisor_util import get_or_create_supervisord_process, start_worker, stop_worker, watch_worker
+
+        print()
+        supervisor = get_or_create_supervisord_process(daemonize=False)
+
+        bg_workers = [
+            {
+                "name": "worker_system_tasks",
+                "command": "archivebox manage djangohuey --queue system_tasks",
+                "autostart": "true",
+                "autorestart": "true",
+                "stdout_logfile": "logs/worker_system_tasks.log",
+                "redirect_stderr": "true",
+            },
+        ]
+        fg_worker = {
+            "name": "worker_daphne",
+            "command": f"daphne --bind={host} --port={port} --application-close-timeout=600 archivebox.core.asgi:application",
+            "autostart": "false",
+            "autorestart": "true",
+            "stdout_logfile": "logs/worker_daphne.log",
+            "redirect_stderr": "true",
+        }
+
+        print()
+        for worker in bg_workers:
+            start_worker(supervisor, worker)
+
+        print()
+        start_worker(supervisor, fg_worker)
+        print()
+
+        try:
+            watch_worker(supervisor, "worker_daphne")
+        except KeyboardInterrupt:
+            print("\n[🛑] Got Ctrl+C, stopping gracefully...")
+        except SystemExit:
+            pass
+        except BaseException as e:
+            print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping web server gracefully...")
+            raise
+        finally:
+            stop_worker(supervisor, "worker_daphne")
+            time.sleep(0.5)
+
+        print("\n[🟩] ArchiveBox server shut down gracefully.")
 
 
 @enforce_types

archivebox/monkey_patches.py

@@ -14,3 +14,45 @@ timezone.utc = datetime.timezone.utc
 # monkey patch django-signals-webhooks to change how it shows up in Admin UI
 # from signal_webhooks.apps import DjangoSignalWebhooksConfig
 # DjangoSignalWebhooksConfig.verbose_name = 'API'
+
+
+# Install rich for pretty tracebacks in console logs
+# https://rich.readthedocs.io/en/stable/traceback.html#traceback-handler
+from rich.traceback import install
+
+install(show_locals=True)
+
+
+from daphne import access
+
+class ModifiedAccessLogGenerator(access.AccessLogGenerator):
+    """Clutge workaround until daphne uses the Python logging framework. https://github.com/django/daphne/pull/473/files"""
+    
+    def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None):
+        
+        # Ignore noisy requests to staticfiles / favicons / etc.
+        if 'GET /static/' in request:
+            return
+        if "GET /health/" in request:
+            return
+        if 'GET /admin/jsi18n/' in request:
+            return
+        if request.endswith("/favicon.ico") or request.endswith("/robots.txt") or request.endswith("/screenshot.png"):
+            return
+        if request.endswith('.css') or request.endswith('.js') or request.endswith('.woff') or request.endswith('.ttf'):
+            return
+        if str(status) in ('404', '304'):
+            return
+        
+        # clean up the log format to mostly match the same format as django.conf.settings.LOGGING rich formats
+        self.stream.write(
+            "[%s] HTTP     %s (%s) %s\n"
+            % (
+                date.strftime("%Y-%m-%d %H:%M:%S"),
+                request,
+                status or "-",
+                "localhost" if host.startswith("127.") else host.split(":")[0],
+            )
+        )
+        
+access.AccessLogGenerator.write_entry = ModifiedAccessLogGenerator.write_entry

archivebox/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "archivebox",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "archivebox",
-      "version": "0.8.2",
+      "version": "0.8.4",
       "license": "MIT",
       "dependencies": {
         "@postlight/parser": "^2.2.3",
@@ -26,9 +26,9 @@
       }
     },
     "node_modules/@babel/runtime-corejs2": {
-      "version": "7.25.4",
-      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.25.4.tgz",
-      "integrity": "sha512-6IxkDkxN13FQAB8FGF+vrvZVA77EqG4UzCwVGc9x9Ylerj70W3nMUwoXDbj2LtJxXlFKM256APcDQIItT2OFYA==",
+      "version": "7.25.6",
+      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.25.6.tgz",
+      "integrity": "sha512-24uCmOJPrsnS7HtRamCibYabHRV0bscPJNFFcyKgj7FqUA0V5XcbZUmz9PVNDW4L+euMsZtCIetU1LxTmUaIlA==",
       "license": "MIT",
       "dependencies": {
         "core-js": "^2.6.12",
@@ -229,6 +229,12 @@
         }
       }
     },
+    "node_modules/@puppeteer/browsers/node_modules/ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "license": "MIT"
+    },
     "node_modules/@tootallnate/quickjs-emscripten": {
       "version": "0.23.0",
       "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz",
@@ -236,9 +242,9 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "22.5.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.0.tgz",
-      "integrity": "sha512-DkFrJOe+rfdHTqqMg0bSNlGlQ85hSoh2TPzZyhHsXnMtligRWpxUySiyw8FY14ITt24HVCiQPWxS3KO/QlGmWg==",
+      "version": "22.5.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.4.tgz",
+      "integrity": "sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==",
       "license": "MIT",
       "optional": true,
       "dependencies": {
@@ -353,9 +359,9 @@
       }
     },
     "node_modules/aws4": {
-      "version": "1.13.1",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz",
-      "integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz",
+      "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==",
       "license": "MIT"
     },
     "node_modules/b4a": {
@@ -629,17 +635,23 @@
       }
     },
     "node_modules/cssstyle": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.0.1.tgz",
-      "integrity": "sha512-8ZYiJ3A/3OkDd093CBT/0UKDWry7ak4BdPTFP2+QEP7cmhouyq/Up709ASSj2cK02BbZiMgk7kYjZNS4QP5qrQ==",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.1.0.tgz",
+      "integrity": "sha512-h66W1URKpBS5YMI/V8PyXvTMFT8SupJ1IzoIV8IeBC/ji8WVmrO8dGlTi+2dh6whmdk6BiKJLD/ZBkhWbcg6nA==",
       "license": "MIT",
       "dependencies": {
-        "rrweb-cssom": "^0.6.0"
+        "rrweb-cssom": "^0.7.1"
       },
       "engines": {
         "node": ">=18"
       }
     },
+    "node_modules/cssstyle/node_modules/rrweb-cssom": {
+      "version": "0.7.1",
+      "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz",
+      "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==",
+      "license": "MIT"
+    },
     "node_modules/dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -700,12 +712,12 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.3.6",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz",
-      "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==",
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
       "license": "MIT",
       "dependencies": {
-        "ms": "2.1.2"
+        "ms": "^2.1.3"
       },
       "engines": {
         "node": ">=6.0"
@@ -860,9 +872,9 @@
       "license": "BSD-2-Clause"
     },
     "node_modules/escalade": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
-      "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==",
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
+      "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
       "license": "MIT",
       "engines": {
         "node": ">=6"
@@ -1585,9 +1597,9 @@
       "license": "MIT"
     },
     "node_modules/ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
     "node_modules/netmask": {
@@ -1727,9 +1739,9 @@
       "license": "MIT"
     },
     "node_modules/postman-request": {
-      "version": "2.88.1-postman.39",
-      "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.39.tgz",
-      "integrity": "sha512-rsncxxDlbn1YpygXSgJqbJzIjGlHFcZjbYDzeBPTQHMDfLuSTzZz735JHV8i1+lOROuJ7MjNap4eaSD3UijHzQ==",
+      "version": "2.88.1-postman.40",
+      "resolved": "https://registry.npmjs.org/postman-request/-/postman-request-2.88.1-postman.40.tgz",
+      "integrity": "sha512-uE4AiIqhjtHKp4pj9ei7fkdfNXEX9IqDBlK1plGAQne6y79UUlrTdtYLhwXoO0AMOvqyl9Ar+BU6Eo6P/MPgfg==",
       "license": "Apache-2.0",
       "dependencies": {
         "@postman/form-data": "~3.1.1",
@@ -1858,6 +1870,12 @@
         }
       }
     },
+    "node_modules/puppeteer-core/node_modules/ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+      "license": "MIT"
+    },
     "node_modules/puppeteer-core/node_modules/ws": {
       "version": "8.16.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz",
@@ -2172,9 +2190,9 @@
       }
     },
     "node_modules/source-map-js": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz",
-      "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==",
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
+      "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
       "license": "BSD-3-Clause",
       "engines": {
         "node": ">=0.10.0"
@@ -2227,9 +2245,9 @@
       }
     },
     "node_modules/streamx": {
-      "version": "2.19.0",
-      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.19.0.tgz",
-      "integrity": "sha512-5z6CNR4gtkPbwlxyEqoDGDmWIzoNJqCBt4Eac1ICP9YaIT08ct712cFj0u1rx4F8luAuL+3Qc+RFIdI4OX00kg==",
+      "version": "2.20.0",
+      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.0.tgz",
+      "integrity": "sha512-ZGd1LhDeGFucr1CUCTBOS58ZhEendd0ttpGT3usTvosS4ntIwKN9LJFp+OeCSprsCPL14BXVRZlHGRY1V9PVzQ==",
       "license": "MIT",
       "dependencies": {
         "fast-fifo": "^1.3.2",

archivebox/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "ArchiveBox: The self-hosted internet archive",
   "author": "Nick Sweeting <archivebox-npm@sweeting.me>",
   "repository": "github:ArchiveBox/ArchiveBox",

archivebox/pkgs/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

archivebox/pkgs/apps.py

@@ -1,14 +0,0 @@
-__package__ = 'archivebox.pkgs'
-
-from django.apps import AppConfig
-
-
-class PkgsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'pkgs'
-
-    def ready(self):
-        from .settings import LOADED_DEPENDENCIES
-
-        # print(LOADED_DEPENDENCIES)
-        

archivebox/pkgs/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/pkgs/settings.py

@@ -1,76 +0,0 @@
-__package__ = 'archivebox.pkgs'
-
-import sys
-import shutil
-import inspect
-from pathlib import Path
-
-import django
-from django.conf import settings
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer
-from pydantic_pkgr.binprovider import bin_abspath
-
-
-env = EnvProvider()
-
-
-LOADED_DEPENDENCIES = {}
-
-for bin_key, dependency in settings.CONFIG.DEPENDENCIES.items():
-    # 'PYTHON_BINARY': {
-    #     'path': bin_path(config['PYTHON_BINARY']),
-    #     'version': config['PYTHON_VERSION'],
-    #     'hash': bin_hash(config['PYTHON_BINARY']),
-    #     'enabled': True,
-    #     'is_valid': bool(config['PYTHON_VERSION']),
-    # },
-    
-
-    bin_name = settings.CONFIG[bin_key]
-
-    if bin_name.endswith('django/__init__.py'):
-        binary_spec = Binary(name='django', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: Path(inspect.getfile(django)),
-                'version': lambda: SemVer('{}.{}.{} {} ({})'.format(*django.VERSION)),
-            }
-        })
-    elif bin_name.endswith('sqlite3/dbapi2.py'):
-        binary_spec = Binary(name='sqlite3', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: Path(inspect.getfile(sqlite3)),
-                'version': lambda: SemVer(sqlite3.version),
-            }
-        })
-    elif bin_name.endswith('archivebox'):
-        binary_spec = Binary(name='archivebox', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: shutil.which(str(Path('archivebox').expanduser())),
-                'version': lambda: settings.CONFIG.VERSION,
-            }
-        })
-    else:
-        binary_spec = Binary(name=bin_name, providers=[env])
-    
-    try:
-        binary = binary_spec.load()
-    except Exception as e:
-        print(f"- ❌ Binary {bin_name} failed to load with error: {e}")
-        continue
-
-    assert isinstance(binary.loaded_version, SemVer)
-
-    try:
-        assert str(binary.loaded_version) == dependency['version'], f"Expected {bin_name} version {dependency['version']}, got {binary.loaded_version}"
-        assert str(binary.loaded_respath) == str(bin_abspath(dependency['path']).resolve()), f"Expected {bin_name} abspath {bin_abspath(dependency['path']).resolve()}, got {binary.loaded_respath}"
-        assert binary.is_valid == dependency['is_valid'], f"Expected {bin_name} is_valid={dependency['is_valid']}, got {binary.is_valid}"
-    except Exception as e:
-        print(f"Assertion error for {bin_name}: {e}")
-        import ipdb; ipdb.set_trace()
-    
-    print(f"- ✅ Binary {bin_name} loaded successfully")
-    LOADED_DEPENDENCIES[bin_key] = binary
-
-

archivebox/pkgs/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/pkgs/views.py

@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.

archivebox/plugantic/__init__.py

@@ -1,16 +1,9 @@
 __package__ = 'archivebox.plugantic'
 
-from .binaries import Binary
-from .extractors import Extractor
-from .replayers import Replayer
-from .configs import ConfigSet
-from .plugins import Plugin
+from .base_plugin import BasePlugin
+from .base_configset import BaseConfigSet
+from .base_binary import BaseBinary
+from .base_extractor import BaseExtractor
+from .base_replayer import BaseReplayer
+from .base_check import BaseCheck
 
-# __all__ = [
-#     'BinProvider',
-#     'Binary',
-#     'Extractor',
-#     'Replayer',
-#     'ConfigSet',
-#     'Plugin',
-# ]

archivebox/plugantic/apps.py

@@ -1,17 +1,12 @@
-import importlib
-from django.apps import AppConfig
+__package__ = 'archivebox.plugantic'
 
+from django.apps import AppConfig
 
 class PluganticConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'plugantic'
 
     def ready(self) -> None:
-        from django.conf import settings
-        from .plugins import PLUGINS
-
-        for plugin_name in settings.INSTALLED_PLUGINS.keys():
-            lib = importlib.import_module(f'{plugin_name}.apps')
-            if hasattr(lib, 'PLUGINS'):
-                for plugin_instance in lib.PLUGINS:
-                    PLUGINS.append(plugin_instance)
+        pass
+        # from django.conf import settings
+        # print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')

archivebox/plugantic/base_admindataview.py

@@ -0,0 +1,38 @@
+__package__ = 'archivebox.plugantic'
+
+from typing import Dict
+
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
+
+class BaseAdminDataView(BaseHook):
+    hook_type: HookType = "ADMINDATAVIEW"
+    
+    verbose_name: str = 'NPM Installed Packages'
+    route: str = '/npm/installed/'
+    view: str = 'builtin_plugins.npm.admin.installed_list_view'
+    items: Dict[str, str] = {
+        "name": "installed_npm_pkg",
+        'route': '<str:key>/',
+        'view': 'builtin_plugins.npm.admin.installed_detail_view',
+    }
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                          # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
+
+        self.register_route_in_admin_data_view_urls(settings)
+
+        settings.ADMINDATAVIEWS = getattr(settings, "ADMINDATAVIEWS", None) or AttrDict({})
+        settings.ADMINDATAVIEWS[self.id] = self
+
+        super().register(settings, parent_plugin)
+
+    def register_route_in_admin_data_view_urls(self, settings):
+        route = {
+            "route": self.route,
+            "view": self.view,
+            "name": self.verbose_name,
+            "items": self.items,
+        }
+        if route not in settings.ADMIN_DATA_VIEWS.URLS:
+            settings.ADMIN_DATA_VIEWS.URLS += [route]  # append our route (update in place)

archivebox/plugantic/base_binary.py

@@ -0,0 +1,55 @@
+__package__ = 'archivebox.plugantic'
+
+import os
+from typing import Dict, List
+
+from pydantic import Field, InstanceOf
+from pydantic_pkgr import Binary, BinProvider, BinProviderName, ProviderLookupDict, AptProvider, BrewProvider, EnvProvider
+
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
+
+
+class BaseBinProvider(BaseHook, BinProvider):
+    hook_type: HookType = 'BINPROVIDER'
+
+    # def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
+    #     Class = super()
+    #     get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
+    #     # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func)
+    #     return get_abspath_func()
+
+    # def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None:
+    #     Class = super()
+    #     get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context)
+    #     # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func)
+    #     return get_version_func()
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+
+        settings.BINPROVIDERS = getattr(settings, "BINPROVIDERS", None) or AttrDict({})
+        settings.BINPROVIDERS[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
+
+
+class BaseBinary(BaseHook, Binary):
+    hook_type: HookType = "BINARY"
+    
+    binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders')
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = Field(default_factory=dict, alias='overrides')
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+
+        settings.BINARIES = getattr(settings, "BINARIES", None) or AttrDict({})
+        settings.BINARIES[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
+
+
+
+apt = AptProvider()
+brew = BrewProvider()
+env = EnvProvider(PATH=os.environ.get("PATH", "/bin"))

archivebox/plugantic/base_check.py

@@ -0,0 +1,51 @@
+__package__ = "archivebox.plugantic"
+
+from typing import List
+
+from django.core.checks import Warning, Tags, register
+
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
+
+class BaseCheck(BaseHook):
+    hook_type: HookType = "CHECK"
+    
+    tag: str = Tags.database
+    
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        """Override this method to implement your custom runtime check."""
+        errors = []
+        # if not hasattr(settings, 'SOME_KEY'):
+        #     errors.extend(Error(
+        #         'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
+        #         id='core.C001',
+        #         hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
+        #     ))
+        # logger.debug('[√] Loaded settings.PLUGINS succesfully.')
+        return errors
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin  # backref to parent is for debugging only, never rely on this!
+
+        self.register_with_django_check_system()  # (SIDE EFFECT)
+
+        # install hook into settings.CHECKS
+        settings.CHECKS = getattr(settings, "CHECKS", None) or AttrDict({})
+        settings.CHECKS[self.id] = self
+
+        # record installed hook in settings.HOOKS
+        super().register(settings, parent_plugin=parent_plugin)
+
+    def register_with_django_check_system(self):
+
+        def run_check(app_configs, **kwargs) -> List[Warning]:
+            from django.conf import settings
+            import logging
+
+            return self.check(settings, logging.getLogger("checks"))
+
+        run_check.__name__ = self.id
+        run_check.tags = [self.tag]
+        register(self.tag)(run_check)
+

archivebox/plugantic/base_configset.py

@@ -0,0 +1,77 @@
+__package__ = 'archivebox.plugantic'
+
+
+from typing import List, Literal
+
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
+
+
+ConfigSectionName = Literal[
+    'GENERAL_CONFIG',
+    'ARCHIVE_METHOD_TOGGLES',
+    'ARCHIVE_METHOD_OPTIONS',
+    'DEPENDENCY_CONFIG',
+]
+ConfigSectionNames: List[ConfigSectionName] = [
+    'GENERAL_CONFIG',
+    'ARCHIVE_METHOD_TOGGLES',
+    'ARCHIVE_METHOD_OPTIONS',
+    'DEPENDENCY_CONFIG',
+]
+
+
+class BaseConfigSet(BaseHook):
+    hook_type: HookType = 'CONFIG'
+
+    section: ConfigSectionName = 'GENERAL_CONFIG'
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+
+        settings.CONFIGS = getattr(settings, "CONFIGS", None) or AttrDict({})
+        settings.CONFIGS[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
+
+
+
+# class WgetToggleConfig(ConfigSet):
+#     section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+#     SAVE_WGET: bool = True
+#     SAVE_WARC: bool = True
+
+# class WgetDependencyConfig(ConfigSet):
+#     section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+#     WGET_BINARY: str = Field(default='wget')
+#     WGET_ARGS: Optional[List[str]] = Field(default=None)
+#     WGET_EXTRA_ARGS: List[str] = []
+#     WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+# class WgetOptionsConfig(ConfigSet):
+#     section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+#     # loaded from shared config
+#     WGET_AUTO_COMPRESSION: bool = Field(default=True)
+#     SAVE_WGET_REQUISITES: bool = Field(default=True)
+#     WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+#     WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+#     WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+#     WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+#     WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+# CONFIG = {
+#     'CHECK_SSL_VALIDITY': False,
+#     'SAVE_WARC': False,
+#     'TIMEOUT': 999,
+# }
+
+
+# WGET_CONFIG = [
+#     WgetToggleConfig(**CONFIG),
+#     WgetDependencyConfig(**CONFIG),
+#     WgetOptionsConfig(**CONFIG),
+# ]

archivebox/plugantic/base_extractor.py

@@ -3,27 +3,13 @@ __package__ = 'archivebox.plugantic'
 from typing import Optional, List, Literal, Annotated, Dict, Any
 from typing_extensions import Self
 
-from abc import ABC
 from pathlib import Path
 
-from pydantic import BaseModel, model_validator, field_serializer, AfterValidator
+from pydantic import model_validator, AfterValidator
+from pydantic_pkgr import BinName
 
-from .binaries import (
-    Binary,
-    YtdlpBinary,
-    WgetBinary,
-)
-
-
-# stubs
-class Snapshot:
-    pass
-
-class ArchiveResult:
-    pass
-
-def get_wget_output_path(*args, **kwargs) -> Path:
-    return Path('.').resolve()
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
 
 
 
@@ -31,15 +17,17 @@ def no_empty_args(args: List[str]) -> List[str]:
     assert all(len(arg) for arg in args)
     return args
 
-ExtractorName = Literal['wget', 'warc', 'media']
+ExtractorName = Literal['wget', 'warc', 'media', 'singlefile'] | str
 
 HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
 CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)]
 
 
-class Extractor(ABC, BaseModel):
+class BaseExtractor(BaseHook):
+    hook_type: HookType = 'EXTRACTOR'
+    
     name: ExtractorName
-    binary: Binary
+    binary: BinName
 
     output_path_func: HandlerFuncStr = 'self.get_output_path'
     should_extract_func: HandlerFuncStr = 'self.should_extract'
@@ -56,12 +44,19 @@ class Extractor(ABC, BaseModel):
             self.args = [*self.default_args, *self.extra_args]
         return self
 
-    @field_serializer('binary', when_used='json')
-    def dump_binary(binary) -> str:
-        return binary.name
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+
+        settings.EXTRACTORS = getattr(settings, "EXTRACTORS", None) or AttrDict({})
+        settings.EXTRACTORS[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
+
+
 
     def get_output_path(self, snapshot) -> Path:
-        return Path(self.name)
+        return Path(self.id.lower())
 
     def should_extract(self, snapshot) -> bool:
         output_dir = self.get_output_path(snapshot)
@@ -86,33 +81,37 @@ class Extractor(ABC, BaseModel):
             'returncode': proc.returncode,
         }
 
-    def exec(self, args: CmdArgsList, pwd: Optional[Path]=None):
+    def exec(self, args: CmdArgsList, pwd: Optional[Path]=None, settings=None):
         pwd = pwd or Path('.')
-        assert self.binary.loaded_provider
-        return self.binary.exec(args, pwd=pwd)
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+        
+        binary = settings.BINARIES[self.binary]
+        return binary.exec(args, pwd=pwd)
 
 
-class YtdlpExtractor(Extractor):
-    name: ExtractorName = 'media'
-    binary: Binary = YtdlpBinary()
+# class YtdlpExtractor(Extractor):
+#     name: ExtractorName = 'media'
+#     binary: Binary = YtdlpBinary()
 
-    def get_output_path(self, snapshot) -> Path:
-        return Path(self.name)
+#     def get_output_path(self, snapshot) -> Path:
+#         return 'media/'
 
 
-class WgetExtractor(Extractor):
-    name: ExtractorName = 'wget'
-    binary: Binary = WgetBinary()
+# class WgetExtractor(Extractor):
+#     name: ExtractorName = 'wget'
+#     binary: Binary = WgetBinary()
 
-    def get_output_path(self, snapshot) -> Path:
-        return get_wget_output_path(snapshot)
+#     def get_output_path(self, snapshot) -> Path:
+#         return get_wget_output_path(snapshot)
 
 
-class WarcExtractor(Extractor):
-    name: ExtractorName = 'warc'
-    binary: Binary = WgetBinary()
+# class WarcExtractor(Extractor):
+#     name: ExtractorName = 'warc'
+#     binary: Binary = WgetBinary()
 
-    def get_output_path(self, snapshot) -> Path:
-        return get_wget_output_path(snapshot)
+#     def get_output_path(self, snapshot) -> Path:
+#         return get_wget_output_path(snapshot)
 
 

archivebox/plugantic/base_hook.py

@@ -0,0 +1,120 @@
+__package__ = 'archivebox.plugantic'
+
+import inspect
+from huey.api import TaskWrapper
+
+from pathlib import Path
+from typing import List, Literal
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+
+
+HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
+hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
+
+class BaseHook(BaseModel):
+    """
+    A Plugin consists of a list of Hooks, applied to django.conf.settings when AppConfig.read() -> Plugin.register() is called.
+    Plugin.register() then calls each Hook.register() on the provided settings.
+    each Hook.regsiter() function (ideally pure) takes a django.conf.settings as input and returns a new one back.
+    or 
+    it modifies django.conf.settings in-place to add changes corresponding to its HookType.
+    e.g. for a HookType.CONFIG, the Hook.register() function places the hook in settings.CONFIG (and settings.HOOKS)
+    An example of an impure Hook would be a CHECK that modifies settings but also calls django.core.checks.register(check).
+    In practice any object that subclasses BaseHook and provides a .register() function can behave as a Hook.
+
+    setup_django() -> imports all settings.INSTALLED_APPS...
+        # django imports AppConfig, models, migrations, admins, etc. for all installed apps
+        # django then calls AppConfig.ready() on each installed app...
+
+        builtin_plugins.npm.NpmPlugin().AppConfig.ready()                    # called by django
+            builtin_plugins.npm.NpmPlugin().register(settings) ->
+                builtin_plugins.npm.NpmConfigSet().register(settings)
+                    plugantic.base_configset.BaseConfigSet().register(settings)
+                        plugantic.base_hook.BaseHook().register(settings, parent_plugin=builtin_plugins.npm.NpmPlugin())
+
+                ...
+        ...
+
+    Both core ArchiveBox code and plugin code depend on python >= 3.10 and django >= 5.0 w/ sqlite and a filesystem.
+    Core ArchiveBox code can depend only on python and the pip libraries it ships with, and can never depend on plugin code / node / other binaries.
+    Plugin code can depend on archivebox core, other django apps, other pip libraries, and other plugins.
+    Plugins can provide BinProviders + Binaries which can depend on arbitrary other binaries / package managers like curl / wget / yt-dlp / etc.
+
+    The execution interface between plugins is simply calling builtinplugins.npm.... functions directly, django handles
+    importing all plugin code. There is no need to manually register methods/classes, only register to call
+    impure setup functions or provide runtime state.
+    settings.CONFIGS / settings.BINPROVIDERS / settings.BINARIES /... etc. are reserved for dynamic runtime state only.
+    This state is exposed to the broader system in a flat namespace, e.g. CONFIG.IS_DOCKER=True, or BINARIES = [
+        ..., Binary('node', abspath='/usr/local/bin/node', version='22.2.0'), ...
+    ]
+
+    """
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+        validate_defaults=True,
+        validate_assignment=False,
+        revalidate_instances="subclass-instances",
+        ignored_types=(TaskWrapper, ),
+    )
+    
+    # verbose_name: str = Field()
+    
+    is_registered: bool = False
+    is_ready: bool = False
+
+
+    @computed_field
+    @property
+    def id(self) -> str:
+        return self.__class__.__name__
+
+    @computed_field
+    @property
+    def hook_module(self) -> str:
+        """e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
+        return f'{self.__module__}.{self.__class__.__name__}'
+
+    @property
+    def hook_file(self) -> Path:
+        """e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
+        return Path(inspect.getfile(self.__class__))
+
+    @property
+    def plugin_module(self) -> str:
+        """e.g. builtin_plugins.singlefile"""
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
+
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
+
+    hook_type: HookType = Field()
+
+    def register(self, settings, parent_plugin=None):
+        """Load a record of an installed hook into global Django settings.HOOKS at runtime."""
+        self._plugin = parent_plugin         # for debugging only, never rely on this!
+
+        # assert json.dumps(self.model_json_schema(), indent=4), f"Hook {self.hook_module} has invalid JSON schema."
+
+        # record installed hook in settings.HOOKS
+        settings.HOOKS[self.id] = self
+
+        if settings.HOOKS[self.id].is_registered:
+            raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!")
+
+        settings.HOOKS[self.id].is_registered = True
+
+        # print("REGISTERED HOOK:", self.hook_module)
+
+    def ready(self, settings):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        assert self.id in settings.HOOKS, f"Tried to ready hook {self.hook_module} but it is not registered in settings.HOOKS."
+
+        if settings.HOOKS[self.id].is_ready:
+            raise Exception(f"Tried to run {self.hook_module}.ready() but its already been called!")
+
+        settings.HOOKS[self.id].is_ready = True

archivebox/plugantic/base_plugin.py

@@ -0,0 +1,186 @@
+__package__ = 'archivebox.plugantic'
+
+import json
+import inspect
+from pathlib import Path
+
+from django.apps import AppConfig
+
+from typing import List, Type, Dict
+from typing_extensions import Self
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    model_validator,
+    InstanceOf,
+    computed_field,
+    validate_call,
+)
+
+from .base_hook import BaseHook, HookType
+
+from ..config import AttrDict
+
+
+class BasePlugin(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
+
+    # Required by AppConfig:
+    app_label: str = Field()                      # e.g. 'singlefile'                  (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name)
+    verbose_name: str = Field()                   # e.g. 'SingleFile'                  (human-readable *short* label, for use in column names, form labels, etc.)
+    
+    # All the hooks the plugin will install:
+    hooks: List[InstanceOf[BaseHook]] = Field(default=[])
+    
+    is_registered: bool = False
+    is_ready: bool = False
+    
+    @computed_field
+    @property
+    def id(self) -> str:
+        return self.__class__.__name__
+    
+    @property
+    def name(self) -> str:
+        return self.app_label
+    
+    @computed_field
+    @property
+    def plugin_module(self) -> str:  # DottedImportPath
+        """ "
+        Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
+        e.g. 'archivebox.builtin_plugins.npm.apps.NpmPlugin' -> 'builtin_plugins.npm'
+        """
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit('.apps.', 1)[0]
+
+    @computed_field
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
+    
+    @model_validator(mode='after')
+    def validate(self) -> Self:
+        """Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
+        
+        assert self.app_label and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
+        
+        assert json.dumps(self.model_json_schema(), indent=4), f"Plugin {self.plugin_module} has invalid JSON schema."
+        return self
+    
+    @property
+    def AppConfig(plugin_self) -> Type[AppConfig]:
+        """Generate a Django AppConfig class for this plugin."""
+
+
+        class PluginAppConfig(AppConfig):
+            """Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS."""
+            name = plugin_self.plugin_module
+            app_label = plugin_self.app_label
+            verbose_name = plugin_self.verbose_name
+
+            default_auto_field = 'django.db.models.AutoField'
+
+            def ready(self):
+                from django.conf import settings
+                plugin_self.ready(settings)
+
+        return PluginAppConfig
+
+    @property
+    def HOOKS_BY_ID(self) -> Dict[str, InstanceOf[BaseHook]]:
+        return AttrDict({hook.id: hook for hook in self.hooks})
+
+    @property
+    def HOOKS_BY_TYPE(self) -> Dict[HookType, Dict[str, InstanceOf[BaseHook]]]:
+        hooks = AttrDict({})
+        for hook in self.hooks:
+            hooks[hook.hook_type] = hooks.get(hook.hook_type) or AttrDict({})
+            hooks[hook.hook_type][hook.id] = hook
+        return hooks
+
+    def register(self, settings=None):
+        """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at import time (before models are imported or any AppConfig.ready() are called)."""
+
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.plugin_module} has invalid JSON schema.'
+
+        assert self.id not in settings.PLUGINS, f'Tried to register plugin {self.plugin_module} but it conflicts with existing plugin of the same name ({self.app_label}).'
+
+        ### Mutate django.conf.settings... values in-place to include plugin-provided overrides
+        settings.PLUGINS[self.id] = self
+
+        if settings.PLUGINS[self.id].is_registered:
+            raise Exception(f"Tried to run {self.plugin_module}.register() but its already been called!")
+
+        for hook in self.hooks:
+            hook.register(settings, parent_plugin=self)
+
+        settings.PLUGINS[self.id].is_registered = True
+        # print('√ REGISTERED PLUGIN:', self.plugin_module)
+
+    def ready(self, settings=None):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        assert (
+            self.id in settings.PLUGINS and settings.PLUGINS[self.id].is_registered
+        ), f"Tried to run plugin.ready() for {self.plugin_module} but plugin is not yet registered in settings.PLUGINS."
+
+        if settings.PLUGINS[self.id].is_ready:
+            raise Exception(f"Tried to run {self.plugin_module}.ready() but its already been called!")
+
+        for hook in self.hooks:
+            hook.ready(settings)
+        
+        settings.PLUGINS[self.id].is_ready = True
+
+    # @validate_call
+    # def install_binaries(self) -> Self:
+    #     new_binaries = []
+    #     for idx, binary in enumerate(self.binaries):
+    #         new_binaries.append(binary.install() or binary)
+    #     return self.model_copy(update={
+    #         'binaries': new_binaries,
+    #     })
+
+    @validate_call
+    def load_binaries(self, cache=True) -> Self:
+        new_binaries = []
+        for idx, binary in enumerate(self.HOOKS_BY_TYPE['BINARY'].values()):
+            new_binaries.append(binary.load(cache=cache) or binary)
+        return self.model_copy(update={
+            'binaries': new_binaries,
+        })
+
+    # @validate_call
+    # def load_or_install_binaries(self, cache=True) -> Self:
+    #     new_binaries = []
+    #     for idx, binary in enumerate(self.binaries):
+    #         new_binaries.append(binary.load_or_install(cache=cache) or binary)
+    #     return self.model_copy(update={
+    #         'binaries': new_binaries,
+    #     })
+
+
+
+
+# class YtdlpPlugin(BasePlugin):
+#     name: str = 'ytdlp'
+#     configs: List[SerializeAsAny[BaseConfigSet]] = []
+#     binaries: List[SerializeAsAny[BaseBinary]] = [YtdlpBinary()]
+#     extractors: List[SerializeAsAny[BaseExtractor]] = [YtdlpExtractor()]
+#     replayers: List[SerializeAsAny[BaseReplayer]] = [MEDIA_REPLAYER]
+
+# class WgetPlugin(BasePlugin):
+#     name: str = 'wget'
+#     configs: List[SerializeAsAny[BaseConfigSet]] = [*WGET_CONFIG]
+#     binaries: List[SerializeAsAny[BaseBinary]] = [WgetBinary()]
+#     extractors: List[SerializeAsAny[BaseExtractor]] = [WgetExtractor(), WarcExtractor()]

archivebox/plugantic/base_queue.py

@@ -0,0 +1,148 @@
+__package__ = 'archivebox.plugantic'
+
+import importlib
+
+from typing import Dict, List, TYPE_CHECKING
+from pydantic import Field, InstanceOf
+
+if TYPE_CHECKING:
+    from huey.api import TaskWrapper
+
+from .base_hook import BaseHook, HookType
+from .base_binary import BaseBinary
+from ..config_stubs import AttrDict
+
+
+
+class BaseQueue(BaseHook):
+    hook_type: HookType = 'QUEUE'
+
+    name: str = Field()       # e.g. 'singlefile'
+
+    binaries: List[InstanceOf[BaseBinary]] = Field()
+
+    @property
+    def tasks(self) -> Dict[str, 'TaskWrapper']:
+        """Return an AttrDict of all the background worker tasks defined in the plugin's tasks.py file."""
+        tasks = importlib.import_module(f"{self.plugin_module}.tasks")
+
+        all_tasks = {}
+
+        for task_name, task in tasks.__dict__.items():
+            # if attr is a Huey task and its queue_name matches our hook's queue name
+            if hasattr(task, "task_class") and task.huey.name == self.name:
+                all_tasks[task_name] = task
+
+        return AttrDict(all_tasks)
+
+    def get_huey_config(self, settings) -> dict:
+        return {
+            "huey_class": "huey.SqliteHuey",
+            "filename": settings.QUEUE_DATABASE_NAME,
+            "name": self.name,
+            "results": True,
+            "store_none": True,
+            "immediate": False,
+            "utc": True,
+            "consumer": {
+                "workers": 1,
+                "worker_type": "thread",
+                "initial_delay": 0.1,  # Smallest polling interval, same as -d.
+                "backoff": 1.15,  # Exponential backoff using this rate, -b.
+                "max_delay": 10.0,  # Max possible polling interval, -m.
+                "scheduler_interval": 1,  # Check schedule every second, -s.
+                "periodic": True,  # Enable crontab feature.
+                "check_worker_health": True,  # Enable worker health checks.
+                "health_check_interval": 1,  # Check worker health every second.
+            },
+        }
+        
+    def get_supervisor_config(self, settings) -> dict:
+        return {
+            "name": f"worker_{self.name}",
+            "command": f"archivebox manage djangohuey --queue {self.name}",
+            "stdout_logfile": f"logs/worker_{self.name}.log",
+            "redirect_stderr": "true",
+            "autorestart": "true",
+            "autostart": "false",
+        }
+        
+    def start_supervisord_worker(self, settings, lazy=True):
+        from queues.supervisor_util import get_or_create_supervisord_process, start_worker
+        print()
+        try:
+            supervisor = get_or_create_supervisord_process(daemonize=False)
+        except Exception as e:
+            print(f"Error starting worker for queue {self.name}: {e}")
+            return None
+        print()
+        worker = start_worker(supervisor, self.get_supervisor_config(settings), lazy=lazy)
+
+        # Update settings.WORKERS to include this worker
+        settings.WORKERS = getattr(settings, "WORKERS", None) or AttrDict({})
+        settings.WORKERS[self.id] = self.start_supervisord_worker(settings, lazy=True)
+
+        return worker
+
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+
+        # Side effect: register queue with django-huey multiqueue dict
+        settings.DJANGO_HUEY = getattr(settings, "DJANGO_HUEY", None) or AttrDict({"queues": {}})
+        settings.DJANGO_HUEY["queues"][self.name] = self.get_huey_config(settings)
+
+        # Side effect: register some extra tasks with huey
+        # on_startup(queue=self.name)(self.on_startup_task)
+        # db_periodic_task(crontab(minute='*/5'))(self.on_periodic_task)
+
+        # Install queue into settings.QUEUES
+        settings.QUEUES = getattr(settings, "QUEUES", None) or AttrDict({})
+        settings.QUEUES[self.id] = self
+
+        # Record installed hook into settings.HOOKS
+        super().register(settings, parent_plugin=parent_plugin)
+
+    # def ready(self, settings):
+    #     self.start_supervisord_worker(settings, lazy=True)
+    #     super().ready(settings)
+
+
+# class WgetToggleConfig(ConfigSet):
+#     section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+#     SAVE_WGET: bool = True
+#     SAVE_WARC: bool = True
+
+# class WgetDependencyConfig(ConfigSet):
+#     section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+#     WGET_BINARY: str = Field(default='wget')
+#     WGET_ARGS: Optional[List[str]] = Field(default=None)
+#     WGET_EXTRA_ARGS: List[str] = []
+#     WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+# class WgetOptionsConfig(ConfigSet):
+#     section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+#     # loaded from shared config
+#     WGET_AUTO_COMPRESSION: bool = Field(default=True)
+#     SAVE_WGET_REQUISITES: bool = Field(default=True)
+#     WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+#     WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+#     WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+#     WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+#     WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+# CONFIG = {
+#     'CHECK_SSL_VALIDITY': False,
+#     'SAVE_WARC': False,
+#     'TIMEOUT': 999,
+# }
+
+
+# WGET_CONFIG = [
+#     WgetToggleConfig(**CONFIG),
+#     WgetDependencyConfig(**CONFIG),
+#     WgetOptionsConfig(**CONFIG),
+# ]

archivebox/plugantic/base_replayer.py

@@ -1,14 +1,15 @@
 __package__ = 'archivebox.plugantic'
 
 
-from pydantic import BaseModel
-
-# from .binproviders import LazyImportStr
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
 
 
-class Replayer(BaseModel):
+class BaseReplayer(BaseHook):
     """Describes how to render an ArchiveResult in several contexts"""
-    name: str = 'GenericReplayer'
+    
+    hook_type: HookType = 'REPLAYER'
+    
     url_pattern: str = '*'
 
     row_template: str = 'plugins/generic_replayer/templates/row.html'
@@ -21,5 +22,16 @@ class Replayer(BaseModel):
     # icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
     # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
 
+    def register(self, settings, parent_plugin=None):
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
 
-MEDIA_REPLAYER = Replayer(name='media')
+        settings.REPLAYERS = getattr(settings, 'REPLAYERS', None) or AttrDict({})
+        settings.REPLAYERS[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
+
+# class MediaReplayer(BaseReplayer):
+#     name: str = 'MediaReplayer'
+
+
+# MEDIA_REPLAYER = MediaReplayer()

archivebox/plugantic/binaries.py

@@ -1,65 +0,0 @@
-__package__ = 'archivebox.plugantic'
-
-import sys
-import inspect
-import importlib
-from pathlib import Path
-
-
-from typing import Any, Optional, Dict, List
-from typing_extensions import Self
-from subprocess import run, PIPE
-
-from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
-
-import django
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-
-
-
-def get_ytdlp_version() -> str:
-    import yt_dlp
-    return yt_dlp.version.__version__
-
-
-
-
-class YtdlpBinary(Binary):
-    name: BinName = 'yt-dlp'
-    providers_supported: List[BinProvider] = [
-        EnvProvider(),
-        PipProvider(),
-        BrewProvider(),
-        AptProvider(),
-    ]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'pip': {
-            'version': get_ytdlp_version,
-        },
-        'brew': {
-            'subdeps': lambda: 'yt-dlp ffmpeg',
-        },
-        'apt': {
-            'subdeps': lambda: 'yt-dlp ffmpeg',
-        }
-    }
-
-class WgetBinary(Binary):
-    name: BinName = 'wget'
-    providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
-
-
-# if __name__ == '__main__':
-#     PYTHON_BINARY = PythonBinary()
-#     SQLITE_BINARY = SqliteBinary()
-#     DJANGO_BINARY = DjangoBinary()
-#     WGET_BINARY = WgetBinary()
-#     YTDLP_BINARY = YtdlpPBinary()
-
-#     print('-------------------------------------DEFINING BINARIES---------------------------------')
-#     print(PYTHON_BINARY)
-#     print(SQLITE_BINARY)
-#     print(DJANGO_BINARY)
-#     print(WGET_BINARY)
-#     print(YTDLP_BINARY)

archivebox/plugantic/configs.py

@@ -1,53 +0,0 @@
-__package__ = 'archivebox.plugantic'
-
-
-from typing import Optional, List, Literal
-from pathlib import Path
-from pydantic import BaseModel, Field
-
-
-ConfigSectionName = Literal['GENERAL_CONFIG', 'ARCHIVE_METHOD_TOGGLES', 'ARCHIVE_METHOD_OPTIONS', 'DEPENDENCY_CONFIG']
-
-
-class ConfigSet(BaseModel):
-    section: ConfigSectionName = 'GENERAL_CONFIG'
-
-class WgetToggleConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
-
-    SAVE_WGET: bool = True
-    SAVE_WARC: bool = True
-
-class WgetDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    WGET_BINARY: str = Field(default='wget')
-    WGET_ARGS: Optional[List[str]] = Field(default=None)
-    WGET_EXTRA_ARGS: List[str] = []
-    WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class WgetOptionsConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
-
-    # loaded from shared config
-    WGET_AUTO_COMPRESSION: bool = Field(default=True)
-    SAVE_WGET_REQUISITES: bool = Field(default=True)
-    WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
-    WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
-    WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
-    WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
-    WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
-
-
-CONFIG = {
-    'CHECK_SSL_VALIDITY': False,
-    'SAVE_WARC': False,
-    'TIMEOUT': 999,
-}
-
-
-WGET_CONFIG = [
-    WgetToggleConfig(**CONFIG),
-    WgetDependencyConfig(**CONFIG),
-    WgetOptionsConfig(**CONFIG),
-]

archivebox/plugantic/management/commands/pkg.py

@@ -0,0 +1,72 @@
+__package__ = 'archivebox.plugantic.management.commands'
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer
+from pydantic_pkgr.binprovider import bin_abspath
+
+from ....config import NODE_BIN_PATH, bin_path
+from ...base_binary import env
+
+
+class Command(BaseCommand):
+    def handle(self, *args, method, **options):
+        method(*args, **options)
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(title="sub-commands", required=True)
+
+        list_parser = subparsers.add_parser("list", help="List archivebox runtime dependencies.")
+        list_parser.set_defaults(method=self.list)
+
+        install_parser = subparsers.add_parser("install", help="Install archivebox runtime dependencies.")
+        install_parser.add_argument("--update", action="store_true", help="Update dependencies to latest versions.")
+        install_parser.add_argument("package_names", nargs="+", type=str)
+        install_parser.set_defaults(method=self.install)
+
+    def list(self, *args, **options):
+        self.stdout.write('################# PLUGINS ####################')
+        for plugin in settings.PLUGINS.values():
+            self.stdout.write(f'{plugin.name}:')
+            for binary in plugin.binaries:
+                try:
+                    binary = binary.load()
+                except Exception as e:
+                    # import ipdb; ipdb.set_trace()
+                    raise
+                self.stdout.write(f'    {binary.name.ljust(14)} {str(binary.version).ljust(11)} {binary.binprovider.INSTALLER_BIN.ljust(5)}  {binary.abspath}')
+
+        self.stdout.write('\n################# LEGACY ####################')
+        for bin_key, dependency in settings.CONFIG.DEPENDENCIES.items():
+            bin_name = settings.CONFIG[bin_key]
+
+            self.stdout.write(f'{bin_key}:     {bin_name}')
+
+            # binary = Binary(name=package_name, providers=[env])
+            # print(binary)
+
+            # try:
+            #     loaded_bin = binary.load()
+            #     self.stdout.write(
+            #         self.style.SUCCESS(f'Successfully loaded {package_name}:') + str(loaded_bin)
+            #     )
+            # except Exception as e:
+            #     self.stderr.write(
+            #         self.style.ERROR(f"Error loading {package_name}: {e}")
+            #     )
+
+    def install(self, *args, bright, **options):
+        for package_name in options["package_names"]:
+            binary = Binary(name=package_name, providers=[env])
+            print(binary)
+
+            try:
+                loaded_bin = binary.load()
+                self.stdout.write(
+                    self.style.SUCCESS(f'Successfully loaded {package_name}:') + str(loaded_bin)
+                )
+            except Exception as e:
+                self.stderr.write(
+                    self.style.ERROR(f"Error loading {package_name}: {e}")
+                )

archivebox/plugantic/migrations/0001_initial.py

@@ -1,38 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 00:16
-
-import abid_utils.models
-import archivebox.plugantic.plugins
-import charidfield.fields
-import django.core.serializers.json
-import django.db.models.deletion
-import django_pydantic_field.fields
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Plugin',
-            fields=[
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('uuid', models.UUIDField(blank=True, null=True, unique=True)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
-                ('schema', django_pydantic_field.fields.PydanticSchemaField(config=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
-    ]

archivebox/plugantic/migrations/0002_alter_plugin_schema.py

@@ -1,21 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:16
-
-import archivebox.plugantic.plugins
-import django.core.serializers.json
-import django_pydantic_field.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='plugin',
-            name='schema',
-            field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin),
-        ),
-    ]

archivebox/plugantic/migrations/0003_alter_plugin_schema.py

@@ -1,21 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:25
-
-import archivebox.plugantic.replayers
-import django.core.serializers.json
-import django_pydantic_field.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0002_alter_plugin_schema'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='plugin',
-            name='schema',
-            field=django_pydantic_field.fields.PydanticSchemaField(config=None, default={'embed_template': 'plugins/generic_replayer/templates/embed.html', 'fullpage_template': 'plugins/generic_replayer/templates/fullpage.html', 'name': 'GenericReplayer', 'row_template': 'plugins/generic_replayer/templates/row.html', 'url_pattern': '*'}, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.replayers.Replayer),
-        ),
-    ]

archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py

@@ -1,32 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:28
-
-import archivebox.plugantic.configs
-import django.core.serializers.json
-import django_pydantic_field.compat.django
-import django_pydantic_field.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0003_alter_plugin_schema'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='plugin',
-            name='schema',
-        ),
-        migrations.AddField(
-            model_name='plugin',
-            name='configs',
-            field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=[], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=django_pydantic_field.compat.django.GenericContainer(list, (archivebox.plugantic.configs.ConfigSet,))),
-        ),
-        migrations.AddField(
-            model_name='plugin',
-            name='name',
-            field=models.CharField(default='name', max_length=64, unique=True),
-            preserve_default=False,
-        ),
-    ]

archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py

@@ -1,39 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:42
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import pathlib
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0004_remove_plugin_schema_plugin_configs_plugin_name'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='CustomPlugin',
-            fields=[
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('uuid', models.UUIDField(blank=True, null=True, unique=True)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
-                ('name', models.CharField(max_length=64, unique=True)),
-                ('path', models.FilePathField(path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'))),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
-        migrations.DeleteModel(
-            name='Plugin',
-        ),
-    ]

archivebox/plugantic/migrations/0006_alter_customplugin_path.py

@@ -1,19 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:45
-
-import pathlib
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0005_customplugin_delete_plugin'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'), recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0007_alter_customplugin_path.py

@@ -1,19 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:46
-
-import pathlib
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0006_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins'), recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0008_alter_customplugin_path.py

@@ -1,19 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:47
-
-import pathlib
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0007_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data'), recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0009_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0008_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0010_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0009_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, match='/plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0011_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0010_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0012_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:49
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0011_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, default='example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0013_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:49
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0012_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]