rename datetime fields to _at, massively improve ABID generation safety and determinism

archivebox/abid_utils/abid.py

@@ -208,6 +208,7 @@ def abid_hashes_from_values(prefix: str, ts: datetime, uri: str, subtype: str |
         'uri': abid_part_from_uri(uri, salt=salt),
         'subtype': abid_part_from_subtype(subtype),
         'rand': abid_part_from_rand(rand),
+        # 'salt': don't add this, salt combined with uri above to form a single hash
     }
 
 @enforce_types

archivebox/abid_utils/admin.py

@@ -1,58 +1,61 @@
 __package__ = 'archivebox.abid_utils'
-from django.contrib import admin
+
+from typing import Any
 from datetime import datetime
+
+from django.contrib import admin, messages
+from django.core.exceptions import ValidationError
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
+from django.shortcuts import redirect
 
-from abid_utils.abid import abid_part_from_ts, abid_part_from_uri, abid_part_from_rand, abid_part_from_subtype
+from abid_utils.abid import ABID, abid_part_from_ts, abid_part_from_uri, abid_part_from_rand, abid_part_from_subtype
 
 from api.auth import get_or_create_api_token
 
 from ..util import parse_date
 
-def highlight_diff(display_val, compare_val):
+def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
     """highlight each character in red that differs with the char at the same index in compare_val"""
 
     display_val = str(display_val)
     compare_val = str(compare_val)
 
+    if len(compare_val) < len(display_val):
+        compare_val += ' ' * (len(display_val) - len(compare_val))
+
+    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
+    if invert:
+        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
+
     return mark_safe(''.join(
-        format_html('<span style="color: red;">{}</span>', display_val[i])
+        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
         if display_val[i] != compare_val[i] else
-        format_html('<span display="color: black">{}</span>', display_val[i])
+        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
         for i in range(len(display_val))
     ))
 
 def get_abid_info(self, obj, request=None):
     try:
-        abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
+        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
 
-        fresh_abid = obj.ABID
-        fresh_abid_diff = f' != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)} ❌' if str(fresh_abid) != str(obj.ABID) else '✅'
-        fresh_uuid_diff = f' != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)} ❌' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
+        fresh_abid = ABID(**obj.ABID_FRESH_HASHES)
+        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
+        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
 
-        id_fresh_abid_diff = f' != .fresh_abid ❌' if str(fresh_abid.uuid) != str(obj.id) else ' == .fresh_abid ✅'
-        id_abid_diff = f' !=  .abid.uuid: {highlight_diff(obj.ABID.uuid, obj.id)} ❌' if str(obj.id) != str(obj.ABID.uuid) else ' == .abid ✅'
-        id_pk_diff = f' !=  .pk: {highlight_diff(obj.pk, obj.id)} ❌' if str(obj.pk) != str(obj.id) else ' ==  .pk ✅'
+        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
 
         fresh_ts = parse_date(obj.ABID_FRESH_VALUES['ts']) or None
-        derived_ts = abid_part_from_ts(fresh_ts) if fresh_ts else None
-        ts_diff = f'!= {highlight_diff(derived_ts, obj.ABID.ts)} ❌' if derived_ts != obj.ABID.ts else '✅'
+        ts_diff = f'❌ != {highlight_diff( obj.ABID_FRESH_HASHES["ts"], obj.ABID.ts)}' if  obj.ABID_FRESH_HASHES["ts"] != obj.ABID.ts else '✅'
 
-        derived_uri = abid_part_from_uri(obj.ABID_FRESH_VALUES['uri'])
-        uri_diff = f'!= {highlight_diff(derived_uri, obj.ABID.uri)} ❌' if derived_uri != obj.ABID.uri else '✅'
+        derived_uri = obj.ABID_FRESH_HASHES['uri']
+        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
 
-        derived_subtype = abid_part_from_subtype(obj.ABID_FRESH_VALUES['subtype'])
-        subtype_diff = f'!= {highlight_diff(derived_subtype, obj.ABID.subtype)} ❌' if derived_subtype != obj.ABID.subtype else '✅'
+        derived_subtype = obj.ABID_FRESH_HASHES['subtype']
+        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
 
-        derived_rand = abid_part_from_rand(obj.ABID_FRESH_VALUES['rand'])
-        rand_diff = f'!= {highlight_diff(derived_rand, obj.ABID.rand)} ❌' if derived_rand != obj.ABID.rand else '✅'
-
-        # any_abid_discrepancies = any(
-        #     '❌' in diff or '!=' in diff
-        #     for diff in (abid_diff, fresh_abid_diff, id_abid_diff, id_pk_diff, ts_diff, uri_diff, subtype_diff, rand_diff)
-        # )
-        # total_diff = f' != .generate_abid() -> {fresh_abid} ❌' if any_abid_discrepancies else '✅'
+        derived_rand = obj.ABID_FRESH_HASHES['rand']
+        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
 
         return format_html(
             # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
@@ -69,30 +72,34 @@ def get_abid_info(self, obj, request=None):
             &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
             &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
             <br/><hr/>
+            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code>
             </div>
             ''',
             obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
-            highlight_diff(obj.id, obj.ABID.uuid), mark_safe(id_pk_diff + id_abid_diff),
-            highlight_diff(obj.ABID.uuid, obj.id), mark_safe(fresh_uuid_diff),
+            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
+            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
             highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
             # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
             # str(fresh_abid), mark_safe(fresh_abid_diff),
-            highlight_diff(obj.ABID.ts, derived_ts), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
+            highlight_diff(obj.ABID.ts,  obj.ABID_FRESH_HASHES['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
             highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(obj.ABID_FRESH_VALUES['uri']),
             highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(obj.ABID_FRESH_VALUES['subtype']),
             highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(obj.ABID_FRESH_VALUES['rand'])[-7:],
+            f'Some values the ABID depends on have changed since the ABID was issued:' if obj.ABID_FRESH_DIFFS else '',
+            ", ".join(diff['abid_src'] for diff in obj.ABID_FRESH_DIFFS.values()),
         )
     except Exception as e:
+        # import ipdb; ipdb.set_trace()
         return str(e)
 
 
 class ABIDModelAdmin(admin.ModelAdmin):
-    list_display = ('created', 'created_by', 'abid', '__str__')
-    sort_fields = ('created', 'created_by', 'abid', '__str__')
-    readonly_fields = ('created', 'modified', '__str__', 'API')
+    list_display = ('created_at', 'created_by', 'abid', '__str__')
+    sort_fields = ('created_at', 'created_by', 'abid', '__str__')
+    readonly_fields = ('created_at', 'modified_at', '__str__', 'abid_info')
 
     @admin.display(description='API Identifiers')
-    def API(self, obj):
+    def abid_info(self, obj):
         return get_abid_info(self, obj, request=self.request)
 
     def queryset(self, request):

archivebox/abid_utils/models.py

@@ -11,7 +11,7 @@ from datetime import datetime, timedelta
 from functools import partial
 from charidfield import CharIDField  # type: ignore[import-untyped]
 
-from django.conf import settings
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils import timezone
 from django.db.utils import OperationalError
@@ -59,7 +59,7 @@ def get_or_create_system_user_pk(username='system'):
         return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
 
     # otherwise, create a dedicated "system" user
-    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
     return user.pk
 
 
@@ -68,69 +68,166 @@ class AutoDateTimeField(models.DateTimeField):
     #     return timezone.now()
     pass
 
+class ABIDError(Exception):
+    pass
+
+class ABIDFieldsCannotBeChanged(ValidationError, ABIDError):
+    """
+    Properties used as unique identifiers (to generate ABID) cannot be edited after an object is created.
+    Create a new object instead with your desired changes (and it will be issued a new ABID).
+    """
+    def __init__(self, ABID_FRESH_DIFFS, obj):
+        self.ABID_FRESH_DIFFS = ABID_FRESH_DIFFS
+        self.obj = obj
+
+    def __str__(self):
+        keys_changed = ', '.join(diff['abid_src'] for diff in self.ABID_FRESH_DIFFS.values())
+        return (
+            f"This {self.obj.__class__.__name__}(abid={str(self.obj.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
+            f'\nThe following changes cannot be made because they would alter the ABID:' +
+            '\n  ' + "\n    ".join(f'  - {diff["summary"]}' for diff in self.ABID_FRESH_DIFFS.values()) +
+            f"\nYou must reduce your changes to not affect these fields, or create a new {self.obj.__class__.__name__} object instead."
+        )
+
 
 class ABIDModel(models.Model):
     """
     Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
     """
     abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
-    abid_ts_src = 'self.created'                     # e.g. 'self.created'
-    abid_uri_src = 'None'                            # e.g. 'self.uri'
+    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
+    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
     abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
     abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
-    abid_salt: str = DEFAULT_ABID_URI_SALT
+    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
+    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
 
     # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     # abid = ABIDField(prefix=abid_prefix)
 
     # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    # created = AutoDateTimeField(default=None, null=False, db_index=True)
-    # modified = models.DateTimeField(auto_now=True)
+    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    # modified_at = models.DateTimeField(auto_now=True)
 
     class Meta(TypedModelMeta):
         abstract = True
 
-    def save(self, *args: Any, **kwargs: Any) -> None:
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
+        super().__init__(*args, **kwargs)
+        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
+        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
+        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
+        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
+        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
+
+    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
+        """Overriden save method ensures new ABID is generated while a new object is first saving."""
+
         if self._state.adding:
-            self.pk = self.id = self.id or uuid4()
-            self.created = ts_from_abid(abid_part_from_ts(timezone.now()))  # cut off precision to match precision of TS component
-            self.modified = self.created
-            self.created_by = self.created_by or get_or_create_system_user_pk()
+            # only runs once when a new object is first saved to the DB
+            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
             self.abid = str(self.issue_new_abid())
+
+        else:
+            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
+            if self.ABID_FRESH_DIFFS:
+                ovewrite_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
+
+                change_error = ABIDFieldsCannotBeChanged(self.ABID_FRESH_DIFFS, obj=self)
+                if ovewrite_abid:
+                    print(f'#### DANGER: Changing ABID of existing record ({self.__class__.__name__}.abid_drift_allowed={abid_drift_allowed}), this will break any references to its previous ABID!')
+                    print(change_error)
+                    self.abid = str(self.issue_new_abid(force_new=True))
+                    print(f'#### DANGER: OVERWROTE OLD ABID. NEW ABID=', self.abid)
+                else:
+                    raise change_error
+
         return super().save(*args, **kwargs)
 
-        # assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
-        # assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
-        # assert str(self.uuid) == str(self.ABID.uuid), f'self.uuid ({self.uuid}) does not match .ABID.uuid ({self.ABID.uuid})'
+    @property
+    def ABID_SOURCES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        assert self.abid_prefix
+        return {
+            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
+            'ts': self.abid_ts_src,
+            'uri': self.abid_uri_src,
+            'subtype': self.abid_subtype_src,
+            'rand': self.abid_rand_src,
+            'salt': 'self.abid_salt',               # defined as static class vars at build time
+        }
 
     @property
     def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
-        assert self.abid_ts_src != 'None'
-        assert self.abid_uri_src != 'None'
-        assert self.abid_rand_src != 'None'
-        assert self.abid_subtype_src != 'None'
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        abid_sources = self.ABID_SOURCES
+        assert all(src != 'None' for src in abid_sources.values())
         return {
-            'prefix': self.abid_prefix,
-            'ts': eval(self.abid_ts_src),
-            'uri': eval(self.abid_uri_src),
-            'subtype': eval(self.abid_subtype_src),
-            'rand': eval(self.abid_rand_src),
-            'salt': self.abid_salt,
+            'prefix': eval(abid_sources['prefix']),
+            'ts': eval(abid_sources['ts']),
+            'uri': eval(abid_sources['uri']),
+            'subtype': eval(abid_sources['subtype']),
+            'rand': eval(abid_sources['rand']),
+            'salt': eval(abid_sources['salt']),
         }
     
     @property
     def ABID_FRESH_HASHES(self) -> Dict[str, str]:
-        return abid_hashes_from_values(**self.ABID_FRESH_VALUES)
+        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
+        abid_values = self.ABID_FRESH_VALUES
+        assert all(val for val in abid_values.values())
+        return abid_hashes_from_values(
+            prefix=abid_values['prefix'],
+            ts=abid_values['ts'],
+            uri=abid_values['uri'],
+            subtype=abid_values['subtype'],
+            rand=abid_values['rand'],
+            salt=abid_values['salt'],
+        )
+    
+    @property
+    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
+        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
+        existing_abid = self.ABID
+        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
+        abid_sources = self.ABID_SOURCES
+        fresh_values = self.ABID_FRESH_VALUES
+        fresh_hashes = self.ABID_FRESH_HASHES
+        return {
+            key: {
+                'model': self.__class__.__name__,
+                'pk': self.pk,
+                'abid_src': abid_sources[key],
+                'abid_section': key,
+                'old_val': existing_values.get(key, None),
+                'old_hash': getattr(existing_abid, key),
+                'new_val': fresh_values[key],
+                'new_hash': new_hash,
+                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
+            }
+            for key, new_hash in fresh_hashes.items()
+            if getattr(existing_abid, key) != new_hash
+        }
 
-    def issue_new_abid(self):
-        assert self.abid is None, f'Can only issue new ABID for new objects that dont already have one {self.abid}'
-        assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
+    def issue_new_abid(self, force_new=False) -> ABID:
+        """
+        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
+        """
+        if not force_new:
+            assert self.abid is None, f'Can only issue new ABID for new objects that dont already have one {self.abid}'
+            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
         assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
 
+        # Setup Field defaults to be ready for ABID generation
         self.abid = None
-        self.pk = self.id = self.id or uuid4()
-        self.created = ts_from_abid(abid_part_from_ts(timezone.now()))  # cut off precision to match precision of TS component
+        self.id = self.id or uuid4()
+        self.pk = self.id
+        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
+        self.modified_at = self.modified_at or self.created_at
+        self.created_by = self.created_by or get_or_create_system_user_pk()
 
+        # Compute fresh ABID values & hashes based on object's live properties
         abid_fresh_values = self.ABID_FRESH_VALUES
         assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
         abid_fresh_hashes = self.ABID_FRESH_HASHES
@@ -140,64 +237,63 @@ class ABIDModel(models.Model):
         
         assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
 
-        # store stable ABID on local fields, overwrite them because we are adding a new entry and existing defaults havent touched db yet
-        self.abid = str(new_abid)
-        assert str(self.ABID.uuid) == str(new_abid.uuid)
         return new_abid
 
-
     @property
     def ABID(self) -> ABID:
         """
-        aka get_or_generate_abid -> ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
+        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
+        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
         """
 
-        if not self.abid:
-            pre_save_abid = self.issue_new_abid()
-            self.abid = str(pre_save_abid)
-            return pre_save_abid
+        if self.abid:
+            return ABID.parse(cast(str, self.abid))
+        
+        return self.issue_new_abid()
 
-        return ABID.parse(cast(str, self.abid))
-
-    @property
-    def ULID(self) -> ULID:
-        """
-        Get a ulid.ULID representation of the object's ABID.
-        """
-        return self.ABID.ulid
-
-    @property
-    def UUID(self) -> UUID:
-        """
-        Get a uuid.UUID (v4) representation of the object's ABID.
-        """
-        return self.ABID.uuid
+    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
+    # @property
+    # def UUID(self) -> UUID:
+    #     """
+    #     Get a uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return self.ABID.uuid
     
-    @property
-    def uuid(self) -> str:
-        """
-        Get a str uuid.UUID (v4) representation of the object's ABID.
-        """
-        return str(self.ABID.uuid)
-
-    @property
-    def TypeID(self) -> TypeID:
-        """
-        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-        """
-        return self.ABID.typeid
+    # @property
+    # def uuid(self) -> str:
+    #     """
+    #     Get a str uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return str(self.ABID.uuid)
     
-    @property
-    def abid_uri(self) -> str:
-        return eval(self.abid_uri_src)
+    # @property
+    # def ULID(self) -> ULID:
+    #     """
+    #     Get a ulid.ULID representation of the object's ABID.
+    #     """
+    #     return self.ABID.ulid
+
+    # @property
+    # def TypeID(self) -> TypeID:
+    #     """
+    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+    #     """
+    #     return self.ABID.typeid
     
     @property
     def api_url(self) -> str:
-        # /api/v1/core/any/{abid}
+        """
+        Compute the REST API URL to access this object.
+        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
+        """
         return reverse_lazy('api-1:get_any', args=[self.abid])
 
     @property
     def api_docs_url(self) -> str:
+        """
+        Compute the REST API Documentation URL to learn about accessing this object.
+        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
+        """
         return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
 
 
@@ -311,7 +407,7 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
                 )
 
             for obj in qs:
-                if abid in (str(obj.ABID_FRESH), str(obj.id), str(obj.abid)):
+                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
                     # found exact match, no need to keep iterating
                     return [obj]
                 partial_matches.append(obj)

archivebox/api/models.py

@@ -27,7 +27,7 @@ class APIToken(ABIDModel):
     """
     # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
     abid_prefix = 'apt_'
-    abid_ts_src = 'self.created'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.token'
     abid_subtype_src = 'self.created_by_id'
     abid_rand_src = 'self.id'
@@ -36,8 +36,8 @@ class APIToken(ABIDModel):
     abid = ABIDField(prefix=abid_prefix)
 
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
     expires = models.DateTimeField(null=True, blank=True)
@@ -59,7 +59,7 @@ class APIToken(ABIDModel):
             "abid":             str(self.ABID),
             "created_by_id":    str(self.created_by_id),
             "token":            self.token,
-            "created":          self.created.isoformat(),
+            "created_at":       self.created_at.isoformat(),
             "expires":          self.expires_as_iso8601,
         }
 
@@ -95,7 +95,7 @@ class OutboundWebhook(ABIDModel, WebhookBase):
         settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
     """
     abid_prefix = 'whk_'
-    abid_ts_src = 'self.created'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.endpoint'
     abid_subtype_src = 'self.ref'
     abid_rand_src = 'self.id'
@@ -104,8 +104,8 @@ class OutboundWebhook(ABIDModel, WebhookBase):
     abid = ABIDField(prefix=abid_prefix)
 
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     # More fields here: WebhookBase...
 

archivebox/api/v1_cli.py

@@ -94,7 +94,7 @@ class ListCommandSchema(Schema):
     status: Optional[StatusChoices] = StatusChoices.indexed
     after: Optional[float] = 0
     before: Optional[float] = 999999999999999
-    sort: str = 'added'
+    sort: str = 'bookmarked_at'
     as_json: bool = True
     as_html: bool = False
     as_csv: str | bool = 'timestamp,url'

archivebox/api/v1_core.py

@@ -60,22 +60,17 @@ class CustomPagination(PaginationBase):
 
 ### ArchiveResult #########################################################################
 
-class ArchiveResultSchema(Schema):
+class MinimalArchiveResultSchema(Schema):
     TYPE: str = 'core.models.ArchiveResult'
 
     id: UUID
     abid: str
 
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
 
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: str
-
     extractor: str
     cmd_version: Optional[str]
     cmd: List[str]
@@ -92,20 +87,12 @@ class ArchiveResultSchema(Schema):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
 
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
     @staticmethod
     def resolve_abid(obj):
         return str(obj.ABID)
 
     @staticmethod
-    def resolve_created(obj):
+    def resolve_created_at(obj):
         return obj.start_ts
 
     @staticmethod
@@ -116,13 +103,28 @@ class ArchiveResultSchema(Schema):
     def resolve_snapshot_url(obj):
         return obj.snapshot.url
 
+    @staticmethod
+    def resolve_snapshot_id(obj):
+        return str(obj.snapshot_id)
+    
     @staticmethod
     def resolve_snapshot_abid(obj):
         return str(obj.snapshot.ABID)
 
     @staticmethod
     def resolve_snapshot_tags(obj):
-        return obj.snapshot.tags_str()
+        return sorted(tag.name for tag in obj.snapshot.tags.all())
+
+class ArchiveResultSchema(MinimalArchiveResultSchema):
+    TYPE: str = 'core.models.ArchiveResult'
+
+    # ... Extends MinimalArchiveResultSchema fields ...
+
+    snapshot_id: UUID
+    snapshot_abid: str
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: List[str]
 
 
 class ArchiveResultFilterSchema(FilterSchema):
@@ -140,9 +142,9 @@ class ArchiveResultFilterSchema(FilterSchema):
     pwd: Optional[str] = Field(None, q='pwd__icontains')
     cmd_version: Optional[str] = Field(None, q='cmd_version')
 
-    created: Optional[datetime] = Field(None, q='updated')
-    created__gte: Optional[datetime] = Field(None, q='updated__gte')
-    created__lt: Optional[datetime] = Field(None, q='updated__lt')
+    created_at: Optional[datetime] = Field(None, q='created_at')
+    created_at__gte: Optional[datetime] = Field(None, q='created_at__gte')
+    created_at__lt: Optional[datetime] = Field(None, q='created_at__lt')
 
 
 @router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
@@ -194,23 +196,25 @@ class SnapshotSchema(Schema):
     id: UUID
     abid: str
 
-    modified: datetime
-    created: datetime
     created_by_id: str
     created_by_username: str
+    created_at: datetime
+    modified_at: datetime
+
+    bookmarked_at: datetime
+    downloaded_at: Optional[datetime]
 
     url: str
-    tags: str
+    tags: List[str]
     title: Optional[str]
     timestamp: str
     archive_path: str
 
-    bookmarked: datetime
-    added: datetime
-    updated: Optional[datetime]
+    # url_for_admin: str
+    # url_for_view: str
 
     num_archiveresults: int
-    archiveresults: List[ArchiveResultSchema]
+    archiveresults: List[MinimalArchiveResultSchema]
 
     @staticmethod
     def resolve_created_by_id(obj):
@@ -221,21 +225,21 @@ class SnapshotSchema(Schema):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
 
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
     @staticmethod
     def resolve_abid(obj):
         return str(obj.ABID)
 
     @staticmethod
     def resolve_tags(obj):
-        return obj.tags_str()
+        return sorted(tag.name for tag in obj.tags.all())
+
+    # @staticmethod
+    # def resolve_url_for_admin(obj):
+    #     return f"/admin/core/snapshot/{obj.id}/change/"
+    
+    # @staticmethod
+    # def resolve_url_for_view(obj):
+    #     return f"/{obj.archive_path}"
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
@@ -255,12 +259,12 @@ class SnapshotFilterSchema(FilterSchema):
     created_by_id: str = Field(None, q='created_by_id')
     created_by_username: str = Field(None, q='created_by__username__icontains')
 
-    created__gte: datetime = Field(None, q='created__gte')
-    created__lt: datetime = Field(None, q='created__lt')
-    created: datetime = Field(None, q='created')
-    modified: datetime = Field(None, q='modified')
-    modified__gte: datetime = Field(None, q='modified__gte')
-    modified__lt: datetime = Field(None, q='modified__lt')
+    created_at__gte: datetime = Field(None, q='created_at__gte')
+    created_at__lt: datetime = Field(None, q='created_at__lt')
+    created_at: datetime = Field(None, q='created_at')
+    modified_at: datetime = Field(None, q='modified_at')
+    modified_at__gte: datetime = Field(None, q='modified_at__gte')
+    modified_at__lt: datetime = Field(None, q='modified_at__lt')
 
     search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'timestamp__startswith'])
     url: Optional[str] = Field(None, q='url')
@@ -268,8 +272,8 @@ class SnapshotFilterSchema(FilterSchema):
     title: Optional[str] = Field(None, q='title__icontains')
     timestamp: Optional[str] = Field(None, q='timestamp__startswith')
     
-    added__gte: Optional[datetime] = Field(None, q='added__gte')
-    added__lt: Optional[datetime] = Field(None, q='added__lt')
+    bookmarked_at__gte: Optional[datetime] = Field(None, q='bookmarked_at__gte')
+    bookmarked_at__lt: Optional[datetime] = Field(None, q='bookmarked_at__lt')
 
 
 
@@ -285,7 +289,7 @@ def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_arch
 
 @router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
 def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid, uuid, or pk."""
+    """Get a specific Snapshot by abid or id."""
     request.with_archiveresults = with_archiveresults
     snapshot = None
     try:
@@ -311,7 +315,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
 # def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
 #
 #     for attr, value in payload.dict().items():
 #         setattr(snapshot, attr, value)
@@ -321,7 +325,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.delete("/snapshot/{snapshot_id}")
 # def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
 #     snapshot.delete()
 #     return {"success": True}
 
@@ -336,8 +340,8 @@ class TagSchema(Schema):
     id: UUID
     abid: str
 
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
 

archivebox/core/admin.py

@@ -127,10 +127,10 @@ class CustomUserAdmin(UserAdmin):
                 '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> {}',
                 snap.pk,
                 snap.abid,
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
                 snap.url[:64],
             )
-            for snap in obj.snapshot_set.order_by('-modified')[:10]
+            for snap in obj.snapshot_set.order_by('-modified_at')[:10]
         ) + f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
     @admin.display(description='Archive Result Logs')
@@ -141,11 +141,11 @@ class CustomUserAdmin(UserAdmin):
                 '<code><a href="/admin/core/archiveresult/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> <b>📄 {}</b> {}',
                 result.pk,
                 result.abid,
-                result.snapshot.updated.strftime('%Y-%m-%d %H:%M') if result.snapshot.updated else 'pending...',
+                result.snapshot.downloaded_at.strftime('%Y-%m-%d %H:%M') if result.snapshot.downloaded_at else 'pending...',
                 result.extractor,
                 result.snapshot.url[:64],
             )
-            for result in obj.archiveresult_set.order_by('-modified')[:10]
+            for result in obj.archiveresult_set.order_by('-modified_at')[:10]
         ) + f'<br/><a href="/admin/core/archiveresult/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
     @admin.display(description='Tags')
@@ -157,7 +157,7 @@ class CustomUserAdmin(UserAdmin):
                 tag.pk,
                 tag.name,
             )
-            for tag in obj.tag_set.order_by('-modified')[:10]
+            for tag in obj.tag_set.order_by('-modified_at')[:10]
         ) + f'<br/><a href="/admin/core/tag/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
     @admin.display(description='API Tokens')
@@ -171,7 +171,7 @@ class CustomUserAdmin(UserAdmin):
                 apitoken.token_redacted[:64],
                 apitoken.expires,
             )
-            for apitoken in obj.apitoken_set.order_by('-modified')[:10]
+            for apitoken in obj.apitoken_set.order_by('-modified_at')[:10]
         ) + f'<br/><a href="/admin/api/apitoken/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
     @admin.display(description='API Outbound Webhooks')
@@ -185,7 +185,7 @@ class CustomUserAdmin(UserAdmin):
                 outboundwebhook.referenced_model,
                 outboundwebhook.endpoint,
             )
-            for outboundwebhook in obj.outboundwebhook_set.order_by('-modified')[:10]
+            for outboundwebhook in obj.outboundwebhook_set.order_by('-modified_at')[:10]
         ) + f'<br/><a href="/admin/api/outboundwebhook/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
 
 
@@ -351,13 +351,13 @@ class SnapshotActionForm(ActionForm):
 
 @admin.register(Snapshot, site=archivebox_admin)
 class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
-    list_display = ('created', 'title_str', 'files', 'size', 'url_str')
-    sort_fields = ('title_str', 'url_str', 'created')
-    readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'created', 'created', 'updated', 'modified', 'API', 'link_dir')
+    list_display = ('created_at', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'created_at')
+    readonly_fields = ('admin_actions', 'status_info', 'tags_str', 'imported_timestamp', 'created_at', 'modified_at', 'downloaded_at', 'abid_info', 'link_dir')
     search_fields = ('id', 'url', 'abid', 'timestamp', 'title', 'tags__name')
-    list_filter = ('created', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
-    fields = ('url', 'created_by', 'title',*readonly_fields)
-    ordering = ['-created']
+    list_filter = ('created_at', 'downloaded_at', 'archiveresult__status', 'created_by', 'tags__name')
+    fields = ('url', 'title', 'created_by', 'bookmarked_at', *readonly_fields)
+    ordering = ['-created_at']
     actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
     inlines = [TagInline, ArchiveResultInline]
     list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
@@ -377,30 +377,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
             self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
             return super().changelist_view(request, GLOBAL_CONTEXT)
 
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        snapshot = None
-
-        try:
-            snapshot = snapshot or Snapshot.objects.get(id=object_id)
-        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
-            pass
-        
-        try:
-            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
-        except (Snapshot.DoesNotExist, ValidationError):
-            pass
-
-        if snapshot:
-            object_id = str(snapshot.id)
-
-
-        return super().change_view(
-            request,
-            object_id,
-            form_url,
-            extra_context=extra_context,
-        )
 
     def get_urls(self):
         urls = super().get_urls()
@@ -416,8 +392,20 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
     #     self.request = request
     #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
 
-    def tag_list(self, obj):
-        return ', '.join(tag.name for tag in obj.tags.all())
+    @admin.action(
+        description="Imported Timestamp"
+    )
+    def imported_timestamp(self, obj):
+        context = RequestContext(self.request, {
+            'bookmarked_date': obj.bookmarked,
+            'timestamp': obj.timestamp,
+        })
+
+        html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
+        return mark_safe(html.render(context))
+    
+        # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
+        # return f'{pretty_time} ({obj.timestamp})'
 
     # TODO: figure out a different way to do this, you cant nest forms so this doenst work
     # def action(self, obj):
@@ -647,14 +635,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
 
 @admin.register(Tag, site=archivebox_admin)
 class TagAdmin(ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
-    list_filter = ('created', 'created_by')
-    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
-    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'snapshots')
+    list_display = ('created_at', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
+    list_filter = ('created_at', 'created_by')
+    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created_at')
+    readonly_fields = ('slug', 'abid', 'created_at', 'modified_at', 'abid_info', 'snapshots')
     search_fields = ('abid', 'name', 'slug')
     fields = ('name', 'created_by', *readonly_fields)
     actions = ['delete_selected']
-    ordering = ['-created']
+    ordering = ['-created_at']
 
     paginator = AccelleratedPaginator
 
@@ -672,10 +660,10 @@ class TagAdmin(ABIDModelAdmin):
             format_html(
                 '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
                 snap.pk,
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
                 snap.url[:64],
             )
-            for snap in tag.snapshot_set.order_by('-updated')[:10]
+            for snap in tag.snapshot_set.order_by('-downloaded_at')[:10]
         ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">{total_count} total snapshots...<a>'))
 
 
@@ -683,7 +671,7 @@ class TagAdmin(ABIDModelAdmin):
 class ArchiveResultAdmin(ABIDModelAdmin):
     list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
+    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created_at', 'modified_at', 'abid_info', 'output_summary')
     search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
     fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
     autocomplete_fields = ['snapshot']
@@ -706,7 +694,7 @@ class ArchiveResultAdmin(ABIDModelAdmin):
             '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
             result.snapshot.timestamp,
             result.snapshot.abid,
-            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
+            result.snapshot.bookmarked_at.strftime('%Y-%m-%d %H:%M'),
             result.snapshot.url[:128],
         )
 
@@ -765,18 +753,18 @@ class ArchiveResultAdmin(ABIDModelAdmin):
 
 @admin.register(APIToken, site=archivebox_admin)
 class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created', 'created_by', 'expires')
-    readonly_fields = ('created', 'modified', 'API')
+    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
     search_fields = ('id', 'abid', 'created_by__username', 'token')
     fields = ('created_by', 'token', 'expires', *readonly_fields)
 
     list_filter = ('created_by',)
-    ordering = ['-created']
+    ordering = ['-created_at']
     list_per_page = 100
 
 @admin.register(get_webhook_model(), site=archivebox_admin)
 class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('created', 'modified', 'API', *WebhookAdmin.readonly_fields)
+    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
+    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)

archivebox/core/migrations/0032_alter_archiveresult_id.py

@@ -1,9 +1,13 @@
 # Generated by Django 5.0.6 on 2024-08-18 05:20
 
 import core.models
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -14,6 +18,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
+            field=models.BigIntegerField(default=rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
         ),
     ]

archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py

@@ -2,11 +2,16 @@
 
 import core.models
 import uuid
+import random
 from django.db import migrations, models
 
 from abid_utils.abid import ABID
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
+
 def update_archiveresult_ids(apps, schema_editor):
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     num_total = ArchiveResult.objects.all().count()
@@ -30,7 +35,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='ID'),
         ),
         migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
         migrations.AlterField(

archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py

@@ -2,9 +2,13 @@
 
 import core.models
 import uuid
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -20,6 +24,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='archiveresult',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0058_alter_tag_old_id.py

@@ -1,9 +1,12 @@
 # Generated by Django 5.0.6 on 2024-08-20 03:30
 
-import core.models
+import random
 from django.db import migrations, models
 
 
+def rand_int_id():
+    return random.getrandbits(32)
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -14,6 +17,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='tag',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
         ),
     ]

archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py

@@ -3,8 +3,11 @@
 import core.models
 import django.db.models.deletion
 import uuid
+import random
 from django.db import migrations, models
 
+def rand_int_id():
+    return random.getrandbits(32)
 
 class Migration(migrations.Migration):
 
@@ -26,6 +29,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='tag',
             name='old_id',
-            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
+            field=models.BigIntegerField(default=rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
         ),
     ]

archivebox/core/models.py

@@ -5,10 +5,7 @@ from typing import Optional, List, Dict, Iterable
 from django_stubs_ext.db.models import TypedModelMeta
 
 import json
-import random
 
-import uuid
-from uuid import uuid4
 from pathlib import Path
 
 from django.db import models
@@ -18,9 +15,10 @@ from django.utils.text import slugify
 from django.core.cache import cache
 from django.urls import reverse, reverse_lazy
 from django.db.models import Case, When, Value, IntegerField
+from django.contrib import admin
 from django.conf import settings
 
-from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, get_or_create_system_user_pk
+from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
 
 from ..system import get_dir_size
 from ..util import parse_date, base_url
@@ -29,13 +27,10 @@ from ..index.html import snapshot_icons
 from ..extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE, EXTRACTORS
 
 
-def rand_int_id():
-    return random.getrandbits(32)
-
 
 # class BaseModel(models.Model):
 #     # TODO: migrate all models to a shared base class with all our standard fields and helpers:
-#     #       ulid/created/modified/owner/is_deleted/as_json/from_json/etc.
+#     #       ulid/created_at/modified_at/created_by/is_deleted/as_json/from_json/etc.
 #     #
 #     # id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
 #     # ulid = models.CharField(max_length=26, null=True, blank=True, db_index=True, unique=True)
@@ -51,17 +46,18 @@ class Tag(ABIDModel):
     Based on django-taggit model + ABID base.
     """
     abid_prefix = 'tag_'
-    abid_ts_src = 'self.created'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.slug'
     abid_subtype_src = '"03"'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
     id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='tag_set')
-    created = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     name = models.CharField(unique=True, blank=False, max_length=100)
     slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
@@ -131,33 +127,41 @@ class SnapshotManager(models.Manager):
 
 class Snapshot(ABIDModel):
     abid_prefix = 'snp_'
-    abid_ts_src = 'self.created'
+    abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.url'
     abid_subtype_src = '"01"'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = False
 
     id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set')
-    created = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)  # loaded from self._init_timestamp
+    modified_at = models.DateTimeField(auto_now=True)
 
     # legacy ts fields
-    added = AutoDateTimeField(default=None, null=False, editable=True, db_index=True)
-    updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
+    bookmarked_at = AutoDateTimeField(default=None, null=False, editable=True, db_index=True)
+    downloaded_at = models.DateTimeField(default=None, null=True, editable=False, db_index=True, blank=True)
 
     url = models.URLField(unique=True, db_index=True)
     timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
     tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
     title = models.CharField(max_length=512, null=True, blank=True, db_index=True)    
 
-    keys = ('url', 'timestamp', 'title', 'tags', 'updated')
+    keys = ('url', 'timestamp', 'title', 'tags', 'downloaded_at')
 
     archiveresult_set: models.Manager['ArchiveResult']
 
     objects = SnapshotManager()
 
+    def save(self, *args, **kwargs):
+        if not self.bookmarked_at:
+            self.bookmarked_at = self.created_at or self._init_timestamp
+        
+        super().save(*args, **kwargs)
+
+
     def __repr__(self) -> str:
         title = (self.title_stripped or '-')[:64]
         return f'[{self.timestamp}] {self.url[:64]} ({title})'
@@ -185,9 +189,10 @@ class Snapshot(ABIDModel):
         from ..index import load_link_details
         return load_link_details(self.as_link())
 
+    @admin.display(description='Tags')
     def tags_str(self, nocache=True) -> str | None:
         calc_tags_str = lambda: ','.join(sorted(tag.name for tag in self.tags.all()))
-        cache_key = f'{self.pk}-{(self.updated or self.added).timestamp()}-tags'
+        cache_key = f'{self.pk}-{(self.downloaded_at or self.bookmarked_at).timestamp()}-tags'
         
         if hasattr(self, '_prefetched_objects_cache') and 'tags' in self._prefetched_objects_cache:
             # tags are pre-fetched already, use them directly (best because db is always freshest)
@@ -255,7 +260,7 @@ class Snapshot(ABIDModel):
 
     @cached_property
     def archive_size(self):
-        cache_key = f'{str(self.pk)[:12]}-{(self.updated or self.added).timestamp()}-size'
+        cache_key = f'{str(self.pk)[:12]}-{(self.downloaded_at or self.bookmarked_at).timestamp()}-size'
 
         def calc_dir_size():
             try:
@@ -274,7 +279,7 @@ class Snapshot(ABIDModel):
                     for result in self.archiveresult_set.all()
                     if result.extractor == 'screenshot' and result.status =='succeeded' and result.output
                 ),
-                key=lambda result: result.created,
+                key=lambda result: result.created_at,
             ) or [None])[-1]
         else:
             result = self.archiveresult_set.filter(
@@ -359,7 +364,7 @@ class Snapshot(ABIDModel):
 
 
     # def get_storage_dir(self, create=True, symlink=True) -> Path:
-    #     date_str = self.added.strftime('%Y%m%d')
+    #     date_str = self.bookmarked_at.strftime('%Y%m%d')
     #     domain_str = domain(self.url)
     #     abs_storage_dir = Path(settings.CONFIG.ARCHIVE_DIR) / 'snapshots' / date_str / domain_str / str(self.ulid)
 
@@ -407,10 +412,11 @@ class ArchiveResultManager(models.Manager):
 
 class ArchiveResult(ABIDModel):
     abid_prefix = 'res_'
-    abid_ts_src = 'self.snapshot.added'
+    abid_ts_src = 'self.snapshot.created_at'
     abid_uri_src = 'self.snapshot.url'
     abid_subtype_src = 'self.extractor'
     abid_rand_src = 'self.id'
+    abid_drift_allowed = True
 
     EXTRACTOR_CHOICES = (
         ('htmltotext', 'htmltotext'),
@@ -438,8 +444,8 @@ class ArchiveResult(ABIDModel):
     abid = ABIDField(prefix=abid_prefix)
 
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='archiveresult_set')
-    created = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified = models.DateTimeField(auto_now=True)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')
 
@@ -460,6 +466,7 @@ class ArchiveResult(ABIDModel):
         
 
     def __str__(self):
+        # return f'[{self.abid}] 📅 {self.start_ts.strftime("%Y-%m-%d %H:%M")} 📄 {self.extractor} {self.snapshot.url}'
         return self.extractor
 
     @cached_property
@@ -503,7 +510,7 @@ class ArchiveResult(ABIDModel):
 
 
     # def get_storage_dir(self, create=True, symlink=True):
-    #     date_str = self.snapshot.added.strftime('%Y%m%d')
+    #     date_str = self.snapshot.bookmarked_at.strftime('%Y%m%d')
     #     domain_str = domain(self.snapshot.url)
     #     abs_storage_dir = Path(settings.CONFIG.ARCHIVE_DIR) / 'results' / date_str / domain_str / self.extractor / str(self.ulid)
 

archivebox/core/views.py

@@ -211,7 +211,7 @@ class SnapshotView(View):
                     format_html(
                         (
                             '<center><br/><br/><br/>'
-                            'No Snapshot directories match the given timestamp or UUID: <code>{}</code><br/><br/>'
+                            'No Snapshot directories match the given timestamp/ID/ABID: <code>{}</code><br/><br/>'
                             'You can <a href="/add/" target="_top">add a new Snapshot</a>, or return to the <a href="/" target="_top">Main Index</a>'
                             '</center>'
                         ),
@@ -225,18 +225,18 @@ class SnapshotView(View):
                 snapshot_hrefs = mark_safe('<br/>').join(
                     format_html(
                         '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                        snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                        snap.bookmarked_at.strftime('%Y-%m-%d %H:%M:%S'),
                         snap.timestamp,
                         snap.timestamp,
                         snap.url,
                         snap.title_stripped[:64] or '',
                     )
-                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
+                    for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'bookmarked_at').order_by('-bookmarked_at')
                 )
                 return HttpResponse(
                     format_html(
                         (
-                            'Multiple Snapshots match the given timestamp/UUID <code>{}</code><br/><pre>'
+                            'Multiple Snapshots match the given timestamp/ID/ABID <code>{}</code><br/><pre>'
                         ),
                         slug,
                     ) + snapshot_hrefs + format_html(
@@ -257,12 +257,12 @@ class SnapshotView(View):
                         (
                             '<center><br/><br/><br/>'
                             f'Snapshot <a href="/archive/{snapshot.timestamp}/index.html" target="_top"><b><code>[{snapshot.timestamp}]</code></b></a>: <a href="{snapshot.url}" target="_blank" rel="noreferrer">{snapshot.url}</a><br/>'
-                            f'was queued on {str(snapshot.added).split(".")[0]}, '
+                            f'was queued on {str(snapshot.bookmarked_at).split(".")[0]}, '
                             f'but no files have been saved yet in:<br/><b><a href="/archive/{snapshot.timestamp}/" target="_top"><code>{snapshot.timestamp}</code></a><code>/'
                             '{}'
                             f'</code></b><br/><br/>'
                             'It\'s possible {} '
-                            f'during the last capture on {str(snapshot.added).split(".")[0]},<br/>or that the archiving process has not completed yet.<br/>'
+                            f'during the last capture on {str(snapshot.bookmarked_at).split(".")[0]},<br/>or that the archiving process has not completed yet.<br/>'
                             f'<pre><code># run this cmd to finish/retry archiving this Snapshot</code><br/>'
                             f'<code style="user-select: all; color: #333">archivebox update -t timestamp {snapshot.timestamp}</code></pre><br/><br/>'
                             '<div class="text-align: left; width: 100%; max-width: 400px">'
@@ -270,7 +270,7 @@ class SnapshotView(View):
                             f'- list all the <a href="/archive/{snapshot.timestamp}/" target="_top">Snapshot files <code>.*</code></a><br/>'
                             f'- view the <a href="/archive/{snapshot.timestamp}/index.html" target="_top">Snapshot <code>./index.html</code></a><br/>'
                             f'- go to the <a href="/admin/core/snapshot/{snapshot.pk}/change/" target="_top">Snapshot admin</a> to edit<br/>'
-                            f'- go to the <a href="/admin/core/snapshot/?uuid__startswith={snapshot.uuid}" target="_top">Snapshot actions</a> to re-archive<br/>'
+                            f'- go to the <a href="/admin/core/snapshot/?id__exact={snapshot.id}" target="_top">Snapshot actions</a> to re-archive<br/>'
                             '- or return to <a href="/" target="_top">the main index...</a></div>'
                             '</center>'
                         ),
@@ -343,7 +343,7 @@ class SnapshotView(View):
             snapshot_hrefs = mark_safe('<br/>').join(
                 format_html(
                     '{} <code style="font-size: 0.8em">{}</code> <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
-                    snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                    snap.bookmarked_at.strftime('%Y-%m-%d %H:%M:%S'),
                     snap.abid,
                     snap.timestamp,
                     snap.timestamp,
@@ -353,7 +353,7 @@ class SnapshotView(View):
                 for snap in Snapshot.objects.filter(
                     Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
                     | Q(abid__icontains=path) | Q(id__icontains=path)
-                ).only('url', 'timestamp', 'title', 'added').order_by('-added')
+                ).only('url', 'timestamp', 'title', 'bookmarked_at').order_by('-bookmarked_at')
             )
             return HttpResponse(
                 format_html(
@@ -376,7 +376,7 @@ class PublicIndexView(ListView):
     template_name = 'public_index.html'
     model = Snapshot
     paginate_by = SNAPSHOTS_PER_PAGE
-    ordering = ['-added']
+    ordering = ['-bookmarked_at', '-created_at']
 
     def get_context_data(self, **kwargs):
         return {

archivebox/extractors/__init__.py

@@ -134,7 +134,7 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
         link = load_link_details(link, out_dir=out_dir)
         write_link_details(link, out_dir=out_dir, skip_sql_index=False)
         log_link_archiving_started(link, str(out_dir), is_new)
-        link = link.overwrite(updated=datetime.now(timezone.utc))
+        link = link.overwrite(downloaded_at=datetime.now(timezone.utc))
         stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
         start_ts = datetime.now(timezone.utc)
 
@@ -157,11 +157,11 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
                                                  output=result.output, pwd=result.pwd, start_ts=result.start_ts, end_ts=result.end_ts, status=result.status, created_by_id=snapshot.created_by_id)
 
 
-                    # bump the updated time on the main Snapshot here, this is critical
+                    # bump the downloaded_at time on the main Snapshot here, this is critical
                     # to be able to cache summaries of the ArchiveResults for a given
                     # snapshot without having to load all the results from the DB each time.
-                    # (we use {Snapshot.pk}-{Snapshot.updated} as the cache key and assume
-                    # ArchiveResults are unchanged as long as the updated timestamp is unchanged)
+                    # (we use {Snapshot.pk}-{Snapshot.downloaded_at} as the cache key and assume
+                    # ArchiveResults are unchanged as long as the downloaded_at timestamp is unchanged)
                     snapshot.save()
                 else:
                     # print('{black}      X {}{reset}'.format(method_name, **ANSI))

archivebox/extractors/wget.py

@@ -245,7 +245,7 @@ def wget_output_path(link: Link, nocache: bool=False) -> Optional[str]:
     #    https://example.com/abc/test/?v=zzVa_tX1OiI
     #       > example.com/abc/test/index.html@v=zzVa_tX1OiI.html
 
-    cache_key = f'{link.url_hash}:{link.timestamp}-{link.updated and link.updated.timestamp()}-wget-output-path'
+    cache_key = f'{link.url_hash}:{link.timestamp}-{link.downloaded_at and link.downloaded_at.timestamp()}-wget-output-path'
     
     if not nocache:
         from django.core.cache import cache

archivebox/index/html.py

@@ -118,7 +118,7 @@ def render_django_template(template: str, context: Mapping[str, str]) -> str:
 
 
 def snapshot_icons(snapshot) -> str:
-    cache_key = f'result_icons:{snapshot.pk}:{(snapshot.modified or snapshot.created or snapshot.added).timestamp()}'
+    cache_key = f'result_icons:{snapshot.pk}:{(snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at or snapshot.bookmarked_at).timestamp()}'
     
     def calc_snapshot_icons():
         from core.models import ArchiveResult

archivebox/index/schema.py

@@ -132,7 +132,7 @@ class Link:
     tags: Optional[str]
     sources: List[str]
     history: Dict[str, List[ArchiveResult]] = field(default_factory=lambda: {})
-    updated: Optional[datetime] = None
+    downloaded_at: Optional[datetime] = None
     schema: str = 'Link'
 
     def __str__(self) -> str:
@@ -164,7 +164,7 @@ class Link:
             assert isinstance(self.timestamp, str) and self.timestamp
             assert self.timestamp.replace('.', '').isdigit()
             assert isinstance(self.url, str) and '://' in self.url
-            assert self.updated is None or isinstance(self.updated, datetime)
+            assert self.downloaded_at is None or isinstance(self.downloaded_at, datetime)
             assert self.title is None or (isinstance(self.title, str) and self.title)
             assert self.tags is None or isinstance(self.tags, str)
             assert isinstance(self.sources, list)
@@ -184,7 +184,7 @@ class Link:
             'url': self.url,
             'title': self.title or None,
             'timestamp': self.timestamp,
-            'updated': self.updated or None,
+            'downloaded_at': self.downloaded_at or None,
             'tags': self.tags or None,
             'sources': self.sources or [],
             'history': self.history or {},
@@ -210,7 +210,7 @@ class Link:
                 'icons': None,           # only used to render static index in index/html.py, remove if no longer needed there
 
                 'bookmarked_date': self.bookmarked_date,
-                'updated_date': self.updated_date,
+                'downloaded_datestr': self.downloaded_datestr,
                 'oldest_archive_date': self.oldest_archive_date,
                 'newest_archive_date': self.newest_archive_date,
         
@@ -236,7 +236,7 @@ class Link:
             for key, val in json_info.items()
             if key in cls.field_names()
         }
-        info['updated'] = parse_date(info.get('updated'))
+        info['downloaded_at'] = parse_date(info.get('updated') or info.get('downloaded_at'))
         info['sources'] = info.get('sources') or []
 
         json_history = info.get('history') or {}
@@ -347,8 +347,8 @@ class Link:
 
 
     @property
-    def updated_date(self) -> Optional[str]:
-        return ts_to_date_str(self.updated) if self.updated else None
+    def downloaded_datestr(self) -> Optional[str]:
+        return ts_to_date_str(self.downloaded_at) if self.downloaded_at else None
 
     @property
     def archive_dates(self) -> List[datetime]:

archivebox/main.py

@@ -540,9 +540,9 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
     last_login = User.objects.order_by('last_login').last()
     if last_login:
         print(f'    Last UI login: {last_login.username} @ {str(last_login.last_login)[:16]}')
-    last_updated = Snapshot.objects.order_by('updated').last()
-    if last_updated:
-        print(f'    Last changes: {str(last_updated.updated)[:16]}')
+    last_downloaded = Snapshot.objects.order_by('downloaded_at').last()
+    if last_downloaded:
+        print(f'    Last changes: {str(last_downloaded.downloaded_at)[:16]}')
 
     if not users:
         print()
@@ -550,13 +550,13 @@ def status(out_dir: Path=OUTPUT_DIR) -> None:
         print('        archivebox manage createsuperuser')
 
     print()
-    for snapshot in links.order_by('-updated')[:10]:
-        if not snapshot.updated:
+    for snapshot in links.order_by('-downloaded_at')[:10]:
+        if not snapshot.downloaded_at:
             continue
         print(
             ANSI['black'],
             (
-                f'   > {str(snapshot.updated)[:16]} '
+                f'   > {str(snapshot.downloaded_at)[:16]} '
                 f'[{snapshot.num_outputs} {("X", "√")[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] '
                 f'"{snapshot.title}": {snapshot.url}'
             )[:TERM_WIDTH()],

archivebox/plugantic/models.py

@@ -1,50 +1 @@
 __package__ = 'archivebox.plugantic'
-
-
-# import uuid
-# from django.db import models
-# from typing_extensions import Self
-
-# from django_pydantic_field import SchemaField
-# from django.conf import settings
-
-# from abid_utils.models import ABIDModel, ABIDField
-
-# # from .plugins import Plugin as PluginSchema, CORE_PLUGIN
-# from .binproviders import BinProvider
-# from .binaries import Binary
-# from .configs import WgetOptionsConfig
-# from .extractors import Extractor
-# from .replayers import Replayer
-
-
-# PLUGINS_ROOT = settings.CONFIG['OUTPUT_DIR'] / 'plugins'
-# PLUGINS_ROOT.mkdir(exist_ok=True)
-
-
-# class CustomPlugin(ABIDModel):
-#     abid_prefix = 'plg_'
-#     abid_ts_src = 'self.added'
-#     abid_uri_src = 'self.name'
-#     abid_subtype_src = '"09"'
-#     abid_rand_src = 'self.id'
-
-#     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)  # legacy pk
-#     uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
-#     abid = ABIDField(prefix=abid_prefix)
-
-#     name = models.CharField(max_length=64, blank=False, unique=True)
-
-#     path = models.FilePathField(path=str(PLUGINS_ROOT), match='*', recursive=True, allow_folders=True, allow_files=False)
-
-#     # replayers: list[Replayer] = SchemaField()
-#     # binaries: list[Replayer] = SchemaField()
-#     # extractors: list[Replayer] = SchemaField()
-
-
-#     # @classmethod
-#     # def from_loaded_plugin(cls, plugin: PluginSchema) -> Self:
-#     #     new_obj = cls(
-#     #         schema=plugin,
-#     #     )
-#     #     return new_obj

archivebox/templates/admin/snapshots_grid.html

@@ -148,7 +148,7 @@
       <div class="card">
           <div class="card-info">
             <a href="{% url 'admin:core_snapshot_change' obj.pk %}">
-              <span class="timestamp">{{obj.added}}</span>
+              <span class="timestamp">{{obj.bookmarked_at}}</span>
             </a>
             <label>
               <span class="num_outputs">📄 &nbsp; {{obj.num_outputs}}</span> &nbsp; &nbsp;

archivebox/templates/core/index_row.html

@@ -1,8 +1,8 @@
 {% load static tz core_tags %}
 
 <tr>
-    <td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.added.timestamp}}">
-        {{ link.added|localtime }}
+    <td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.bookmarked_at.timestamp}}">
+        {{ link.bookmarked_at|localtime }}
     </td>
     <td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}" title="{{link.title|default:'Not yet archived...'}}">
         {% if link.is_archived %}

archivebox/templates/core/snapshot.html

@@ -317,7 +317,7 @@
                         </div>
                         <div title="Date last checked" class="info-chunk" title="UTC Timezone">
                             <h5>Last Checked</h5>
-                            {{updated_date}}
+                            {{downloaded_datestr}}
                         </div>
                     </div>
                     <div class="col-lg-4">

archivebox/templates/core/snapshot_live.html

@@ -379,8 +379,8 @@
                         </small>
                     </div>
                     <div class="col-lg-2" style="padding-top: 4px">
-                        <a href="/archive/{{url}}" title="Date Added: {{bookmarked_date}}  |  First Archived: {{oldest_archive_date|default:updated_date}}  |  Last Checked: {{updated_date}}   (UTC)">
-                            {{oldest_archive_date|default:updated_date|default:bookmarked_date}}
+                        <a href="/archive/{{url}}" title="Date Added: {{bookmarked_date}}  |  First Archived: {{oldest_archive_date|default:downloaded_datestr}}  |  Last Checked: {{downloaded_datestr}}   (UTC)">
+                            {{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date}}
                         </a>
                         <br/>
                         <div class="external-links">

archivebox/util.py

@@ -200,7 +200,7 @@ def parse_date(date: Any) -> Optional[datetime]:
         date = str(date)
 
     if isinstance(date, str):
-        return dateparser(date, settings={'TIMEZONE': 'UTC'}).replace(tzinfo=timezone.utc)
+        return dateparser(date, settings={'TIMEZONE': 'UTC'}).astimezone(timezone.utc)
 
     raise ValueError('Tried to parse invalid date! {}'.format(date))