mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2024-09-19 07:28:49 -04:00
remove .old_id entirely and make ABID generation only happen once on initial save
Some checks are pending
CodeQL / Analyze (python) (push) Waiting to run
Build Debian package / build (push) Waiting to run
Build Docker image / buildx (push) Waiting to run
Build Homebrew package / build (push) Waiting to run
Build GitHub Pages website / build (push) Waiting to run
Build GitHub Pages website / deploy (push) Blocked by required conditions
Run linters / lint (push) Waiting to run
Build Pip package / build (push) Waiting to run
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Waiting to run
Run tests / docker_tests (push) Waiting to run
Some checks are pending
CodeQL / Analyze (python) (push) Waiting to run
Build Debian package / build (push) Waiting to run
Build Docker image / buildx (push) Waiting to run
Build Homebrew package / build (push) Waiting to run
Build GitHub Pages website / build (push) Waiting to run
Build GitHub Pages website / deploy (push) Blocked by required conditions
Run linters / lint (push) Waiting to run
Build Pip package / build (push) Waiting to run
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Waiting to run
Run tests / docker_tests (push) Waiting to run
This commit is contained in:
Nick Sweeting
parent
22a6666bab
commit
68a39b7392
11 changed files with 242 additions and 142 deletions
|
|
@ -69,7 +69,6 @@ def get_abid_info(self, obj, request=None):
|
|||
SUBTYPE: <code style="font-size: 10px;"><b style="user-select: all">{}</b> {}</code> <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
|
||||
RAND: <code style="font-size: 10px;"><b style="user-select: all">{}</b> {}</code> <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
|
||||
<br/><hr/>
|
||||
<small style="opacity: 0.5">.old_id: <code style="font-size: 10px; user-select: all">{}</code></small><br/>
|
||||
</div>
|
||||
''',
|
||||
obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
|
||||
|
|
@ -82,7 +81,6 @@ def get_abid_info(self, obj, request=None):
|
|||
highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(obj.ABID_FRESH_VALUES['uri']),
|
||||
highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(obj.ABID_FRESH_VALUES['subtype']),
|
||||
highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(obj.ABID_FRESH_VALUES['rand'])[-7:],
|
||||
highlight_diff(getattr(obj, 'old_id', ''), obj.pk),
|
||||
)
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
|
|
|||
|
|
@ -73,27 +73,30 @@ class ABIDModel(models.Model):
|
|||
"""
|
||||
Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
|
||||
"""
|
||||
abid_prefix: str = DEFAULT_ABID_PREFIX # e.g. 'tag_'
|
||||
abid_ts_src = 'None' # e.g. 'self.created'
|
||||
abid_uri_src = 'None' # e.g. 'self.uri'
|
||||
abid_subtype_src = 'None' # e.g. 'self.extractor'
|
||||
abid_rand_src = 'None' # e.g. 'self.uuid' or 'self.id'
|
||||
abid_prefix: str = DEFAULT_ABID_PREFIX # e.g. 'tag_'
|
||||
abid_ts_src = 'self.created' # e.g. 'self.created'
|
||||
abid_uri_src = 'None' # e.g. 'self.uri'
|
||||
abid_subtype_src = 'self.__class__.__name__' # e.g. 'self.extractor'
|
||||
abid_rand_src = 'self.id' # e.g. 'self.uuid' or 'self.id'
|
||||
abid_salt: str = DEFAULT_ABID_URI_SALT
|
||||
|
||||
# id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
|
||||
# uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
# id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||
# abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||
created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
# created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
|
||||
# created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
# modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
def save(self, *args: Any, **kwargs: Any) -> None:
|
||||
if self._state.adding:
|
||||
self.issue_new_abid()
|
||||
self.pk = self.id = self.id or uuid4()
|
||||
self.created = ts_from_abid(abid_part_from_ts(timezone.now())) # cut off precision to match precision of TS component
|
||||
self.modified = self.created
|
||||
self.created_by = self.created_by or get_or_create_system_user_pk()
|
||||
self.abid = str(self.issue_new_abid())
|
||||
return super().save(*args, **kwargs)
|
||||
|
||||
# assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
|
||||
|
|
@ -119,48 +122,28 @@ class ABIDModel(models.Model):
|
|||
def ABID_FRESH_HASHES(self) -> Dict[str, str]:
|
||||
return abid_hashes_from_values(**self.ABID_FRESH_VALUES)
|
||||
|
||||
|
||||
@property
|
||||
def ABID_FRESH(self) -> ABID:
|
||||
"""
|
||||
Return a pure freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
|
||||
"""
|
||||
def issue_new_abid(self):
|
||||
assert self.abid is None, f'Can only issue new ABID for new objects that dont already have one {self.abid}'
|
||||
assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
|
||||
assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
|
||||
|
||||
self.abid = None
|
||||
self.pk = self.id = self.id or uuid4()
|
||||
self.created = ts_from_abid(abid_part_from_ts(timezone.now())) # cut off precision to match precision of TS component
|
||||
|
||||
abid_fresh_values = self.ABID_FRESH_VALUES
|
||||
assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
|
||||
abid_fresh_hashes = self.ABID_FRESH_HASHES
|
||||
assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
|
||||
|
||||
abid = ABID(**abid_fresh_hashes)
|
||||
new_abid = ABID(**abid_fresh_hashes)
|
||||
|
||||
assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
|
||||
return abid
|
||||
|
||||
|
||||
def issue_new_abid(self):
|
||||
assert self.abid is None, f'Can only issue new ABID for new objects that dont already have one {self.abid}'
|
||||
assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
|
||||
assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
|
||||
|
||||
self.old_id = getattr(self, 'old_id', None) or self.id or uuid4()
|
||||
self.abid = None
|
||||
self.created = ts_from_abid(abid_part_from_ts(getattr(self, 'bookmarked', None) or timezone.now())) # cut off precision to match precision of TS component
|
||||
self.added = getattr(self, 'added', None) or self.created
|
||||
self.modified = self.created
|
||||
|
||||
assert all(self.ABID_FRESH_VALUES.values()), f'Can only issue new ABID if all self.ABID_FRESH_VALUES are defined {self.ABID_FRESH_VALUES}'
|
||||
|
||||
new_abid = self.ABID_FRESH
|
||||
assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
|
||||
|
||||
# store stable ABID on local fields, overwrite them because we are adding a new entry and existing defaults havent touched db yet
|
||||
self.abid = str(new_abid)
|
||||
self.id = new_abid.uuid
|
||||
self.pk = new_abid.uuid
|
||||
|
||||
assert self.ABID == new_abid
|
||||
assert str(self.ABID.uuid) == str(self.id) == str(self.pk) == str(ABID.parse(self.abid).uuid)
|
||||
|
||||
self._ready_to_save_as_new = True
|
||||
assert str(self.ABID.uuid) == str(new_abid.uuid)
|
||||
return new_abid
|
||||
|
||||
|
||||
@property
|
||||
|
|
@ -169,27 +152,12 @@ class ABIDModel(models.Model):
|
|||
aka get_or_generate_abid -> ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
|
||||
"""
|
||||
|
||||
# otherwise DB is single source of truth, load ABID from existing db pk
|
||||
abid: ABID | None = None
|
||||
try:
|
||||
abid = abid or ABID.parse(cast(str, self.abid))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
abid = abid or ABID.parse(cast(str, self.id))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
abid = abid or ABID.parse(cast(str, self.pk))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
abid = abid or self.ABID_FRESH
|
||||
|
||||
return abid
|
||||
if not self.abid:
|
||||
pre_save_abid = self.issue_new_abid()
|
||||
self.abid = str(pre_save_abid)
|
||||
return pre_save_abid
|
||||
|
||||
return ABID.parse(cast(str, self.abid))
|
||||
|
||||
@property
|
||||
def ULID(self) -> ULID:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
# Generated by Django 5.1 on 2024-09-04 23:32
|
||||
|
||||
import abid_utils.models
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('api', '0007_alter_apitoken_created_by'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='created',
|
||||
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='apitoken',
|
||||
name='id',
|
||||
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='outboundwebhook',
|
||||
name='created',
|
||||
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='outboundwebhook',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='outboundwebhook',
|
||||
name='id',
|
||||
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
|
||||
),
|
||||
]
|
||||
|
|
@ -12,7 +12,7 @@ from signal_webhooks.models import WebhookBase
|
|||
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
|
||||
from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
|
||||
|
||||
|
||||
|
||||
|
|
@ -32,11 +32,12 @@ class APIToken(ABIDModel):
|
|||
abid_subtype_src = 'self.created_by_id'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
|
||||
created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
|
||||
expires = models.DateTimeField(null=True, blank=True)
|
||||
|
|
@ -99,11 +100,11 @@ class OutboundWebhook(ABIDModel, WebhookBase):
|
|||
abid_subtype_src = 'self.ref'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
|
||||
created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
# More fields here: WebhookBase...
|
||||
|
|
|
|||
|
|
@ -64,7 +64,6 @@ class ArchiveResultSchema(Schema):
|
|||
TYPE: str = 'core.models.ArchiveResult'
|
||||
|
||||
id: UUID
|
||||
old_id: int
|
||||
abid: str
|
||||
|
||||
modified: datetime
|
||||
|
|
@ -127,9 +126,9 @@ class ArchiveResultSchema(Schema):
|
|||
|
||||
|
||||
class ArchiveResultFilterSchema(FilterSchema):
|
||||
id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||
id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||
|
||||
search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||
search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||
snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
|
||||
snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
|
||||
snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
|
||||
|
|
@ -157,8 +156,8 @@ def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...))
|
|||
|
||||
@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
|
||||
def get_archiveresult(request, archiveresult_id: str):
|
||||
"""Get a specific ArchiveResult by pk, abid, or old_id."""
|
||||
return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
|
||||
"""Get a specific ArchiveResult by id or abid."""
|
||||
return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id))
|
||||
|
||||
|
||||
# @router.post("/archiveresult", response=ArchiveResultSchema)
|
||||
|
|
@ -193,7 +192,6 @@ class SnapshotSchema(Schema):
|
|||
TYPE: str = 'core.models.Snapshot'
|
||||
|
||||
id: UUID
|
||||
old_id: UUID
|
||||
abid: str
|
||||
|
||||
modified: datetime
|
||||
|
|
@ -251,9 +249,7 @@ class SnapshotSchema(Schema):
|
|||
|
||||
|
||||
class SnapshotFilterSchema(FilterSchema):
|
||||
id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
|
||||
|
||||
old_id: Optional[str] = Field(None, q='old_id__icontains')
|
||||
id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'timestamp__startswith'])
|
||||
abid: Optional[str] = Field(None, q='abid__icontains')
|
||||
|
||||
created_by_id: str = Field(None, q='created_by_id')
|
||||
|
|
@ -266,7 +262,7 @@ class SnapshotFilterSchema(FilterSchema):
|
|||
modified__gte: datetime = Field(None, q='modified__gte')
|
||||
modified__lt: datetime = Field(None, q='modified__lt')
|
||||
|
||||
search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
|
||||
search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'timestamp__startswith'])
|
||||
url: Optional[str] = Field(None, q='url')
|
||||
tag: Optional[str] = Field(None, q='tags__name')
|
||||
title: Optional[str] = Field(None, q='title__icontains')
|
||||
|
|
@ -293,12 +289,12 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
|
|||
request.with_archiveresults = with_archiveresults
|
||||
snapshot = None
|
||||
try:
|
||||
snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
|
||||
snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
|
||||
except Snapshot.DoesNotExist:
|
||||
pass
|
||||
|
||||
try:
|
||||
snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
|
||||
snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
|
||||
except Snapshot.DoesNotExist:
|
||||
pass
|
||||
|
||||
|
|
@ -338,7 +334,6 @@ class TagSchema(Schema):
|
|||
TYPE: str = 'core.models.Tag'
|
||||
|
||||
id: UUID
|
||||
old_id: str
|
||||
abid: str
|
||||
|
||||
modified: datetime
|
||||
|
|
@ -351,10 +346,6 @@ class TagSchema(Schema):
|
|||
num_snapshots: int
|
||||
snapshots: List[SnapshotSchema]
|
||||
|
||||
@staticmethod
|
||||
def resolve_old_id(obj):
|
||||
return str(obj.old_id)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by_id)
|
||||
|
|
@ -386,11 +377,6 @@ def get_tag(request, tag_id: str, with_snapshots: bool=True):
|
|||
request.with_snapshots = with_snapshots
|
||||
request.with_archiveresults = False
|
||||
tag = None
|
||||
try:
|
||||
tag = tag or Tag.objects.get(old_id__icontains=tag_id)
|
||||
except (Tag.DoesNotExist, ValidationError, ValueError):
|
||||
pass
|
||||
|
||||
try:
|
||||
tag = Tag.objects.get(abid__icontains=tag_id)
|
||||
except (Tag.DoesNotExist, ValidationError):
|
||||
|
|
|
|||
|
|
@ -240,8 +240,8 @@ class ArchiveResultInline(admin.TabularInline):
|
|||
# fk_name = 'snapshot'
|
||||
extra = 0
|
||||
sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
|
||||
readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
|
||||
fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
|
||||
readonly_fields = ('id', 'result_id', 'completed', 'command', 'version')
|
||||
fields = ('start_ts', 'end_ts', *readonly_fields, 'extractor', 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
|
||||
# exclude = ('id',)
|
||||
ordering = ('end_ts',)
|
||||
show_change_link = True
|
||||
|
|
@ -273,7 +273,7 @@ class ArchiveResultInline(admin.TabularInline):
|
|||
snapshot = self.get_parent_object_from_request(request)
|
||||
|
||||
# import ipdb; ipdb.set_trace()
|
||||
formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
|
||||
# formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
|
||||
|
||||
# default values for new entries
|
||||
formset.form.base_fields['status'].initial = 'succeeded'
|
||||
|
|
@ -351,13 +351,13 @@ class SnapshotActionForm(ActionForm):
|
|||
|
||||
@admin.register(Snapshot, site=archivebox_admin)
|
||||
class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
|
||||
list_display = ('added', 'title_str', 'files', 'size', 'url_str')
|
||||
sort_fields = ('title_str', 'url_str', 'added')
|
||||
readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'created', 'added', 'updated', 'modified', 'API', 'link_dir')
|
||||
search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
|
||||
list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
|
||||
list_display = ('created', 'title_str', 'files', 'size', 'url_str')
|
||||
sort_fields = ('title_str', 'url_str', 'created')
|
||||
readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'created', 'created', 'updated', 'modified', 'API', 'link_dir')
|
||||
search_fields = ('id', 'url', 'abid', 'timestamp', 'title', 'tags__name')
|
||||
list_filter = ('created', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
|
||||
fields = ('url', 'created_by', 'title',*readonly_fields)
|
||||
ordering = ['-added']
|
||||
ordering = ['-created']
|
||||
actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
|
||||
inlines = [TagInline, ArchiveResultInline]
|
||||
list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
|
||||
|
|
@ -391,12 +391,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
|
|||
except (Snapshot.DoesNotExist, ValidationError):
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
|
||||
except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
|
||||
pass
|
||||
|
||||
if snapshot:
|
||||
object_id = str(snapshot.id)
|
||||
|
||||
|
|
@ -690,7 +684,7 @@ class ArchiveResultAdmin(ABIDModelAdmin):
|
|||
list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
|
||||
sort_fields = ('start_ts', 'extractor', 'status')
|
||||
readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
|
||||
search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
|
||||
search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
|
||||
fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
|
||||
autocomplete_fields = ['snapshot']
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
# Generated by Django 5.1 on 2024-09-04 09:00
|
||||
|
||||
import abid_utils.models
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0069_alter_archiveresult_created_alter_snapshot_added_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='archiveresult',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='added',
|
||||
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='created',
|
||||
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='id',
|
||||
field=models.UUIDField(default=None, primary_key=True, serialize=False, unique=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='old_id',
|
||||
field=models.UUIDField(default=None, editable=False, unique=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='tag',
|
||||
name='created',
|
||||
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
# Generated by Django 5.1 on 2024-09-04 23:23
|
||||
|
||||
import abid_utils.models
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0070_alter_archiveresult_created_by_alter_snapshot_added_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='archiveresult',
|
||||
name='old_id',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='snapshot',
|
||||
name='old_id',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='tag',
|
||||
name='old_id',
|
||||
),
|
||||
|
||||
migrations.AlterField(
|
||||
model_name='archiveresult',
|
||||
name='created',
|
||||
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='archiveresult',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='archiveresult',
|
||||
name='id',
|
||||
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='snapshot',
|
||||
name='id',
|
||||
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='tag',
|
||||
name='created_by',
|
||||
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='tag_set', to=settings.AUTH_USER_MODEL),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='tag',
|
||||
name='id',
|
||||
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
|
||||
),
|
||||
]
|
||||
|
|
@ -54,13 +54,15 @@ class Tag(ABIDModel):
|
|||
abid_ts_src = 'self.created'
|
||||
abid_uri_src = 'self.slug'
|
||||
abid_subtype_src = '"03"'
|
||||
abid_rand_src = 'self.old_id'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
old_id = models.BigIntegerField(unique=True, default=rand_int_id, serialize=False, verbose_name='Old ID') # legacy PK
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True)
|
||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='tag_set')
|
||||
created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
name = models.CharField(unique=True, blank=False, max_length=100)
|
||||
slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
|
||||
# slug is autoset on save from name, never set it manually
|
||||
|
|
@ -74,10 +76,6 @@ class Tag(ABIDModel):
|
|||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
# @property
|
||||
# def old_id(self):
|
||||
# return self.id
|
||||
|
||||
def slugify(self, tag, i=None):
|
||||
slug = slugify(tag)
|
||||
if i is not None:
|
||||
|
|
@ -133,16 +131,15 @@ class SnapshotManager(models.Manager):
|
|||
|
||||
class Snapshot(ABIDModel):
|
||||
abid_prefix = 'snp_'
|
||||
abid_ts_src = 'self.added'
|
||||
abid_ts_src = 'self.created'
|
||||
abid_uri_src = 'self.url'
|
||||
abid_subtype_src = '"01"'
|
||||
abid_rand_src = 'self.old_id'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
old_id = models.UUIDField(default=None, null=False, editable=False, unique=True) # legacy pk
|
||||
id = models.UUIDField(default=None, null=False, primary_key=True, editable=True, unique=True)
|
||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, related_name='snapshot_set')
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set')
|
||||
created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
|
|
@ -161,7 +158,6 @@ class Snapshot(ABIDModel):
|
|||
|
||||
objects = SnapshotManager()
|
||||
|
||||
|
||||
def __repr__(self) -> str:
|
||||
title = (self.title_stripped or '-')[:64]
|
||||
return f'[{self.timestamp}] {self.url[:64]} ({title})'
|
||||
|
|
@ -414,7 +410,7 @@ class ArchiveResult(ABIDModel):
|
|||
abid_ts_src = 'self.snapshot.added'
|
||||
abid_uri_src = 'self.snapshot.url'
|
||||
abid_subtype_src = 'self.extractor'
|
||||
abid_rand_src = 'self.old_id'
|
||||
abid_rand_src = 'self.id'
|
||||
|
||||
EXTRACTOR_CHOICES = (
|
||||
('htmltotext', 'htmltotext'),
|
||||
|
|
@ -438,13 +434,11 @@ class ArchiveResult(ABIDModel):
|
|||
("skipped", "skipped")
|
||||
]
|
||||
|
||||
old_id = models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID')
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True, verbose_name='ID')
|
||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||
abid = ABIDField(prefix=abid_prefix)
|
||||
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, related_name='archiveresult_set')
|
||||
created = AutoDateTimeField(default=timezone.now, db_index=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='archiveresult_set')
|
||||
created = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
modified = models.DateTimeField(auto_now=True)
|
||||
|
||||
snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')
|
||||
|
|
|
|||
|
|
@ -285,17 +285,17 @@ class SnapshotView(View):
|
|||
# ulid = slug.split('_', 1)[-1]
|
||||
# try:
|
||||
# try:
|
||||
# snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid) | Q(old_id=ulid))
|
||||
# snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid))
|
||||
# except Snapshot.DoesNotExist:
|
||||
# pass
|
||||
|
||||
# try:
|
||||
# snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug) | Q(old_id__startswith=slug))
|
||||
# snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug))
|
||||
# except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned):
|
||||
# pass
|
||||
|
||||
# try:
|
||||
# snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
|
||||
# snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
|
||||
# except Snapshot.DoesNotExist:
|
||||
# pass
|
||||
# return redirect(f'/archive/{snapshot.timestamp}/index.html')
|
||||
|
|
@ -308,7 +308,7 @@ class SnapshotView(View):
|
|||
# try exact match on full url / ABID first
|
||||
snapshot = Snapshot.objects.get(
|
||||
Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
|
||||
| Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
|
||||
| Q(abid__icontains=path) | Q(id__icontains=path)
|
||||
)
|
||||
except Snapshot.DoesNotExist:
|
||||
# fall back to match on exact base_url
|
||||
|
|
@ -352,7 +352,7 @@ class SnapshotView(View):
|
|||
)
|
||||
for snap in Snapshot.objects.filter(
|
||||
Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
|
||||
| Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
|
||||
| Q(abid__icontains=path) | Q(id__icontains=path)
|
||||
).only('url', 'timestamp', 'title', 'added').order_by('-added')
|
||||
)
|
||||
return HttpResponse(
|
||||
|
|
|
|||
|
|
@ -192,7 +192,6 @@ class Link:
|
|||
if extended:
|
||||
info.update({
|
||||
'snapshot_id': self.snapshot_id,
|
||||
'snapshot_old_id': self.snapshot_old_id,
|
||||
'snapshot_abid': self.snapshot_abid,
|
||||
|
||||
'link_dir': self.link_dir,
|
||||
|
|
@ -266,16 +265,12 @@ class Link:
|
|||
@cached_property
|
||||
def snapshot(self):
|
||||
from core.models import Snapshot
|
||||
return Snapshot.objects.only('id', 'old_id', 'abid').get(url=self.url)
|
||||
return Snapshot.objects.only('id', 'abid').get(url=self.url)
|
||||
|
||||
@cached_property
|
||||
def snapshot_id(self):
|
||||
return str(self.snapshot.pk)
|
||||
|
||||
@cached_property
|
||||
def snapshot_old_id(self):
|
||||
return str(self.snapshot.old_id)
|
||||
|
||||
@cached_property
|
||||
def snapshot_abid(self):
|
||||
return str(self.snapshot.ABID)
|
||||
|
|
|
|||
Loading…
Reference in a new issue