diff --git a/archivebox/core/migrations/0027_update_snapshot_ids.py b/archivebox/core/migrations/0027_update_snapshot_ids.py new file mode 100644 index 00000000..9b97782d --- /dev/null +++ b/archivebox/core/migrations/0027_update_snapshot_ids.py @@ -0,0 +1,46 @@ +# Generated by Django 5.0.6 on 2024-08-18 02:48 + +from django.db import migrations + +from django.db import migrations +from datetime import datetime +from abid_utils.abid import ABID + + +def update_snapshot_ids(apps, schema_editor): + Snapshot = apps.get_model("core", "Snapshot") + num_total = Snapshot.objects.all().count() + print(f' Updating {num_total} Snapshot.id, Snapshot.uuid values in place...') + for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator()): + assert snapshot.abid + snapshot.uuid = ABID.parse(snapshot.abid).uuid + snapshot.save(update_fields=["uuid"]) + assert str(ABID.parse(snapshot.abid).uuid) == str(snapshot.uuid) + if idx % 1000 == 0: + print(f'Migrated {idx}/{num_total} Snapshot objects...') + +def update_archiveresult_ids(apps, schema_editor): + ArchiveResult = apps.get_model("core", "ArchiveResult") + num_total = ArchiveResult.objects.all().count() + print(f' Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)') + for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()): + assert result.abid + result.uuid = ABID.parse(result.abid).uuid + result.save(update_fields=["uuid"]) + assert str(ABID.parse(result.abid).uuid) == str(result.uuid) + if idx % 5000 == 0: + print(f'Migrated {idx}/{num_total} ArchiveResult objects...') + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0026_archiveresult_created_archiveresult_created_by_and_more'), + ] + + operations = [ + migrations.RunPython(update_snapshot_ids, reverse_code=migrations.RunPython.noop), + migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop), + ] + + diff --git a/archivebox/core/migrations/0028_alter_archiveresult_uuid.py b/archivebox/core/migrations/0028_alter_archiveresult_uuid.py new file mode 100644 index 00000000..9b10f044 --- /dev/null +++ b/archivebox/core/migrations/0028_alter_archiveresult_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-18 04:28 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0027_update_snapshot_ids'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='uuid', + field=models.UUIDField(default=uuid.uuid4), + ), + ] diff --git a/archivebox/core/migrations/0029_alter_archiveresult_id.py b/archivebox/core/migrations/0029_alter_archiveresult_id.py new file mode 100644 index 00000000..7464a670 --- /dev/null +++ b/archivebox/core/migrations/0029_alter_archiveresult_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-18 04:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0028_alter_archiveresult_uuid'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='id', + field=models.BigIntegerField(primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/archivebox/core/migrations/0030_alter_archiveresult_uuid.py b/archivebox/core/migrations/0030_alter_archiveresult_uuid.py new file mode 100644 index 00000000..3c1ad788 --- /dev/null +++ b/archivebox/core/migrations/0030_alter_archiveresult_uuid.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0029_alter_archiveresult_id'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='uuid', + field=models.UUIDField(unique=True), + ), + ] diff --git a/archivebox/core/migrations/0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more.py b/archivebox/core/migrations/0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more.py new file mode 100644 index 00000000..64fd6cbe --- /dev/null +++ b/archivebox/core/migrations/0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:09 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0030_alter_archiveresult_uuid'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='id', + field=models.IntegerField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='archiveresult', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='snapshot', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, unique=True), + ), + migrations.AlterField( + model_name='tag', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, null=True, unique=True), + ), + ] diff --git a/archivebox/core/migrations/0032_alter_archiveresult_id.py b/archivebox/core/migrations/0032_alter_archiveresult_id.py new file mode 100644 index 00000000..98299a31 --- /dev/null +++ b/archivebox/core/migrations/0032_alter_archiveresult_id.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:20 + +import core.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='id', + field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/archivebox/core/migrations/0033_rename_id_archiveresult_old_id.py b/archivebox/core/migrations/0033_rename_id_archiveresult_old_id.py new file mode 100644 index 00000000..ebced58e --- /dev/null +++ b/archivebox/core/migrations/0033_rename_id_archiveresult_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0032_alter_archiveresult_id'), + ] + + operations = [ + migrations.RenameField( + model_name='archiveresult', + old_name='id', + new_name='old_id', + ), + ] diff --git a/archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py b/archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py new file mode 100644 index 00000000..121a2154 --- /dev/null +++ b/archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py @@ -0,0 +1,41 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:37 + +import core.models +import uuid +from django.db import migrations, models + +from abid_utils.abid import ABID + + +def update_archiveresult_ids(apps, schema_editor): + ArchiveResult = apps.get_model("core", "ArchiveResult") + num_total = ArchiveResult.objects.all().count() + print(f' Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)') + for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()): + assert result.abid + result.uuid = ABID.parse(result.abid).uuid + result.save(update_fields=["uuid"]) + assert str(ABID.parse(result.abid).uuid) == str(result.uuid) + if idx % 2500 == 0: + print(f'Migrated {idx}/{num_total} ArchiveResult objects...') + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0033_rename_id_archiveresult_old_id'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='old_id', + field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'), + ), + migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop), + migrations.AlterField( + model_name='archiveresult', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/archivebox/core/migrations/0035_remove_archiveresult_uuid_archiveresult_id.py b/archivebox/core/migrations/0035_remove_archiveresult_uuid_archiveresult_id.py new file mode 100644 index 00000000..26287e3c --- /dev/null +++ b/archivebox/core/migrations/0035_remove_archiveresult_uuid_archiveresult_id.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:49 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0034_alter_archiveresult_old_id_alter_archiveresult_uuid'), + ] + + operations = [ + migrations.RenameField( + model_name='archiveresult', + old_name='uuid', + new_name='id', + ), + ] diff --git a/archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py b/archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py new file mode 100644 index 00000000..10b4f9c6 --- /dev/null +++ b/archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py @@ -0,0 +1,25 @@ +# Generated by Django 5.0.6 on 2024-08-18 05:59 + +import core.models +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0035_remove_archiveresult_uuid_archiveresult_id'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='id', + field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='ID'), + ), + migrations.AlterField( + model_name='archiveresult', + name='old_id', + field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'), + ), + ] diff --git a/archivebox/core/migrations/0037_rename_id_snapshot_old_id.py b/archivebox/core/migrations/0037_rename_id_snapshot_old_id.py new file mode 100644 index 00000000..7d901d96 --- /dev/null +++ b/archivebox/core/migrations/0037_rename_id_snapshot_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0036_alter_archiveresult_id_alter_archiveresult_old_id'), + ] + + operations = [ + migrations.RenameField( + model_name='snapshot', + old_name='id', + new_name='old_id', + ), + ] diff --git a/archivebox/core/migrations/0038_rename_uuid_snapshot_id.py b/archivebox/core/migrations/0038_rename_uuid_snapshot_id.py new file mode 100644 index 00000000..d22a8fc4 --- /dev/null +++ b/archivebox/core/migrations/0038_rename_uuid_snapshot_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0037_rename_id_snapshot_old_id'), + ] + + operations = [ + migrations.RenameField( + model_name='snapshot', + old_name='uuid', + new_name='id', + ), + ] diff --git a/archivebox/core/migrations/0039_rename_snapshot_archiveresult_snapshot_old.py b/archivebox/core/migrations/0039_rename_snapshot_archiveresult_snapshot_old.py new file mode 100644 index 00000000..7c2a4e29 --- /dev/null +++ b/archivebox/core/migrations/0039_rename_snapshot_archiveresult_snapshot_old.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0038_rename_uuid_snapshot_id'), + ] + + operations = [ + migrations.RenameField( + model_name='archiveresult', + old_name='snapshot', + new_name='snapshot_old', + ), + ] diff --git a/archivebox/core/migrations/0040_archiveresult_snapshot.py b/archivebox/core/migrations/0040_archiveresult_snapshot.py new file mode 100644 index 00000000..fa04a9d4 --- /dev/null +++ b/archivebox/core/migrations/0040_archiveresult_snapshot.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:46 + +import django.db.models.deletion +from django.db import migrations, models + +def update_archiveresult_snapshot_ids(apps, schema_editor): + ArchiveResult = apps.get_model("core", "ArchiveResult") + Snapshot = apps.get_model("core", "Snapshot") + num_total = ArchiveResult.objects.all().count() + print(f' Updating {num_total} ArchiveResult.snapshot_id values in place... (may take an hour or longer for large collections...)') + for idx, result in enumerate(ArchiveResult.objects.all().only('snapshot_old_id').iterator()): + assert result.snapshot_old_id + snapshot = Snapshot.objects.get(old_id=result.snapshot_old_id) + result.snapshot_id = snapshot.id + result.save(update_fields=["snapshot_id"]) + assert str(result.snapshot_id) == str(snapshot.id) + if idx % 5000 == 0: + print(f'Migrated {idx}/{num_total} ArchiveResult objects...') + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0039_rename_snapshot_archiveresult_snapshot_old'), + ] + + operations = [ + migrations.AddField( + model_name='archiveresult', + name='snapshot', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults', to='core.snapshot', to_field='id'), + ), + migrations.RunPython(update_archiveresult_snapshot_ids, reverse_code=migrations.RunPython.noop), + ] diff --git a/archivebox/core/migrations/0041_alter_archiveresult_snapshot_and_more.py b/archivebox/core/migrations/0041_alter_archiveresult_snapshot_and_more.py new file mode 100644 index 00000000..d4be8875 --- /dev/null +++ b/archivebox/core/migrations/0041_alter_archiveresult_snapshot_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:50 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0040_archiveresult_snapshot'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='snapshot', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'), + ), + migrations.AlterField( + model_name='archiveresult', + name='snapshot_old', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults_old', to='core.snapshot'), + ), + ] diff --git a/archivebox/core/migrations/0042_remove_archiveresult_snapshot_old.py b/archivebox/core/migrations/0042_remove_archiveresult_snapshot_old.py new file mode 100644 index 00000000..3fe9f316 --- /dev/null +++ b/archivebox/core/migrations/0042_remove_archiveresult_snapshot_old.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:51 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0041_alter_archiveresult_snapshot_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='archiveresult', + name='snapshot_old', + ), + ] diff --git a/archivebox/core/migrations/0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py b/archivebox/core/migrations/0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py new file mode 100644 index 00000000..c0acddb0 --- /dev/null +++ b/archivebox/core/migrations/0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 5.0.6 on 2024-08-18 06:52 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0042_remove_archiveresult_snapshot_old'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='snapshot', + field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'), + ), + ] diff --git a/archivebox/core/migrations/0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more.py b/archivebox/core/migrations/0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more.py new file mode 100644 index 00000000..d981dca9 --- /dev/null +++ b/archivebox/core/migrations/0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 5.0.6 on 2024-08-19 23:01 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + # No-op, SnapshotTag model already exists in DB + ], + state_operations=[ + migrations.CreateModel( + name='SnapshotTag', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('snapshot', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot')), + ('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.tag')), + ], + options={ + 'db_table': 'core_snapshot_tags', + 'unique_together': {('snapshot', 'tag')}, + }, + ), + migrations.AlterField( + model_name='snapshot', + name='tags', + field=models.ManyToManyField(blank=True, related_name='snapshot_set', through='core.SnapshotTag', to='core.tag'), + ), + ], + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 09ad94fa..9ccd6145 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -61,7 +61,7 @@ class Tag(ABIDModel): # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True) id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID') - uuid = models.UUIDField(default=uuid.uuid4, editable=True, unique=True) + uuid = models.UUIDField(default=uuid.uuid4, null=True, unique=True) abid = ABIDField(prefix=abid_prefix) @@ -77,6 +77,10 @@ class Tag(ABIDModel): def __str__(self): return self.name + @property + def old_id(self): + return self.id + def slugify(self, tag, i=None): slug = slugify(tag) if i is not None: @@ -115,9 +119,15 @@ class Tag(ABIDModel): return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag' class SnapshotTag(models.Model): - snapshot = models.OneToOneField('Snapshot', primary_key=True, on_delete=models.CASCADE, to_field='id') + id = models.AutoField(primary_key=True) + + snapshot = models.OneToOneField('Snapshot', on_delete=models.CASCADE, to_field='old_id') tag = models.ForeignKey(Tag, on_delete=models.CASCADE, to_field='id') + class Meta: + db_table = 'core_snapshot_tags' + unique_together = [('snapshot', 'tag')] + class Snapshot(ABIDModel): abid_prefix = 'snp_' abid_ts_src = 'self.added' @@ -133,10 +143,11 @@ class Snapshot(ABIDModel): timestamp = models.CharField(max_length=32, unique=True, db_index=True) title = models.CharField(max_length=512, null=True, blank=True, db_index=True) + + tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag')) added = models.DateTimeField(auto_now_add=True, db_index=True) updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True) - tags = models.ManyToManyField(Tag, blank=True) keys = ('url', 'timestamp', 'title', 'tags', 'updated')