From 44669fab735bf01767bf1b175429bd578cd39f3a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 5 Sep 2024 03:36:18 -0700 Subject: [PATCH] add BaseHook concept to underlie all Plugin hooks --- archivebox/abid_utils/abid.py | 7 ++- archivebox/abid_utils/admin.py | 49 +++++++++------ archivebox/abid_utils/models.py | 83 ++++++++++++++++---------- archivebox/api/models.py | 9 ++- archivebox/config.py | 2 +- archivebox/core/admin.py | 3 +- archivebox/core/models.py | 12 +++- archivebox/core/settings.py | 1 + archivebox/plugantic/base_check.py | 4 +- archivebox/plugantic/base_configset.py | 15 +++-- archivebox/plugantic/base_hook.py | 71 ++++++++++++++++++++++ archivebox/plugantic/base_plugin.py | 35 +++++++---- 12 files changed, 212 insertions(+), 79 deletions(-) create mode 100644 archivebox/plugantic/base_hook.py diff --git a/archivebox/abid_utils/abid.py b/archivebox/abid_utils/abid.py index 8863e61c..317eae02 100644 --- a/archivebox/abid_utils/abid.py +++ b/archivebox/abid_utils/abid.py @@ -148,11 +148,12 @@ def abid_part_from_prefix(prefix: str) -> str: return prefix + '_' @enforce_types -def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str: +def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str: """ 'E4A5CCD9' # takes first 8 characters of sha256(url) """ - uri = str(uri) + uri = str(uri).strip() + assert uri not in ('None', '') return uri_hash(uri, salt=salt)[:ABID_URI_LEN] @enforce_types @@ -201,7 +202,7 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str: @enforce_types -def abid_hashes_from_values(prefix: str, ts: datetime, uri: str, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]: +def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]: return { 'prefix': abid_part_from_prefix(prefix), 'ts': abid_part_from_ts(ts), diff --git a/archivebox/abid_utils/admin.py b/archivebox/abid_utils/admin.py index 6b8e949c..f74493fc 100644 --- a/archivebox/abid_utils/admin.py +++ b/archivebox/abid_utils/admin.py @@ -9,7 +9,7 @@ from django.utils.html import format_html from django.utils.safestring import mark_safe from django.shortcuts import redirect -from abid_utils.abid import ABID, abid_part_from_ts, abid_part_from_uri, abid_part_from_rand, abid_part_from_subtype +from .abid import ABID from api.auth import get_or_create_api_token @@ -94,29 +94,25 @@ def get_abid_info(self, obj, request=None): class ABIDModelAdmin(admin.ModelAdmin): - list_display = ('created_at', 'created_by', 'abid', '__str__') - sort_fields = ('created_at', 'created_by', 'abid', '__str__') - readonly_fields = ('created_at', 'modified_at', '__str__', 'abid_info') - - @admin.display(description='API Identifiers') - def abid_info(self, obj): - return get_abid_info(self, obj, request=self.request) + list_display = ('created_at', 'created_by', 'abid') + sort_fields = ('created_at', 'created_by', 'abid') + readonly_fields = ('created_at', 'modified_at', 'abid_info') + # fields = [*readonly_fields] + def _get_obj_does_not_exist_redirect(self, request, opts, object_id): + try: + object_pk = self.model.id_from_abid(object_id) + return redirect(self.request.path.replace(object_id, object_pk), permanent=False) + except (self.model.DoesNotExist, ValidationError): + pass + return super()._get_obj_does_not_exist_redirect(request, opts, object_id) # type: ignore + def queryset(self, request): self.request = request return super().queryset(request) def change_view(self, request, object_id, form_url="", extra_context=None): self.request = request - - if object_id: - try: - object_uuid = str(self.model.objects.only('pk').get(abid=self.model.abid_prefix + object_id.split('_', 1)[-1]).pk) - if object_id != object_uuid: - return redirect(self.request.path.replace(object_id, object_uuid), permanent=False) - except (self.model.DoesNotExist, ValidationError): - pass - return super().change_view(request, object_id, form_url, extra_context) def get_form(self, request, obj=None, **kwargs): @@ -126,9 +122,24 @@ class ABIDModelAdmin(admin.ModelAdmin): form.base_fields['created_by'].initial = request.user return form + def get_formset(self, request, formset=None, obj=None, **kwargs): + formset = super().get_formset(request, formset, obj, **kwargs) + formset.form.base_fields['created_at'].disabled = True + return formset + def save_model(self, request, obj, form, change): - old_abid = obj.abid + self.request = request + + old_abid = getattr(obj, '_previous_abid', None) or obj.abid + super().save_model(request, obj, form, change) + obj.refresh_from_db() + new_abid = obj.abid if new_abid != old_abid: - messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any references to the old ABID will need to be updated)") + messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)") + # import ipdb; ipdb.set_trace() + + @admin.display(description='API Identifiers') + def abid_info(self, obj): + return get_abid_info(self, obj, request=self.request) diff --git a/archivebox/abid_utils/models.py b/archivebox/abid_utils/models.py index 38ad57f7..c5ba8c25 100644 --- a/archivebox/abid_utils/models.py +++ b/archivebox/abid_utils/models.py @@ -11,7 +11,8 @@ from datetime import datetime, timedelta from functools import partial from charidfield import CharIDField # type: ignore[import-untyped] -from django.core.exceptions import ValidationError +from django.contrib import admin +from django.core.exceptions import ValidationError, NON_FIELD_ERRORS from django.db import models from django.utils import timezone from django.db.utils import OperationalError @@ -71,24 +72,6 @@ class AutoDateTimeField(models.DateTimeField): class ABIDError(Exception): pass -class ABIDFieldsCannotBeChanged(ValidationError, ABIDError): - """ - Properties used as unique identifiers (to generate ABID) cannot be edited after an object is created. - Create a new object instead with your desired changes (and it will be issued a new ABID). - """ - def __init__(self, ABID_FRESH_DIFFS, obj): - self.ABID_FRESH_DIFFS = ABID_FRESH_DIFFS - self.obj = obj - - def __str__(self): - keys_changed = ', '.join(diff['abid_src'] for diff in self.ABID_FRESH_DIFFS.values()) - return ( - f"This {self.obj.__class__.__name__}(abid={str(self.obj.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " + - f'\nThe following changes cannot be made because they would alter the ABID:' + - '\n ' + "\n ".join(f' - {diff["summary"]}' for diff in self.ABID_FRESH_DIFFS.values()) + - f"\nYou must reduce your changes to not affect these fields, or create a new {self.obj.__class__.__name__} object instead." - ) - class ABIDModel(models.Model): """ @@ -112,6 +95,10 @@ class ABIDModel(models.Model): class Meta(TypedModelMeta): abstract = True + @admin.display(description='Summary') + def __str__(self) -> str: + return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}' + def __init__(self, *args: Any, **kwargs: Any) -> None: """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB.""" super().__init__(*args, **kwargs) @@ -121,29 +108,59 @@ class ABIDModel(models.Model): # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled) self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now())) - def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None: - """Overriden save method ensures new ABID is generated while a new object is first saving.""" - + def clean(self, abid_drift_allowed: bool | None=None) -> None: if self._state.adding: # only runs once when a new object is first saved to the DB # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at + self._previous_abid = None self.abid = str(self.issue_new_abid()) else: # otherwise if updating, make sure none of the field changes would invalidate existing ABID - if self.ABID_FRESH_DIFFS: - ovewrite_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed + abid_diffs = self.ABID_FRESH_DIFFS + if abid_diffs: - change_error = ABIDFieldsCannotBeChanged(self.ABID_FRESH_DIFFS, obj=self) - if ovewrite_abid: - print(f'#### DANGER: Changing ABID of existing record ({self.__class__.__name__}.abid_drift_allowed={abid_drift_allowed}), this will break any references to its previous ABID!') + keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values()) + full_summary = ( + f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " + + f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead." + ) + + change_error = ValidationError({ + NON_FIELD_ERRORS: ValidationError(full_summary), + **{ + # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...') + diff['abid_src'].replace('self.', '') if diff['old_val'] != diff['new_val'] else NON_FIELD_ERRORS + : ValidationError( + 'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)', + code='ABIDConflict', + params=diff, + ) + for diff in abid_diffs.values() + }, + }) + + should_ovewrite_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed + if should_ovewrite_abid: + print(f'\n#### DANGER: Changing ABID of existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed}), this will break any references to its previous ABID!') print(change_error) + self._previous_abid = self.abid self.abid = str(self.issue_new_abid(force_new=True)) print(f'#### DANGER: OVERWROTE OLD ABID. NEW ABID=', self.abid) else: - raise change_error + print(f'\n#### WARNING: ABID of existing record is outdated and has not been updated ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})') + print(change_error) + + def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None: + """Overriden save method ensures new ABID is generated while a new object is first saving.""" + + self.clean(abid_drift_allowed=abid_drift_allowed) return super().save(*args, **kwargs) + + @classmethod + def id_from_abid(cls, abid: str) -> str: + return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk) @property def ABID_SOURCES(self) -> Dict[str, str]: @@ -196,10 +213,10 @@ class ABIDModel(models.Model): fresh_hashes = self.ABID_FRESH_HASHES return { key: { + 'key': key, 'model': self.__class__.__name__, 'pk': self.pk, 'abid_src': abid_sources[key], - 'abid_section': key, 'old_val': existing_values.get(key, None), 'old_hash': getattr(existing_abid, key), 'new_val': fresh_values[key], @@ -215,7 +232,6 @@ class ABIDModel(models.Model): Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB). """ if not force_new: - assert self.abid is None, f'Can only issue new ABID for new objects that dont already have one {self.abid}' assert self._state.adding, 'Can only issue new ABID when model._state.adding is True' assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})' @@ -286,7 +302,7 @@ class ABIDModel(models.Model): Compute the REST API URL to access this object. e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP """ - return reverse_lazy('api-1:get_any', args=[self.abid]) + return reverse_lazy('api-1:get_any', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}' @property def api_docs_url(self) -> str: @@ -296,7 +312,12 @@ class ABIDModel(models.Model): """ return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}' + @property + def admin_change_url(self) -> str: + return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/" + def get_absolute_url(self): + return self.api_docs_url #################################################### diff --git a/archivebox/api/models.py b/archivebox/api/models.py index 9f6b8395..8dd90116 100644 --- a/archivebox/api/models.py +++ b/archivebox/api/models.py @@ -28,9 +28,10 @@ class APIToken(ABIDModel): # ABID: apt____ abid_prefix = 'apt_' abid_ts_src = 'self.created_at' - abid_uri_src = 'self.token' - abid_subtype_src = 'self.created_by_id' + abid_uri_src = 'self.created_by_id' + abid_subtype_src = '"01"' abid_rand_src = 'self.id' + abid_drift_allowed = True id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID') abid = ABIDField(prefix=abid_prefix) @@ -99,6 +100,7 @@ class OutboundWebhook(ABIDModel, WebhookBase): abid_uri_src = 'self.endpoint' abid_subtype_src = 'self.ref' abid_rand_src = 'self.id' + abid_drift_allowed = True id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID') abid = ABIDField(prefix=abid_prefix) @@ -121,3 +123,6 @@ class OutboundWebhook(ABIDModel, WebhookBase): class Meta(WebhookBase.Meta): verbose_name = 'API Outbound Webhook' + + def __str__(self) -> str: + return f'[{self.abid}] {self.ref} -> {self.endpoint}' diff --git a/archivebox/config.py b/archivebox/config.py index 8f22dd8f..0bf24efa 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -103,7 +103,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = { 'PUBLIC_SNAPSHOTS': {'type': bool, 'default': True}, 'PUBLIC_ADD_VIEW': {'type': bool, 'default': False}, 'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'}, - 'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 100}, + 'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40}, 'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None}, 'TIME_ZONE': {'type': str, 'default': 'UTC'}, 'TIMEZONE': {'type': str, 'default': 'UTC'}, diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py index 46abfa07..e2124fce 100644 --- a/archivebox/core/admin.py +++ b/archivebox/core/admin.py @@ -254,7 +254,7 @@ class ArchiveResultInline(admin.TabularInline): try: return self.parent_model.objects.get(pk=resolved.kwargs['object_id']) except (self.parent_model.DoesNotExist, ValidationError): - return self.parent_model.objects.get(abid=self.parent_model.abid_prefix + resolved.kwargs['object_id'].split('_', 1)[-1]) + return self.parent_model.objects.get(pk=self.parent_model.id_from_abid(resolved.kwargs['object_id'])) @admin.display( description='Completed', @@ -685,6 +685,7 @@ class ArchiveResultAdmin(ABIDModelAdmin): list_per_page = CONFIG.SNAPSHOTS_PER_PAGE paginator = AccelleratedPaginator + save_on_top = True def change_view(self, request, object_id, form_url="", extra_context=None): self.request = request diff --git a/archivebox/core/models.py b/archivebox/core/models.py index aa224e88..7a975b38 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -103,7 +103,7 @@ class Tag(ABIDModel): @property def api_url(self) -> str: # /api/v1/core/snapshot/{uulid} - return reverse_lazy('api-1:get_tag', args=[self.abid]) + return reverse_lazy('api-1:get_tag', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}' @property def api_docs_url(self) -> str: @@ -211,12 +211,15 @@ class Snapshot(ABIDModel): @property def api_url(self) -> str: # /api/v1/core/snapshot/{uulid} - return reverse_lazy('api-1:get_snapshot', args=[self.abid]) + return reverse_lazy('api-1:get_snapshot', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}' @property def api_docs_url(self) -> str: return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot' + def get_absolute_url(self): + return f'/{self.archive_path}' + @cached_property def title_stripped(self) -> str: return (self.title or '').replace("\n", " ").replace("\r", "") @@ -476,11 +479,14 @@ class ArchiveResult(ABIDModel): @property def api_url(self) -> str: # /api/v1/core/archiveresult/{uulid} - return reverse_lazy('api-1:get_archiveresult', args=[self.abid]) + return reverse_lazy('api-1:get_archiveresult', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}' @property def api_docs_url(self) -> str: return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult' + + def get_absolute_url(self): + return f'/{self.snapshot.archive_path}/{self.output_path()}' @property def extractor_module(self): diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 962b48d1..1a765499 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -40,6 +40,7 @@ INSTALLED_PLUGINS = { ### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup) PLUGINS = AttrDict({}) +HOOKS = AttrDict({}) CONFIGS = AttrDict({}) BINPROVIDERS = AttrDict({}) diff --git a/archivebox/plugantic/base_check.py b/archivebox/plugantic/base_check.py index 542b1957..fb07a386 100644 --- a/archivebox/plugantic/base_check.py +++ b/archivebox/plugantic/base_check.py @@ -1,14 +1,14 @@ from typing import List, Type, Any from pydantic_core import core_schema -from pydantic import GetCoreSchemaHandler +from pydantic import GetCoreSchemaHandler, BaseModel from django.utils.functional import classproperty from django.core.checks import Warning, Tags, register class BaseCheck: label: str = '' - tag = Tags.database + tag: str = Tags.database @classmethod def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: diff --git a/archivebox/plugantic/base_configset.py b/archivebox/plugantic/base_configset.py index 5edd1407..31b07455 100644 --- a/archivebox/plugantic/base_configset.py +++ b/archivebox/plugantic/base_configset.py @@ -5,6 +5,7 @@ from typing import Optional, List, Literal from pathlib import Path from pydantic import BaseModel, Field, ConfigDict, computed_field +from .base_hook import BaseHook, HookType ConfigSectionName = Literal[ 'GENERAL_CONFIG', @@ -20,24 +21,26 @@ ConfigSectionNames: List[ConfigSectionName] = [ ] -class BaseConfigSet(BaseModel): +class BaseConfigSet(BaseHook): model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True) + hook_type: HookType = 'CONFIG' section: ConfigSectionName = 'GENERAL_CONFIG' - @computed_field - @property - def name(self) -> str: - return self.__class__.__name__ - def register(self, settings, parent_plugin=None): + """Installs the ConfigSet into Django settings.CONFIGS (and settings.HOOKS).""" if settings is None: from django.conf import settings as django_settings settings = django_settings self._plugin = parent_plugin # for debugging only, never rely on this! + + # install hook into settings.CONFIGS settings.CONFIGS[self.name] = self + # record installed hook in settings.HOOKS + super().register(settings, parent_plugin=parent_plugin) + # class WgetToggleConfig(ConfigSet): diff --git a/archivebox/plugantic/base_hook.py b/archivebox/plugantic/base_hook.py new file mode 100644 index 00000000..3a5c81a8 --- /dev/null +++ b/archivebox/plugantic/base_hook.py @@ -0,0 +1,71 @@ +__package__ = 'archivebox.plugantic' + +import json +from typing import Optional, List, Literal, ClassVar +from pathlib import Path +from pydantic import BaseModel, Field, ConfigDict, computed_field + + +HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW'] +hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW'] + + + +class BaseHook(BaseModel): + """ + A Plugin consists of a list of Hooks, applied to django.conf.settings when AppConfig.read() -> Plugin.register() is called. + Plugin.register() then calls each Hook.register() on the provided settings. + each Hook.regsiter() function (ideally pure) takes a django.conf.settings as input and returns a new one back. + or + it modifies django.conf.settings in-place to add changes corresponding to its HookType. + e.g. for a HookType.CONFIG, the Hook.register() function places the hook in settings.CONFIG (and settings.HOOKS) + An example of an impure Hook would be a CHECK that modifies settings but also calls django.core.checks.register(check). + + + setup_django() -> imports all settings.INSTALLED_APPS... + # django imports AppConfig, models, migrations, admins, etc. for all installed apps + # django then calls AppConfig.ready() on each installed app... + + builtin_plugins.npm.NpmPlugin().AppConfig.ready() # called by django + builtin_plugins.npm.NpmPlugin().register(settings) -> + builtin_plugins.npm.NpmConfigSet().register(settings) + plugantic.base_configset.BaseConfigSet().register(settings) + plugantic.base_hook.BaseHook().register(settings, parent_plugin=builtin_plugins.npm.NpmPlugin()) + + ... + ... + + + """ + model_config = ConfigDict( + extra='allow', + arbitrary_types_allowed=True, + from_attributes=True, + populate_by_name=True, + validate_defaults=True, + validate_assignment=True, + ) + + hook_type: HookType = 'CONFIG' + + @property + def name(self) -> str: + return f'{self.__module__}.{__class__.__name__}' + + def register(self, settings, parent_plugin=None): + """Load a record of an installed hook into global Django settings.HOOKS at runtime.""" + + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + assert json.dumps(self.model_json_schema(), indent=4), f'Hook {self.name} has invalid JSON schema.' + + self._plugin = parent_plugin # for debugging only, never rely on this! + + # record installed hook in settings.HOOKS + settings.HOOKS[self.name] = self + + hook_prefix, plugin_shortname = self.name.split('.', 1) + + print('REGISTERED HOOK:', self.name) diff --git a/archivebox/plugantic/base_plugin.py b/archivebox/plugantic/base_plugin.py index cdad499c..26c12af7 100644 --- a/archivebox/plugantic/base_plugin.py +++ b/archivebox/plugantic/base_plugin.py @@ -1,6 +1,8 @@ __package__ = 'archivebox.plugantic' import json +import inspect +from pathlib import Path from django.apps import AppConfig from django.core.checks import register @@ -32,12 +34,11 @@ class BasePlugin(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True) # Required by AppConfig: - name: str = Field() # e.g. 'builtin_plugins.singlefile' - app_label: str = Field() # e.g. 'singlefile' - verbose_name: str = Field() # e.g. 'SingleFile' - default_auto_field: ClassVar[str] = 'django.db.models.AutoField' + name: str = Field() # e.g. 'builtin_plugins.singlefile' (DottedImportPath) + app_label: str = Field() # e.g. 'singlefile' (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name) + verbose_name: str = Field() # e.g. 'SingleFile' (human-readable *short* label, for use in column names, form labels, etc.) - # Required by Plugantic: + # All the hooks the plugin will install: configs: List[InstanceOf[BaseConfigSet]] = Field(default=[]) binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')] binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')] @@ -53,20 +54,23 @@ class BasePlugin(BaseModel): assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name' assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.' + return self @property def AppConfig(plugin_self) -> Type[AppConfig]: """Generate a Django AppConfig class for this plugin.""" class PluginAppConfig(AppConfig): + """Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS.""" name = plugin_self.name app_label = plugin_self.app_label verbose_name = plugin_self.verbose_name + default_auto_field = 'django.db.models.AutoField' def ready(self): from django.conf import settings - plugin_self.validate() + # plugin_self.validate() plugin_self.register(settings) return PluginAppConfig @@ -105,11 +109,6 @@ class BasePlugin(BaseModel): @property def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]: return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews}) - - @computed_field - @property - def PLUGIN_KEYS(self) -> List[str]: - return def register(self, settings=None): """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime.""" @@ -185,6 +184,20 @@ class BasePlugin(BaseModel): # 'binaries': new_binaries, # }) + @computed_field + @property + def module_dir(self) -> Path: + return Path(inspect.getfile(self.__class__)).parent.resolve() + + @computed_field + @property + def module_path(self) -> str: # DottedImportPath + """" + Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS). + e.g. 'archivebox.builtin_plugins.npm' + """ + return self.name.strip('archivebox.') +