diff --git a/archivebox/builtin_plugins/npm/apps.py b/archivebox/builtin_plugins/npm/apps.py index 7ffed0c1..cd3f5826 100644 --- a/archivebox/builtin_plugins/npm/apps.py +++ b/archivebox/builtin_plugins/npm/apps.py @@ -3,6 +3,7 @@ __package__ = 'archivebox.builtin_plugins.npm' from typing import List, Optional from pydantic import InstanceOf, Field +from django.conf import settings from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr from plugantic.base_plugin import BasePlugin @@ -65,4 +66,5 @@ class NpmPlugin(BasePlugin): PLUGIN = NpmPlugin() +PLUGIN.register(settings) DJANGO_APP = PLUGIN.AppConfig diff --git a/archivebox/builtin_plugins/pip/apps.py b/archivebox/builtin_plugins/pip/apps.py index 7cc359a1..965f370e 100644 --- a/archivebox/builtin_plugins/pip/apps.py +++ b/archivebox/builtin_plugins/pip/apps.py @@ -9,6 +9,7 @@ import django from django.db.backends.sqlite3.base import Database as sqlite3 # type: ignore[import-type] from django.core.checks import Error, Tags +from django.conf import settings from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer from plugantic.base_plugin import BasePlugin @@ -139,4 +140,5 @@ class PipPlugin(BasePlugin): ] PLUGIN = PipPlugin() +PLUGIN.register(settings) DJANGO_APP = PLUGIN.AppConfig diff --git a/archivebox/builtin_plugins/singlefile/apps.py b/archivebox/builtin_plugins/singlefile/apps.py index 0c92c267..2eb0de05 100644 --- a/archivebox/builtin_plugins/singlefile/apps.py +++ b/archivebox/builtin_plugins/singlefile/apps.py @@ -1,6 +1,8 @@ from pathlib import Path from typing import List, Dict, Optional +from django.conf import settings + # Depends on other PyPI/vendor packages: from pydantic import InstanceOf, Field from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName @@ -101,9 +103,11 @@ class SinglefilePlugin(BasePlugin): SINGLEFILE_CONFIG, SINGLEFILE_BINARY, SINGLEFILE_EXTRACTOR, + SINGLEFILE_QUEUE, ] PLUGIN = SinglefilePlugin() +PLUGIN.register(settings) DJANGO_APP = PLUGIN.AppConfig diff --git a/archivebox/builtin_plugins/ytdlp/apps.py b/archivebox/builtin_plugins/ytdlp/apps.py index 087054a8..31985687 100644 --- a/archivebox/builtin_plugins/ytdlp/apps.py +++ b/archivebox/builtin_plugins/ytdlp/apps.py @@ -2,6 +2,7 @@ from typing import List, Dict from subprocess import run, PIPE from pydantic import InstanceOf, Field +from django.conf import settings from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict from plugantic.base_plugin import BasePlugin @@ -74,4 +75,5 @@ class YtdlpPlugin(BasePlugin): PLUGIN = YtdlpPlugin() +PLUGIN.register(settings) DJANGO_APP = PLUGIN.AppConfig diff --git a/archivebox/plugantic/base_hook.py b/archivebox/plugantic/base_hook.py index f714bf1d..3495ff25 100644 --- a/archivebox/plugantic/base_hook.py +++ b/archivebox/plugantic/base_hook.py @@ -1,14 +1,15 @@ __package__ = 'archivebox.plugantic' -import json +import inspect +from huey.api import TaskWrapper + +from pathlib import Path from typing import List, Literal from pydantic import BaseModel, ConfigDict, Field, computed_field -HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW'] -hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW'] - - +HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE'] +hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE'] class BaseHook(BaseModel): """ @@ -56,24 +57,37 @@ class BaseHook(BaseModel): validate_defaults=True, validate_assignment=False, revalidate_instances="subclass-instances", + ignored_types=(TaskWrapper, ), ) # verbose_name: str = Field() + + is_registered: bool = False + is_ready: bool = False @computed_field @property def id(self) -> str: return self.__class__.__name__ - + @computed_field @property def hook_module(self) -> str: + """e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet""" return f'{self.__module__}.{self.__class__.__name__}' - + + @property + def plugin_module(self) -> str: + """e.g. builtin_plugins.singlefile""" + return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0] + + @computed_field + @property + def plugin_dir(self) -> Path: + return Path(inspect.getfile(self.__class__)).parent.resolve() + hook_type: HookType = Field() - - def register(self, settings, parent_plugin=None): """Load a record of an installed hook into global Django settings.HOOKS at runtime.""" @@ -83,5 +97,20 @@ class BaseHook(BaseModel): # record installed hook in settings.HOOKS settings.HOOKS[self.id] = self + + if settings.HOOKS[self.id].is_registered: + raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!") + + settings.HOOKS[self.id].is_registered = True # print("REGISTERED HOOK:", self.hook_module) + + def ready(self, settings): + """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported).""" + + assert self.id in settings.HOOKS, f"Tried to ready hook {self.hook_module} but it is not registered in settings.HOOKS." + + if settings.HOOKS[self.id].is_ready: + raise Exception(f"Tried to run {self.hook_module}.ready() but its already been called!") + + settings.HOOKS[self.id].is_ready = True diff --git a/archivebox/plugantic/base_plugin.py b/archivebox/plugantic/base_plugin.py index 009baa67..22d52f01 100644 --- a/archivebox/plugantic/base_plugin.py +++ b/archivebox/plugantic/base_plugin.py @@ -34,6 +34,9 @@ class BasePlugin(BaseModel): # All the hooks the plugin will install: hooks: List[InstanceOf[BaseHook]] = Field(default=[]) + is_registered: bool = False + is_ready: bool = False + @computed_field @property def id(self) -> str: @@ -81,7 +84,7 @@ class BasePlugin(BaseModel): def ready(self): from django.conf import settings - plugin_self.register(settings) + plugin_self.ready(settings) return PluginAppConfig @@ -97,9 +100,8 @@ class BasePlugin(BaseModel): hooks[hook.hook_type][hook.id] = hook return hooks - def register(self, settings=None): - """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime.""" + """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at import time (before models are imported or any AppConfig.ready() are called).""" if settings is None: from django.conf import settings as django_settings @@ -112,11 +114,34 @@ class BasePlugin(BaseModel): ### Mutate django.conf.settings... values in-place to include plugin-provided overrides settings.PLUGINS[self.id] = self + if settings.PLUGINS[self.id].is_registered: + raise Exception(f"Tried to run {self.plugin_module}.register() but its already been called!") + for hook in self.hooks: hook.register(settings, parent_plugin=self) + settings.PLUGINS[self.id].is_registered = True # print('√ REGISTERED PLUGIN:', self.plugin_module) + def ready(self, settings=None): + """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported).""" + + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + assert ( + self.id in settings.PLUGINS and settings.PLUGINS[self.id].is_registered + ), f"Tried to run plugin.ready() for {self.plugin_module} but plugin is not yet registered in settings.PLUGINS." + + if settings.PLUGINS[self.id].is_ready: + raise Exception(f"Tried to run {self.plugin_module}.ready() but its already been called!") + + for hook in self.hooks: + hook.ready(settings) + + settings.PLUGINS[self.id].is_ready = True + # @validate_call # def install_binaries(self) -> Self: # new_binaries = []