change plugins to have both a .register that runs at import and .ready that runs later

This commit is contained in:
Nick Sweeting 2024-09-10 00:00:41 -07:00
parent f1cca5bbba
commit 4df90fbb40
No known key found for this signature in database
6 changed files with 76 additions and 12 deletions

View file

@ -3,6 +3,7 @@ __package__ = 'archivebox.builtin_plugins.npm'
from typing import List, Optional from typing import List, Optional
from pydantic import InstanceOf, Field from pydantic import InstanceOf, Field
from django.conf import settings
from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
from plugantic.base_plugin import BasePlugin from plugantic.base_plugin import BasePlugin
@ -65,4 +66,5 @@ class NpmPlugin(BasePlugin):
PLUGIN = NpmPlugin() PLUGIN = NpmPlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig DJANGO_APP = PLUGIN.AppConfig

View file

@ -9,6 +9,7 @@ import django
from django.db.backends.sqlite3.base import Database as sqlite3 # type: ignore[import-type] from django.db.backends.sqlite3.base import Database as sqlite3 # type: ignore[import-type]
from django.core.checks import Error, Tags from django.core.checks import Error, Tags
from django.conf import settings
from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
from plugantic.base_plugin import BasePlugin from plugantic.base_plugin import BasePlugin
@ -139,4 +140,5 @@ class PipPlugin(BasePlugin):
] ]
PLUGIN = PipPlugin() PLUGIN = PipPlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig DJANGO_APP = PLUGIN.AppConfig

View file

@ -1,6 +1,8 @@
from pathlib import Path from pathlib import Path
from typing import List, Dict, Optional from typing import List, Dict, Optional
from django.conf import settings
# Depends on other PyPI/vendor packages: # Depends on other PyPI/vendor packages:
from pydantic import InstanceOf, Field from pydantic import InstanceOf, Field
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
@ -101,9 +103,11 @@ class SinglefilePlugin(BasePlugin):
SINGLEFILE_CONFIG, SINGLEFILE_CONFIG,
SINGLEFILE_BINARY, SINGLEFILE_BINARY,
SINGLEFILE_EXTRACTOR, SINGLEFILE_EXTRACTOR,
SINGLEFILE_QUEUE,
] ]
PLUGIN = SinglefilePlugin() PLUGIN = SinglefilePlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig DJANGO_APP = PLUGIN.AppConfig

View file

@ -2,6 +2,7 @@ from typing import List, Dict
from subprocess import run, PIPE from subprocess import run, PIPE
from pydantic import InstanceOf, Field from pydantic import InstanceOf, Field
from django.conf import settings
from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
from plugantic.base_plugin import BasePlugin from plugantic.base_plugin import BasePlugin
@ -74,4 +75,5 @@ class YtdlpPlugin(BasePlugin):
PLUGIN = YtdlpPlugin() PLUGIN = YtdlpPlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig DJANGO_APP = PLUGIN.AppConfig

View file

@ -1,14 +1,15 @@
__package__ = 'archivebox.plugantic' __package__ = 'archivebox.plugantic'
import json import inspect
from huey.api import TaskWrapper
from pathlib import Path
from typing import List, Literal from typing import List, Literal
from pydantic import BaseModel, ConfigDict, Field, computed_field from pydantic import BaseModel, ConfigDict, Field, computed_field
HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW'] HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW'] hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
class BaseHook(BaseModel): class BaseHook(BaseModel):
""" """
@ -56,24 +57,37 @@ class BaseHook(BaseModel):
validate_defaults=True, validate_defaults=True,
validate_assignment=False, validate_assignment=False,
revalidate_instances="subclass-instances", revalidate_instances="subclass-instances",
ignored_types=(TaskWrapper, ),
) )
# verbose_name: str = Field() # verbose_name: str = Field()
is_registered: bool = False
is_ready: bool = False
@computed_field @computed_field
@property @property
def id(self) -> str: def id(self) -> str:
return self.__class__.__name__ return self.__class__.__name__
@computed_field @computed_field
@property @property
def hook_module(self) -> str: def hook_module(self) -> str:
"""e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
return f'{self.__module__}.{self.__class__.__name__}' return f'{self.__module__}.{self.__class__.__name__}'
@property
def plugin_module(self) -> str:
"""e.g. builtin_plugins.singlefile"""
return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
@computed_field
@property
def plugin_dir(self) -> Path:
return Path(inspect.getfile(self.__class__)).parent.resolve()
hook_type: HookType = Field() hook_type: HookType = Field()
def register(self, settings, parent_plugin=None): def register(self, settings, parent_plugin=None):
"""Load a record of an installed hook into global Django settings.HOOKS at runtime.""" """Load a record of an installed hook into global Django settings.HOOKS at runtime."""
@ -83,5 +97,20 @@ class BaseHook(BaseModel):
# record installed hook in settings.HOOKS # record installed hook in settings.HOOKS
settings.HOOKS[self.id] = self settings.HOOKS[self.id] = self
if settings.HOOKS[self.id].is_registered:
raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!")
settings.HOOKS[self.id].is_registered = True
# print("REGISTERED HOOK:", self.hook_module) # print("REGISTERED HOOK:", self.hook_module)
def ready(self, settings):
"""Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
assert self.id in settings.HOOKS, f"Tried to ready hook {self.hook_module} but it is not registered in settings.HOOKS."
if settings.HOOKS[self.id].is_ready:
raise Exception(f"Tried to run {self.hook_module}.ready() but its already been called!")
settings.HOOKS[self.id].is_ready = True

View file

@ -34,6 +34,9 @@ class BasePlugin(BaseModel):
# All the hooks the plugin will install: # All the hooks the plugin will install:
hooks: List[InstanceOf[BaseHook]] = Field(default=[]) hooks: List[InstanceOf[BaseHook]] = Field(default=[])
is_registered: bool = False
is_ready: bool = False
@computed_field @computed_field
@property @property
def id(self) -> str: def id(self) -> str:
@ -81,7 +84,7 @@ class BasePlugin(BaseModel):
def ready(self): def ready(self):
from django.conf import settings from django.conf import settings
plugin_self.register(settings) plugin_self.ready(settings)
return PluginAppConfig return PluginAppConfig
@ -97,9 +100,8 @@ class BasePlugin(BaseModel):
hooks[hook.hook_type][hook.id] = hook hooks[hook.hook_type][hook.id] = hook
return hooks return hooks
def register(self, settings=None): def register(self, settings=None):
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime.""" """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at import time (before models are imported or any AppConfig.ready() are called)."""
if settings is None: if settings is None:
from django.conf import settings as django_settings from django.conf import settings as django_settings
@ -112,11 +114,34 @@ class BasePlugin(BaseModel):
### Mutate django.conf.settings... values in-place to include plugin-provided overrides ### Mutate django.conf.settings... values in-place to include plugin-provided overrides
settings.PLUGINS[self.id] = self settings.PLUGINS[self.id] = self
if settings.PLUGINS[self.id].is_registered:
raise Exception(f"Tried to run {self.plugin_module}.register() but its already been called!")
for hook in self.hooks: for hook in self.hooks:
hook.register(settings, parent_plugin=self) hook.register(settings, parent_plugin=self)
settings.PLUGINS[self.id].is_registered = True
# print('√ REGISTERED PLUGIN:', self.plugin_module) # print('√ REGISTERED PLUGIN:', self.plugin_module)
def ready(self, settings=None):
"""Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
assert (
self.id in settings.PLUGINS and settings.PLUGINS[self.id].is_registered
), f"Tried to run plugin.ready() for {self.plugin_module} but plugin is not yet registered in settings.PLUGINS."
if settings.PLUGINS[self.id].is_ready:
raise Exception(f"Tried to run {self.plugin_module}.ready() but its already been called!")
for hook in self.hooks:
hook.ready(settings)
settings.PLUGINS[self.id].is_ready = True
# @validate_call # @validate_call
# def install_binaries(self) -> Self: # def install_binaries(self) -> Self:
# new_binaries = [] # new_binaries = []