mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2024-09-19 15:38:48 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

add django-object-actions to provide Regenerate ABID button

Browse source
This commit is contained in:
Nick Sweeting 2024-09-05 23:19:21 -07:00
parent 00aa7dc19f
commit 2e1e1945f2
No known key found for this signature in database
7 changed files with 144 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
archivebox/abid_utils/admin.py
View file

@ -9,11 +9,13 @@ from django.utils.html import format_html
from django.utils.safestring import mark_safe from django.utils.safestring import mark_safe
from django.shortcuts import redirect from django.shortcuts import redirect
from .abid import ABID from django_object_actions import DjangoObjectActions, action
from api.auth import get_or_create_api_token from api.auth import get_or_create_api_token
from ..util import parse_date from ..util import parse_date
from .abid import ABID
def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None): def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
"""highlight each character in red that differs with the char at the same index in compare_val""" """highlight each character in red that differs with the char at the same index in compare_val"""
@ -39,22 +41,26 @@ def get_abid_info(self, obj, request=None):
try: try:
#abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅' #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
fresh_abid = ABID(**obj.ABID_FRESH_HASHES) fresh_values = obj.ABID_FRESH_VALUES
fresh_hashes = obj.ABID_FRESH_HASHES
fresh_diffs = obj.ABID_FRESH_DIFFS
fresh_abid = ABID(**fresh_hashes)
fresh_abid_diff = f'❌ !=   .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅' fresh_abid_diff = f'❌ !=   .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
fresh_uuid_diff = f'❌ !=   .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅' fresh_uuid_diff = f'❌ !=   .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
id_pk_diff = f'❌ != .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅' id_pk_diff = f'❌ != .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
fresh_ts = parse_date(obj.ABID_FRESH_VALUES['ts']) or None fresh_ts = parse_date(fresh_values['ts']) or None
ts_diff = f'❌ != {highlight_diff( obj.ABID_FRESH_HASHES["ts"], obj.ABID.ts)}' if obj.ABID_FRESH_HASHES["ts"] != obj.ABID.ts else '✅' ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if fresh_hashes["ts"] != obj.ABID.ts else '✅'
derived_uri = obj.ABID_FRESH_HASHES['uri'] derived_uri = fresh_hashes['uri']
uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅' uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
derived_subtype = obj.ABID_FRESH_HASHES['subtype'] derived_subtype = fresh_hashes['subtype']
subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅' subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
derived_rand = obj.ABID_FRESH_HASHES['rand'] derived_rand = fresh_hashes['rand']
rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅' rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
return format_html( return format_html(
@ -72,7 +78,7 @@ def get_abid_info(self, obj, request=None):
&nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/> &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
&nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code> &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
<br/><hr/> <br/><hr/>
<span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
</div> </div>
''', ''',
obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url, obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
@ -81,24 +87,28 @@ def get_abid_info(self, obj, request=None):
highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff), highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
# str(fresh_abid.uuid), mark_safe(fresh_uuid_diff), # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
# str(fresh_abid), mark_safe(fresh_abid_diff), # str(fresh_abid), mark_safe(fresh_abid_diff),
highlight_diff(obj.ABID.ts, obj.ABID_FRESH_HASHES['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(), highlight_diff(obj.ABID.ts, fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(obj.ABID_FRESH_VALUES['uri']), highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(obj.ABID_FRESH_VALUES['subtype']), highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(obj.ABID_FRESH_VALUES['rand'])[-7:], highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
f'Some values the ABID depends on have changed since the ABID was issued:' if obj.ABID_FRESH_DIFFS else '', 'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
", ".join(diff['abid_src'] for diff in obj.ABID_FRESH_DIFFS.values()), ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
'(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
) )
except Exception as e: except Exception as e:
# import ipdb; ipdb.set_trace() # import ipdb; ipdb.set_trace()
return str(e) return str(e)
class ABIDModelAdmin(admin.ModelAdmin): class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
list_display = ('created_at', 'created_by', 'abid') list_display = ('created_at', 'created_by', 'abid')
sort_fields = ('created_at', 'created_by', 'abid') sort_fields = ('created_at', 'created_by', 'abid')
readonly_fields = ('created_at', 'modified_at', 'abid_info') readonly_fields = ('created_at', 'modified_at', 'abid_info')
# fields = [*readonly_fields] # fields = [*readonly_fields]
change_actions = ("regenerate_abid",)
# changelist_actions = ("regenerate_abid",)
def _get_obj_does_not_exist_redirect(self, request, opts, object_id): def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
try: try:
object_pk = self.model.id_from_abid(object_id) object_pk = self.model.id_from_abid(object_id)
@ -120,11 +130,17 @@ class ABIDModelAdmin(admin.ModelAdmin):
form = super().get_form(request, obj, **kwargs) form = super().get_form(request, obj, **kwargs)
if 'created_by' in form.base_fields: if 'created_by' in form.base_fields:
form.base_fields['created_by'].initial = request.user form.base_fields['created_by'].initial = request.user
if obj:
if obj.ABID_FRESH_DIFFS:
messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
return form return form
def get_formset(self, request, formset=None, obj=None, **kwargs): def get_formset(self, request, formset=None, obj=None, **kwargs):
formset = super().get_formset(request, formset, obj, **kwargs) formset = super().get_formset(request, formset, obj, **kwargs)
formset.form.base_fields['created_at'].disabled = True formset.form.base_fields['created_at'].disabled = True
return formset return formset
def save_model(self, request, obj, form, change): def save_model(self, request, obj, form, change):
@ -143,3 +159,16 @@ class ABIDModelAdmin(admin.ModelAdmin):
@admin.display(description='API Identifiers') @admin.display(description='API Identifiers')
def abid_info(self, obj): def abid_info(self, obj):
return get_abid_info(self, obj, request=self.request) return get_abid_info(self, obj, request=self.request)
@action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
def regenerate_abid(self, request, obj):
old_abid = str(obj.abid)
obj.abid = obj.issue_new_abid(overwrite=True)
obj.save()
obj.refresh_from_db()
new_abid = str(obj.abid)
if new_abid != old_abid:
messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
else:
messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")

27
archivebox/abid_utils/models.py
View file

@ -2,12 +2,10 @@
This file provides the Django ABIDField and ABIDModel base model to inherit from. This file provides the Django ABIDField and ABIDModel base model to inherit from.
""" """
from typing import Any, Dict, Union, List, Set, NamedTuple, cast
from ulid import ULID from typing import Any, Dict, Union, List, Set, cast
from uuid import uuid4, UUID
from typeid import TypeID # type: ignore[import-untyped] from uuid import uuid4
from datetime import datetime, timedelta
from functools import partial from functools import partial
from charidfield import CharIDField # type: ignore[import-untyped] from charidfield import CharIDField # type: ignore[import-untyped]
@ -30,7 +28,6 @@ from .abid import (
DEFAULT_ABID_URI_SALT, DEFAULT_ABID_URI_SALT,
abid_part_from_prefix, abid_part_from_prefix,
abid_hashes_from_values, abid_hashes_from_values,
abid_from_values,
ts_from_abid, ts_from_abid,
abid_part_from_ts, abid_part_from_ts,
) )
@ -119,6 +116,7 @@ class ABIDModel(models.Model):
# otherwise if updating, make sure none of the field changes would invalidate existing ABID # otherwise if updating, make sure none of the field changes would invalidate existing ABID
abid_diffs = self.ABID_FRESH_DIFFS abid_diffs = self.ABID_FRESH_DIFFS
if abid_diffs: if abid_diffs:
# change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values()) keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
full_summary = ( full_summary = (
@ -142,16 +140,15 @@ class ABIDModel(models.Model):
NON_FIELD_ERRORS: ValidationError(full_summary), NON_FIELD_ERRORS: ValidationError(full_summary),
}) })
should_ovewrite_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
if should_ovewrite_abid: if allowed_to_invalidate_abid:
print(f'\n#### DANGER: Changing ABID of existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed}), this will break any references to its previous ABID!') print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
print(change_error) print(change_error)
self._previous_abid = self.abid print('--------------------------------------------------------------------------------------------------')
self.abid = str(self.issue_new_abid(force_new=True))
print(f'#### DANGER: OVERWROTE OLD ABID. NEW ABID=', self.abid)
else: else:
print(f'\n#### WARNING: ABID of existing record is outdated and has not been updated ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})') print(f'\n#### ERROR: Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
print(change_error) print(change_error)
print('--------------------------------------------------------------------------------------------------')
raise change_error raise change_error
def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None: def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
@ -230,11 +227,11 @@ class ABIDModel(models.Model):
if getattr(existing_abid, key) != new_hash if getattr(existing_abid, key) != new_hash
} }
def issue_new_abid(self, force_new=False) -> ABID: def issue_new_abid(self, overwrite=False) -> ABID:
""" """
Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB). Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
""" """
if not force_new: if not overwrite:
assert self._state.adding, 'Can only issue new ABID when model._state.adding is True' assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})' assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'

2
archivebox/core/models.py
View file

@ -131,7 +131,7 @@ class Snapshot(ABIDModel):
abid_uri_src = 'self.url' abid_uri_src = 'self.url'
abid_subtype_src = '"01"' abid_subtype_src = '"01"'
abid_rand_src = 'self.id' abid_rand_src = 'self.id'
abid_drift_allowed = False abid_drift_allowed = True
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID') id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix) abid = ABIDField(prefix=abid_prefix)

91
archivebox/core/settings.py
View file

@ -6,9 +6,11 @@ import re
import logging import logging
import inspect import inspect
import tempfile import tempfile
from typing import Any, Dict
from typing import Dict
from pathlib import Path from pathlib import Path
import django
from django.utils.crypto import get_random_string from django.utils.crypto import get_random_string
from ..config import CONFIG from ..config import CONFIG
@ -89,8 +91,9 @@ INSTALLED_APPS = [
'django.contrib.admin', 'django.contrib.admin',
# 3rd-party apps from PyPI # 3rd-party apps from PyPI
'django_jsonform', # handles rendering Pydantic models to Django HTML widgets/forms 'django_jsonform', # handles rendering Pydantic models to Django HTML widgets/forms https://github.com/bhch/django-jsonform
'signal_webhooks', # handles REST API outbound webhooks 'signal_webhooks', # handles REST API outbound webhooks https://github.com/MrThearMan/django-signal-webhooks
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
# our own apps # our own apps
'abid_utils', # handles ABID ID creation, handling, and models 'abid_utils', # handles ABID ID creation, handling, and models
@ -384,15 +387,11 @@ class NoisyRequestsFilter(logging.Filter):
return True return True
def add_extra_logging_attrs(record):
record.username = '' class CustomOutboundWebhookLogFormatter(logging.Formatter):
try: def format(self, record):
record.username = record.request.user.username result = super().format(record)
except AttributeError: return result.replace('HTTP Request: ', 'OutboundWebhook: ')
record.username = "Anonymous"
if hasattr(record, 'request'):
import ipdb; ipdb.set_trace()
return True
ERROR_LOG = tempfile.NamedTemporaryFile().name ERROR_LOG = tempfile.NamedTemporaryFile().name
@ -416,21 +415,13 @@ LOGGING = {
"disable_existing_loggers": False, "disable_existing_loggers": False,
"formatters": { "formatters": {
"rich": { "rich": {
"datefmt": "[%X]", "datefmt": "[%Y-%m-%d %H:%M:%S]",
# "format": "{asctime} {levelname} {module} {name} {message} {username}", # "format": "{asctime} {levelname} {module} {name} {message} {username}",
# "format": "%(message)s (user=%(username)s", "format": "%(name)s %(message)s",
}, },
"verbose": { "outbound_webhooks": {
"style": "{", "()": CustomOutboundWebhookLogFormatter,
}, "datefmt": "[%Y-%m-%d %H:%M:%S]",
"simple": {
"format": "{name} {message}",
"style": "{",
},
"django.server": {
"()": "django.utils.log.ServerFormatter",
# "format": "{message} (user={username})",
"style": "{",
}, },
}, },
"filters": { "filters": {
@ -443,10 +434,6 @@ LOGGING = {
"require_debug_true": { "require_debug_true": {
"()": "django.utils.log.RequireDebugTrue", "()": "django.utils.log.RequireDebugTrue",
}, },
# "add_extra_logging_attrs": {
# "()": "django.utils.log.CallbackFilter",
# "callback": add_extra_logging_attrs,
# },
}, },
"handlers": { "handlers": {
# "console": { # "console": {
@ -455,7 +442,7 @@ LOGGING = {
# "class": "logging.StreamHandler", # "class": "logging.StreamHandler",
# 'filters': ['noisyrequestsfilter', 'add_extra_logging_attrs'], # 'filters': ['noisyrequestsfilter', 'add_extra_logging_attrs'],
# }, # },
"console": { "default": {
"class": "rich.logging.RichHandler", "class": "rich.logging.RichHandler",
"formatter": "rich", "formatter": "rich",
"level": "DEBUG", "level": "DEBUG",
@ -463,19 +450,25 @@ LOGGING = {
"rich_tracebacks": True, "rich_tracebacks": True,
"filters": ["noisyrequestsfilter"], "filters": ["noisyrequestsfilter"],
"tracebacks_suppress": [ "tracebacks_suppress": [
django,
pydantic, pydantic,
django.template,
], ],
}, },
"logfile": { "logfile": {
"level": "ERROR", "level": "INFO",
"class": "logging.handlers.RotatingFileHandler", "class": "logging.handlers.RotatingFileHandler",
"filename": ERROR_LOG, "filename": ERROR_LOG,
"maxBytes": 1024 * 1024 * 25, # 25 MB "maxBytes": 1024 * 1024 * 25, # 25 MB
"backupCount": 10, "backupCount": 10,
"formatter": "verbose", "formatter": "rich",
"filters": ["noisyrequestsfilter"], "filters": ["noisyrequestsfilter"],
}, },
"outbound_webhooks": {
"class": "rich.logging.RichHandler",
"markup": False,
"rich_tracebacks": True,
"formatter": "outbound_webhooks",
},
# "mail_admins": { # "mail_admins": {
# "level": "ERROR", # "level": "ERROR",
# "filters": ["require_debug_false"], # "filters": ["require_debug_false"],
@ -486,29 +479,35 @@ LOGGING = {
}, },
}, },
"root": { "root": {
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "INFO", "level": "INFO",
"formatter": "verbose", "formatter": "rich",
}, },
"loggers": { "loggers": {
"api": { "api": {
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "DEBUG", "level": "DEBUG",
}, },
"checks": { "checks": {
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "DEBUG", "level": "DEBUG",
}, },
"core": { "core": {
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "DEBUG", "level": "DEBUG",
}, },
"builtin_plugins": { "builtin_plugins": {
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "DEBUG", "level": "DEBUG",
}, },
"httpx": {
"handlers": ["outbound_webhooks"],
"level": "INFO",
"formatter": "outbound_webhooks",
"propagate": False,
},
"django": { "django": {
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "INFO", "level": "INFO",
"filters": ["noisyrequestsfilter"], "filters": ["noisyrequestsfilter"],
}, },
@ -518,29 +517,27 @@ LOGGING = {
"level": "ERROR", "level": "ERROR",
}, },
"django.channels.server": { "django.channels.server": {
# see archivebox.monkey_patches.ModifiedAccessLogGenerator for dedicated daphne server logging settings
"propagate": False, "propagate": False,
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "INFO", "level": "INFO",
"filters": ["noisyrequestsfilter"], "filters": ["noisyrequestsfilter"],
"formatter": "django.server",
}, },
"django.server": { # logs all requests (2xx, 3xx, 4xx) "django.server": { # logs all requests (2xx, 3xx, 4xx)
"propagate": False, "propagate": False,
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "INFO", "level": "INFO",
"filters": ["noisyrequestsfilter"], "filters": ["noisyrequestsfilter"],
"formatter": "django.server",
}, },
"django.request": { # only logs 4xx and 5xx errors "django.request": { # only logs 4xx and 5xx errors
"propagate": False, "propagate": False,
"handlers": ["console", "logfile"], "handlers": ["default", "logfile"],
"level": "INFO", "level": "INFO",
"filters": ["noisyrequestsfilter"], "filters": ["noisyrequestsfilter"],
"formatter": "django.server",
}, },
"django.db.backends": { "django.db.backends": {
"propagate": False, "propagate": False,
"handlers": ["console"], "handlers": ["default"],
"level": LOG_LEVEL_DATABASE, "level": LOG_LEVEL_DATABASE,
}, },
}, },

29
archivebox/monkey_patches.py
View file

@ -21,3 +21,32 @@ timezone.utc = datetime.timezone.utc
from rich.traceback import install from rich.traceback import install
install(show_locals=True) install(show_locals=True)
from daphne import access
class ModifiedAccessLogGenerator(access.AccessLogGenerator):
"""Clutge workaround until daphne uses the Python logging framework. https://github.com/django/daphne/pull/473/files"""
def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None):
# Ignore noisy requests to staticfiles / favicons / etc.
if 'GET /static/' in request:
return
if 'GET /admin/jsi18n/' in request:
return
if request.endswith("/favicon.ico") or request.endswith("/robots.txt") or request.endswith("/screenshot.png"):
return
# clean up the log format to mostly match the same format as django.conf.settings.LOGGING rich formats
self.stream.write(
"[%s] HTTP %s (%s) %s\n"
% (
date.strftime("%Y-%m-%d %H:%M:%S"),
request,
status or "-",
"localhost" if host.startswith("127.") else host.split(":")[0],
)
)
access.AccessLogGenerator.write_entry = ModifiedAccessLogGenerator.write_entry

14
pdm.lock
View file

@ -5,7 +5,7 @@
groups = ["default", "ldap", "sonic"] groups = ["default", "ldap", "sonic"]
strategy = ["inherit_metadata"] strategy = ["inherit_metadata"]
lock_version = "4.5.0" lock_version = "4.5.0"
content_hash = "sha256:f940c4c0a330b7b0bcff68a006b29ea3b1292ad6aadd3cfc909de0622f2963ac" content_hash = "sha256:61d53c8fbfcdaaf18e04d7aab12887caf9260b803db7e5b66a22e37b88824c55"
[[metadata.targets]] [[metadata.targets]]
requires_python = "==3.10.*" requires_python = "==3.10.*"
@ -456,6 +456,18 @@ files = [
{file = "django_ninja-1.3.0.tar.gz", hash = "sha256:5b320e2dc0f41a6032bfa7e1ebc33559ae1e911a426f0c6be6674a50b20819be"}, {file = "django_ninja-1.3.0.tar.gz", hash = "sha256:5b320e2dc0f41a6032bfa7e1ebc33559ae1e911a426f0c6be6674a50b20819be"},
] ]
[[package]]
name = "django-object-actions"
version = "4.2.0"
requires_python = ">=3.7,<4.0"
summary = "A Django app for adding object tools for models in the admin"
groups = ["default"]
marker = "python_version == \"3.10\""
files = [
{file = "django_object_actions-4.2.0-py3-none-any.whl", hash = "sha256:ae0df9984c68a4f42f219a391b71fa0630fe44a2983b39b8064378ebddcff30c"},
{file = "django_object_actions-4.2.0.tar.gz", hash = "sha256:e24befedf01b6fcdccbb03c33c0e2c855fd1a88f352a66dc7e2170ba31e80128"},
]
[[package]] [[package]]
name = "django-pydantic-field" name = "django-pydantic-field"
version = "0.3.10" version = "0.3.10"

1
pyproject.toml
View file

@ -50,6 +50,7 @@ dependencies = [
"base32-crockford==0.3.0", "base32-crockford==0.3.0",
"rich>=13.8.0", "rich>=13.8.0",
"channels[daphne]>=4.1.0", "channels[daphne]>=4.1.0",
"django-object-actions>=4.2.0",
] ]
homepage = "https://github.com/ArchiveBox/ArchiveBox" homepage = "https://github.com/ArchiveBox/ArchiveBox"