mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-13 18:46:17 +05:00
77 lines
2.8 KiB
Python
77 lines
2.8 KiB
Python
__package__ = "archivebox"
|
|
|
|
|
|
import datetime
|
|
import re
|
|
import warnings
|
|
|
|
from daphne import access
|
|
import django_stubs_ext
|
|
from django.utils import timezone
|
|
|
|
django_stubs_ext.monkeypatch()
|
|
|
|
|
|
# monkey patch django timezone to add back utc (it was removed in Django 5.0)
|
|
setattr(timezone, "utc", datetime.UTC)
|
|
|
|
# Hide site-packages/sonic/client.py:115: SyntaxWarning
|
|
# https://github.com/xmonader/python-sonic-client/pull/18
|
|
warnings.filterwarnings("ignore", category=SyntaxWarning, module="sonic")
|
|
|
|
|
|
SENSITIVE_QUERY_PARAM_RE = re.compile(r"(?i)([?&](?:api_key|token|access_token|password|secret)=)([^&#\s]+)")
|
|
|
|
|
|
# Make daphne log requests quieter and easier to read
|
|
class ModifiedAccessLogGenerator(access.AccessLogGenerator):
|
|
"""Clutge workaround until daphne uses the Python logging framework. https://github.com/django/daphne/pull/473/files"""
|
|
|
|
def __call__(self, protocol, action, details):
|
|
if protocol == "http" and action == "complete":
|
|
self.write_entry(
|
|
host=details["client"],
|
|
date=datetime.datetime.now(),
|
|
request="%(method)s %(path)s" % details,
|
|
status=details["status"],
|
|
length=details["size"],
|
|
time_taken=details.get("time_taken"),
|
|
)
|
|
return
|
|
return super().__call__(protocol, action, details)
|
|
|
|
def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None, time_taken=None):
|
|
request = SENSITIVE_QUERY_PARAM_RE.sub(r"\1[REDACTED]", request)
|
|
|
|
# Ignore noisy requests to staticfiles / favicons / etc.
|
|
if "GET /static/" in request:
|
|
return
|
|
if "GET /health/" in request:
|
|
return
|
|
if "GET /progress.json" in request and (time_taken is None or time_taken < 1.0):
|
|
return
|
|
if "GET /api/v1/crawls/crawl/" in request and "/files/chrome_screencast/latest.jpg" in request:
|
|
return
|
|
if "GET /admin/jsi18n/" in request:
|
|
return
|
|
if request.endswith("/favicon.ico") or request.endswith("/robots.txt") or request.endswith("/screenshot.png"):
|
|
return
|
|
if request.endswith(".css") or request.endswith(".js") or request.endswith(".woff") or request.endswith(".ttf"):
|
|
return
|
|
if str(status) in ("404", "304"):
|
|
return
|
|
|
|
# clean up the log format to mostly match the same format as django.conf.settings.LOGGING rich formats
|
|
self.stream.write(
|
|
"%s HTTP %s %s %s\n"
|
|
% (
|
|
date.strftime("%Y-%m-%d %H:%M:%S"),
|
|
request,
|
|
status or "-",
|
|
"localhost" if host.startswith("127.") else host.split(":")[0],
|
|
),
|
|
)
|
|
|
|
|
|
access.AccessLogGenerator.write_entry = ModifiedAccessLogGenerator.write_entry # type: ignore
|