The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Build default USER_AGENT from cached chrome Binary.version

Browse Source 
ArchivingConfig.USER_AGENT hardcoded Chrome/131, which went stale every
time Chrome shipped a new major. Replace the static default with a
default_factory that reads the chrome Binary record (populated by the
on_BinaryRequest chrome hook via abxpkg) and parses its .version with
abxpkg.SemVer — no subprocesses, no manual regex, no new module.

When nothing supplied USER_AGENT via the get_config() scope chain
(persona/env/conf/machine/crawl/snapshot/overrides) and no chrome
Binary record exists yet, fall back to a generic ArchiveBox bot UA
that doesn't claim a Chrome version.

Drop the stale Chrome/83 example UAs from ArchiveBox.conf.default.
This commit is contained in:
Claude 2026-05-22 00:59:27 +00:00
parent 81d6d36fb3
commit e569a6c216
No known key found for this signature in database
2 changed files with 26 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
archivebox/config/common.py
View File

@ -39,6 +39,23 @@ def rprint(*args, file=None, **kwargs):
console.print(*args, **kwargs)
def _default_user_agent() -> str:
"""Default USER_AGENT built from the cached chrome Binary.version
populated by the chrome on_BinaryRequest hook (via abxpkg). Used only
when no scope (persona/env/conf/machine/crawl/snapshot) supplies one —
see get_config(). Reads cached version, never subprocesses."""
try:
from abxpkg import SemVer
from archivebox.machine.models import Binary
installed = SemVer((Binary.objects.get_valid_binary("chrome") or Binary()).version)
except Exception:
installed = None
suffix = f"ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)"
if installed:
return f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{installed.major}.0.0.0 Safari/537.36 {suffix}"
return f"Mozilla/5.0 (compatible; {suffix})"
class ShellConfig(BaseConfigSet):
toml_section_header: str = "SHELL_CONFIG"
@ -254,9 +271,7 @@ class ArchivingConfig(BaseConfigSet):
RESOLUTION: str = Field(default="1440,2000")
CHECK_SSL_VALIDITY: bool = Field(default=True)
USER_AGENT: str = Field(
default=f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
)
USER_AGENT: str = Field(default_factory=lambda: _default_user_agent())
COOKIES_FILE: Path | None = Field(default=None)
URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")

11
etc/ArchiveBox.conf.default
View File

@ -40,9 +40,14 @@
# SAVE_ARCHIVE_DOT_ORG = True
# CURL_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.3683.75 Safari/537.36"
# WGET_USER_AGENT = Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.3683.75 Safari/537.36
# CHROME_USER_AGENT = Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.3683.75 Safari/537.36
# Leave USER_AGENT (and CURL_/WGET_/CHROME_USER_AGENT) unset to let ArchiveBox
# build a fresh User-Agent from the installed Chrome version at runtime. Only
# override if you have a specific reason to pin a UA — ArchiveBox will warn if
# a configured Chrome UA falls more than a few major versions behind installed.
# USER_AGENT = <your-custom-user-agent>
# CURL_USER_AGENT = <your-custom-user-agent>
# WGET_USER_AGENT = <your-custom-user-agent>
# CHROME_USER_AGENT = <your-custom-user-agent>
[DEPENDENCY_CONFIG]