From e4df2d6bf20ba4e2a5e3712bf8e5d8583f0dc705 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 5 Jun 2026 04:10:41 +0000 Subject: [PATCH] Keep MAX_URL_LENGTH at 65535; only switch url storage to TextField MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The column change to a variable-length, still-indexed TextField is what matters for supporting long URLs efficiently — no reason to lower the supported limit. Adjust the long-URL tests to the real 65535 boundary. --- archivebox/misc/util.py | 13 ++++++------- archivebox/tests/test_snapshot_url_length.py | 10 +++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py index 17f5b2d7..f8ff1e68 100644 --- a/archivebox/misc/util.py +++ b/archivebox/misc/util.py @@ -86,13 +86,12 @@ URL_REGEX = re.compile( re.IGNORECASE | re.UNICODE, ) -# Maximum supported URL length. Very long URLs are rare but must be supported -# correctly (e.g. data: URLs, deeply nested query strings). 8000 chars matches the -# practical upper bound enforced by most web servers/proxies. The Snapshot.url column is -# stored as a variable-length TextField (so short URLs don't reserve space and long URLs -# still fit) while keeping a normal index on the field so exact, prefix, and substring -# lookups all keep working. -MAX_URL_LENGTH = 8000 +# Maximum supported URL length. Very long URLs are rare but must be supported correctly +# (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a +# variable-length TextField (so short URLs don't reserve space and very long URLs still +# fit) while keeping a normal index on the field so exact, prefix, and substring lookups +# all keep working. +MAX_URL_LENGTH = 65535 QUOTE_DELIMITERS = ( '"', diff --git a/archivebox/tests/test_snapshot_url_length.py b/archivebox/tests/test_snapshot_url_length.py index 3c21944f..9dcc7d6f 100644 --- a/archivebox/tests/test_snapshot_url_length.py +++ b/archivebox/tests/test_snapshot_url_length.py @@ -1,8 +1,8 @@ """Tests for very long Snapshot URLs. -ArchiveBox supports URLs up to MAX_URL_LENGTH (8000) chars. The url column is stored as a -variable-length TextField (so short URLs don't reserve 8000 chars) but stays fully indexed, -so exact, prefix, and substring lookups all keep working for long URLs. +ArchiveBox supports URLs up to MAX_URL_LENGTH chars. The url column is stored as a +variable-length TextField (so short URLs don't reserve the full max length) but stays fully +indexed, so exact, prefix, and substring lookups all keep working for long URLs. """ import pytest @@ -23,8 +23,8 @@ def _make_long_url(length: int, *, needle: str = "", prefix: str = "https://exam return url -def test_max_url_length_is_8000(): - assert MAX_URL_LENGTH == 8000 +def test_max_url_length_constant(): + assert MAX_URL_LENGTH == 65535 def test_validate_url_length_accepts_exactly_max():