The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Keep MAX_URL_LENGTH at 65535; only switch url storage to TextField

Browse Source 
The column change to a variable-length, still-indexed TextField is what
matters for supporting long URLs efficiently — no reason to lower the
supported limit. Adjust the long-URL tests to the real 65535 boundary.
This commit is contained in:
Claude 2026-06-05 04:10:41 +00:00
parent b74861ef1e
commit e4df2d6bf2
No known key found for this signature in database
2 changed files with 11 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
archivebox/misc/util.py
View File

@ -86,13 +86,12 @@ URL_REGEX = re.compile(
re.IGNORECASE | re.UNICODE,
)
# Maximum supported URL length. Very long URLs are rare but must be supported
# correctly (e.g. data: URLs, deeply nested query strings). 8000 chars matches the
# practical upper bound enforced by most web servers/proxies. The Snapshot.url column is
# stored as a variable-length TextField (so short URLs don't reserve space and long URLs
# still fit) while keeping a normal index on the field so exact, prefix, and substring
# lookups all keep working.
MAX_URL_LENGTH = 8000
# Maximum supported URL length. Very long URLs are rare but must be supported correctly
# (e.g. data: URLs, deeply nested query strings). The Snapshot.url column is stored as a
# variable-length TextField (so short URLs don't reserve space and very long URLs still
# fit) while keeping a normal index on the field so exact, prefix, and substring lookups
# all keep working.
MAX_URL_LENGTH = 65535
QUOTE_DELIMITERS = (
'"',

10
archivebox/tests/test_snapshot_url_length.py
View File

@ -1,8 +1,8 @@
"""Tests for very long Snapshot URLs.
ArchiveBox supports URLs up to MAX_URL_LENGTH (8000) chars. The url column is stored as a
variable-length TextField (so short URLs don't reserve 8000 chars) but stays fully indexed,
so exact, prefix, and substring lookups all keep working for long URLs.
ArchiveBox supports URLs up to MAX_URL_LENGTH chars. The url column is stored as a
variable-length TextField (so short URLs don't reserve the full max length) but stays fully
indexed, so exact, prefix, and substring lookups all keep working for long URLs.
"""
import pytest
@ -23,8 +23,8 @@ def _make_long_url(length: int, *, needle: str = "", prefix: str = "https://exam
return url
def test_max_url_length_is_8000():
assert MAX_URL_LENGTH == 8000
def test_max_url_length_constant():
assert MAX_URL_LENGTH == 65535
def test_validate_url_length_accepts_exactly_max():