The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Stabilize rewritten preview screenshots

Browse Source
This commit is contained in:
Nick Sweeting 2026-07-25 22:54:35 -07:00
parent 9129e41c76
commit 412bd144f0
No known key found for this signature in database
4 changed files with 77 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
archivebox/misc/serve_static.py
View File

@ -531,6 +531,14 @@ def _rewrite_html_image_sources_for_request(
)
def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat_result, encoding: str | None, config) -> None:
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
if encoding:
response.headers["Content-Encoding"] = encoding
def _render_markdown_fallback(text: str) -> str:
if _markdown is not None and not HTML_TAG_RE.search(text):
try:
@ -871,15 +879,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
decoded = fullpath.read_text(encoding="utf-8", errors="replace")
wrapped = _render_text_preview_document(decoded, fullpath.name)
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
if etag:
response.headers["ETag"] = etag
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
if encoding:
response.headers["Content-Encoding"] = encoding
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
return _apply_archive_replay_headers(
response,
fullpath=fullpath,
@ -899,15 +899,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
raw_image_url = f"{raw_image_url}?{urlencode(list(preview_query.lists()), doseq=True)}"
wrapped = _render_image_preview_document(raw_image_url, fullpath.name)
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
if etag:
response.headers["ETag"] = etag
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
if encoding:
response.headers["Content-Encoding"] = encoding
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
return _apply_archive_replay_headers(
response,
fullpath=fullpath,
@ -972,15 +964,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
wrapped = _render_markdown_document(markdown_candidate)
wrapped, _rewrite_count = _rewrite_html_image_sources_for_request(request, wrapped, document_root, rel_path)
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
if etag:
response.headers["ETag"] = etag
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
if encoding:
response.headers["Content-Encoding"] = encoding
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
return _apply_archive_replay_headers(
response,
fullpath=fullpath,
@ -990,15 +974,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
)
if rewritten_count:
response = HttpResponse(rewritten_html, content_type=content_type)
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
if etag:
response.headers["ETag"] = etag
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
if encoding:
response.headers["Content-Encoding"] = encoding
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
return _apply_archive_replay_headers(
response,
fullpath=fullpath,
@ -1008,15 +984,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
)
if escaped_count and escaped_count > tag_count * 2:
response = HttpResponse(decoded, content_type=content_type)
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
if etag:
response.headers["ETag"] = etag
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"'
if encoding:
response.headers["Content-Encoding"] = encoding
_set_transformed_response_headers(response, fullpath, statobj, encoding, config)
return _apply_archive_replay_headers(
response,
fullpath=fullpath,

48
archivebox/tests/test_urls.py
View File

@ -27,7 +27,7 @@ def test_html_image_sources_rewrite_to_captured_responses(tmp_path):
rewritten, count = _rewrite_html_image_sources_to_responses(
'<img src="images/twitter.png"><img src="https://a.sweeting.me/matomo.php?idsite=1&rec=1">',
tmp_path,
"defuddle/content.html",
"extractor/content.html",
"https://sweeting.me/",
)
@ -46,6 +46,52 @@ def test_html_image_sources_rewrite_to_captured_responses(tmp_path):
assert 'src="responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in rewritten_root
def test_static_html_and_markdown_preview_images_rewrite_to_captured_responses(tmp_path):
from django.test import RequestFactory
from archivebox.misc.serve_static import serve_static_with_byterange_support
responses_dir = tmp_path / "responses" / "all"
responses_dir.mkdir(parents=True)
(responses_dir / "20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png").write_bytes(b"png")
html_path = tmp_path / "extractor" / "content.html"
html_path.parent.mkdir()
html_path.write_text('<img src="images/twitter.png">', encoding="utf-8")
request = RequestFactory().get("/web/20260722/sweeting.me/snapshot/extractor/content.html")
request.archivebox_snapshot_url = "https://sweeting.me/"
response = serve_static_with_byterange_support(request, "extractor/content.html", document_root=tmp_path)
assert response.status_code == 200
assert "ETag" not in response.headers
assert "max-age=60" in response.headers["Cache-Control"]
assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content
text_path = tmp_path / "article" / "content.txt"
text_path.parent.mkdir()
text_path.write_text(
"# Title\n\n"
"![Twitter](images/twitter.png)\n\n"
"- One\n"
"- Two\n"
"- Three\n"
"[A](https://example.com) [B](https://example.com/b) [C](https://example.com/c)\n",
encoding="utf-8",
)
request = RequestFactory().get("/web/20260722/sweeting.me/snapshot/article/content.txt")
request.archivebox_snapshot_url = "https://sweeting.me/"
response = serve_static_with_byterange_support(request, "article/content.txt", document_root=tmp_path)
assert response.status_code == 200
assert "ETag" not in response.headers
assert "max-age=60" in response.headers["Cache-Control"]
assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content
@pytest.fixture
def checked_in_static_site():
handler = partial(SimpleHTTPRequestHandler, directory=str(REPO_ROOT))

16
bin/take_screenshot.js
View File

@ -112,6 +112,7 @@ async function main() {
// admin requests and overwhelm the server being documented.
const restoredPages = await browser.pages();
const page = await browser.newPage();
await page.setCacheEnabled(false);
await Promise.all(restoredPages.map((restoredPage) => restoredPage.close()));
page.setDefaultTimeout(45000);
@ -189,6 +190,21 @@ async function main() {
&& frame.getAttribute('src')
&& frame.getAttribute('src') !== 'about:blank';
}, { timeout: 45000 }, expectedPlugin);
await page.waitForFunction(() => {
const frames = [document.querySelector('#main-frame'), document.querySelector('.thumb-card.selected-card iframe')]
.filter(Boolean);
if (!frames.length) return true;
return frames.every((frame) => {
try {
const doc = frame.contentDocument || frame.contentWindow?.document;
if (!doc || doc.readyState === 'loading') return false;
const images = Array.from(doc.images || []);
return images.every((img) => img.complete && (img.naturalWidth > 0 || img.currentSrc.startsWith('data:')));
} catch (err) {
return true;
}
});
}, { timeout: 45000, polling: 250 }).catch(() => {});
}
if (process.env.SCREENSHOT_EXPECT_LIVE_PROGRESS === '1') {

2
docs/Screenshots.md
View File

@ -541,7 +541,7 @@ View: [`/#defuddle`](http://snap-f9864c2b39b4.archivebox.localhost:9292#defuddle
<table><thead><tr>
<th>Desktop</th><th>Tablet</th><th>Mobile</th>
</tr></thead><tbody><tr>
<td align="center"><strong>Desktop (1600x1000)</strong><br><img src="screenshots/54-snapshot-view-defuddle-desktop.png?v=ca9a2bef8e63" alt="Snapshot View (defuddle) — desktop" width="1600"></td><td align="center"><strong>Tablet (1024x1366)</strong><br><img src="screenshots/54-snapshot-view-defuddle-tablet.png?v=cfa00139ef74" alt="Snapshot View (defuddle) — tablet" width="1024"></td><td align="center"><strong>Mobile (390x844)</strong><br><img src="screenshots/54-snapshot-view-defuddle-mobile.png?v=619982c774d6" alt="Snapshot View (defuddle) — mobile" width="390"></td>
<td align="center"><strong>Desktop (1600x1000)</strong><br><img src="screenshots/54-snapshot-view-defuddle-desktop.png?v=890389a6022f" alt="Snapshot View (defuddle) — desktop" width="1600"></td><td align="center"><strong>Tablet (1024x1366)</strong><br><img src="screenshots/54-snapshot-view-defuddle-tablet.png?v=4e68ffdec070" alt="Snapshot View (defuddle) — tablet" width="1024"></td><td align="center"><strong>Mobile (390x844)</strong><br><img src="screenshots/54-snapshot-view-defuddle-mobile.png?v=d8cd40f58043" alt="Snapshot View (defuddle) — mobile" width="390"></td>
</tr></tbody></table>
## Snapshot View (mercury)