From 412bd144f073b2b0020a2bf564b3b008e8ca3800 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 25 Jul 2026 22:54:35 -0700 Subject: [PATCH] Stabilize rewritten preview screenshots --- archivebox/misc/serve_static.py | 58 ++++++++------------------------- archivebox/tests/test_urls.py | 48 ++++++++++++++++++++++++++- bin/take_screenshot.js | 16 +++++++++ docs/Screenshots.md | 2 +- 4 files changed, 77 insertions(+), 47 deletions(-) diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py index 8b9f18a8..9e36e77d 100644 --- a/archivebox/misc/serve_static.py +++ b/archivebox/misc/serve_static.py @@ -531,6 +531,14 @@ def _rewrite_html_image_sources_for_request( ) +def _set_transformed_response_headers(response, fullpath: Path, statobj: os.stat_result, encoding: str | None, config) -> None: + response.headers["Last-Modified"] = http_date(statobj.st_mtime) + response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" + response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"' + if encoding: + response.headers["Content-Encoding"] = encoding + + def _render_markdown_fallback(text: str) -> str: if _markdown is not None and not HTML_TAG_RE.search(text): try: @@ -871,15 +879,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ decoded = fullpath.read_text(encoding="utf-8", errors="replace") wrapped = _render_text_preview_document(decoded, fullpath.name) response = HttpResponse(wrapped, content_type="text/html; charset=utf-8") - response.headers["Last-Modified"] = http_date(statobj.st_mtime) - if etag: - response.headers["ETag"] = etag - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" - response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"' - if encoding: - response.headers["Content-Encoding"] = encoding + _set_transformed_response_headers(response, fullpath, statobj, encoding, config) return _apply_archive_replay_headers( response, fullpath=fullpath, @@ -899,15 +899,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ raw_image_url = f"{raw_image_url}?{urlencode(list(preview_query.lists()), doseq=True)}" wrapped = _render_image_preview_document(raw_image_url, fullpath.name) response = HttpResponse(wrapped, content_type="text/html; charset=utf-8") - response.headers["Last-Modified"] = http_date(statobj.st_mtime) - if etag: - response.headers["ETag"] = etag - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" - response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"' - if encoding: - response.headers["Content-Encoding"] = encoding + _set_transformed_response_headers(response, fullpath, statobj, encoding, config) return _apply_archive_replay_headers( response, fullpath=fullpath, @@ -972,15 +964,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ wrapped = _render_markdown_document(markdown_candidate) wrapped, _rewrite_count = _rewrite_html_image_sources_for_request(request, wrapped, document_root, rel_path) response = HttpResponse(wrapped, content_type="text/html; charset=utf-8") - response.headers["Last-Modified"] = http_date(statobj.st_mtime) - if etag: - response.headers["ETag"] = etag - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" - response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"' - if encoding: - response.headers["Content-Encoding"] = encoding + _set_transformed_response_headers(response, fullpath, statobj, encoding, config) return _apply_archive_replay_headers( response, fullpath=fullpath, @@ -990,15 +974,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ ) if rewritten_count: response = HttpResponse(rewritten_html, content_type=content_type) - response.headers["Last-Modified"] = http_date(statobj.st_mtime) - if etag: - response.headers["ETag"] = etag - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" - response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"' - if encoding: - response.headers["Content-Encoding"] = encoding + _set_transformed_response_headers(response, fullpath, statobj, encoding, config) return _apply_archive_replay_headers( response, fullpath=fullpath, @@ -1008,15 +984,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ ) if escaped_count and escaped_count > tag_count * 2: response = HttpResponse(decoded, content_type=content_type) - response.headers["Last-Modified"] = http_date(statobj.st_mtime) - if etag: - response.headers["ETag"] = etag - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" - response.headers["Content-Disposition"] = f'inline; filename="{fullpath.name}"' - if encoding: - response.headers["Content-Encoding"] = encoding + _set_transformed_response_headers(response, fullpath, statobj, encoding, config) return _apply_archive_replay_headers( response, fullpath=fullpath, diff --git a/archivebox/tests/test_urls.py b/archivebox/tests/test_urls.py index 91c0c0fd..316107ef 100644 --- a/archivebox/tests/test_urls.py +++ b/archivebox/tests/test_urls.py @@ -27,7 +27,7 @@ def test_html_image_sources_rewrite_to_captured_responses(tmp_path): rewritten, count = _rewrite_html_image_sources_to_responses( '', tmp_path, - "defuddle/content.html", + "extractor/content.html", "https://sweeting.me/", ) @@ -46,6 +46,52 @@ def test_html_image_sources_rewrite_to_captured_responses(tmp_path): assert 'src="responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in rewritten_root +def test_static_html_and_markdown_preview_images_rewrite_to_captured_responses(tmp_path): + from django.test import RequestFactory + + from archivebox.misc.serve_static import serve_static_with_byterange_support + + responses_dir = tmp_path / "responses" / "all" + responses_dir.mkdir(parents=True) + (responses_dir / "20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png").write_bytes(b"png") + + html_path = tmp_path / "extractor" / "content.html" + html_path.parent.mkdir() + html_path.write_text('', encoding="utf-8") + + request = RequestFactory().get("/web/20260722/sweeting.me/snapshot/extractor/content.html") + request.archivebox_snapshot_url = "https://sweeting.me/" + + response = serve_static_with_byterange_support(request, "extractor/content.html", document_root=tmp_path) + + assert response.status_code == 200 + assert "ETag" not in response.headers + assert "max-age=60" in response.headers["Cache-Control"] + assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content + + text_path = tmp_path / "article" / "content.txt" + text_path.parent.mkdir() + text_path.write_text( + "# Title\n\n" + "![Twitter](images/twitter.png)\n\n" + "- One\n" + "- Two\n" + "- Three\n" + "[A](https://example.com) [B](https://example.com/b) [C](https://example.com/c)\n", + encoding="utf-8", + ) + + request = RequestFactory().get("/web/20260722/sweeting.me/snapshot/article/content.txt") + request.archivebox_snapshot_url = "https://sweeting.me/" + + response = serve_static_with_byterange_support(request, "article/content.txt", document_root=tmp_path) + + assert response.status_code == 200 + assert "ETag" not in response.headers + assert "max-age=60" in response.headers["Cache-Control"] + assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content + + @pytest.fixture def checked_in_static_site(): handler = partial(SimpleHTTPRequestHandler, directory=str(REPO_ROOT)) diff --git a/bin/take_screenshot.js b/bin/take_screenshot.js index d61a6226..a5959e42 100755 --- a/bin/take_screenshot.js +++ b/bin/take_screenshot.js @@ -112,6 +112,7 @@ async function main() { // admin requests and overwhelm the server being documented. const restoredPages = await browser.pages(); const page = await browser.newPage(); + await page.setCacheEnabled(false); await Promise.all(restoredPages.map((restoredPage) => restoredPage.close())); page.setDefaultTimeout(45000); @@ -189,6 +190,21 @@ async function main() { && frame.getAttribute('src') && frame.getAttribute('src') !== 'about:blank'; }, { timeout: 45000 }, expectedPlugin); + await page.waitForFunction(() => { + const frames = [document.querySelector('#main-frame'), document.querySelector('.thumb-card.selected-card iframe')] + .filter(Boolean); + if (!frames.length) return true; + return frames.every((frame) => { + try { + const doc = frame.contentDocument || frame.contentWindow?.document; + if (!doc || doc.readyState === 'loading') return false; + const images = Array.from(doc.images || []); + return images.every((img) => img.complete && (img.naturalWidth > 0 || img.currentSrc.startsWith('data:'))); + } catch (err) { + return true; + } + }); + }, { timeout: 45000, polling: 250 }).catch(() => {}); } if (process.env.SCREENSHOT_EXPECT_LIVE_PROGRESS === '1') { diff --git a/docs/Screenshots.md b/docs/Screenshots.md index 58230e15..056f4a31 100644 --- a/docs/Screenshots.md +++ b/docs/Screenshots.md @@ -541,7 +541,7 @@ View: [`/#defuddle`](http://snap-f9864c2b39b4.archivebox.localhost:9292#defuddle - +
DesktopTabletMobile
Desktop (1600x1000)
Snapshot View (defuddle) — desktop		Tablet (1024x1366)
Snapshot View (defuddle) — tablet		Mobile (390x844)
Snapshot View (defuddle) — mobile		Desktop (1600x1000)
Snapshot View (defuddle) — desktop		Tablet (1024x1366)
Snapshot View (defuddle) — tablet		Mobile (390x844)
Snapshot View (defuddle) — mobile
## Snapshot View (mercury)