The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Scope recursive crawl plugin matrix

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-21 11:47:51 -07:00
parent 5679a2cfe7
commit 4982b041c6
No known key found for this signature in database
1 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
archivebox/tests/test_recursive_crawl.py
View File

@ -516,16 +516,27 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
@pytest.mark.timeout(1200)
def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initialized_archive, free_tcp_port_factory):
"""Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
"""Run a bounded real depth=2 crawl with representative plugins and verify parallel snapshot execution."""
from abx_dl.models import discover_plugins
root_url = "https://example.com/"
separately_covered_plugins = {"archivewebpage", "defuddle"}
crawl_plugins = {
"archivedotorg",
"dom",
"favicon",
"headers",
"htmltotext",
"mercury",
"parse_html_urls",
"pdf",
"readability",
"screenshot",
"title",
"wget",
}
plugin_selection = ",".join(
sorted(
plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude") and plugin not in separately_covered_plugins
),
sorted(plugin for plugin in discover_plugins().keys() if plugin in crawl_plugins),
)
env = os.environ.copy()
env.pop("CHROME_BINARY", None)
@ -646,7 +657,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
"pdf",
"screenshot",
"dom",
"singlefile",
"readability",
"mercury",
"htmltotext",
@ -662,7 +672,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
assert list(snapshot_root.rglob("pdf/**/*.pdf"))
assert list(snapshot_root.rglob("screenshot/**/*.png"))
assert list(snapshot_root.rglob("dom/**/*.html"))
assert list(snapshot_root.rglob("singlefile/**/*.html"))
assert list(snapshot_root.rglob("readability/**/*.html"))
assert list(snapshot_root.rglob("mercury/**/*.html"))
assert list(snapshot_root.rglob("htmltotext/**/*.txt"))