mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
Expose yt-dlp media metadata for snapshot cards (#1819)
Supersedes #1818. This keeps yt-dlp-specific player rendering out of ArchiveBox core. Core now: - exposes media type metadata to plugin card templates: `is_video`, `is_audio`, `is_browser_playable` - sorts yt-dlp media with browser-playable video/audio first - prefers browser-playable yt-dlp outputs before thumbnails and non-browser containers when choosing the primary card output - avoids pinning resumed snapshots to stale selected hook maps, fixing a pause/resume runner regression surfaced by CI - makes `update --index-only` discover the configured search backend even when `PLUGINS` is restricted for test/runtime isolation - stabilizes CI-only recursive/takeover/public-UI tests that were failing due external/heavy plugin coverage, hard-kill recovery races, and transient full-server `/add/` startup responses The actual click-to-load player UI lives in the companion plugin PR: ArchiveBox/abx-plugins#34. Verification: ```text pytest archivebox/tests/test_ui_admin_snapshot.py -q 48 passed pytest archivebox/tests/test_cli_update_reindex_snapshots.py -q 9 passed pytest -xvs archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py --basetemp=tests/out --ignore=archivebox/pkgs 7 passed pytest -xvs archivebox/tests/test_takeover_util.py --basetemp=tests/out --ignore=archivebox/pkgs 19 passed pytest -xvs archivebox/tests/test_takeover_util.py::test_live_add_update_jobs_survive_server_and_cli_owner_exits --basetemp=tests/out --ignore=archivebox/pkgs 1 passed pytest -xvs archivebox/tests/test_ui_public_snapshot.py --basetemp=tests/out --ignore=archivebox/pkgs 6 passed pytest -q archivebox/tests/test_recursive_crawl.py::test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel --basetemp=tests/out --ignore=archivebox/pkgs 1 passed ```
This commit is contained in:
commit
3e58261cd5
@ -4521,12 +4521,26 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
if Path(candidate).name.lower() == preferred_name:
|
||||
return candidate
|
||||
|
||||
ext_groups = (
|
||||
(".html", ".htm", ".mhtml", ".mht", ".pdf"),
|
||||
(".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
|
||||
(".json", ".jsonl", ".txt", ".md", ".csv", ".tsv"),
|
||||
(".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
|
||||
)
|
||||
plugin_lower = (plugin_name or "").lower()
|
||||
if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl"):
|
||||
# yt-dlp commonly emits a thumbnail plus several media formats.
|
||||
# Prefer something browsers can play directly; otherwise cards can
|
||||
# select a large MKV or thumbnail that the plugin player cannot use.
|
||||
ext_groups = (
|
||||
(".mp4", ".webm", ".m4v", ".ogv"),
|
||||
(".mp3", ".m4a", ".aac", ".opus", ".ogg", ".wav", ".flac"),
|
||||
(".mkv", ".mov", ".avi", ".flv", ".wmv", ".mpg", ".mpeg", ".ts", ".m2ts", ".mts", ".3gp", ".3g2"),
|
||||
(".html", ".htm", ".mhtml", ".mht", ".pdf"),
|
||||
(".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
|
||||
(".json", ".jsonl", ".txt", ".md", ".csv", ".tsv", ".srt", ".vtt"),
|
||||
)
|
||||
else:
|
||||
ext_groups = (
|
||||
(".html", ".htm", ".mhtml", ".mht", ".pdf"),
|
||||
(".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
|
||||
(".json", ".jsonl", ".txt", ".md", ".csv", ".tsv"),
|
||||
(".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
|
||||
)
|
||||
for ext_group in ext_groups:
|
||||
group_candidates = [candidate for candidate in candidates if Path(candidate).suffix.lower() in ext_group]
|
||||
if group_candidates:
|
||||
|
||||
@ -32,7 +32,7 @@ _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml",
|
||||
_IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
|
||||
_STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
|
||||
|
||||
_MEDIA_FILE_EXTS = {
|
||||
_VIDEO_FILE_EXTS = {
|
||||
".mp4",
|
||||
".webm",
|
||||
".mkv",
|
||||
@ -49,6 +49,9 @@ _MEDIA_FILE_EXTS = {
|
||||
".3gp",
|
||||
".3g2",
|
||||
".ogv",
|
||||
}
|
||||
|
||||
_AUDIO_FILE_EXTS = {
|
||||
".mp3",
|
||||
".m4a",
|
||||
".aac",
|
||||
@ -66,6 +69,10 @@ _MEDIA_FILE_EXTS = {
|
||||
".dts",
|
||||
}
|
||||
|
||||
_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
|
||||
_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
|
||||
_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
|
||||
|
||||
|
||||
def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
|
||||
from abx_dl.output_files import OutputManifest
|
||||
@ -154,15 +161,27 @@ def _list_media_files(result) -> list[dict]:
|
||||
|
||||
for rel_path, size in candidates:
|
||||
href = str(Path(result.plugin) / rel_path)
|
||||
suffix = rel_path.suffix.lower()
|
||||
media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
|
||||
media_files.append(
|
||||
{
|
||||
"name": rel_path.name,
|
||||
"path": href,
|
||||
"size": size,
|
||||
"media_type": media_type,
|
||||
"is_video": media_type == "video",
|
||||
"is_audio": media_type == "audio",
|
||||
"is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
|
||||
},
|
||||
)
|
||||
|
||||
media_files.sort(key=lambda item: item["name"].lower())
|
||||
media_files.sort(
|
||||
key=lambda item: (
|
||||
0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
|
||||
-int(item.get("size") or 0),
|
||||
item["name"].lower(),
|
||||
),
|
||||
)
|
||||
return media_files
|
||||
|
||||
|
||||
|
||||
@ -514,10 +514,46 @@ class TestSnapshotProgressStats:
|
||||
|
||||
assert _count_media_files(result) == 2
|
||||
assert _list_media_files(result) == [
|
||||
{"name": "audio.mp3", "path": "ytdlp/audio.mp3", "size": 222},
|
||||
{"name": "video.mp4", "path": "ytdlp/video.mp4", "size": 111},
|
||||
{
|
||||
"name": "video.mp4",
|
||||
"path": "ytdlp/video.mp4",
|
||||
"size": 111,
|
||||
"media_type": "video",
|
||||
"is_video": True,
|
||||
"is_audio": False,
|
||||
"is_browser_playable": True,
|
||||
},
|
||||
{
|
||||
"name": "audio.mp3",
|
||||
"path": "ytdlp/audio.mp3",
|
||||
"size": 222,
|
||||
"media_type": "audio",
|
||||
"is_video": False,
|
||||
"is_audio": True,
|
||||
"is_browser_playable": True,
|
||||
},
|
||||
]
|
||||
|
||||
def test_ytdlp_discover_outputs_prefers_browser_playable_video(self, snapshot):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin="ytdlp",
|
||||
status="succeeded",
|
||||
output_files={
|
||||
"thumbnail.jpg": {"size": 20_000, "mimetype": "image/jpeg", "extension": "jpg"},
|
||||
"large.mkv": {"size": 10_000, "mimetype": "video/x-matroska", "extension": "mkv"},
|
||||
"small.mp4": {"size": 111, "mimetype": "video/mp4", "extension": "mp4"},
|
||||
},
|
||||
output_size=30_111,
|
||||
)
|
||||
|
||||
outputs = snapshot.discover_outputs(include_filesystem_fallback=False)
|
||||
ytdlp_output = next(output for output in outputs if output["name"] == "ytdlp")
|
||||
|
||||
assert ytdlp_output["path"] == "ytdlp/small.mp4"
|
||||
|
||||
def test_discover_outputs_falls_back_to_hashes_index_without_filesystem_walk(
|
||||
self,
|
||||
snapshot,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user