"""Snapshot model and admin UI tests.""" import json import os import re import shutil import warnings from pathlib import Path from threading import Thread from types import SimpleNamespace import pytest from django.contrib.auth.models import AnonymousUser from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME from django.core.paginator import UnorderedObjectListWarning from django.test import RequestFactory from django.urls import reverse from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE from archivebox.tests.conftest import ADMIN_TEST_HOST from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook pytestmark = pytest.mark.django_db(transaction=True) REPO_ROOT = Path(__file__).resolve().parents[2] def test_current_snapshot_layout_has_no_top_level_timestamp_projection(snapshot): from archivebox.config import CONSTANTS legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp assert Path(snapshot.output_dir).is_relative_to(CONSTANTS.ARCHIVE_DIR / "users") assert not legacy_path.exists() assert not legacy_path.is_symlink() @pytest.fixture def real_hash_projection(snapshot, cached_abxpkg_lib_dir): snapshot.output_dir.mkdir(parents=True, exist_ok=True) (snapshot.output_dir / "source.txt").write_text("real snapshot admin input", encoding="utf-8") return _run_shipped_snapshot_hook( snapshot, plugin="hashes", hook_name="on_Snapshot__93_hashes.py", lib_dir=cached_abxpkg_lib_dir, ) @pytest.fixture def real_failed_title_projection(snapshot, cached_abxpkg_lib_dir): return _run_shipped_snapshot_hook( snapshot, plugin="title", hook_name="on_Snapshot__54_title.js", lib_dir=cached_abxpkg_lib_dir, expected_exit_codes=(1,), ) @pytest.fixture def real_noresults_projection(snapshot, cached_abxpkg_lib_dir): staticfile_dir = snapshot.output_dir / "staticfile" staticfile_dir.mkdir(parents=True, exist_ok=True) (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8") return _run_shipped_snapshot_hook( snapshot, plugin="parse_txt_urls", hook_name="on_Snapshot__71_parse_txt_urls.py", lib_dir=cached_abxpkg_lib_dir, ) @pytest.fixture def real_parse_projection(snapshot, cached_abxpkg_lib_dir): staticfile_dir = snapshot.output_dir / "staticfile" staticfile_dir.mkdir(parents=True, exist_ok=True) (staticfile_dir / "input.txt").write_text("https://example.org/parsed\n", encoding="utf-8") return _run_shipped_snapshot_hook( snapshot, plugin="parse_txt_urls", hook_name="on_Snapshot__71_parse_txt_urls.py", lib_dir=cached_abxpkg_lib_dir, ) @pytest.fixture def running_wget_projection(snapshot, blocking_http_server): from django.utils import timezone from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot now = timezone.now() Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now, modified_at=now) Snapshot.objects.filter(pk=snapshot.pk).update( status=Snapshot.StatusChoices.QUEUED, retry_at=now, downloaded_at=None, url=blocking_http_server.url, config={"PLUGINS": "wget"}, ) snapshot.refresh_from_db() errors = [] def run_snapshot(): try: assert run_due_snapshot(snapshot, lock_seconds=60) is True except BaseException as err: errors.append(err) finally: blocking_http_server.request_started.set() runner = Thread(target=run_snapshot, name="archivebox-test-admin-wget-runner") runner.start() blocking_http_server.request_started.wait() assert errors == [] result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget") assert result.status == ArchiveResult.StatusChoices.STARTED yield result blocking_http_server.release_response.set() runner.join() assert errors == [] @pytest.fixture def real_skipped_hash_projection(snapshot, cached_abxpkg_lib_dir): return _run_shipped_snapshot_hook( snapshot, plugin="hashes", hook_name="on_Snapshot__93_hashes.py", lib_dir=cached_abxpkg_lib_dir, env={"HASHES_ENABLED": "False"}, ) def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_warning(admin_client, snapshot): url = reverse("admin:core_snapshot_changelist") with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST) assert response.status_code == 200 assert not any(issubclass(warning.category, UnorderedObjectListWarning) for warning in caught) assert response.context["cl"].queryset.ordered is True assert response.context["cl"].queryset.query.order_by[0] == "-created_at" assert b"archivebox-search-stream-status" in response.content assert b"Searching matching snapshots..." in response.content def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, snapshot, real_hash_projection): from archivebox.core.models import ArchiveResult _process, result = real_hash_projection ArchiveResult.objects.filter(pk=result.pk).update( plugin="screenshot", hook_name="on_Snapshot__40_screenshot.js", output_size=128, output_files={"screenshot.png": {"size": 128, "root_relative": True}}, ) response = admin_client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST) assert response.status_code == 200 assert b'class="snapshot-preview' in response.content assert b"screenshot.png" in response.content def test_snapshot_result_health_filter_uses_live_status_rows(admin_client, snapshot): from archivebox.core.models import ArchiveResult for plugin, status in ( ("title", ArchiveResult.StatusChoices.FAILED), ("wget", ArchiveResult.StatusChoices.FAILED), ("hashes", ArchiveResult.StatusChoices.SUCCEEDED), ): ArchiveResult.objects.create( snapshot=snapshot, plugin=plugin, hook_name=f"on_Snapshot__50_{plugin}.py", status=status, ) response = admin_client.get( reverse("admin:core_snapshot_changelist"), {"archiveresult_status": "failed"}, HTTP_HOST=ADMIN_TEST_HOST, ) assert response.status_code == 200 assert snapshot in response.context["cl"].queryset def test_snapshot_icons_reflect_live_results_without_stale_html_cache(snapshot): from archivebox.core.models import ArchiveResult result = ArchiveResult.objects.create( snapshot=snapshot, plugin="hashes", hook_name="on_Snapshot__93_hashes.py", status=ArchiveResult.StatusChoices.SUCCEEDED, output_files={"hashes.json": {"size": 10}}, ) assert "hashes" in str(snapshot.icons()) ArchiveResult.objects.filter(pk=result.pk).update(status=ArchiveResult.StatusChoices.FAILED) assert "hashes" not in str(snapshot.icons()) def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client, snapshot): from archivebox.core.models import Tag tag = Tag.objects.create(name="safe-tag") snapshot.tags.add(tag) malicious_name = '' Tag.objects.filter(pk=tag.pk).update(name=malicious_name) response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST) body = response.content assert response.status_code == 200 assert malicious_name.encode() not in body assert b'') type(process).objects.filter(pk=process.pk).update(pwd='/tmp/archivebox">') ArchiveResult.objects.filter(pk=result.pk).update( output_files={'evil">.txt': {"size": 12, "mimetype": "text/plain"}}, output_str='hashes/evil">.txt', plugin='', ) response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST) body = response.content assert response.status_code == 200 assert b'