"""Snapshot model and admin UI tests."""
import json
import os
import re
import shutil
import warnings
from pathlib import Path
from threading import Thread
from types import SimpleNamespace
import pytest
from django.contrib.auth.models import AnonymousUser
from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
from django.core.paginator import UnorderedObjectListWarning
from django.test import RequestFactory
from django.urls import reverse
from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
from archivebox.tests.conftest import ADMIN_TEST_HOST
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
pytestmark = pytest.mark.django_db(transaction=True)
REPO_ROOT = Path(__file__).resolve().parents[2]
def test_current_snapshot_layout_has_no_top_level_timestamp_projection(snapshot):
from archivebox.config import CONSTANTS
legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
assert Path(snapshot.output_dir).is_relative_to(CONSTANTS.ARCHIVE_DIR / "users")
assert not legacy_path.exists()
assert not legacy_path.is_symlink()
@pytest.fixture
def real_hash_projection(snapshot, cached_abxpkg_lib_dir):
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
(snapshot.output_dir / "source.txt").write_text("real snapshot admin input", encoding="utf-8")
return _run_shipped_snapshot_hook(
snapshot,
plugin="hashes",
hook_name="on_Snapshot__93_hashes.py",
lib_dir=cached_abxpkg_lib_dir,
)
@pytest.fixture
def real_failed_title_projection(snapshot, cached_abxpkg_lib_dir):
return _run_shipped_snapshot_hook(
snapshot,
plugin="title",
hook_name="on_Snapshot__54_title.js",
lib_dir=cached_abxpkg_lib_dir,
expected_exit_codes=(1,),
)
@pytest.fixture
def real_noresults_projection(snapshot, cached_abxpkg_lib_dir):
staticfile_dir = snapshot.output_dir / "staticfile"
staticfile_dir.mkdir(parents=True, exist_ok=True)
(staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8")
return _run_shipped_snapshot_hook(
snapshot,
plugin="parse_txt_urls",
hook_name="on_Snapshot__71_parse_txt_urls.py",
lib_dir=cached_abxpkg_lib_dir,
)
@pytest.fixture
def real_parse_projection(snapshot, cached_abxpkg_lib_dir):
staticfile_dir = snapshot.output_dir / "staticfile"
staticfile_dir.mkdir(parents=True, exist_ok=True)
(staticfile_dir / "input.txt").write_text("https://example.org/parsed\n", encoding="utf-8")
return _run_shipped_snapshot_hook(
snapshot,
plugin="parse_txt_urls",
hook_name="on_Snapshot__71_parse_txt_urls.py",
lib_dir=cached_abxpkg_lib_dir,
)
@pytest.fixture
def running_wget_projection(snapshot, blocking_http_server):
from django.utils import timezone
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
now = timezone.now()
Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now, modified_at=now)
Snapshot.objects.filter(pk=snapshot.pk).update(
status=Snapshot.StatusChoices.QUEUED,
retry_at=now,
downloaded_at=None,
url=blocking_http_server.url,
config={"PLUGINS": "wget"},
)
snapshot.refresh_from_db()
errors = []
def run_snapshot():
try:
assert run_due_snapshot(snapshot, lock_seconds=60) is True
except BaseException as err:
errors.append(err)
finally:
blocking_http_server.request_started.set()
runner = Thread(target=run_snapshot, name="archivebox-test-admin-wget-runner")
runner.start()
blocking_http_server.request_started.wait()
assert errors == []
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget")
assert result.status == ArchiveResult.StatusChoices.STARTED
yield result
blocking_http_server.release_response.set()
runner.join()
assert errors == []
@pytest.fixture
def real_skipped_hash_projection(snapshot, cached_abxpkg_lib_dir):
return _run_shipped_snapshot_hook(
snapshot,
plugin="hashes",
hook_name="on_Snapshot__93_hashes.py",
lib_dir=cached_abxpkg_lib_dir,
env={"HASHES_ENABLED": "False"},
)
def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_warning(admin_client, snapshot):
url = reverse("admin:core_snapshot_changelist")
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert not any(issubclass(warning.category, UnorderedObjectListWarning) for warning in caught)
assert response.context["cl"].queryset.ordered is True
assert response.context["cl"].queryset.query.order_by[0] == "-created_at"
assert b"archivebox-search-stream-status" in response.content
assert b"Searching matching snapshots..." in response.content
def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, snapshot, real_hash_projection):
from archivebox.core.models import ArchiveResult
_process, result = real_hash_projection
ArchiveResult.objects.filter(pk=result.pk).update(
plugin="screenshot",
hook_name="on_Snapshot__40_screenshot.js",
output_size=128,
output_files={"screenshot.png": {"size": 128, "root_relative": True}},
)
response = admin_client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert b'class="snapshot-preview' in response.content
assert b"screenshot.png" in response.content
def test_snapshot_result_health_filter_uses_live_status_rows(admin_client, snapshot):
from archivebox.core.models import ArchiveResult
for plugin, status in (
("title", ArchiveResult.StatusChoices.FAILED),
("wget", ArchiveResult.StatusChoices.FAILED),
("hashes", ArchiveResult.StatusChoices.SUCCEEDED),
):
ArchiveResult.objects.create(
snapshot=snapshot,
plugin=plugin,
hook_name=f"on_Snapshot__50_{plugin}.py",
status=status,
)
response = admin_client.get(
reverse("admin:core_snapshot_changelist"),
{"archiveresult_status": "failed"},
HTTP_HOST=ADMIN_TEST_HOST,
)
assert response.status_code == 200
assert snapshot in response.context["cl"].queryset
def test_snapshot_icons_reflect_live_results_without_stale_html_cache(snapshot):
from archivebox.core.models import ArchiveResult
result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="hashes",
hook_name="on_Snapshot__93_hashes.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_files={"hashes.json": {"size": 10}},
)
assert "hashes" in str(snapshot.icons())
ArchiveResult.objects.filter(pk=result.pk).update(status=ArchiveResult.StatusChoices.FAILED)
assert "hashes" not in str(snapshot.icons())
def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client, snapshot):
from archivebox.core.models import Tag
tag = Tag.objects.create(name="safe-tag")
snapshot.tags.add(tag)
malicious_name = ''
Tag.objects.filter(pk=tag.pk).update(name=malicious_name)
response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST)
body = response.content
assert response.status_code == 200
assert malicious_name.encode() not in body
assert b'')
type(process).objects.filter(pk=process.pk).update(pwd='/tmp/archivebox">')
ArchiveResult.objects.filter(pk=result.pk).update(
output_files={'evil">.txt': {"size": 12, "mimetype": "text/plain"}},
output_str='hashes/evil">.txt',
plugin='',
)
response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST)
body = response.content
assert response.status_code == 200
assert b'