mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
feat: add snapshot output batch deletion
This commit is contained in:
Nick Sweeting
parent
10cd7ad768
commit
e548608149
@ -284,16 +284,20 @@ class ModelWithOutputDir(ModelWithUUID):
|
||||
elif path.is_dir():
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
|
||||
"""Capture output paths before DB deletion and remove them after commit."""
|
||||
paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
|
||||
transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
|
||||
|
||||
@classmethod
|
||||
def register_delete_signal(cls) -> None:
|
||||
if cls._delete_signal_registered:
|
||||
return
|
||||
|
||||
def schedule_output_dir_cleanup(sender, instance, **kwargs):
|
||||
def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
|
||||
if not isinstance(instance, ModelWithOutputDir):
|
||||
return
|
||||
paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
|
||||
transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
|
||||
instance.schedule_delete_cleanup(using=using)
|
||||
|
||||
pre_delete.connect(
|
||||
schedule_output_dir_cleanup,
|
||||
|
||||
@ -3300,7 +3300,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
fallback_ts = ts_to_date_str(self.downloaded_at or self.created_at)
|
||||
for root, root_entries in grouped_hash_outputs.items():
|
||||
fallback_path = ArchiveResult._fallback_output_file_path(list(root_entries.keys()), root, root_entries)
|
||||
if not fallback_path:
|
||||
if not fallback_path or not (snap_dir / root / fallback_path).exists():
|
||||
continue
|
||||
fallback_meta = root_entries.get(fallback_path, {})
|
||||
outputs.append(
|
||||
@ -4112,12 +4112,19 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
),
|
||||
)
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
|
||||
"""Remove shared plugin output and refresh persisted Snapshot metadata after commit."""
|
||||
snapshot_id = self.snapshot_id
|
||||
deleted = super().delete(*args, **kwargs)
|
||||
if snapshot_id:
|
||||
transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id}))
|
||||
return deleted
|
||||
paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
|
||||
|
||||
def cleanup() -> None:
|
||||
type(self).delete_output_paths(paths)
|
||||
type(self).refresh_snapshot_output_sizes({snapshot_id})
|
||||
snapshot = Snapshot.objects.filter(pk=snapshot_id).first()
|
||||
if snapshot:
|
||||
snapshot.write_index_jsonl()
|
||||
|
||||
transaction.on_commit(cleanup, using=using)
|
||||
|
||||
@staticmethod
|
||||
def refresh_snapshot_output_sizes(snapshot_ids):
|
||||
|
||||
@ -353,10 +353,15 @@ class SnapshotView(View):
|
||||
if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
|
||||
]
|
||||
archiveresults = {}
|
||||
result_ids_by_name = {}
|
||||
for output in outputs:
|
||||
if output.get("result"):
|
||||
result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id))
|
||||
current = archiveresults.get(output["name"])
|
||||
if current is None or (output.get("size") or 0) > (current.get("size") or 0):
|
||||
archiveresults[output["name"]] = output
|
||||
for name, output in archiveresults.items():
|
||||
output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
|
||||
hash_index = snapshot.hashes_index
|
||||
loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(
|
||||
outputs,
|
||||
@ -479,6 +484,7 @@ class SnapshotView(View):
|
||||
"related_years": related_years,
|
||||
"loose_items": loose_items,
|
||||
"failed_items": failed_items,
|
||||
"can_delete_outputs": bool(request.user.is_authenticated and request.user.is_active and request.user.is_superuser),
|
||||
"title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)],
|
||||
}
|
||||
return render(template_name="core/snapshot.html", request=request, context=context)
|
||||
|
||||
@ -1117,7 +1117,8 @@
|
||||
line-height: 1;
|
||||
opacity: 0.8;
|
||||
}
|
||||
.thumb-actions a {
|
||||
.thumb-actions a,
|
||||
.thumb-actions button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
@ -1127,11 +1128,20 @@
|
||||
background: #e7ebef;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
border: 0;
|
||||
cursor: pointer;
|
||||
}
|
||||
.thumb-actions a:hover {
|
||||
.thumb-actions a:hover,
|
||||
.thumb-actions button:hover {
|
||||
background: #d9e0e7;
|
||||
opacity: 1;
|
||||
}
|
||||
.thumb-actions .delete-pending {
|
||||
width: auto;
|
||||
min-width: 22px;
|
||||
color: #b91c1c;
|
||||
font-weight: 700;
|
||||
}
|
||||
.thumb-card .thumb-body > a:not(.thumb-actions a),
|
||||
.thumb-card .thumb-body > h4 {
|
||||
grid-column: 1;
|
||||
@ -1246,11 +1256,15 @@
|
||||
font-size: 12px;
|
||||
max-height: 24px;
|
||||
}
|
||||
.thumb-card:has([data-compact]) .thumb-actions a {
|
||||
.thumb-card:has([data-compact]) .thumb-actions a,
|
||||
.thumb-card:has([data-compact]) .thumb-actions button {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
.thumb-card:has([data-compact]) .thumb-actions .delete-pending {
|
||||
width: auto;
|
||||
}
|
||||
.thumb-card:has([data-compact]) .thumb-body h4 {
|
||||
font-size: 0.9em;
|
||||
margin-bottom: 0px;
|
||||
@ -1697,6 +1711,9 @@
|
||||
{% if display_path %}
|
||||
<a href="{{display_url}}" data-no-preview="1" title="Download output file" download>⬇️</a>
|
||||
{% endif %}
|
||||
{% if can_delete_outputs and result.result %}
|
||||
<button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" title="Delete this output">❌</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if display_path %}
|
||||
<a href="{{preview_url}}" target="preview" title="./{{display_path}} (downloaded {{result.ts}})">
|
||||
@ -1782,12 +1799,80 @@
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{% if can_delete_outputs %}<input type="hidden" id="delete-output-csrf" value="{{csrf_token}}">{% endif %}
|
||||
<script src="{% static 'jquery.min.js' %}" type="text/javascript"></script>
|
||||
|
||||
<script>
|
||||
const snapshotBaseUrl = "{% snapshot_base_url snapshot %}";
|
||||
const snapshotFilesUrl = `${snapshotBaseUrl}/?files=1`;
|
||||
|
||||
{% if can_delete_outputs %}
|
||||
const queuedOutputIds = new Set()
|
||||
const deleteOutputButtons = [...document.querySelectorAll('[data-archive-result-ids]')]
|
||||
let deleteOutputTimer = null
|
||||
|
||||
function renderDeleteCountdown(label='') {
|
||||
for (const button of deleteOutputButtons) {
|
||||
const queued = button.dataset.archiveResultIds.split(',').every((id) => queuedOutputIds.has(id))
|
||||
button.textContent = queued ? label : '❌'
|
||||
button.classList.toggle('delete-pending', queued)
|
||||
}
|
||||
}
|
||||
|
||||
function runDeleteCountdown(count=3) {
|
||||
if (count) {
|
||||
renderDeleteCountdown(`[${count}]`)
|
||||
deleteOutputTimer = window.setTimeout(() => runDeleteCountdown(count - 1), 1000)
|
||||
return
|
||||
}
|
||||
renderDeleteCountdown('[deleting]')
|
||||
deleteOutputTimer = window.setTimeout(deleteQueuedOutputs, 1000)
|
||||
}
|
||||
|
||||
function toggleOutputDeletion(button) {
|
||||
if (button.disabled) return
|
||||
const resultIds = button.dataset.archiveResultIds.split(',')
|
||||
const remove = resultIds.every((id) => queuedOutputIds.has(id))
|
||||
resultIds.forEach((id) => remove ? queuedOutputIds.delete(id) : queuedOutputIds.add(id))
|
||||
window.clearTimeout(deleteOutputTimer)
|
||||
if (!queuedOutputIds.size) {
|
||||
renderDeleteCountdown()
|
||||
return
|
||||
}
|
||||
renderDeleteCountdown('[4]')
|
||||
deleteOutputTimer = window.setTimeout(runDeleteCountdown, 1000)
|
||||
}
|
||||
|
||||
function deleteQueuedOutputs() {
|
||||
deleteOutputButtons.forEach((button) => button.disabled = true)
|
||||
const body = new URLSearchParams({
|
||||
action: 'delete_selected',
|
||||
post: 'yes',
|
||||
csrfmiddlewaretoken: document.getElementById('delete-output-csrf').value,
|
||||
})
|
||||
queuedOutputIds.forEach((id) => body.append('_selected_action', id))
|
||||
fetch("{% admin_base_url %}/admin/core/archiveresult/", {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: {'Content-Type': 'application/x-www-form-urlencoded'},
|
||||
body,
|
||||
}).then((response) => {
|
||||
if (!response.ok) throw new Error(`Delete failed (${response.status})`)
|
||||
window.location.reload()
|
||||
}).catch((error) => {
|
||||
deleteOutputButtons.forEach((button) => button.disabled = false)
|
||||
queuedOutputIds.clear()
|
||||
renderDeleteCountdown()
|
||||
window.alert(error.message)
|
||||
})
|
||||
}
|
||||
|
||||
for (const button of deleteOutputButtons) {
|
||||
button.addEventListener('click', () => toggleOutputDeletion(button))
|
||||
}
|
||||
window.addEventListener('pagehide', () => window.clearTimeout(deleteOutputTimer))
|
||||
{% endif %}
|
||||
|
||||
function tryCenterImageFrame(frame) {
|
||||
try {
|
||||
const doc = frame.contentDocument || frame.contentWindow.document
|
||||
|
||||
@ -113,3 +113,31 @@ class TestArchiveResultAdminListView:
|
||||
assert response.status_code == 200
|
||||
assert b"output-json-loaded-once" in response.content
|
||||
assert len(result_queries) == 1
|
||||
|
||||
def test_admin_delete_removes_output_directory_and_refreshes_snapshot_size(self, client, admin_user, snapshot):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
output_dir = Path(snapshot.output_dir) / "screenshot"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "output.png").write_bytes(b"archive output")
|
||||
result = ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin="screenshot",
|
||||
hook_name="on_Snapshot__50_screenshot.py",
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
output_files={"output.png": {"size": 14, "mimetype": "image/png"}},
|
||||
output_size=14,
|
||||
)
|
||||
client.force_login(admin_user)
|
||||
|
||||
response = client.post(
|
||||
reverse("admin:core_archiveresult_delete", args=[result.pk]),
|
||||
{"post": "yes"},
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert not ArchiveResult.objects.filter(pk=result.pk).exists()
|
||||
assert not output_dir.exists()
|
||||
snapshot.refresh_from_db()
|
||||
assert snapshot.output_size == 0
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
"""Snapshot model and admin UI tests."""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
@ -7,6 +8,7 @@ from threading import Thread
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
|
||||
from django.core.paginator import UnorderedObjectListWarning
|
||||
from django.test import RequestFactory
|
||||
@ -744,6 +746,92 @@ class TestSnapshotProgressStats:
|
||||
assert rendered.count(".on('click', handleSnapshotHeaderToggle)") == 1
|
||||
|
||||
|
||||
class TestSnapshotOutputDeletion:
|
||||
@staticmethod
|
||||
def _create_output(snapshot, *, plugin="screenshot", hook_name="on_Snapshot__50_screenshot.py", size=11):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
output_dir = Path(snapshot.output_dir) / plugin
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_dir / "output.png"
|
||||
output_path.write_bytes(b"x" * size)
|
||||
return ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin=plugin,
|
||||
hook_name=hook_name,
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
output_str="output.png",
|
||||
output_files={"output.png": {"size": size, "mimetype": "image/png"}},
|
||||
output_size=size,
|
||||
)
|
||||
|
||||
def test_snapshot_detail_only_shows_delete_controls_to_superusers(self, snapshot, admin_user):
|
||||
from archivebox.core.views import SnapshotView
|
||||
|
||||
result = self._create_output(snapshot)
|
||||
request = RequestFactory().get(f"/{snapshot.url_path}/index.html", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
request.user = admin_user
|
||||
|
||||
html = SnapshotView.render_live_index(request, snapshot).content.decode()
|
||||
|
||||
assert f'data-archive-result-ids="{result.id}"' in html
|
||||
assert 'title="Delete this output"' in html
|
||||
assert "const queuedOutputIds = new Set()" in html
|
||||
assert "action: 'delete_selected'" in html
|
||||
assert "/admin/core/archiveresult/" in html
|
||||
assert "[deleting]" in html
|
||||
|
||||
request.user = AnonymousUser()
|
||||
anonymous_html = SnapshotView.render_live_index(request, snapshot).content.decode()
|
||||
assert "data-archive-result-ids" not in anonymous_html
|
||||
assert 'title="Delete this output"' not in anonymous_html
|
||||
|
||||
def test_batch_delete_removes_plugin_rows_files_and_refreshes_snapshot_size(self, client, snapshot, admin_user):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
first = self._create_output(snapshot, size=11)
|
||||
second = self._create_output(snapshot, hook_name="on_Snapshot__51_screenshot_retry.py", size=13)
|
||||
kept = self._create_output(snapshot, plugin="pdf", hook_name="on_Snapshot__60_pdf.py", size=7)
|
||||
deleted_dir = Path(first.output_dir)
|
||||
kept_dir = Path(kept.output_dir)
|
||||
hashes_dir = Path(snapshot.output_dir) / "hashes"
|
||||
hashes_dir.mkdir(parents=True, exist_ok=True)
|
||||
(hashes_dir / "hashes.json").write_text(
|
||||
json.dumps({"files": [{"path": "screenshot/output.png", "size": 13}]}),
|
||||
)
|
||||
|
||||
delete_url = reverse("admin:core_archiveresult_changelist")
|
||||
delete_data = {
|
||||
"action": "delete_selected",
|
||||
"post": "yes",
|
||||
ACTION_CHECKBOX_NAME: [str(first.id), str(second.id)],
|
||||
}
|
||||
denied = client.post(
|
||||
delete_url,
|
||||
delete_data,
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
)
|
||||
assert denied.status_code == 302
|
||||
assert deleted_dir.exists()
|
||||
|
||||
client.force_login(admin_user)
|
||||
response = client.post(
|
||||
delete_url,
|
||||
delete_data,
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert not ArchiveResult.objects.filter(pk__in=[first.pk, second.pk]).exists()
|
||||
assert ArchiveResult.objects.filter(pk=kept.pk).exists()
|
||||
assert not deleted_dir.exists()
|
||||
assert kept_dir.exists()
|
||||
snapshot.refresh_from_db()
|
||||
assert snapshot.output_size == 7
|
||||
assert "screenshot" not in {output["name"] for output in snapshot.discover_outputs()}
|
||||
assert '"plugin": "screenshot"' not in (Path(snapshot.output_dir) / "index.jsonl").read_text()
|
||||
|
||||
|
||||
class TestAdminSnapshotListView:
|
||||
"""Tests for the admin snapshot list view."""
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user