mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
release: archivebox 0.9.32rc26
This commit is contained in:
Nick Sweeting
parent
a21765f366
commit
e8bb2351d9
@ -312,7 +312,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
|
||||
&& apt-get update -qq \
|
||||
&& if [ "$TARGETARCH" = "arm64" ]; then \
|
||||
abxpkg install --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
|
||||
abxpkg install --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" --overrides='{"playwright":{"install_args":["chrome-beta"]}}' chromium; \
|
||||
fi \
|
||||
&& PUID=0 PGID=0 abx-dl plugins --install \
|
||||
&& find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
|
||||
@ -54,7 +54,7 @@ def list_processes(
|
||||
|
||||
is_tty = sys.stdout.isatty()
|
||||
|
||||
queryset = Process.objects.all().select_related("binary", "machine").order_by("-start_ts")
|
||||
queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
|
||||
|
||||
# Apply filters
|
||||
filter_kwargs = {}
|
||||
|
||||
@ -483,15 +483,16 @@ def get_config(
|
||||
|
||||
Priority (highest to lowest):
|
||||
1. Explicit overrides
|
||||
2. Per-snapshot config and output path
|
||||
3. Per-crawl config and output path
|
||||
4. Per-user config
|
||||
5. Per-persona derived config
|
||||
6. Current machine derived config
|
||||
7. Environment variables
|
||||
8. Config file (ArchiveBox.conf)
|
||||
9. Plugin schema defaults
|
||||
10. Core config defaults
|
||||
2. Per-ArchiveResult config
|
||||
3. Per-snapshot config and output path
|
||||
4. Per-crawl config and output path
|
||||
5. Per-user config
|
||||
6. Per-persona derived config
|
||||
7. Current machine derived config
|
||||
8. Environment variables
|
||||
9. Config file (ArchiveBox.conf)
|
||||
10. Plugin schema defaults
|
||||
11. Core config defaults
|
||||
"""
|
||||
if snapshot is None and archiveresult is not None:
|
||||
snapshot = archiveresult.snapshot
|
||||
@ -559,6 +560,9 @@ def get_config(
|
||||
if snapshot is not None:
|
||||
scope_overrides["SNAP_DIR"] = snapshot.output_dir
|
||||
|
||||
if archiveresult is not None and archiveresult.config:
|
||||
scope_overrides.update(archiveresult.config)
|
||||
|
||||
if overrides:
|
||||
scope_overrides.update(overrides)
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
|
||||
|
||||
|
||||
def _get_replay_source_url(result: ArchiveResult) -> str:
|
||||
process = getattr(result, "process", None)
|
||||
process = result.process_record
|
||||
return str(getattr(process, "url", None) or result.snapshot.url or "")
|
||||
|
||||
|
||||
@ -134,20 +134,21 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
|
||||
# Format timestamp
|
||||
end_time = result.end_ts.strftime("%Y-%m-%d %H:%M:%S") if result.end_ts else "-"
|
||||
|
||||
process = result.process_record
|
||||
process_display = "-"
|
||||
if result.process_id and result.process:
|
||||
if process:
|
||||
process_display = f'''
|
||||
<a href="{reverse("admin:machine_process_change", args=[result.process_id])}"
|
||||
<a href="{reverse("admin:machine_process_change", args=[process.id])}"
|
||||
style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
|
||||
title="View process">{result.process.pid or "-"}</a>
|
||||
title="View process">{process.pid or "-"}</a>
|
||||
'''
|
||||
|
||||
machine_display = "-"
|
||||
if result.process_id and result.process and result.process.machine_id:
|
||||
if process and process.machine_id:
|
||||
machine_display = f'''
|
||||
<a href="{reverse("admin:machine_machine_change", args=[result.process.machine_id])}"
|
||||
<a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
|
||||
style="color: #2563eb; text-decoration: none; font-size: 12px;"
|
||||
title="View machine">{result.process.machine.hostname}</a>
|
||||
title="View machine">{process.machine.hostname}</a>
|
||||
'''
|
||||
|
||||
# Truncate output for display
|
||||
@ -660,20 +661,22 @@ class ArchiveResultAdmin(BaseModelAdmin):
|
||||
|
||||
@admin.display(description="Process", ordering="process__pid")
|
||||
def process_link(self, result):
|
||||
if not result.process_id:
|
||||
process = result.process_record
|
||||
if not process:
|
||||
return "-"
|
||||
process_label = result.process.pid if result.process and result.process.pid else "-"
|
||||
process_label = process.pid or "-"
|
||||
return format_html(
|
||||
'<a href="{}"><code>{}</code></a>',
|
||||
reverse("admin:machine_process_change", args=[result.process_id]),
|
||||
reverse("admin:machine_process_change", args=[process.id]),
|
||||
process_label,
|
||||
)
|
||||
|
||||
@admin.display(description="Machine", ordering="process__machine__hostname")
|
||||
def machine_link(self, result):
|
||||
if not result.process_id or not result.process or not result.process.machine_id:
|
||||
process = result.process_record
|
||||
if not process or not process.machine_id:
|
||||
return "-"
|
||||
machine = result.process.machine
|
||||
machine = process.machine
|
||||
return format_html(
|
||||
'<a href="{}"><code>{}</code> {}</a>',
|
||||
reverse("admin:machine_machine_change", args=[machine.id]),
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
# Generated by Django 6.0.5 on 2026-05-27 20:11
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("core", "0038_snapshot_progress_idx"),
|
||||
("machine", "0015_process_progress_indexes"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="archiveresult",
|
||||
name="process",
|
||||
field=models.OneToOneField(
|
||||
blank=True,
|
||||
help_text="Process execution details for this archive result",
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="archiveresult",
|
||||
to="machine.process",
|
||||
),
|
||||
),
|
||||
]
|
||||
@ -1341,15 +1341,16 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
|
||||
# Write ArchiveResult records with their associated Binary and Process
|
||||
# Use select_related to optimize queries
|
||||
for ar in self.archiveresult_set.select_related("process__binary").order_by("start_ts"):
|
||||
process = ar.process_record
|
||||
# Write Binary record if not already written
|
||||
if ar.process and ar.process.binary and ar.process.binary_id not in binaries_seen:
|
||||
binaries_seen.add(ar.process.binary_id)
|
||||
f.write(json.dumps(ar.process.binary.to_json()) + "\n")
|
||||
if process and process.binary and process.binary_id not in binaries_seen:
|
||||
binaries_seen.add(process.binary_id)
|
||||
f.write(json.dumps(process.binary.to_json()) + "\n")
|
||||
|
||||
# Write Process record if not already written
|
||||
if ar.process and ar.process_id not in processes_seen:
|
||||
processes_seen.add(ar.process_id)
|
||||
f.write(json.dumps(ar.process.to_json()) + "\n")
|
||||
if process and process.id not in processes_seen:
|
||||
processes_seen.add(process.id)
|
||||
f.write(json.dumps(process.to_json()) + "\n")
|
||||
|
||||
# Write ArchiveResult record
|
||||
f.write(json.dumps(ar.to_json()) + "\n")
|
||||
@ -3065,7 +3066,7 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
# Added POST-v0.9.0, will be added in a separate migration
|
||||
process = models.OneToOneField(
|
||||
"machine.Process",
|
||||
on_delete=models.PROTECT,
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
related_name="archiveresult",
|
||||
@ -3137,12 +3138,15 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
record["output_size"] = self.output_size
|
||||
if self.output_mimetypes:
|
||||
record["output_mimetypes"] = self.output_mimetypes
|
||||
if self.pwd:
|
||||
record["pwd"] = self.pwd
|
||||
if self.cmd:
|
||||
record["cmd"] = self.cmd
|
||||
if self.cmd_version:
|
||||
record["cmd_version"] = self.cmd_version
|
||||
if self.process_id:
|
||||
record["process_id"] = str(self.process_id)
|
||||
process = self.process_record
|
||||
if process:
|
||||
record["process_id"] = str(process.id)
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
@ -3607,40 +3611,56 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
# these properties provide seamless access to Process data through ArchiveResult
|
||||
|
||||
# Uncommented after migration 3 completed - properties now active
|
||||
@property
|
||||
def process_record(self):
|
||||
if not self.process_id:
|
||||
return None
|
||||
try:
|
||||
return self.process
|
||||
except ObjectDoesNotExist:
|
||||
return None
|
||||
|
||||
@property
|
||||
def pwd(self) -> str:
|
||||
"""Working directory (from Process)."""
|
||||
return self.process.pwd if self.process_id else ""
|
||||
"""Working directory, derived from the snapshot/plugin path if the Process row is gone."""
|
||||
process = self.process_record
|
||||
return process.pwd if process and process.pwd else str(self.output_dir)
|
||||
|
||||
@property
|
||||
def cmd(self) -> list:
|
||||
"""Command array (from Process)."""
|
||||
return self.process.cmd if self.process_id else []
|
||||
process = self.process_record
|
||||
return process.cmd if process else []
|
||||
|
||||
@property
|
||||
def cmd_version(self) -> str:
|
||||
"""Command version (from Process.binary)."""
|
||||
return self.process.cmd_version if self.process_id else ""
|
||||
process = self.process_record
|
||||
return process.cmd_version if process else ""
|
||||
|
||||
@property
|
||||
def binary(self):
|
||||
"""Binary FK (from Process)."""
|
||||
return self.process.binary if self.process_id else None
|
||||
process = self.process_record
|
||||
return process.binary if process else None
|
||||
|
||||
@property
|
||||
def iface(self):
|
||||
"""Network interface FK (from Process)."""
|
||||
return self.process.iface if self.process_id else None
|
||||
process = self.process_record
|
||||
return process.iface if process else None
|
||||
|
||||
@property
|
||||
def machine(self):
|
||||
"""Machine FK (from Process)."""
|
||||
return self.process.machine if self.process_id else None
|
||||
process = self.process_record
|
||||
return process.machine if process else None
|
||||
|
||||
@property
|
||||
def timeout(self) -> int:
|
||||
"""Timeout in seconds (from Process)."""
|
||||
return self.process.timeout if self.process_id else 120
|
||||
process = self.process_record
|
||||
return process.timeout if process else 120
|
||||
|
||||
def save_search_index(self):
|
||||
pass
|
||||
@ -3676,8 +3696,9 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
# Read and parse JSONL output from stdout.log
|
||||
stdout_file = plugin_dir / "stdout.log"
|
||||
records = []
|
||||
if self.process_id and self.process:
|
||||
records = extract_records_from_process(self.process)
|
||||
process = self.process_record
|
||||
if process:
|
||||
records = extract_records_from_process(process)
|
||||
|
||||
if not records:
|
||||
stdout = stdout_file.read_text() if stdout_file.exists() else ""
|
||||
@ -3703,9 +3724,9 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
|
||||
# Update cmd fields
|
||||
if hook_data.get("cmd"):
|
||||
if self.process_id:
|
||||
self.process.cmd = hook_data["cmd"]
|
||||
self.process.save()
|
||||
if process:
|
||||
process.cmd = hook_data["cmd"]
|
||||
process.save()
|
||||
self._set_binary_from_cmd(hook_data["cmd"])
|
||||
# Note: cmd_version is derived from binary.version, not stored on Process
|
||||
else:
|
||||
@ -3718,7 +3739,7 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if is_background or (self.process_id and self.process and self.process.exit_code == 0):
|
||||
if is_background or (process and process.exit_code == 0):
|
||||
self.status = self.StatusChoices.SKIPPED
|
||||
self.output_str = "Hook did not output ArchiveResult record"
|
||||
else:
|
||||
@ -3822,9 +3843,10 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
).first()
|
||||
|
||||
if binary:
|
||||
if self.process_id:
|
||||
self.process.binary = binary
|
||||
self.process.save()
|
||||
process = self.process_record
|
||||
if process:
|
||||
process.binary = binary
|
||||
process.save()
|
||||
return
|
||||
|
||||
# Fallback: match by binary name
|
||||
@ -3835,9 +3857,10 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
|
||||
).first()
|
||||
|
||||
if binary:
|
||||
if self.process_id:
|
||||
self.process.binary = binary
|
||||
self.process.save()
|
||||
process = self.process_record
|
||||
if process:
|
||||
process.binary = binary
|
||||
process.save()
|
||||
|
||||
def _url_passes_filters(self, url: str) -> bool:
|
||||
"""Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.
|
||||
|
||||
@ -917,13 +917,13 @@ class PublicIndexView(ListView):
|
||||
def get_paginate_by(self, queryset):
|
||||
runtime_config = getattr(self, "runtime_config", None)
|
||||
if runtime_config is None:
|
||||
self.runtime_config = runtime_config = get_config()
|
||||
self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=True)
|
||||
return runtime_config.SNAPSHOTS_PER_PAGE
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
runtime_config = getattr(self, "runtime_config", None)
|
||||
if runtime_config is None:
|
||||
self.runtime_config = runtime_config = get_config()
|
||||
self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=True)
|
||||
context = {
|
||||
**super().get_context_data(**kwargs),
|
||||
"VERSION": VERSION,
|
||||
@ -990,7 +990,7 @@ class PublicIndexView(ListView):
|
||||
def get(self, *args, **kwargs):
|
||||
if self.request.user.is_authenticated:
|
||||
return redirect("/admin/core/snapshot/")
|
||||
if get_config().PUBLIC_INDEX:
|
||||
if _get_request_config(self.request).PUBLIC_INDEX:
|
||||
response = super().get(*args, **kwargs)
|
||||
return response
|
||||
else:
|
||||
@ -1012,7 +1012,7 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
return super().get_initial()
|
||||
|
||||
def test_func(self):
|
||||
return get_config().PUBLIC_ADD_VIEW or self.request.user.is_authenticated
|
||||
return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
|
||||
|
||||
def _can_override_crawl_config(self) -> bool:
|
||||
user = self.request.user
|
||||
@ -1032,7 +1032,8 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
def get_context_data(self, **kwargs):
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip()
|
||||
request_config = _get_request_config(self.request, resolve_plugins=True)
|
||||
required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
|
||||
plugin_configs = discover_plugin_configs()
|
||||
sensitive_keys = {
|
||||
str(config_key)
|
||||
@ -1062,7 +1063,7 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
# We can't just call request.build_absolute_uri in the template, because it would include query parameters
|
||||
"absolute_add_path": self.request.build_absolute_uri(self.request.path),
|
||||
"VERSION": VERSION,
|
||||
"FOOTER_INFO": get_config().FOOTER_INFO,
|
||||
"FOOTER_INFO": request_config.FOOTER_INFO,
|
||||
"required_search_plugin": required_search_plugin,
|
||||
"plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True),
|
||||
"persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
|
||||
@ -1820,6 +1821,7 @@ def live_progress_view(request):
|
||||
|
||||
for ar in sorted(snapshot_results, key=plugin_sort_key):
|
||||
status = ar.status
|
||||
process = ar.process_record
|
||||
progress_value = 0
|
||||
if status in (
|
||||
ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
@ -1829,8 +1831,8 @@ def live_progress_view(request):
|
||||
):
|
||||
progress_value = 100
|
||||
elif status == ArchiveResult.StatusChoices.STARTED:
|
||||
started_at = ar.start_ts or (ar.process.started_at if ar.process_id and ar.process else None)
|
||||
timeout = ar.process.timeout if ar.process_id and ar.process else 120
|
||||
started_at = ar.start_ts or (process.started_at if process else None)
|
||||
timeout = process.timeout if process else 120
|
||||
if started_at and timeout:
|
||||
elapsed = max(0.0, (now - started_at).total_seconds())
|
||||
progress_value = int(min(99, max(1, (elapsed / float(timeout)) * 100)))
|
||||
@ -1849,21 +1851,21 @@ def live_progress_view(request):
|
||||
"hook_name": hook_name,
|
||||
"phase": phase,
|
||||
"status": status,
|
||||
"process_id": str(ar.process_id) if ar.process_id else None,
|
||||
"process_id": str(process.id) if process else None,
|
||||
"admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
|
||||
}
|
||||
output_path = archiveresult_output_path(ar)
|
||||
if output_path:
|
||||
plugin_payload["output_path"] = output_path
|
||||
plugin_payload["output_url"] = snapshot_view_url(snapshot, output_path)
|
||||
if status == ArchiveResult.StatusChoices.STARTED and ar.process_id and ar.process:
|
||||
plugin_payload["pid"] = ar.process.pid
|
||||
if status == ArchiveResult.StatusChoices.STARTED and process:
|
||||
plugin_payload["pid"] = process.pid
|
||||
if status == ArchiveResult.StatusChoices.STARTED:
|
||||
plugin_payload["progress"] = progress_value
|
||||
plugin_payload["timeout"] = ar.process.timeout if ar.process_id and ar.process else 120
|
||||
plugin_payload["timeout"] = process.timeout if process else 120
|
||||
plugin_payload["source"] = "archiveresult"
|
||||
all_plugins.append(plugin_payload)
|
||||
seen_plugin_keys.add(str(ar.process_id) if ar.process_id else f"{ar.plugin}:{hook_name}")
|
||||
seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
|
||||
|
||||
for proc_payload, proc_started_at in process_records_by_snapshot.get(str(snapshot["id"]), []):
|
||||
if not is_current_run_timestamp(proc_started_at, snapshot_run_started_at):
|
||||
@ -2158,15 +2160,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
|
||||
assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
|
||||
|
||||
# Get merged config that includes Machine.config overrides
|
||||
try:
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
Machine.current()
|
||||
merged_config = get_config()
|
||||
except Exception:
|
||||
# Fallback if Machine model not available
|
||||
merged_config = get_config()
|
||||
merged_config = get_config()
|
||||
|
||||
rows = {
|
||||
"Section": [],
|
||||
@ -2231,7 +2225,6 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
|
||||
|
||||
assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
|
||||
|
||||
# Get merged config
|
||||
merged_config = get_config()
|
||||
|
||||
# Determine all sources for this config value
|
||||
|
||||
@ -96,6 +96,69 @@ def test_archiveresult_admin_links_plugin_and_process():
|
||||
assert f"/admin/machine/process/{process.id}/change" in process_html
|
||||
|
||||
|
||||
def test_deleting_binary_and_process_records_preserves_results():
|
||||
from archivebox.core.admin_archiveresults import ArchiveResultAdmin, build_abx_dl_replay_command, render_archiveresults_list
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.machine.admin import ProcessAdmin
|
||||
from archivebox.machine.models import Binary, Process
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
machine = _create_machine()
|
||||
binary = Binary.objects.create(
|
||||
machine=machine,
|
||||
name="wget",
|
||||
abspath="/usr/bin/wget",
|
||||
version="1.21.2",
|
||||
binprovider="env",
|
||||
binproviders="env",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
)
|
||||
process = Process.objects.create(
|
||||
machine=machine,
|
||||
binary=binary,
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
pwd=str(snapshot.output_dir / "wget"),
|
||||
cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"],
|
||||
status=Process.StatusChoices.EXITED,
|
||||
)
|
||||
result = ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg.py",
|
||||
process=process,
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
)
|
||||
|
||||
binary.delete()
|
||||
process.refresh_from_db()
|
||||
assert process.binary_id is None
|
||||
assert process.cmd_version == ""
|
||||
assert process.bin_abspath == ""
|
||||
assert "binary_id" not in process.to_json()
|
||||
assert ProcessAdmin(Process, AdminSite()).binary_link(process) == "-"
|
||||
|
||||
process.delete()
|
||||
result.refresh_from_db()
|
||||
assert result.process_id is None
|
||||
assert ArchiveResult.objects.filter(id=result.id).exists()
|
||||
assert result.pwd == str(result.output_dir)
|
||||
assert result.cmd == []
|
||||
assert result.cmd_version == ""
|
||||
assert result.binary is None
|
||||
assert result.iface is None
|
||||
assert result.machine is None
|
||||
assert result.timeout == 120
|
||||
result_json = result.to_json()
|
||||
assert result_json["pwd"] == str(result.output_dir)
|
||||
assert "process_id" not in result_json
|
||||
|
||||
admin = ArchiveResultAdmin(ArchiveResult, AdminSite())
|
||||
assert admin.process_link(result) == "-"
|
||||
assert admin.machine_link(result) == "-"
|
||||
assert "cd " in build_abx_dl_replay_command(result)
|
||||
assert "wget" in render_archiveresults_list(ArchiveResult.objects.filter(id=result.id))
|
||||
|
||||
|
||||
def test_snapshot_admin_zip_links():
|
||||
from archivebox.core.admin_snapshots import SnapshotAdmin
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
@ -217,3 +217,91 @@ def test_get_config_raises_for_missing_persona_id(initialized_archive):
|
||||
payload = json.loads(stdout.strip().splitlines()[-1])
|
||||
assert payload["raised"] is True
|
||||
assert "references missing Persona" in payload["message"]
|
||||
|
||||
|
||||
def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(initialized_archive):
|
||||
script = textwrap.dedent(
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.core.settings')
|
||||
os.environ['TIMEOUT'] = '22'
|
||||
os.environ['CHROME_BINARY'] = 'env-chrome'
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
CONSTANTS.CONFIG_FILE.write_text('[ARCHIVING_CONFIG]\\nTIMEOUT=11\\nCHROME_BINARY=file-chrome\\n')
|
||||
|
||||
machine = Machine.current()
|
||||
machine.config = {'CHROME_BINARY': 'machine-chrome'}
|
||||
machine.save(update_fields=['config'])
|
||||
|
||||
persona = Persona.objects.create(
|
||||
name='StackPersona',
|
||||
config={'TIMEOUT': 33, 'CHROME_BINARY': 'persona-chrome'},
|
||||
)
|
||||
persona.ensure_dirs()
|
||||
crawl = Crawl.objects.create(
|
||||
urls='https://example.com',
|
||||
persona_id=persona.id,
|
||||
config={'TIMEOUT': 44, 'CHROME_BINARY': 'crawl-chrome'},
|
||||
)
|
||||
snapshot = Snapshot.objects.create(
|
||||
url='https://example.com',
|
||||
crawl=crawl,
|
||||
config={'TIMEOUT': 55, 'CHROME_BINARY': 'snapshot-chrome'},
|
||||
)
|
||||
result = ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin='title',
|
||||
config={'TIMEOUT': 66, 'CHROME_BINARY': 'archiveresult-chrome'},
|
||||
)
|
||||
|
||||
env_config = get_config(include_machine=False)
|
||||
machine_config = get_config(machine=machine)
|
||||
persona_config = get_config(persona=persona)
|
||||
crawl_config = get_config(crawl=crawl)
|
||||
snapshot_config = get_config(snapshot=snapshot)
|
||||
result_config = get_config(archiveresult=result)
|
||||
override_config = get_config(archiveresult=result, overrides={'TIMEOUT': 77, 'CHROME_BINARY': 'override-chrome'})
|
||||
|
||||
print(json.dumps({
|
||||
'env': [env_config.TIMEOUT, env_config.CHROME_BINARY],
|
||||
'machine': [machine_config.TIMEOUT, machine_config.CHROME_BINARY],
|
||||
'persona': [persona_config.TIMEOUT, persona_config.CHROME_BINARY],
|
||||
'crawl': [crawl_config.TIMEOUT, crawl_config.CHROME_BINARY],
|
||||
'snapshot': [snapshot_config.TIMEOUT, snapshot_config.CHROME_BINARY],
|
||||
'archiveresult': [result_config.TIMEOUT, result_config.CHROME_BINARY],
|
||||
'override': [override_config.TIMEOUT, override_config.CHROME_BINARY],
|
||||
'snap_dir': str(result_config.SNAP_DIR),
|
||||
'expected_snap_dir': str(snapshot.output_dir),
|
||||
'crawl_dir': str(result_config.CRAWL_DIR),
|
||||
'expected_crawl_dir': str(crawl.output_dir),
|
||||
'active_persona': result_config.ACTIVE_PERSONA,
|
||||
}, default=str))
|
||||
""",
|
||||
)
|
||||
|
||||
stdout, stderr, code = run_python_cwd(script, cwd=initialized_archive, timeout=60)
|
||||
assert code == 0, stderr
|
||||
|
||||
payload = json.loads(stdout.strip().splitlines()[-1])
|
||||
assert payload["env"] == [22, "env-chrome"]
|
||||
assert payload["machine"] == [22, "machine-chrome"]
|
||||
assert payload["persona"] == [33, "persona-chrome"]
|
||||
assert payload["crawl"] == [44, "crawl-chrome"]
|
||||
assert payload["snapshot"] == [55, "snapshot-chrome"]
|
||||
assert payload["archiveresult"] == [66, "archiveresult-chrome"]
|
||||
assert payload["override"] == [77, "override-chrome"]
|
||||
assert payload["snap_dir"] == payload["expected_snap_dir"]
|
||||
assert payload["crawl_dir"] == payload["expected_crawl_dir"]
|
||||
assert payload["active_persona"] == "StackPersona"
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.32rc25",
|
||||
"version": "0.9.32rc26",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.32rc25"
|
||||
version = "0.9.32rc26"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@ -80,9 +80,9 @@ dependencies = [
|
||||
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.7", # EventBus API
|
||||
"abxpkg>=1.11.10", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.13", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.13", # shared ArchiveBox downloader package with blocking install preflight
|
||||
"abxpkg>=1.11.11", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.14", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.14", # shared ArchiveBox downloader package with blocking install preflight
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user