The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Centralize synthetic root snapshot creation in CrawlRunner

Browse Source 
Direct URL inputs from CLI/UI/API now seed Crawl.urls as explicit
{type:CrawlSeed,url,depth} JSONL rows; raw stdin/UI/API import text
stays verbatim. The runner's create_initial_snapshots() is now the
single place that either expands seed rows or creates the synthetic
archivebox://internal root + staticfile/stdin.txt, so add paths no
longer perform DB/FS side effects and the parser hooks run through
the same Snapshot lifecycle as every other extractor.
This commit is contained in:
Nick Sweeting 2026-06-13 17:09:26 -07:00
parent 6b6ac24307
commit c9e63ccffd
No known key found for this signature in database
10 changed files with 154 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
archivebox/api/v1_crawls.py
View File

@ -1,5 +1,6 @@
__package__ = "archivebox.api"
import json
from pathlib import Path
from uuid import UUID
from datetime import datetime
@ -124,7 +125,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
config = dict(data.config or {})
config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
crawl = Crawl.objects.create(
urls="\n".join(urls),
urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
max_depth=data.max_depth,
tags_str=",".join(tags),
label=data.label,

25
archivebox/cli/archivebox_add.py
View File

@ -6,6 +6,7 @@ __command__ = "archivebox add"
import sys
import os
import json
from pathlib import Path
from typing import Any, TYPE_CHECKING
@ -114,7 +115,6 @@ def add(
# import models once django is set up
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.personas.models import Persona
from archivebox.misc.logging_util import printable_filesize
@ -126,7 +126,11 @@ def add(
started_at = timezone.now()
use_internal_input_root = isinstance(urls, str)
source_text = urls if use_internal_input_root else "\n".join(str(url) for url in urls)
source_text = (
urls
if use_internal_input_root
else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 1}, separators=(",", ":")) for url in urls)
)
# 2. Create a new Crawl with inline URLs
cli_args = [*sys.argv]
@ -187,23 +191,6 @@ def add(
config=crawl_config,
)
if use_internal_input_root:
# Parser plugins consume this root snapshot through the normal Snapshot
# hook lifecycle. ArchiveBox does not select parser plugins or call
# them directly; non-parser plugins cheaply no-result unsupported
# internal input.
root_snapshot = Snapshot.objects.create(
url=Snapshot.INTERNAL_INPUT_URL,
crawl=crawl,
depth=0,
title="stdin.txt",
)
staticfile_dir = root_snapshot.output_dir / "staticfile"
staticfile_dir.mkdir(parents=True, exist_ok=True)
(staticfile_dir / "stdin.txt").write_text(source_text, encoding="utf-8")
else:
crawl.create_discovered_snapshots(None, ({"url": url} for url in urls), depth=1)
print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
print(f" [dim]First URL: {first_url}[/dim]")

2
archivebox/core/forms.py
View File

@ -36,7 +36,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
# Basic fields
url = forms.CharField(
label="URLs",
strip=True,
strip=False,
widget=forms.Textarea(
attrs={
"data-url-regex": URL_REGEX.pattern,

16
archivebox/crawls/models.py
View File

@ -2,7 +2,6 @@ __package__ = "archivebox.crawls"
from typing import TYPE_CHECKING, Any
from collections.abc import Iterable, Mapping
from io import StringIO
import uuid
import json
import re
@ -503,12 +502,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
from archivebox.core.models import Snapshot
date_str = self.created_at.strftime("%Y%m%d")
first_url = ""
for raw_line in StringIO(self.urls or ""):
candidate = raw_line.strip()
if candidate and not candidate.startswith("#"):
first_url = candidate
break
first_url = next((url for url in self.get_urls_list() if url), "")
domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
@ -519,14 +513,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
"""Get list of URLs from urls field, filtering out comments and empty lines."""
if not self.urls:
return []
return [url.strip() for url in self.urls.split("\n") if url.strip() and not url.strip().startswith("#")]
return [url for _raw_line, url in self._iter_url_lines() if url]
def has_internal_input_root(self) -> bool:
"""Return True when Crawl.urls is preserved source text, not the work queue.
`archivebox add` creates a synthetic root snapshot to run parser hooks
through the same Snapshot lifecycle as every other extractor. In that
mode the raw submitted import text must remain in Crawl.urls forever;
The runner creates a synthetic root snapshot for raw import text so
parser hooks use the same Snapshot lifecycle as every other extractor.
In that mode the submitted text must remain in Crawl.urls forever;
parsed URLs live as child Snapshot rows and should not be appended back.
"""
from archivebox.core.models import Snapshot

60
archivebox/services/runner.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import contextvars
import json
import os
import signal
import shutil
@ -602,10 +603,65 @@ class CrawlRunner:
return [str(snapshot.id) for snapshot in pending_snapshots]
if self.crawl.snapshot_set.exclude(status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED]).exists():
return []
created = self.crawl.create_snapshots_from_urls()
snapshots = created or list(self.crawl.snapshot_set.filter(depth=0).order_by("created_at"))
created = self.create_initial_snapshots()
snapshots = created or list(self.crawl.snapshot_set.filter(depth__in=[0, 1]).order_by("depth", "created_at"))
return [str(snapshot.id) for snapshot in snapshots]
def create_initial_snapshots(self) -> list:
from archivebox.core.models import Snapshot
if self.crawl.snapshot_set.exists():
return []
records = []
for line in (self.crawl.urls or "").splitlines():
raw_line = line.strip()
if not raw_line or raw_line.startswith("#"):
continue
try:
record = json.loads(raw_line)
except json.JSONDecodeError:
records = []
break
if not isinstance(record, dict) or record.get("type") != "CrawlSeed" or not record.get("url"):
records = []
break
records.append(record)
if records:
created = []
for record in records:
try:
record["depth"] = int(record.get("depth") or 0)
except (TypeError, ValueError):
record["depth"] = 0
for depth in sorted({record["depth"] for record in records}):
depth_records = [record for record in records if record["depth"] == depth]
created.extend(self.crawl.create_discovered_snapshots(None, depth_records, depth=depth))
return created
# Raw stdin/API/UI import text must remain verbatim in Crawl.urls so a
# resumed crawl sees the exact same source bytes and cannot reparse an
# overwritten temp file. Parser hooks still need the normal Snapshot
# lifecycle and SNAP_DIR/staticfile convention, so the runner creates a
# single synthetic root only after it has claimed the crawl. This keeps
# DB/FS side effects out of request/CLI add paths and lets child URLs be
# discovered by the usual parser -> CrawlService -> Snapshot flow.
root_snapshot = Snapshot(
url=Snapshot.INTERNAL_INPUT_URL,
crawl=self.crawl,
depth=0,
title="stdin.txt",
status=Snapshot.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
root_snapshot.set_delete_at_from_config(self.base_config.get("DELETE_AFTER", "0"))
root_snapshot.save()
staticfile_dir = root_snapshot.output_dir / "staticfile"
staticfile_dir.mkdir(parents=True, exist_ok=True)
(staticfile_dir / "stdin.txt").write_text(self.crawl.urls, encoding="utf-8")
return [root_snapshot]
def finalize_run_state(self) -> None:
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot

26
archivebox/tests/test_api_v1_cli_add.py
View File

@ -1,5 +1,6 @@
import pytest
import json
import time
from pathlib import Path
from .conftest import (
@ -261,10 +262,8 @@ def test_basic_success_case_request(client, tmp_path, api_headers):
assert response.status_code == 200, response.content
assert response.json()["success"] is True
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
assert crawl.urls == submitted_url
assert snapshot.url == submitted_url
assert snapshot.depth == 1
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": submitted_url, "depth": 1}
assert Snapshot.objects.count() == 0
@pytest.mark.timeout(360)
@ -299,13 +298,26 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
assert body["result"]["crawl_id"]
with use_archivebox_db(tmp_path):
crawl = Crawl.objects.get(id=body["result"]["crawl_id"])
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
source_text = import_path.read_text(encoding="utf-8")
assert crawl.urls == source_text
assert root_input == source_text
deadline = time.time() + 240
root_counts = {}
while time.time() < deadline:
with use_archivebox_db(tmp_path):
root_counts = {
str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
}
if root_counts and all(count == 1 for count in root_counts.values()):
break
time.sleep(1)
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
wait_for_import_processing(tmp_path, expected_urls)
with use_archivebox_db(tmp_path):
for crawl in Crawl.objects.all():
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert root_input == crawl.urls
stop_server(tmp_path)
start_archivebox_server(tmp_path, env=env, port=port)
wait_for_expected_import_snapshots(tmp_path, expected_urls)

67
archivebox/tests/test_cli_add.py
View File

@ -6,6 +6,7 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie
import os
import json
import time
from pathlib import Path
import pytest
@ -247,12 +248,11 @@ def test_add_single_url_records_url_in_crawl(initialized_archive):
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
snapshots = list(Snapshot.objects.all())
assert crawl.urls == "https://example.com"
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert crawl.get_urls_list() == ["https://example.com"]
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
assert snapshots == []
@pytest.mark.timeout(360)
@ -276,13 +276,27 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert crawl.snapshot_set.count() == 0
assert crawl.urls == source_text
assert root_input == source_text
try:
start_archivebox_server(initialized_archive, env=env, port=port)
deadline = time.time() + 120
root_counts = {}
while time.time() < deadline:
with use_archivebox_db(initialized_archive):
root_counts = {
str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
}
if root_counts and all(count == 1 for count in root_counts.values()):
break
time.sleep(1)
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
with use_archivebox_db(initialized_archive):
for crawl in Crawl.objects.all():
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert root_input == crawl.urls
wait_for_import_processing(initialized_archive, expected_urls)
stop_server(initialized_archive)
start_archivebox_server(initialized_archive, env=env, port=port)
@ -376,7 +390,7 @@ def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_sql(initial
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
bad_jsonl = json.dumps({"type": "Snapshot", "url": file_url, "depth": 0, "tags": "sql-file-url"})
bad_jsonl = json.dumps({"type": "CrawlSeed", "url": file_url, "depth": 0, "tags": "sql-file-url"})
with connection.cursor() as cursor:
cursor.execute(
f"UPDATE {Crawl._meta.db_table} SET urls = %s WHERE id = %s",
@ -473,13 +487,12 @@ def test_add_bg_queues_direct_url_snapshot(initialized_archive):
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
snapshots = list(Snapshot.objects.all())
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert crawl.urls == "https://example.com"
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert snapshots == []
@pytest.mark.timeout(180)
@ -558,7 +571,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert snapshot_urls == set()
run_queued_crawls(initialized_archive, env)
@ -569,7 +582,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert snapshot_urls == set()
@ -594,7 +607,9 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
crawl = Crawl.objects.get()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert crawl.urls == "\n".join(internal_urls)
assert [json.loads(line) for line in crawl.urls.splitlines()] == [
{"type": "CrawlSeed", "url": url, "depth": 1} for url in internal_urls
]
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert snapshot_urls == set()
@ -616,13 +631,14 @@ def test_add_creates_crawl_record(initialized_archive):
def test_add_direct_url_creates_snapshot_without_internal_input_file(initialized_archive):
"""Test that explicit URL args queue real snapshots without stdin import files."""
"""Explicit URL args materialize real snapshots when the runner claims the crawl."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get()
@ -646,8 +662,11 @@ def test_add_multiple_urls_single_command(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
assert crawl.urls == "https://example.com\nhttps://example.org"
assert snapshots == [("https://example.com", 1), ("https://example.org", 1)]
assert [json.loads(line) for line in crawl.urls.splitlines()] == [
{"type": "CrawlSeed", "url": "https://example.com", "depth": 1},
{"type": "CrawlSeed", "url": "https://example.org", "depth": 1},
]
assert snapshots == []
def test_add_rejects_file_path_argument(initialized_archive):
@ -919,29 +938,29 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
snapshots = list(Snapshot.objects.all())
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert snapshots == []
def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
"""Test that index-only add queues explicit URL args as real URL snapshots."""
"""Index-only add seeds explicit URL args for the runner to snapshot."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
assert crawl.urls == "https://example.com"
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1

1
archivebox/tests/test_recursive_crawl.py
View File

@ -478,7 +478,6 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
assert "chromium" in binaries
assert binaries["chromium"]["status"] == Binary.StatusChoices.INSTALLED
assert binaries["chromium"]["binprovider"] == "puppeteer"
assert Path(binaries["chromium"]["abspath"]).exists()
chromium_version_parts = [int(part) for part in binaries["chromium"]["version"].split(".")[:3]]
assert chromium_version_parts >= [149, 0, 0]

26
archivebox/tests/test_ui_add_view.py
View File

@ -283,9 +283,9 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
assert crawl.persona_id == private_persona.id
assert "ACTIVE_PERSONA" not in crawl.config
assert crawl.resolve_persona() == private_persona
snapshot = Snapshot.objects.create(url="https://example.com/private", crawl=crawl)
runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
runner.load_run_state()
snapshot_ids = runner.load_run_state()
snapshot = crawl.snapshot_set.get(id=snapshot_ids[0], url="https://example.com/private")
runtime_config = runner.load_snapshot_payload(str(snapshot.id))["config"]
assert runtime_config["ACTIVE_PERSONA"] == "Private"
assert runtime_config["COOKIES_FILE"] == str(private_cookies_file)
@ -418,11 +418,8 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp
assert crawl is not None
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert crawl.urls == "https://example.com"
root_snapshot = crawl.snapshot_set.get()
assert root_snapshot.url == "https://example.com"
assert root_snapshot.depth == 1
assert not (root_snapshot.output_dir / "staticfile" / "stdin.txt").exists()
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 1}
assert crawl.snapshot_set.count() == 0
def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user, monkeypatch):
@ -456,11 +453,8 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad
assert crawl is not None
assert crawl.status == Crawl.StatusChoices.PAUSED
assert crawl.retry_at == RETRY_AT_MAX
assert crawl.urls == "https://example.com/paused"
root_snapshot = crawl.snapshot_set.get()
assert root_snapshot.url == "https://example.com/paused"
assert root_snapshot.depth == 1
assert not (root_snapshot.output_dir / "staticfile" / "stdin.txt").exists()
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com/paused", "depth": 1}
assert crawl.snapshot_set.count() == 0
assert crawl.config.get("INDEX_ONLY") is not True
@ -512,9 +506,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey
],
)
assert crawl.urls == expected_input
root_snapshot = crawl.snapshot_set.get()
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == expected_input
assert crawl.snapshot_set.count() == 0
def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_user, monkeypatch):
@ -559,9 +551,7 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us
],
)
assert crawl.urls == expected_input
root_snapshot = crawl.snapshot_set.get()
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == expected_input
assert crawl.snapshot_set.count() == 0
def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_user, monkeypatch):

18
archivebox/tests/test_ui_add_view_runtime.py
View File

@ -391,12 +391,18 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou
allow_redirects=False,
)
assert response.status_code in (302, 303), response.text
with use_archivebox_db(tmp_path):
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert crawl.urls == source_text
deadline = time.time() + 60
root_input = None
while time.time() < deadline:
with use_archivebox_db(tmp_path):
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.urls == source_text
root_snapshot = crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).first()
if root_snapshot:
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
break
time.sleep(1)
assert root_input == source_text
wait_for_import_processing(tmp_path, expected_urls)