@@ -1139,6 +1214,7 @@
}
// Recent thumbnails removed
+ updateDurationBadges();
}
function fetchProgress() {
@@ -1277,6 +1353,7 @@
// Start polling when page loads
startPolling();
+ setInterval(updateDurationBadges, 1000);
// Pause polling when tab is hidden
document.addEventListener('visibilitychange', function() {
diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py
index ca0a2bff..f2a4f2bf 100644
--- a/archivebox/tests/conftest.py
+++ b/archivebox/tests/conftest.py
@@ -17,9 +17,13 @@ pytest_plugins = ["archivebox.tests.fixtures"]
REPO_ROOT = Path(__file__).resolve().parents[2]
PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
-SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
+SESSION_DATA_DIR = Path(
+ os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
+).resolve()
# Force ArchiveBox imports to see a temp DATA_DIR during test collection.
+os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
os.environ.pop("ARCHIVE_DIR", None)
os.environ.pop("USERS_DIR", None)
os.environ.pop("CRAWL_DIR", None)
@@ -129,10 +133,9 @@ def isolate_test_runtime(tmp_path, monkeypatch):
contract is that every test starts in its own temp directory and any
in-process ``os.environ`` edits are rolled back afterwards.
- We intentionally clear ``DATA_DIR`` for the body of each test so subprocess
- tests that rely on cwd keep working. During collection/import time we still
- seed a separate session-scoped temp ``DATA_DIR`` above so any ArchiveBox
- config imported before this fixture runs never points at the repo root.
+ Each in-process test gets an explicit temp ``DATA_DIR`` so ArchiveBox code
+ never falls back to the repo cwd. Subprocess helpers that intentionally test
+ cwd-based behavior remove ``DATA_DIR`` for the child process themselves.
"""
_assert_not_repo_path(tmp_path, label="tmp_path")
original_cwd = Path.cwd()
@@ -140,11 +143,6 @@ def isolate_test_runtime(tmp_path, monkeypatch):
original_chdir = os.chdir
original_popen = subprocess.Popen
os.chdir(tmp_path)
- os.environ.pop("DATA_DIR", None)
- os.environ.pop("ARCHIVE_DIR", None)
- os.environ.pop("USERS_DIR", None)
- os.environ.pop("CRAWL_DIR", None)
- os.environ.pop("SNAP_DIR", None)
def guarded_chdir(path: os.PathLike[str] | str) -> None:
_assert_not_repo_path(Path(path), label="cwd")
@@ -160,6 +158,11 @@ def isolate_test_runtime(tmp_path, monkeypatch):
monkeypatch.setattr(os, "chdir", guarded_chdir)
monkeypatch.setattr(subprocess, "Popen", guarded_popen)
+ os.environ["DATA_DIR"] = str(tmp_path)
+ os.environ.pop("ARCHIVE_DIR", None)
+ os.environ.pop("USERS_DIR", None)
+ os.environ.pop("CRAWL_DIR", None)
+ os.environ.pop("SNAP_DIR", None)
try:
_assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ)
yield
diff --git a/archivebox/tests/test_add_view.py b/archivebox/tests/test_add_view.py
index ece5bca3..36695338 100644
--- a/archivebox/tests/test_add_view.py
+++ b/archivebox/tests/test_add_view.py
@@ -141,7 +141,6 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
assert crawl.max_urls == 3
assert crawl.crawl_max_size == 45 * 1024 * 1024
assert crawl.snapshot_max_size == 5 * 1024 * 1024
- assert crawl.config.get("DEFAULT_PERSONA") == "Default"
assert crawl.config["CRAWL_MAX_URLS"] == 3
assert crawl.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024
assert crawl.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
@@ -185,7 +184,7 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.persona_id == private_persona.id
- assert crawl.config.get("DEFAULT_PERSONA") == "Private"
+ assert "DEFAULT_PERSONA" not in crawl.config
assert crawl.resolve_persona() == private_persona
runtime_config = get_config(crawl=crawl)
assert runtime_config.ACTIVE_PERSONA == "Private"
diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py
index 14685978..dc6e144a 100644
--- a/archivebox/tests/test_cli_add.py
+++ b/archivebox/tests/test_cli_add.py
@@ -236,7 +236,7 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac
conn.close()
assert persona_id
- assert default_persona == "Default"
+ assert default_persona is None
assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()
diff --git a/archivebox/tests/test_crawl_admin.py b/archivebox/tests/test_crawl_admin.py
index 7d397be6..58964f65 100644
--- a/archivebox/tests/test_crawl_admin.py
+++ b/archivebox/tests/test_crawl_admin.py
@@ -105,6 +105,50 @@ def test_crawl_admin_form_saves_tags_editor_to_tags_str(crawl, admin_user):
assert updated.config["URL_DENYLIST"] == "static.example.com"
+@pytest.mark.django_db(transaction=True)
+def test_crawl_tag_changes_sync_existing_snapshot_tags(crawl):
+ snapshots = crawl.create_snapshots_from_urls()
+ snapshots[0].save_tags(["alpha", "beta", "keep"])
+
+ crawl.tags_str = "beta,gamma"
+ crawl.save(update_fields=["tags_str", "modified_at"])
+
+ assert set(snapshots[0].tags.values_list("name", flat=True)) == {"beta", "gamma", "keep"}
+ assert set(snapshots[1].tags.values_list("name", flat=True)) == {"beta", "gamma"}
+
+
+@pytest.mark.django_db(transaction=True)
+def test_create_snapshots_from_urls_uses_current_crawl_tags_for_stale_crawl_instance(crawl):
+ crawl.create_snapshots_from_urls()
+ fresh_crawl = Crawl.objects.get(pk=crawl.pk)
+ fresh_crawl.tags_str = "midcrawl"
+ fresh_crawl.save(update_fields=["tags_str", "modified_at"])
+
+ crawl.urls = f"{crawl.urls}\nhttps://example.net/new"
+ created = crawl.create_snapshots_from_urls()
+
+ assert [snapshot.url for snapshot in created] == ["https://example.net/new"]
+ assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl"}
+
+
+@pytest.mark.django_db(transaction=True)
+def test_discovered_snapshots_inherit_current_crawl_tags(crawl):
+ crawl.max_depth = 1
+ crawl.save(update_fields=["max_depth", "modified_at"])
+ parent_snapshot = crawl.create_snapshots_from_urls()[0]
+ crawl.tags_str = "midcrawl"
+ crawl.save(update_fields=["tags_str", "modified_at"])
+
+ created = crawl.create_discovered_snapshots(
+ parent_snapshot,
+ [{"url": "https://example.com/child", "tags": "discovered"}],
+ depth=1,
+ )
+
+ assert [snapshot.url for snapshot in created] == ["https://example.com/child"]
+ assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl", "discovered"}
+
+
def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, admin_user):
crawl = Crawl.objects.create(
urls="https://example.com/remove-me",
diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py
index f9b3e815..ee2feda7 100644
--- a/archivebox/tests/test_runner.py
+++ b/archivebox/tests/test_runner.py
@@ -376,6 +376,85 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
assert snapshot.retry_at is None
+@pytest.mark.django_db(transaction=True)
+def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot():
+ from abx_dl.events import SnapshotCompletedEvent
+ from archivebox.base_models.models import get_or_create_system_user_pk
+ from archivebox.crawls.models import Crawl
+ from archivebox.core.models import ArchiveResult, Snapshot
+ from archivebox.machine.models import Process
+ from archivebox.personas.models import Persona
+ from archivebox.services.runner import CrawlRunner
+
+ persona = Persona.objects.create(
+ name="RuntimeConfig",
+ config={
+ "FAVICON_PROVIDER": "https://example.com/persona-first.ico",
+ "FAVICON_TIMEOUT": 10,
+ },
+ )
+ persona.ensure_dirs()
+ crawl = Crawl.objects.create(
+ urls="\n".join(
+ [
+ "https://www.python.org/",
+ "https://www.djangoproject.com/",
+ "https://www.wikipedia.org/",
+ ],
+ ),
+ config={
+ "PLUGINS": "favicon",
+ "CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
+ },
+ persona_id=persona.id,
+ created_by_id=get_or_create_system_user_pk(),
+ )
+ runner = CrawlRunner(crawl)
+ completed_snapshot_ids: list[str] = []
+
+ async def update_config_between_snapshots(event: SnapshotCompletedEvent) -> None:
+ if event.snapshot_id in completed_snapshot_ids:
+ return
+ completed_snapshot_ids.append(event.snapshot_id)
+ if len(completed_snapshot_ids) == 1:
+ persona.config = {
+ **(persona.config or {}),
+ "FAVICON_PROVIDER": "https://example.com/persona-second.ico",
+ }
+ await persona.asave(update_fields=["config"])
+ elif len(completed_snapshot_ids) == 2:
+ fresh_crawl = await Crawl.objects.aget(id=crawl.id)
+ fresh_crawl.config = {
+ **(fresh_crawl.config or {}),
+ "FAVICON_PROVIDER": "https://example.com/crawl-third.ico",
+ }
+ await fresh_crawl.asave(update_fields=["config"])
+
+ runner.bus.on(SnapshotCompletedEvent, update_config_between_snapshots)
+
+ asyncio.run(runner.run())
+
+ favicon_processes = [
+ process
+ for process in Process.objects.filter(process_type=Process.TypeChoices.HOOK).order_by("started_at")
+ if process.cmd and "on_Snapshot__11_favicon.finite.bg.py" in str(process.cmd[0])
+ ]
+ providers = [process.env.get("FAVICON_PROVIDER") for process in favicon_processes]
+
+ crawl.refresh_from_db()
+ assert crawl.status == Crawl.StatusChoices.SEALED
+ assert Snapshot.objects.filter(crawl=crawl, status=Snapshot.StatusChoices.SEALED).count() == 3
+ assert (
+ ArchiveResult.objects.filter(snapshot__crawl=crawl, plugin="favicon").exclude(status=ArchiveResult.StatusChoices.FAILED).count()
+ == 3
+ )
+ assert providers == [
+ "https://example.com/persona-first.ico",
+ "https://example.com/persona-second.ico",
+ "https://example.com/crawl-third.ico",
+ ]
+
+
@pytest.mark.django_db(transaction=True)
def test_run_snapshot_seals_descendant_when_crawl_max_size_is_reached(tmp_path):
from abx_dl.events import CrawlStartEvent, SnapshotEvent
diff --git a/archivebox/tests/test_test_harness.py b/archivebox/tests/test_test_harness.py
index 22aace9d..23c3b249 100644
--- a/archivebox/tests/test_test_harness.py
+++ b/archivebox/tests/test_test_harness.py
@@ -14,6 +14,21 @@ def test_session_data_dir_is_outside_repo_root():
assert test_harness.PYTEST_BASETEMP_ROOT in (Path.cwd(), *Path.cwd().parents)
+def test_in_process_archivebox_config_uses_temp_data_dir():
+ from archivebox.config import CONSTANTS
+ from archivebox.config.common import get_config
+
+ data_dir = Path(os.environ["DATA_DIR"]).resolve()
+ assert data_dir == Path.cwd().resolve()
+ assert test_harness.REPO_ROOT not in data_dir.parents
+ assert CONSTANTS.DATA_DIR != test_harness.REPO_ROOT
+
+ config = get_config(include_machine=False)
+ assert config.DATA_DIR == data_dir
+ assert config.ARCHIVE_DIR == data_dir / "archive"
+ assert config.USERS_DIR == data_dir / "archive" / "users"
+
+
def test_cli_helpers_reject_repo_root_runtime_paths():
with pytest.raises(AssertionError, match="repo root"):
test_harness.run_archivebox_cmd(["version"], data_dir=test_harness.REPO_ROOT)
diff --git a/conftest.py b/conftest.py
new file mode 100644
index 00000000..fea10b51
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,32 @@
+"""Root pytest bootstrap.
+
+This file is intentionally outside the ``archivebox`` package so pytest can
+load it before importing ``archivebox/__init__.py``. ArchiveBox constants are
+computed at import time, so DATA_DIR must already point at a temp collection.
+"""
+
+import os
+import tempfile
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent
+SESSION_DATA_DIR = Path(
+ os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
+).resolve()
+
+os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
+os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
+os.environ.pop("ARCHIVE_DIR", None)
+os.environ.pop("USERS_DIR", None)
+os.environ.pop("CRAWL_DIR", None)
+os.environ.pop("SNAP_DIR", None)
+
+
+def pytest_configure():
+ import django
+ from django.apps import apps
+
+ if not apps.ready:
+ django.setup()
diff --git a/etc/package.json b/etc/package.json
index 92fb681b..c7dc7550 100644
--- a/etc/package.json
+++ b/etc/package.json
@@ -1,6 +1,6 @@
{
"name": "archivebox",
- "version": "0.9.32rc23",
+ "version": "0.9.32rc24",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {
diff --git a/pyproject.toml b/pyproject.toml
index 02538b3f..7a5c8b89 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "archivebox"
-version = "0.9.32rc23"
+version = "0.9.32rc24"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -201,7 +201,6 @@ skip = "*.json,*.min.js,*.min.css,uv.lock,old/*,publicsite/*"
[tool.pytest.ini_options]
testpaths = [ "archivebox/tests" ]
norecursedirs = ["archivebox/tests/data"]
-DJANGO_SETTINGS_MODULE = "archivebox.core.settings"
# Note: Plugin tests under abx_plugins/plugins/ must NOT load Django
# They use a conftest.py to disable Django automatically