The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: archivebox 0.9.33rc68

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-01 03:12:47 -07:00
parent 72a67bd511
commit 3871a4ac7e
No known key found for this signature in database
9 changed files with 149 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py
View File

@ -1,29 +1,59 @@
import json
from django.conf import settings
from django.db import migrations
BATCH_SIZE = 1000
def _config_cache_key(config):
return json.dumps(config or {}, sort_keys=True, separators=(",", ":"), default=str)
def _flush_updates(Crawl, db_alias, pending):
if pending:
Crawl.objects.using(db_alias).bulk_update(pending, ["config"], batch_size=BATCH_SIZE)
pending.clear()
def freeze_existing_crawl_configs(apps, schema_editor):
from archivebox.config.common import build_crawl_config_snapshot
from archivebox.personas.models import Persona
from django.contrib.auth import get_user_model
Crawl = apps.get_model("crawls", "Crawl")
User = get_user_model()
auth_app_label, auth_model_name = settings.AUTH_USER_MODEL.split(".", 1)
User = apps.get_model(auth_app_label, auth_model_name)
db_alias = schema_editor.connection.alias
rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "created_by_id", "config")
persona_ids = {persona_id for _, persona_id, _, _ in rows if persona_id}
user_ids = {user_id for _, _, user_id, _ in rows if user_id}
personas = {persona.pk: persona for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
users = {user.pk: user for user in User.objects.using(db_alias).filter(pk__in=user_ids)}
for crawl in Crawl.objects.using(db_alias).select_related("persona", "created_by").iterator(chunk_size=200):
current_config = dict(crawl.config or {})
persona = Persona.objects.using(db_alias).filter(pk=crawl.persona_id).first()
user = User.objects.using(db_alias).filter(pk=crawl.created_by_id).first()
frozen_config = build_crawl_config_snapshot(
user=user,
persona=persona,
overrides=current_config,
)
frozen_cache = {}
pending = []
for crawl_id, persona_id, user_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
current_config = dict(current_config or {})
cache_key = (persona_id, user_id, _config_cache_key(current_config))
if cache_key not in frozen_cache:
frozen_cache[cache_key] = build_crawl_config_snapshot(
user=users.get(user_id),
persona=personas.get(persona_id),
overrides=current_config,
)
frozen_config = frozen_cache[cache_key]
if frozen_config != current_config:
Crawl.objects.using(db_alias).filter(pk=crawl.pk).update(config=frozen_config)
pending.append(Crawl(id=crawl_id, config=frozen_config))
if len(pending) >= BATCH_SIZE:
_flush_updates(Crawl, db_alias, pending)
_flush_updates(Crawl, db_alias, pending)
class Migration(migrations.Migration):
atomic = False
dependencies = [
("crawls", "0017_drop_stale_crawl_limit_columns"),
]

2
archivebox/personas/admin.py
View File

@ -92,14 +92,12 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
"<div class='abx-persona-path-list'>"
"<div><strong>Persona root</strong><code>{}</code></div>"
"<div><strong>chrome_profile</strong><code>{}</code></div>"
"<div><strong>chrome_extensions</strong><code>{}</code></div>"
"<div><strong>chrome_downloads</strong><code>{}</code></div>"
"<div><strong>cookies.txt</strong><code>{}</code></div>"
"<div><strong>auth.json</strong><code>{}</code></div>"
"</div>",
obj.path,
obj.CHROME_USER_DATA_DIR,
obj.CHROME_EXTENSIONS_DIR,
obj.CHROME_DOWNLOADS_DIR,
obj.COOKIES_FILE or (obj.path / "cookies.txt"),
obj.AUTH_STORAGE_FILE or (obj.path / "auth.json"),

11
archivebox/personas/models.py
View File

@ -4,7 +4,6 @@ Persona management for ArchiveBox.
A Persona represents a browser profile/identity used for archiving.
Each persona has its own:
- Chrome user data directory (for cookies, localStorage, extensions, etc.)
- Chrome extensions directory
- Cookies file
- Config overrides
"""
@ -66,7 +65,6 @@ class Persona(ModelWithConfig):
Each persona provides:
- CHROME_USER_DATA_DIR: Chrome profile directory
- CHROME_EXTENSIONS_DIR: Installed extensions directory
- CHROME_DOWNLOADS_DIR: Chrome downloads directory
- COOKIES_FILE: Cookies file for wget/curl
- config: JSON field with persona-specific config overrides
@ -124,11 +122,6 @@ class Persona(ModelWithConfig):
"""Derived path to Chrome user data directory for this persona."""
return str(self.path / "chrome_profile")
@property
def CHROME_EXTENSIONS_DIR(self) -> str:
"""Derived path to Chrome extensions directory for this persona."""
return str(self.path / "chrome_extensions")
@property
def CHROME_DOWNLOADS_DIR(self) -> str:
"""Derived path to Chrome downloads directory for this persona."""
@ -153,7 +146,6 @@ class Persona(ModelWithConfig):
Returns dict with:
- All values from self.config JSONField
- CHROME_USER_DATA_DIR (derived from persona path)
- CHROME_EXTENSIONS_DIR (derived from persona path)
- CHROME_DOWNLOADS_DIR (derived from persona path)
- COOKIES_FILE (derived from persona path, if file exists)
- AUTH_STORAGE_FILE (derived from persona path, if file exists)
@ -164,8 +156,6 @@ class Persona(ModelWithConfig):
# Add derived paths (don't override if explicitly set in config)
if "CHROME_USER_DATA_DIR" not in derived:
derived["CHROME_USER_DATA_DIR"] = self.CHROME_USER_DATA_DIR
if "CHROME_EXTENSIONS_DIR" not in derived:
derived["CHROME_EXTENSIONS_DIR"] = self.CHROME_EXTENSIONS_DIR
if "CHROME_DOWNLOADS_DIR" not in derived:
derived["CHROME_DOWNLOADS_DIR"] = self.CHROME_DOWNLOADS_DIR
if "COOKIES_FILE" not in derived and self.COOKIES_FILE:
@ -182,7 +172,6 @@ class Persona(ModelWithConfig):
"""Create persona directories if they don't exist."""
self.path.mkdir(parents=True, exist_ok=True)
(self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
(self.path / "chrome_extensions").mkdir(parents=True, exist_ok=True)
(self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
def cleanup_chrome_profile(self, profile_dir: Path) -> bool:

114
archivebox/services/runner.py
View File

@ -1273,8 +1273,6 @@ def run_snapshot_maintenance(snapshot_id: str) -> bool:
snapshot.retry_at = timezone.now() if has_queued_results else None
snapshot.save(update_fields=["retry_at", "modified_at"])
snapshot.write_index_jsonl()
snapshot.write_json_details()
snapshot.write_html_details()
return True
@ -1433,7 +1431,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
snapshot.finalize_completed_upload_results()
maintenance_ran = False
if snapshot.fs_migration_needed:
# Final snapshots can still need filesystem/json maintenance after
# Final snapshots can still need filesystem/index maintenance after
# a data-dir migration, but queued ArchiveResult rows are the actual
# runnable work. Do the metadata rewrite first, then continue into
# the targeted plugin path in the same tick so large migrations do
@ -1442,6 +1440,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
snapshot.refresh_from_db()
selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
if selected_plugins:
search_only_plugins = all(plugin.startswith("search_backend_") for plugin in selected_plugins)
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
run_crawl(
str(snapshot.crawl_id),
@ -1450,6 +1449,21 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
)
if search_only_plugins:
from archivebox.core.models import ArchiveResult
has_queued_results = ArchiveResult.objects.filter(
snapshot_id=snapshot.id,
status=ArchiveResult.StatusChoices.QUEUED,
).exists()
if not has_queued_results:
type(snapshot).objects.filter(
pk=snapshot.pk,
status=snapshot.StatusChoices.SEALED,
).update(
retry_at=None,
modified_at=timezone.now(),
)
return True
if maintenance_ran:
return True
@ -1678,8 +1692,8 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup
return True
def _run_due_queued_download_result(
download_plugin_names: frozenset[str],
def _run_due_queued_plugin_result(
plugin_names: frozenset[str],
*,
crawl_id: str | None,
lock_seconds: int,
@ -1688,11 +1702,11 @@ def _run_due_queued_download_result(
) -> bool:
from archivebox.core.models import ArchiveResult, Snapshot
if not download_plugin_names:
if not plugin_names:
return False
queued_results = ArchiveResult.objects.filter(
status=ArchiveResult.StatusChoices.QUEUED,
plugin__in=download_plugin_names,
plugin__in=plugin_names,
snapshot__status=Snapshot.StatusChoices.SEALED,
snapshot__retry_at__lte=timezone.now(),
)
@ -1726,6 +1740,53 @@ def _run_due_binary() -> bool:
return True
def _fast_forward_same_path_snapshot_fs_versions(batch_size: int = 10000) -> bool:
from django.db import connection
from archivebox.core.models import Snapshot, ArchiveResult
now = timezone.now()
current_version = Snapshot._fs_current_version()
same_path_versions = ("0.9.0", "0.9.1", "0.9.2", "0.9.3")
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE core_snapshot
SET fs_version = %s,
retry_at = CASE
WHEN EXISTS (
SELECT 1
FROM core_archiveresult
WHERE core_archiveresult.snapshot_id = core_snapshot.id
AND core_archiveresult.status = %s
)
THEN retry_at
ELSE NULL
END,
modified_at = %s
WHERE id IN (
SELECT id
FROM core_snapshot
WHERE status = %s
AND retry_at <= %s
AND fs_version IN (%s, %s, %s, %s)
ORDER BY retry_at, created_at
LIMIT %s
)
""",
[
current_version,
ArchiveResult.StatusChoices.QUEUED,
now,
Snapshot.StatusChoices.SEALED,
now,
*same_path_versions,
batch_size,
],
)
return bool(cursor.rowcount)
def run_pending_crawls(
*,
daemon: bool = False,
@ -1747,6 +1808,7 @@ def run_pending_crawls(
for plugin_name, plugin_config in plugin_configs.items()
if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
)
search_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
last_recovery_at = 0.0
last_retention_at = 0.0
last_analyze_at = 0.0
@ -1773,7 +1835,7 @@ def run_pending_crawls(
# Final-state download rows are always first: they have no parent crawl
# scheduler of their own, and leaving them behind makes the global
# counters report stale queued work while new crawls continue.
if _run_due_queued_download_result(
if _run_due_queued_plugin_result(
download_plugin_names,
crawl_id=crawl_id,
lock_seconds=60,
@ -1782,13 +1844,22 @@ def run_pending_crawls(
):
continue
# Other final-state snapshot work comes next: search backfills,
# filesystem/json maintenance, and upload finalization should drain
# before starting or resuming regular crawl work.
if _fast_forward_same_path_snapshot_fs_versions():
continue
# Final-state snapshot maintenance comes before normal crawl work:
# filesystem/index maintenance and upload finalization should drain
# promptly, but pure search backend backfills are deferred below so
# they do not starve live crawls.
sealed_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),
status=Snapshot.StatusChoices.SEALED,
)
if search_plugin_names:
sealed_snapshots = sealed_snapshots.exclude(
archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
archiveresult__plugin__in=search_plugin_names,
)
if crawl_id:
sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)
if _run_due_snapshot_query(
@ -1868,11 +1939,13 @@ def run_pending_crawls(
):
continue
# Final fallback uses only the retry_at scheduler index and selects an
# id first. The active/paused/sealed parent-specific branches above get
# first priority, so this stays broad without hydrating wide rows or
# forcing SQLite into a slow status/join plan.
due_snapshots = Snapshot.objects.filter(retry_at__lte=timezone.now())
# Final active-state fallback uses only the retry_at scheduler index and
# selects an id first. Keep final SEALED rows out of this broad path so
# large filesystem/index backfills cannot starve newly queued crawls.
due_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),
status__in=Snapshot.OPEN_STATES,
)
if maintenance_only:
due_snapshots = due_snapshots.filter(status=Snapshot.StatusChoices.PAUSED)
if crawl_id:
@ -1885,6 +1958,15 @@ def run_pending_crawls(
):
continue
if _run_due_queued_plugin_result(
search_plugin_names,
crawl_id=crawl_id,
lock_seconds=60,
interactive_interrupts=interactive_interrupts,
runtime_config=runtime_config,
):
continue
if not maintenance_only:
if _run_due_crawl_status(
Crawl.StatusChoices.SEALED,

2
archivebox/services/snapshot_service.py
View File

@ -41,8 +41,6 @@ def finalize_completed_snapshot(snapshot_id: str) -> None:
snapshot.refresh_from_db()
snapshot.write_index_jsonl()
snapshot.write_json_details()
snapshot.write_html_details()
def _crawl_limit_stop_reason(crawl) -> str:

2
archivebox/tests/test_cli_add.py
View File

@ -296,7 +296,7 @@ def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_ex
assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com"
assert crawl.config["URL_DENYLIST"] == "static.example.com"
assert (tmp_path / "personas" / "Default" / "chrome_extensions").is_dir()
assert not (tmp_path / "personas" / "Default" / "chrome_extensions").exists()
def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_extractors_dict):

3
archivebox/tests/test_cli_piping.py
View File

@ -272,6 +272,7 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive):
snapshot_stdout, snapshot_stderr, snapshot_code = run_archivebox_cmd(
["snapshot", "create", url],
data_dir=initialized_archive,
env=PIPE_TEST_ENV,
)
assert snapshot_code == 0, snapshot_stderr
@ -279,6 +280,7 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive):
["archiveresult", "create", "--plugin=favicon"],
stdin=snapshot_stdout,
data_dir=initialized_archive,
env=PIPE_TEST_ENV,
)
assert ar_create_code == 0, ar_create_stderr
@ -293,6 +295,7 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive):
list_stdout, list_stderr, list_code = run_archivebox_cmd(
["archiveresult", "list", "--plugin=favicon"],
data_dir=initialized_archive,
env=PIPE_TEST_ENV,
)
assert list_code == 0, list_stderr
_assert_stdout_is_jsonl_only(list_stdout)

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.33rc67",
"version": "0.9.33rc68",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.33rc67"
version = "0.9.33rc68"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.8", # EventBus API
"abxpkg>=1.11.94", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.101", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.101", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.113", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.116", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.116", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]