From 96437e1ffd5c9c5290f8dd1ba1ab54a028ba9443 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 2 Jun 2026 02:25:52 -0700 Subject: [PATCH] Publish local ArchiveBox changes --- archivebox/__main__.py | 2 +- archivebox/api/auth.py | 21 + archivebox/api/v1_cli.py | 178 +-- archivebox/api/v1_core.py | 67 +- archivebox/api/v1_crawls.py | 33 +- archivebox/api/v1_personas.py | 3 + archivebox/cli/__init__.py | 44 +- archivebox/cli/archivebox_add.py | 40 +- archivebox/cli/archivebox_config.py | 31 +- archivebox/cli/archivebox_crawl_compat.py | 27 - archivebox/cli/archivebox_help.py | 8 +- archivebox/cli/archivebox_list.py | 45 +- archivebox/cli/archivebox_remove.py | 102 +- archivebox/cli/archivebox_run.py | 17 +- archivebox/cli/archivebox_search.py | 250 +--- archivebox/cli/archivebox_server.py | 6 +- archivebox/cli/archivebox_snapshot.py | 267 ++-- archivebox/cli/archivebox_snapshot_compat.py | 24 - archivebox/cli/archivebox_update.py | 87 +- archivebox/core/admin_snapshots.py | 4 +- archivebox/core/models.py | 80 +- archivebox/core/recovery_util.py | 19 + archivebox/core/settings.py | 2 + archivebox/core/snapshot_status.py | 25 + archivebox/core/takeover_util.py | 73 +- archivebox/crawls/admin.py | 2 +- archivebox/machine/admin.py | 1 + archivebox/services/runner.py | 20 +- .../admin/crawls/crawl/change_form.html | 7 + archivebox/templates/static/admin.css | 12 +- archivebox/tests/conftest.py | 881 ++++++++++--- archivebox/tests/fixtures.py | 121 -- archivebox/tests/migrations_helpers.py | 42 +- archivebox/tests/test_api_archiveresult.py | 1 + archivebox/tests/test_api_cli.py | 156 +-- archivebox/tests/test_api_cli_schedule.py | 84 +- archivebox/tests/test_api_core.py | 374 ------ archivebox/tests/test_api_crawl.py | 1 + archivebox/tests/test_api_crud.py | 1 + archivebox/tests/test_api_delete_paths.py | 89 +- archivebox/tests/test_api_personas.py | 1 + archivebox/tests/test_api_remove.py | 1 + archivebox/tests/test_api_rss.py | 167 +-- archivebox/tests/test_api_search.py | 1 + .../tests/test_api_v1_auth_check_api_token.py | 19 + .../tests/test_api_v1_auth_get_api_token.py | 22 + archivebox/tests/test_api_v1_cli_add.py | 29 + archivebox/tests/test_api_v1_cli_remove.py | 209 ++++ archivebox/tests/test_api_v1_cli_schedule.py | 71 ++ archivebox/tests/test_api_v1_cli_search.py | 31 + archivebox/tests/test_api_v1_cli_update.py | 137 +++ ...1_cli_workflow_add_search_update_remove.py | 160 +++ archivebox/tests/test_api_v1_core_any_id.py | 16 + ..._v1_core_archiveresult_archiveresult_id.py | 69 ++ .../tests/test_api_v1_core_archiveresults.py | 170 +++ .../test_api_v1_core_snapshot_snapshot_id.py | 497 ++++++++ .../tests/test_api_v1_core_snapshots.py | 46 + .../tests/test_api_v1_core_snapshots_rss.py | 115 ++ .../tests/test_api_v1_core_tag_tag_id.py | 15 + .../test_api_v1_core_tag_tag_id_rename.py | 38 + ..._api_v1_core_tag_tag_id_snapshots_jsonl.py | 36 + .../test_api_v1_core_tag_tag_id_urls_txt.py | 35 + archivebox/tests/test_api_v1_core_tags.py | 14 + .../test_api_v1_core_tags_add_to_snapshot.py | 25 + .../test_api_v1_core_tags_autocomplete.py | 18 + .../tests/test_api_v1_core_tags_create.py | 19 + ...t_api_v1_core_tags_remove_from_snapshot.py | 26 + .../tests/test_api_v1_core_tags_search.py | 83 ++ .../test_api_v1_crawls_crawl_crawl_id.py | 614 ++++++++++ ...v1_crawls_crawl_crawl_id_files_filename.py | 16 + ...ls_crawl_crawl_id_files_folder_filename.py | 17 + ...rawl_id_files_folder_subfolder_filename.py | 17 + archivebox/tests/test_api_v1_crawls_crawls.py | 38 + .../tests/test_api_v1_machine_binaries.py | 22 + .../test_api_v1_machine_binary_binary_id.py | 22 + ...test_api_v1_machine_binary_by_name_name.py | 22 + .../test_api_v1_machine_machine_current.py | 10 + .../test_api_v1_machine_machine_machine_id.py | 14 + .../tests/test_api_v1_machine_machines.py | 14 + .../tests/test_api_v1_personas_personas.py | 57 + archivebox/tests/test_api_v1_personas_sync.py | 25 + ...1_workflow_core_token_auth_side_effects.py | 207 ++++ ...v1_workflow_frozen_crawl_config_sources.py | 66 + archivebox/tests/test_archiveresult_pause.py | 374 +----- archivebox/tests/test_auth_ldap.py | 21 +- archivebox/tests/test_binary_service.py | 43 +- archivebox/tests/test_cli_add.py | 595 ++++++--- archivebox/tests/test_cli_archiveresult.py | 264 +++- archivebox/tests/test_cli_binary.py | 18 + archivebox/tests/test_cli_config.py | 210 ++-- archivebox/tests/test_cli_crawl.py | 290 ++++- archivebox/tests/test_cli_extract.py | 50 +- archivebox/tests/test_cli_extract_input.py | 213 ++-- archivebox/tests/test_cli_help.py | 11 +- archivebox/tests/test_cli_init.py | 105 +- archivebox/tests/test_cli_install.py | 91 +- archivebox/tests/test_cli_list.py | 377 ++++-- archivebox/tests/test_cli_machine.py | 18 + archivebox/tests/test_cli_manage.py | 39 +- archivebox/tests/test_cli_mcp.py | 17 + archivebox/tests/test_cli_persona.py | 18 + archivebox/tests/test_cli_piping.py | 163 ++- archivebox/tests/test_cli_pluginmap.py | 18 + archivebox/tests/test_cli_process.py | 18 + archivebox/tests/test_cli_real_flows.py | 1091 ----------------- archivebox/tests/test_cli_remove.py | 254 ++-- archivebox/tests/test_cli_run.py | 329 +++-- archivebox/tests/test_cli_schedule.py | 126 +- archivebox/tests/test_cli_search.py | 73 ++ archivebox/tests/test_cli_server.py | 227 +++- archivebox/tests/test_cli_shell.py | 21 +- archivebox/tests/test_cli_snapshot.py | 346 +++++- archivebox/tests/test_cli_status.py | 162 +-- archivebox/tests/test_cli_tag.py | 18 + archivebox/tests/test_cli_update.py | 108 +- .../test_cli_update_reindex_snapshots.py | 18 +- archivebox/tests/test_cli_version.py | 77 +- archivebox/tests/test_config.py | 172 --- ...tention.py => test_config_DELETE_AFTER.py} | 39 +- archivebox/tests/test_config_MAX_limits.py | 295 +++++ archivebox/tests/test_config_ONLY_NEW.py | 87 ++ ...est_title.py => test_config_SAVE_TITLE.py} | 50 +- archivebox/tests/test_config_URL_filters.py | 60 + archivebox/tests/test_core_config.py | 5 + archivebox/tests/test_crawl.py | 175 --- archivebox/tests/test_crawl_pause.py | 253 +--- archivebox/tests/test_crawl_runner.py | 214 +--- archivebox/tests/test_crawl_service.py | 34 +- archivebox/tests/test_frozen_crawl_config.py | 39 +- archivebox/tests/test_machine_service.py | 14 +- archivebox/tests/test_migrations_04_to_09.py | 28 +- archivebox/tests/test_migrations_07_to_09.py | 58 +- archivebox/tests/test_migrations_08_to_09.py | 86 +- archivebox/tests/test_migrations_fresh.py | 62 +- archivebox/tests/test_recursive_crawl.py | 114 +- archivebox/tests/test_search.py | 16 +- .../tests/test_server_security_browser.py | 168 +-- archivebox/tests/test_snapshot.py | 180 --- archivebox/tests/test_snapshot_pause.py | 220 +--- archivebox/tests/test_snapshot_service.py | 146 +-- archivebox/tests/test_tag_admin.py | 203 --- archivebox/tests/test_takeover_util.py | 898 +++++++++++++- archivebox/tests/test_test_harness.py | 5 +- archivebox/tests/test_ui_add_view_runtime.py | 28 +- archivebox/tests/test_ui_admin_apitoken.py | 17 + .../tests/test_ui_admin_archiveresult.py | 35 + ..._crawl_admin.py => test_ui_admin_crawl.py} | 349 ++---- archivebox/tests/test_ui_admin_machine.py | 194 +++ ...sona_admin.py => test_ui_admin_persona.py} | 37 +- ...min_views.py => test_ui_admin_snapshot.py} | 841 +------------ archivebox/tests/test_ui_admin_tag.py | 35 + archivebox/tests/test_ui_admin_user.py | 27 + archivebox/tests/test_ui_live_progress.py | 381 ++++++ archivebox/tests/test_ui_public_snapshot.py | 75 ++ archivebox/tests/test_ui_savepagenow.py | 98 +- .../commands/supervisord_watchdog.py | 2 +- archivebox/workers/supervisord_util.py | 6 +- bin/take_screenshot.js | 3 +- pyproject.toml | 1 + 159 files changed, 9896 insertions(+), 8135 deletions(-) delete mode 100644 archivebox/cli/archivebox_crawl_compat.py delete mode 100644 archivebox/cli/archivebox_snapshot_compat.py create mode 100644 archivebox/core/snapshot_status.py delete mode 100644 archivebox/tests/fixtures.py create mode 100644 archivebox/tests/test_api_archiveresult.py delete mode 100644 archivebox/tests/test_api_core.py create mode 100644 archivebox/tests/test_api_crawl.py create mode 100644 archivebox/tests/test_api_crud.py create mode 100644 archivebox/tests/test_api_personas.py create mode 100644 archivebox/tests/test_api_remove.py create mode 100644 archivebox/tests/test_api_search.py create mode 100644 archivebox/tests/test_api_v1_auth_check_api_token.py create mode 100644 archivebox/tests/test_api_v1_auth_get_api_token.py create mode 100644 archivebox/tests/test_api_v1_cli_add.py create mode 100644 archivebox/tests/test_api_v1_cli_remove.py create mode 100644 archivebox/tests/test_api_v1_cli_schedule.py create mode 100644 archivebox/tests/test_api_v1_cli_search.py create mode 100644 archivebox/tests/test_api_v1_cli_update.py create mode 100644 archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py create mode 100644 archivebox/tests/test_api_v1_core_any_id.py create mode 100644 archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py create mode 100644 archivebox/tests/test_api_v1_core_archiveresults.py create mode 100644 archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py create mode 100644 archivebox/tests/test_api_v1_core_snapshots.py create mode 100644 archivebox/tests/test_api_v1_core_snapshots_rss.py create mode 100644 archivebox/tests/test_api_v1_core_tag_tag_id.py create mode 100644 archivebox/tests/test_api_v1_core_tag_tag_id_rename.py create mode 100644 archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py create mode 100644 archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py create mode 100644 archivebox/tests/test_api_v1_core_tags.py create mode 100644 archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py create mode 100644 archivebox/tests/test_api_v1_core_tags_autocomplete.py create mode 100644 archivebox/tests/test_api_v1_core_tags_create.py create mode 100644 archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py create mode 100644 archivebox/tests/test_api_v1_core_tags_search.py create mode 100644 archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py create mode 100644 archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py create mode 100644 archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py create mode 100644 archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py create mode 100644 archivebox/tests/test_api_v1_crawls_crawls.py create mode 100644 archivebox/tests/test_api_v1_machine_binaries.py create mode 100644 archivebox/tests/test_api_v1_machine_binary_binary_id.py create mode 100644 archivebox/tests/test_api_v1_machine_binary_by_name_name.py create mode 100644 archivebox/tests/test_api_v1_machine_machine_current.py create mode 100644 archivebox/tests/test_api_v1_machine_machine_machine_id.py create mode 100644 archivebox/tests/test_api_v1_machine_machines.py create mode 100644 archivebox/tests/test_api_v1_personas_personas.py create mode 100644 archivebox/tests/test_api_v1_personas_sync.py create mode 100644 archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py create mode 100644 archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py create mode 100644 archivebox/tests/test_cli_binary.py create mode 100644 archivebox/tests/test_cli_machine.py create mode 100644 archivebox/tests/test_cli_mcp.py create mode 100644 archivebox/tests/test_cli_persona.py create mode 100644 archivebox/tests/test_cli_pluginmap.py create mode 100644 archivebox/tests/test_cli_process.py delete mode 100644 archivebox/tests/test_cli_real_flows.py create mode 100644 archivebox/tests/test_cli_search.py create mode 100644 archivebox/tests/test_cli_tag.py delete mode 100644 archivebox/tests/test_config.py rename archivebox/tests/{test_retention.py => test_config_DELETE_AFTER.py} (87%) create mode 100644 archivebox/tests/test_config_MAX_limits.py create mode 100644 archivebox/tests/test_config_ONLY_NEW.py rename archivebox/tests/{test_title.py => test_config_SAVE_TITLE.py} (51%) create mode 100644 archivebox/tests/test_config_URL_filters.py create mode 100644 archivebox/tests/test_core_config.py delete mode 100644 archivebox/tests/test_crawl.py delete mode 100644 archivebox/tests/test_snapshot.py delete mode 100644 archivebox/tests/test_tag_admin.py create mode 100644 archivebox/tests/test_ui_admin_apitoken.py create mode 100644 archivebox/tests/test_ui_admin_archiveresult.py rename archivebox/tests/{test_crawl_admin.py => test_ui_admin_crawl.py} (60%) create mode 100644 archivebox/tests/test_ui_admin_machine.py rename archivebox/tests/{test_persona_admin.py => test_ui_admin_persona.py} (86%) rename archivebox/tests/{test_ui_admin_views.py => test_ui_admin_snapshot.py} (51%) create mode 100644 archivebox/tests/test_ui_admin_tag.py create mode 100644 archivebox/tests/test_ui_admin_user.py create mode 100644 archivebox/tests/test_ui_live_progress.py create mode 100644 archivebox/tests/test_ui_public_snapshot.py diff --git a/archivebox/__main__.py b/archivebox/__main__.py index 125ae205..fe4b74f4 100755 --- a/archivebox/__main__.py +++ b/archivebox/__main__.py @@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r""" """ if __name__ == "__main__": - main(args=sys.argv[1:], stdin=sys.stdin) + main(args=sys.argv[1:]) diff --git a/archivebox/api/auth.py b/archivebox/api/auth.py index 68992ca7..70d38766 100644 --- a/archivebox/api/auth.py +++ b/archivebox/api/auth.py @@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U return user +def token_from_request(request: HttpRequest) -> str: + token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or "" + auth_header = request.headers.get("Authorization", "") + if not token and auth_header.lower().startswith("bearer "): + token = auth_header.split(None, 1)[1].strip() + return token + + +def authenticated_user_from_request(request: HttpRequest) -> User | None: + user = request.user + if user.is_authenticated and user.is_active: + return user + + token = token_from_request(request) + token_user = auth_using_token(token=token, request=request) if token else None + if token_user and token_user.is_active: + request.user = token_user + return token_user + return None + + def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None: """Given a username and password, check if they are valid and return the corresponding user""" user: User | None = None diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index 02ce46e0..09af4319 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -8,8 +8,11 @@ from enum import Enum from django.http import HttpRequest from ninja import Router, Schema +from ninja.errors import HttpError +from pydantic import Field from archivebox.misc.util import ansi_to_html +from archivebox.core.models import SnapshotQuerySet # from .auth import API_AUTH_METHODS @@ -21,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"]) # Schemas JSONType = list[Any] | dict[str, Any] | bool | int | str | None +FILTER_PATTERNS_EXAMPLES = [["https://example.com"]] class CLICommandResponseSchema(Schema): @@ -32,26 +36,11 @@ class CLICommandResponseSchema(Schema): stderr: str -class FilterTypeChoices(str, Enum): - exact = "exact" - substring = "substring" - regex = "regex" - domain = "domain" - tag = "tag" - timestamp = "timestamp" - - -class StatusChoices(str, Enum): - indexed = "indexed" - archived = "archived" - unarchived = "unarchived" - present = "present" - valid = "valid" - invalid = "invalid" - duplicate = "duplicate" - orphaned = "orphaned" - corrupted = "corrupted" - unrecognized = "unrecognized" +FilterTypeChoices = Enum( + "FilterTypeChoices", + {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES}, + type=str, +) class AddCommandSchema(Schema): @@ -71,14 +60,27 @@ class AddCommandSchema(Schema): index_only: bool = False -class UpdateCommandSchema(Schema): - resume: str | None = None +class SnapshotFilterCommandSchema(Schema): after: float | None = 0 - before: float | None = 999999999999999 + before: float | None = None filter_type: str | None = FilterTypeChoices.substring - filter_patterns: list[str] | None = ["https://example.com"] + filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES) + status: str | None = None + url__icontains: str | None = None + url__istartswith: str | None = None + tag: str | None = None + crawl_id: str | None = None + limit: int | None = None + sort: str | None = None + search: str | None = None + + +class UpdateCommandSchema(SnapshotFilterCommandSchema): + resume: str | None = None batch_size: int = 100 continuous: bool = False + index_only: bool = False + migrate_only: bool = False class ScheduleCommandSchema(Schema): @@ -97,24 +99,23 @@ class ScheduleCommandSchema(Schema): clear: bool = False -class ListCommandSchema(Schema): - filter_patterns: list[str] | None = ["https://example.com"] - filter_type: str = FilterTypeChoices.substring - status: StatusChoices = StatusChoices.indexed - after: float | None = 0 - before: float | None = 999999999999999 - sort: str = "bookmarked_at" +class ListCommandSchema(SnapshotFilterCommandSchema): as_json: bool = True as_html: bool = False as_csv: str | None = "timestamp,url" with_headers: bool = False -class RemoveCommandSchema(Schema): - after: float | None = 0 - before: float | None = 999999999999999 +class RemoveCommandSchema(SnapshotFilterCommandSchema): filter_type: str = FilterTypeChoices.exact - filter_patterns: list[str] | None = ["https://example.com"] + timeout: float = 60.0 + + +def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]: + kwargs = args.dict() + kwargs["filter_patterns"] = kwargs.get("filter_patterns") or [] + kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type + return kwargs @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]") @@ -165,24 +166,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema): @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]") def cli_update(request: HttpRequest, args: UpdateCommandSchema): - from archivebox.cli.archivebox_update import update + from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update + from archivebox.core.snapshot_status import normalize_snapshot_status - result = update( - filter_patterns=args.filter_patterns or [], - filter_type=args.filter_type or FilterTypeChoices.substring, - after=args.after, - before=args.before, - resume=args.resume, - batch_size=args.batch_size, - continuous=args.continuous, - stop_daemon_stack=False, + try: + status = normalize_snapshot_status(args.status) + except ValueError as err: + raise HttpError(400, str(err)) from err + + update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring) + update_kwargs["status"] = status + update_kwargs["stop_daemon_stack"] = False + + is_filtered_update = any( + (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"), ) + matched_snapshot_ids = [] + if is_filtered_update: + matched_snapshot_ids = [ + str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True) + ] + + update(**update_kwargs) stdout = request.__dict__.get("stdout") stderr = request.__dict__.get("stderr") return { "success": True, "errors": [], - "result": result, + "result": { + "matched_count": len(matched_snapshot_ids), + "snapshot_ids": matched_snapshot_ids, + } + if is_filtered_update + else None, "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "", "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "", } @@ -225,29 +241,35 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema): @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]") def cli_search(request: HttpRequest, args: ListCommandSchema): - from archivebox.cli.archivebox_search import search + from archivebox.cli.archivebox_snapshot import build_snapshot_queryset - result = search( - filter_patterns=args.filter_patterns, - filter_type=args.filter_type, - status=args.status, - after=args.after, - before=args.before, - sort=args.sort, - csv=args.as_csv, - json=args.as_json, - html=args.as_html, - with_headers=args.with_headers, - ) + search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring) + as_json = search_kwargs.pop("as_json") + as_html = search_kwargs.pop("as_html") + as_csv = search_kwargs.pop("as_csv") + with_headers = search_kwargs.pop("with_headers") + try: + snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by") + except ValueError as err: + raise HttpError(400, str(err)) from err result_format = "txt" - if args.as_json: + if as_json: result_format = "json" - result = json.loads(result) - elif args.as_html: + result = [ + json.loads(json.dumps(snapshot.to_dict(extended=True), default=str)) + for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500) + ] + elif as_html: result_format = "html" - elif args.as_csv: + result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500)) + elif as_csv: result_format = "csv" + cols = [col.strip() for col in as_csv.split(",") if col.strip()] + rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)] + result = "\n".join((",".join(cols), *rows) if with_headers else rows) + else: + result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500)) stdout = request.__dict__.get("stdout") stderr = request.__dict__.get("stderr") @@ -264,37 +286,23 @@ def cli_search(request: HttpRequest, args: ListCommandSchema): @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]") def cli_remove(request: HttpRequest, args: RemoveCommandSchema): from archivebox.cli.archivebox_remove import remove - from archivebox.cli.archivebox_search import get_snapshots from archivebox.core.models import Snapshot - filter_patterns = args.filter_patterns or [] - snapshots_to_remove = get_snapshots( - filter_patterns=filter_patterns, - filter_type=args.filter_type, - after=args.after, - before=args.before, - ) - removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)] + remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact) + timeout_arg = remove_kwargs.pop("timeout") + timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0) + snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs) - remove( + result = remove( yes=True, # no way to interactively ask for confirmation via API, so we force yes snapshots=snapshots_to_remove, - before=args.before, - after=args.after, - filter_type=args.filter_type, - filter_patterns=filter_patterns, + timeout=timeout, ) - - result = { - "removed_count": len(removed_snapshot_ids), - "removed_snapshot_ids": removed_snapshot_ids, - "remaining_snapshots": Snapshot.objects.count(), - } stdout = request.__dict__.get("stdout") stderr = request.__dict__.get("stderr") return { - "success": True, - "errors": [], + "success": bool(result["success"]), + "errors": [str(result["error"])] if result["error"] else [], "result": result, "result_format": "json", "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "", diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index b7f806a0..a90ed6ec 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -29,7 +29,7 @@ from ninja.errors import HttpError from archivebox.core.models import Snapshot, ArchiveResult, Tag from archivebox.core.permissions import public_snapshots_queryset -from archivebox.api.auth import auth_using_token +from archivebox.api.auth import authenticated_user_from_request from archivebox.config.common import get_config from archivebox.core.routes_util import build_web_url from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length @@ -49,9 +49,10 @@ from archivebox.core.tag_util import ( rename_tag as rename_tag_record, ) from archivebox.crawls.models import Crawl -from archivebox.api.v1_crawls import CrawlSchema +from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref from archivebox.search.config import get_search_mode, get_search_mode_backend from archivebox.search.query import apply_snapshot_search +from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status router = Router(tags=["Core Models"]) @@ -79,7 +80,7 @@ class CustomPagination(PaginationBase): def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params): limit = min(pagination.limit, 500) offset = pagination.offset or (pagination.page * limit) - total = queryset.count() + total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count() total_pages = math.ceil(total / limit) current_page = math.ceil(offset / (limit + 1)) items = queryset[offset : offset + limit] @@ -212,7 +213,10 @@ class ArchiveResultFilterSchema(FilterSchema): @paginate(CustomPagination) def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]): """List all ArchiveResult entries matching these filters.""" - return filters.filter(ArchiveResult.objects.all()).distinct() + queryset = filters.filter(ArchiveResult.objects.all()) + if filters.search or filters.snapshot_tag: + return queryset.distinct() + return queryset def _uuid_ref_query(field_name: str, ref: str) -> Q: @@ -854,6 +858,7 @@ class SnapshotFilterSchema(FilterSchema): modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None search: str | None = None search_mode: str | None = None + status: str | None = None url: Annotated[str | None, FilterLookup("url")] = None tag: Annotated[str | None, FilterLookup("tags__name")] = None title: Annotated[str | None, FilterLookup("title__icontains")] = None @@ -867,13 +872,20 @@ class SnapshotFilterSchema(FilterSchema): def filter_search_mode(self, value: str | None) -> Q: return Q() + def filter_status(self, value: str | None) -> Q: + return Q() + @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots") @paginate(CustomPagination) def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False): """List all Snapshot entries matching these filters.""" setattr(request, "with_archiveresults", with_archiveresults) - queryset = filters.filter(Snapshot.objects.all()).distinct() + try: + queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status) + except ValueError as err: + raise HttpError(400, str(err)) from err + queryset = filters.filter(queryset).distinct() query = (filters.search or "").strip() if not query: return queryset @@ -916,18 +928,16 @@ def get_snapshots_rss( def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True): """Get a specific Snapshot by id.""" setattr(request, "with_archiveresults", with_archiveresults) - queryset = Snapshot.objects.all() - try: - return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id)) - except Snapshot.DoesNotExist: - return queryset.get(_uuid_ref_query("id", snapshot_id)) + return _get_snapshot_by_ref(snapshot_id) @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot") def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): tags = normalize_tag_list(data.tags) - if data.status is not None and data.status not in Snapshot.StatusChoices.values: - raise HttpError(400, f"Invalid status: {data.status}") + try: + status = normalize_snapshot_status(data.status) + except ValueError as err: + raise HttpError(400, str(err)) from err if not data.url.strip(): raise HttpError(400, "URL is required") try: @@ -938,7 +948,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): raise HttpError(400, "depth must be between 0 and 4") if data.crawl_id: - crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get() + crawl = get_crawl_by_ref(data.crawl_id) crawl_tags = normalize_tag_list(crawl.tags_str.split(",")) tags = tags or crawl_tags else: @@ -955,7 +965,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): "depth": data.depth, "title": data.title, "timestamp": str(timezone.now().timestamp()), - "status": data.status or Snapshot.StatusChoices.QUEUED, + "status": status or Snapshot.StatusChoices.QUEUED, "retry_at": timezone.now(), } snapshot, _ = Snapshot.objects.get_or_create( @@ -968,10 +978,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): if data.title is not None and snapshot.title != data.title: snapshot.title = data.title update_fields.append("title") - if data.status is not None and snapshot.status != data.status: - if data.status not in Snapshot.StatusChoices.values: - raise HttpError(400, f"Invalid status: {data.status}") - snapshot.status = data.status + if status is not None and snapshot.status != status: + snapshot.status = status update_fields.append("status") if update_fields: update_fields.append("modified_at") @@ -992,10 +1000,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot") def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema): """Update a snapshot (e.g., set status=sealed to cancel queued work).""" - try: - snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id)) - except Snapshot.DoesNotExist: - snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get() + snapshot = _get_snapshot_by_ref(snapshot_id) payload = data.dict(exclude_unset=True) update_fields = ["modified_at"] @@ -1018,9 +1023,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS raise HttpError(400, f"Invalid action: {action}") if "status" in payload: - if payload["status"] not in Snapshot.StatusChoices.values: - raise HttpError(400, f"Invalid status: {payload['status']}") - snapshot.status = payload["status"] + try: + snapshot.status = normalize_snapshot_status(payload["status"]) + except ValueError as err: + raise HttpError(400, str(err)) from err if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload: snapshot.retry_at = None update_fields.append("status") @@ -1289,16 +1295,7 @@ def _public_tag_listing_enabled() -> bool: def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool: - user = request.user - if user.is_authenticated: - return True - - token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") - auth_header = request.headers.get("Authorization", "") - if not token and auth_header.lower().startswith("bearer "): - token = auth_header.split(None, 1)[1].strip() - - if token and auth_using_token(token=token, request=request): + if authenticated_user_from_request(request): return True return _public_tag_listing_enabled() diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py index ef043ede..711d7d65 100644 --- a/archivebox/api/v1_crawls.py +++ b/archivebox/api/v1_crawls.py @@ -24,7 +24,7 @@ from archivebox.config.common import get_config from archivebox.crawls.models import Crawl from archivebox.misc.util import filter_queryset_by_uuid_substring -from .auth import API_AUTH_METHODS, auth_using_token +from .auth import API_AUTH_METHODS, authenticated_user_from_request router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS) @@ -137,12 +137,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema): return crawl +def get_crawl_by_ref(crawl_id: str): + return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() + + @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl") def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False): """Get a specific Crawl by id.""" setattr(request, "with_snapshots", with_snapshots) setattr(request, "with_archiveresults", with_archiveresults) - crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() + crawl = get_crawl_by_ref(crawl_id) if crawl and as_rss: query = request.GET.copy() @@ -156,33 +160,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn def crawl_file(request: HttpRequest, crawl_id: str, path: str): # Try to resolve the crawl first; if it doesn't exist, return 404. try: - crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() + crawl = get_crawl_by_ref(crawl_id) except Crawl.DoesNotExist: raise HttpError(404, "Crawl not found") - # Determine the effective viewer: session user takes precedence, otherwise - # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or - # Authorization: Bearer ... (so that programmatic clients still work). - user = request.user - is_authenticated = bool(user.is_authenticated and user.is_active) - if not is_authenticated: - token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") - auth_header = request.headers.get("Authorization", "") - if not token and auth_header.lower().startswith("bearer "): - token = auth_header.split(None, 1)[1].strip() - token_user = auth_using_token(token=token, request=request) if token else None - if token_user and token_user.is_active: - user = token_user - is_authenticated = True - # Re-bind so is_admin_user() / ownership checks below see the token user. - setattr(request, "user", token_user) + user = authenticated_user_from_request(request) # Gate access using the same model as SnapshotView/can_view_snapshot: # admins always pass; owners can see their own crawls; otherwise the crawl # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls. if not is_admin_user(request): permissions = normalize_permissions(crawl.permissions) - is_owner = bool(is_authenticated and crawl.created_by_id == user.id) + is_owner = bool(user and crawl.created_by_id == user.id) if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}: raise HttpError(404, "Crawl not found") @@ -217,7 +206,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl") def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): """Update a crawl (e.g., set status=sealed to cancel queued work).""" - crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() + crawl = get_crawl_by_ref(crawl_id) payload = data.dict(exclude_unset=True) update_fields = ["modified_at"] @@ -259,7 +248,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl") def delete_crawl(request: HttpRequest, crawl_id: str): - crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() + crawl = get_crawl_by_ref(crawl_id) crawl_id_str = str(crawl.id) snapshot_count = crawl.snapshot_set.count() deleted_count, _ = crawl.delete() diff --git a/archivebox/api/v1_personas.py b/archivebox/api/v1_personas.py index 65945762..15036202 100644 --- a/archivebox/api/v1_personas.py +++ b/archivebox/api/v1_personas.py @@ -8,8 +8,10 @@ from uuid import UUID from django.db.models import Q from django.http import HttpRequest from ninja import Router, Schema +from ninja.pagination import paginate from pydantic import Field +from archivebox.api.v1_core import CustomPagination from archivebox.personas.importers import validate_persona_name from archivebox.personas.models import Persona @@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None: @router.get("/personas", response=list[PersonaSchema], url_name="get_personas") +@paginate(CustomPagination) def get_personas(request: HttpRequest): """List personas available on this ArchiveBox server.""" return Persona.objects.all().order_by("name") diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index bbb29ca5..9ea9992a 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -66,10 +66,6 @@ class ArchiveBoxGroup(click.Group): # Introspection commands "pluginmap": "archivebox.cli.archivebox_pluginmap.main", } - legacy_model_commands = { - "crawl": "archivebox.cli.archivebox_crawl_compat.main", - "snapshot": "archivebox.cli.archivebox_snapshot_compat.main", - } all_subcommands = { **meta_commands, **setup_commands, @@ -81,34 +77,29 @@ class ArchiveBoxGroup(click.Group): "import": "add", "archive": "add", } - legacy_model_subcommands = { - "crawl": {"create", "list", "update", "delete"}, - "snapshot": {"create", "list", "update", "delete"}, - } @classmethod def get_canonical_name(cls, cmd_name): return cls.renamed_commands.get(cmd_name, cmd_name) @classmethod - def _should_use_legacy_model_command(cls, cmd_name: str) -> bool: - if cmd_name not in cls.legacy_model_commands: - return False + def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool: + wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:]) + return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands) - try: - arg_idx = sys.argv.index(cmd_name) - except ValueError: - return False + @classmethod + def _setup_django_for_lazy_import(cls, cmd_name: str) -> None: + if not cls._needs_django_for_lazy_import(cmd_name): + return - remaining_args = sys.argv[arg_idx + 1 :] - if not remaining_args: - return False + from django.apps import apps - first_arg = remaining_args[0] - if first_arg in ("-h", "--help"): - return False + if apps.ready: + return - return first_arg not in cls.legacy_model_subcommands[cmd_name] + from archivebox.config.django import setup_django + + setup_django() def get_command(self, ctx, cmd_name): # handle renamed commands @@ -120,11 +111,9 @@ class ArchiveBoxGroup(click.Group): cmd_name = new_name ctx.invoked_subcommand = cmd_name - if self._should_use_legacy_model_command(cmd_name): - return self._lazy_load(self.legacy_model_commands[cmd_name]) - # handle lazy loading of commands if cmd_name in self.all_subcommands: + self._setup_django_for_lazy_import(cmd_name) return self._lazy_load(cmd_name) # fall-back to using click's default command lookup @@ -199,15 +188,12 @@ def cli(ctx, help=False): raise -def main(args=None, prog_name=None, stdin=None): +def main(args=None, prog_name=None): # show `docker run archivebox xyz` in help messages if running in docker IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes") IS_TTY = sys.stdin.isatty() prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox") - # stdin param allows passing input data from caller (used by __main__.py) - # currently not used by click-based CLI, but kept for backwards compatibility - previous_unraisablehook = sys.unraisablehook def ignore_shutdown_unraisable(unraisable): diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 3108fb86..6d2ea4c7 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -108,6 +108,7 @@ def add( # import models once django is set up from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.personas.models import Persona from archivebox.misc.logging_util import printable_filesize @@ -124,16 +125,28 @@ def add( url_list = [str(url).strip() for url in urls if str(url).strip()] if snapshot_ids and len(snapshot_ids) != len(url_list): raise ValueError("snapshot_ids length must match urls length") + admitted_urls: list[str] = [] + admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None + for index, url in enumerate(url_list): + if Snapshot.is_archivebox_internal_url(url): + print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]") + continue + admitted_urls.append(url) + if admitted_snapshot_ids is not None and snapshot_ids: + admitted_snapshot_ids.append(snapshot_ids[index]) # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt" sources_file.parent.mkdir(parents=True, exist_ok=True) - if snapshot_ids: + if admitted_snapshot_ids is not None: sources_file.write_text( - "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)), + "\n".join( + json.dumps({"url": url, "id": admitted_snapshot_ids[index], "tags": tag, "depth": 0}) + for index, url in enumerate(admitted_urls) + ), ) else: - sources_file.write_text("\n".join(url_list)) + sources_file.write_text("\n".join(admitted_urls)) # 2. Create a new Crawl with inline URLs cli_args = [*sys.argv] @@ -143,7 +156,6 @@ def add( timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S") - # Read URLs directly into crawl urls_content = sources_file.read_text() persona_name = (persona or "Default").strip() or "Default" plugins = plugins or "" @@ -214,8 +226,8 @@ def add( # Foreground mode: run full crawl runner until all work is done print("[green]\\[*] Starting crawl runner to process crawl...[/green]") from archivebox.machine.models import Process - from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed - from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process + from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed + from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url) exit_code = 0 @@ -223,15 +235,27 @@ def add( try: with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog(): while True: - standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR) + standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR) exit_code = run_runner_worker( ["--crawl-id", str(crawl.id)], name=f"worker_runner_add_{os.getpid()}", interactive_interrupts=True, ) + crawl.refresh_from_db(fields=["status", "retry_at"]) + if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED: + break + # A shared supervisord can be stopped by another + # foreground owner (for example `archivebox server` + # shutting down) and stop the one-shot add runner while + # this add's crawl is still runnable. Keep the add + # process alive so it can re-enter the normal + # foreground-runner ownership loop and finish its crawl. + supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None + if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone): + continue if exit_code == 0: break - if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR): + if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR): continue raise SystemExit(exit_code) except KeyboardInterrupt: diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py index 86b8632f..5dc50528 100644 --- a/archivebox/cli/archivebox_config.py +++ b/archivebox/cli/archivebox_config.py @@ -6,6 +6,7 @@ import sys import toml import rich_click as click from rich import print +from pathlib import Path from archivebox.misc.util import docstring, enforce_types from archivebox.misc.toml_util import CustomTOMLEncoder @@ -30,12 +31,17 @@ def config( from archivebox.misc.logging_util import printable_config from abx_plugins.plugins.base.utils import resolve_alias from archivebox.config.collection import write_config_file + from archivebox.config import CONSTANTS_CONFIG from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs from archivebox.plugins.discovery import discover_plugin_configs check_data_folder() FLAT_CONFIG = get_config().as_dict() + runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys() + readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)} + writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config} + readable_config = {**writable_config, **readonly_config} CONFIGS = get_all_configs() plugin_schemas = { plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict) @@ -56,18 +62,23 @@ def config( config_options = [ core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options ] - matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG} + matching_config = {key: readable_config[key] for key in config_options if key in readable_config} for config_section in CONFIGS.values(): aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias} for search_key in config_options: # search all aliases in the section for alias_key, key in aliases.items(): - if search_key.lower() in alias_key.lower(): + if key in readable_config and search_key.lower() in alias_key.lower(): matching_config[key] = dict(config_section)[key] # search all keys and values in the section for existing_key, value in dict(config_section).items(): + if existing_key in readable_config and ( + search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower() + ): + matching_config[existing_key] = value + for existing_key, value in readonly_config.items(): if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower(): matching_config[existing_key] = value @@ -79,14 +90,14 @@ def config( config_options = [ core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options ] - matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG} - failed_config = [key for key in config_options if key not in FLAT_CONFIG] + matching_config = {key: readable_config[key] for key in config_options if key in readable_config} + failed_config = [key for key in config_options if key not in readable_config] if failed_config: print("\n[red][X] These options failed to get[/red]") print(" {}".format("\n ".join(config_options))) raise SystemExit(1) else: - matching_config = FLAT_CONFIG + matching_config = readable_config # Display core config sections for config_section in CONFIGS.values(): @@ -100,12 +111,18 @@ def config( print(_format_toml(kv_in_section)) print("[grey53]################################################################[/grey53]") + readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config} + if readonly_keys: + print("[grey53]\\[CONSTANTS] # (read-only)[/grey53]") + print(_format_toml(readonly_keys)) + print("[grey53]################################################################[/grey53]") + plugin_keys = {} # Collect all plugin config keys for schema in plugin_schemas.values(): for key in schema.keys(): - if key in matching_config: + if key in matching_config and key in writable_config: plugin_keys[key] = matching_config[key] # Display all plugin config in single [PLUGINS] section @@ -135,7 +152,7 @@ def config( f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]", ) - if key in FLAT_CONFIG: + if key in writable_config: new_config[key] = val.strip() else: failed_options.append(line) diff --git a/archivebox/cli/archivebox_crawl_compat.py b/archivebox/cli/archivebox_crawl_compat.py deleted file mode 100644 index a9210ff8..00000000 --- a/archivebox/cli/archivebox_crawl_compat.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 - -__package__ = "archivebox.cli" -__command__ = "archivebox crawl" - -import sys - -import rich_click as click - -from archivebox.cli.archivebox_add import add, _collect_input_urls - - -@click.command(context_settings={"ignore_unknown_options": True}) -@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)") -@click.option("--tag", "-t", default="", help="Comma-separated tags to add") -@click.option("--status", "-s", default="queued", help="Initial status (default: queued)") -@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility") -@click.argument("urls", nargs=-1) -def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]): - """Backwards-compatible `archivebox crawl URL...` entrypoint.""" - del status, wait - add(_collect_input_urls(urls), depth=depth, tag=tag, bg=True) - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/archivebox/cli/archivebox_help.py b/archivebox/cli/archivebox_help.py index 812b78d4..390618c2 100755 --- a/archivebox/cli/archivebox_help.py +++ b/archivebox/cli/archivebox_help.py @@ -13,6 +13,10 @@ from rich.panel import Panel def _command_doc(cmd: str, import_path: str) -> str: + def first_doc_line(docstring: str | None) -> str: + lines = (docstring or "").splitlines() + return lines[0] if lines else "" + modname, _ = import_path.rsplit(".", 1) spec = importlib.util.find_spec(modname) if spec is None or spec.origin is None: @@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str: for name in (cmd, "main"): for node in tree.body: if isinstance(node, ast.FunctionDef) and node.name == name: - return (ast.get_docstring(node) or "").splitlines()[0] + return first_doc_line(ast.get_docstring(node)) - return (ast.get_docstring(tree) or "").splitlines()[0] + return first_doc_line(ast.get_docstring(tree)) def help() -> None: diff --git a/archivebox/cli/archivebox_list.py b/archivebox/cli/archivebox_list.py index 8010bade..1a1fe264 100644 --- a/archivebox/cli/archivebox_list.py +++ b/archivebox/cli/archivebox_list.py @@ -7,50 +7,15 @@ import sys import rich_click as click -from archivebox.cli.archivebox_snapshot import list_snapshots +from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options @click.command() -@click.option("--status", "-s", help="Filter by status (queued, started, sealed)") -@click.option("--url__icontains", help="Filter by URL contains") -@click.option("--url__istartswith", help="Filter by URL starts with") -@click.option("--tag", "-t", help="Filter by tag name") -@click.option("--crawl-id", help="Filter by crawl ID") -@click.option("--limit", "-n", type=int, help="Limit number of results") -@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at") -@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title") -@click.option("--with-headers", is_flag=True, help="Include column headers in structured output") -@click.option("--search", help="Search mode to use for the query") -@click.argument("query", nargs=-1) -def main( - status: str | None, - url__icontains: str | None, - url__istartswith: str | None, - tag: str | None, - crawl_id: str | None, - limit: int | None, - sort: str | None, - csv: str | None, - with_headers: bool, - search: str | None, - query: tuple[str, ...], -) -> None: +@snapshot_output_options +@snapshot_filter_options(default_filter_type="substring") +def main(**kwargs) -> None: """List Snapshots.""" - sys.exit( - list_snapshots( - status=status, - url__icontains=url__icontains, - url__istartswith=url__istartswith, - tag=tag, - crawl_id=crawl_id, - limit=limit, - sort=sort, - csv=csv, - with_headers=with_headers, - search=search, - query=" ".join(query), - ), - ) + sys.exit(list_snapshots(**kwargs)) if __name__ == "__main__": diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index d677e3c6..791c69e6 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -23,6 +23,7 @@ from archivebox.misc.logging_util import ( log_removal_finished, TimedProgress, ) +from archivebox.cli.archivebox_snapshot import snapshot_filter_options @enforce_types @@ -34,48 +35,54 @@ def remove( before: float | None = None, yes: bool = False, out_dir: Path = CONSTANTS.DATA_DIR, -) -> QuerySet: + timeout: float | None = None, + **kwargs, +) -> dict[str, object]: """Remove the specified URLs from the archive""" setup_django() check_data_folder() + timeout = float(timeout) if timeout is not None else None - from archivebox.cli.archivebox_search import get_snapshots + from archivebox.core.models import Snapshot + filter_kwargs = { + **kwargs, + "filter_patterns": filter_patterns, + "filter_type": filter_type, + "after": after, + "before": before, + } pattern_list = list(filter_patterns) log_list_started(pattern_list or None, filter_type) timer = TimedProgress(360, prefix=" ") try: - snapshots = get_snapshots( - snapshots=snapshots, - filter_patterns=pattern_list or None, - filter_type=filter_type, - after=after, - before=before, - ) + if snapshots is None: + snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs) + # Freeze the target set up-front so a concurrent daemon writing new + # snapshots can't extend the deletion under us, and so the cursor isn't + # held open across the per-row deletes below. + snapshot_pks = list(snapshots.values_list("pk", flat=True)) finally: timer.end() - if not snapshots.exists(): + if not snapshot_pks: log_removal_finished(0, 0) raise SystemExit(1) - log_list_finished(snapshots) - log_removal_started(snapshots, yes=yes) + if not yes: + log_list_finished(snapshots) + log_removal_started(snapshots, yes=False) - from archivebox.core.models import Snapshot from archivebox.search.query import flush_search_index - # Freeze the target set up-front so a concurrent daemon writing new - # snapshots can't extend the deletion under us, and so the cursor isn't - # held open across the per-row deletes below. - snapshot_pks = list(snapshots.values_list("pk", flat=True)) - to_remove = len(snapshot_pks) + started_at = time.monotonic() + deadline = started_at + timeout if timeout is not None else None # Search-index flush touches a separate backend (FTS / sonic), not the # main index.sqlite3 writer lock, so it's safe to do once up front. - flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks)) + flush_search_index(snapshots=snapshots) # Delete one snapshot at a time. Each ``.delete()`` is its own short # Django-atomic block, so the writer lock is released between rows and @@ -90,40 +97,59 @@ def remove( # own retry loop at this outer (non-atomic) level. Each attempt is a # fresh atomic; an exception cleanly rolls it back before we sleep. retry_interval = 1.0 - retry_timeout = 60.0 - for pk in snapshot_pks: - deadline = time.monotonic() + retry_timeout + deleted_snapshot_pks = [] + timed_out = False + timeout_error = "" + for index, pk in enumerate(snapshot_pks): + if deadline is not None and time.monotonic() >= deadline: + timed_out = True + timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining." + break while True: try: - Snapshot.objects.filter(pk=pk).delete() + deleted_count, _ = Snapshot.objects.filter(pk=pk).delete() + if deleted_count: + deleted_snapshot_pks.append(pk) break except OperationalError as err: - if "database is locked" not in str(err) or time.monotonic() >= deadline: + if "database is locked" not in str(err): raise - time.sleep(retry_interval) + remaining_time = deadline - time.monotonic() if deadline is not None else None + if remaining_time is not None and remaining_time <= 0: + timed_out = True + timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock." + break + time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval) + if timed_out: + break all_snapshots = Snapshot.objects.all() - log_removal_finished(all_snapshots.count(), to_remove) + remaining_count = all_snapshots.count() + deleted_snapshot_id_set = set(deleted_snapshot_pks) + remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set] + log_removal_finished(remaining_count, len(deleted_snapshot_pks)) - return all_snapshots + return { + "removed_count": len(deleted_snapshot_pks), + "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks], + "not_removed_count": len(remaining_snapshot_pks), + "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks], + "success": not timed_out, + "error": timeout_error, + "timeout": timeout, + } @click.command() @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm") -@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp") -@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp") -@click.option( - "--filter-type", - "-f", - type=click.Choice(("exact", "substring", "domain", "regex", "tag")), - default="exact", - help="Type of pattern matching to use when filtering URLs", -) -@click.argument("filter_patterns", nargs=-1) +@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots") +@snapshot_filter_options(default_filter_type="exact") @docstring(remove.__doc__) def main(**kwargs): """Remove the specified URLs from the archive""" - remove(**kwargs) + result = remove(**kwargs) + if not result["success"]: + raise SystemExit(124) if __name__ == "__main__": diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index 709d580f..e0908c1c 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -259,11 +259,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on from archivebox.config import CONSTANTS from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals from archivebox.machine.models import Machine, Process - from archivebox.core.takeover_util import enter_single_runner_gate - from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls + from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed + from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.services.runner import run_pending_crawls Machine.current() current = Process.current() + root_command = current.root + if daemon and root_command.process_type in ( + Process.TypeChoices.SERVER, + Process.TypeChoices.ADD, + Process.TypeChoices.UPDATE, + ): + # Server-owned daemon runners are persistent supervisor workers, but + # foreground add/update commands are allowed to borrow runner/sonic + # leadership without taking down Daphne. Waiting here keeps the worker + # on the normal runner path while preventing a server restart loop from + # immediately stealing the single-runner gate back from the newer CLI. + standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR) if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR): current.mark_exited() return 0 diff --git a/archivebox/cli/archivebox_search.py b/archivebox/cli/archivebox_search.py index e84f3698..c192bed9 100644 --- a/archivebox/cli/archivebox_search.py +++ b/archivebox/cli/archivebox_search.py @@ -3,255 +3,7 @@ __package__ = "archivebox.cli" __command__ = "archivebox search" -import sys -from pathlib import Path -from typing import TYPE_CHECKING -from collections.abc import Callable - -import rich_click as click - -from django.db.models import Q, QuerySet - -from archivebox.config import CONSTANTS -from archivebox.config.common import get_config -from archivebox.misc.logging import stderr -from archivebox.misc.util import enforce_types, docstring - -if TYPE_CHECKING: - from archivebox.core.models import Snapshot - -# Filter types for URL matching -LINK_FILTERS: dict[str, Callable[[str], Q]] = { - "exact": lambda pattern: Q(url=pattern), - "substring": lambda pattern: Q(url__icontains=pattern), - "regex": lambda pattern: Q(url__iregex=pattern), - "domain": lambda pattern: ( - Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}") - ), - "tag": lambda pattern: Q(tags__name=pattern), - "timestamp": lambda pattern: Q(timestamp=pattern), -} - -STATUS_CHOICES = ["indexed", "archived", "unarchived"] - - -def _apply_pattern_filters( - snapshots: QuerySet["Snapshot", "Snapshot"], - filter_patterns: list[str], - filter_type: str, -) -> QuerySet["Snapshot", "Snapshot"]: - filter_builder = LINK_FILTERS.get(filter_type) - if filter_builder is None: - stderr() - stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red") - raise SystemExit(2) - - query = Q() - for pattern in filter_patterns: - query |= filter_builder(pattern) - return snapshots.filter(query) - - -def _snapshots_to_json( - snapshots: QuerySet["Snapshot", "Snapshot"], - *, - with_headers: bool, -) -> str: - from datetime import datetime, timezone as tz - - from archivebox.config import VERSION - from archivebox.misc.util import to_json - - config = get_config() - main_index_header = ( - { - "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.", - "schema": "archivebox.index.json", - "copyright_info": config.FOOTER_INFO, - "meta": { - "project": "ArchiveBox", - "version": VERSION, - "git_sha": VERSION, - "website": "https://ArchiveBox.io", - "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki", - "source": "https://github.com/ArchiveBox/ArchiveBox", - "issues": "https://github.com/ArchiveBox/ArchiveBox/issues", - "dependencies": {}, - }, - } - if with_headers - else {} - ) - - snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)] - output: dict[str, object] | list[dict[str, object]] - if with_headers: - output = { - **main_index_header, - "num_links": len(snapshot_dicts), - "updated": datetime.now(tz.utc), - "last_run_cmd": sys.argv, - "links": snapshot_dicts, - } - else: - output = snapshot_dicts - - return to_json(output, indent=4, sort_keys=True) - - -def _snapshots_to_csv( - snapshots: QuerySet["Snapshot", "Snapshot"], - *, - cols: list[str], - with_headers: bool, -) -> str: - header = ",".join(cols) if with_headers else "" - rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)] - return "\n".join((header, *rows)) - - -def _snapshots_to_html( - snapshots: QuerySet["Snapshot", "Snapshot"], - *, - with_headers: bool, -) -> str: - from datetime import datetime, timezone as tz - - from django.template.loader import render_to_string - - from archivebox.config import VERSION - from archivebox.config.version import get_COMMIT_HASH - - config = get_config() - template = "static_index.html" if with_headers else "minimal_index.html" - snapshot_list = list(snapshots.iterator(chunk_size=500)) - - return render_to_string( - template, - { - "version": VERSION, - "git_sha": get_COMMIT_HASH() or VERSION, - "num_links": str(len(snapshot_list)), - "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"), - "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"), - "links": snapshot_list, - "FOOTER_INFO": config.FOOTER_INFO, - }, - ) - - -def get_snapshots( - snapshots: QuerySet["Snapshot", "Snapshot"] | None = None, - filter_patterns: list[str] | None = None, - filter_type: str = "substring", - after: float | None = None, - before: float | None = None, - out_dir: Path = CONSTANTS.DATA_DIR, -) -> QuerySet["Snapshot", "Snapshot"]: - """Filter and return Snapshots matching the given criteria.""" - from archivebox.core.models import Snapshot - - if snapshots is not None: - result = snapshots - else: - result = Snapshot.objects.all() - - if after is not None: - result = result.filter(timestamp__gte=after) - if before is not None: - result = result.filter(timestamp__lt=before) - if filter_patterns: - result = _apply_pattern_filters(result, filter_patterns, filter_type) - - # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir - result = result.select_related("crawl", "crawl__created_by") - - if not result.exists(): - stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow") - - return result - - -@enforce_types -def search( - filter_patterns: list[str] | None = None, - filter_type: str = "substring", - status: str = "indexed", - before: float | None = None, - after: float | None = None, - sort: str | None = None, - json: bool = False, - html: bool = False, - csv: str | None = None, - with_headers: bool = False, -): - """List, filter, and export information about archive entries""" - - if with_headers and not (json or html or csv): - stderr("[X] --with-headers requires --json, --html or --csv\n", color="red") - raise SystemExit(2) - - # Query DB directly - no filesystem scanning - snapshots = get_snapshots( - filter_patterns=list(filter_patterns) if filter_patterns else None, - filter_type=filter_type, - before=before, - after=after, - ) - - # Apply status filter - if status == "archived": - snapshots = snapshots.filter(downloaded_at__isnull=False) - elif status == "unarchived": - snapshots = snapshots.filter(downloaded_at__isnull=True) - # 'indexed' = all snapshots (no filter) - - if sort: - snapshots = snapshots.order_by(sort) - - # Export to requested format - if json: - output = _snapshots_to_json(snapshots, with_headers=with_headers) - elif html: - output = _snapshots_to_html(snapshots, with_headers=with_headers) - elif csv: - output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers) - else: - from archivebox.misc.logging_util import printable_folders - - # Convert to dict for printable_folders - folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots} - output = printable_folders(folders, with_headers) - - # Structured exports must be written directly to stdout. - # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output. - sys.stdout.write(output) - if not output.endswith("\n"): - sys.stdout.write("\n") - return output - - -@click.command() -@click.option( - "--filter-type", - "-f", - type=click.Choice(["search", *LINK_FILTERS.keys()]), - default="substring", - help="Pattern matching type for filtering URLs", -) -@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status") -@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp") -@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp") -@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at") -@click.option("--json", "-J", is_flag=True, help="Print output in JSON format") -@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)") -@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title") -@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output") -@click.help_option("--help", "-h") -@click.argument("filter_patterns", nargs=-1) -@docstring(search.__doc__) -def main(**kwargs): - return search(**kwargs) +from archivebox.cli.archivebox_list import main if __name__ == "__main__": diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index e6934558..65576ec1 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -313,6 +313,7 @@ def server( from archivebox.core.takeover_util import ( command_owns_runtime_stack, current_command, + foreground_runner_owner, runtime_stack_owner, standby_until_runtime_stack_needed, ) @@ -354,7 +355,10 @@ def server( ): while True: standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR) - older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) + older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner( + data_dir=CONSTANTS.DATA_DIR, + exclude_id=command.id, + ) takeover_components = active_supervisord_runtime_components(config=config) if older_owner and takeover_components: print( diff --git a/archivebox/cli/archivebox_snapshot.py b/archivebox/cli/archivebox_snapshot.py index 4d3a6388..5aaefcb0 100644 --- a/archivebox/cli/archivebox_snapshot.py +++ b/archivebox/cli/archivebox_snapshot.py @@ -35,9 +35,9 @@ from collections.abc import Iterable import rich_click as click from rich import print as rprint -from django.db.models import Case, IntegerField, QuerySet, When +from django.db.models import QuerySet -from archivebox.cli.cli_util import apply_filters +SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp") # ============================================================================= @@ -178,72 +178,63 @@ def create_snapshots( # ============================================================================= +def snapshot_filter_options(*, default_filter_type: str): + def decorate(func): + for decorator in reversed( + ( + click.option("--status", "-s", help="Filter by status (queued, started, sealed)"), + click.option("--url__icontains", help="Filter by URL contains"), + click.option("--url__istartswith", help="Filter by URL starts with"), + click.option("--tag", "-t", help="Filter by tag name"), + click.option("--crawl-id", help="Filter by crawl ID"), + click.option("--limit", "-n", type=int, help="Limit number of results"), + click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"), + click.option("--search", help="Search mode to use for positional query"), + click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"), + click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"), + click.option( + "--filter-type", + "-f", + type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES), + default=default_filter_type, + help="Type of pattern matching to use for positional filters", + ), + click.argument("filter_patterns", nargs=-1), + ), + ): + func = decorator(func) + return func + + return decorate + + +def snapshot_output_options(func): + for decorator in reversed( + ( + click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"), + click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"), + click.option("--html", "as_html", is_flag=True, help="Print output as HTML"), + click.option("--with-headers", is_flag=True, help="Include column headers in structured output"), + ), + ): + func = decorator(func) + return func + + def build_snapshot_queryset( - *, - status: str | None = None, - url__icontains: str | None = None, - url__istartswith: str | None = None, - tag: str | None = None, - crawl_id: str | None = None, - sort: str | None = None, - search: str | None = None, - query: str | None = None, - limit: int | None = None, + **kwargs, ) -> QuerySet: from archivebox.core.models import Snapshot - from archivebox.search.query import apply_snapshot_search - queryset = Snapshot.objects.order_by("-created_at") - queryset = apply_filters( - queryset, - { - "status": status, - "url__icontains": url__icontains, - "url__istartswith": url__istartswith, - "crawl_id": crawl_id, - }, - ) - - if tag: - queryset = queryset.filter(tags__name__iexact=tag) - - query = (query or "").strip() - if query: - try: - queryset = apply_snapshot_search( - queryset, - query, - search_mode=search, - ordering=("-created_at",) if not sort else None, - max_results=limit, - skip_backend_when_metadata_satisfies_limit=True, - include_metadata_for_forced_backend=True, - ) - except Exception as err: - rprint( - f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]", - file=sys.stderr, - ) - queryset = apply_snapshot_search(queryset, query, search_mode="meta") - - if sort: - queryset = queryset.order_by(sort) - - return queryset + return Snapshot.objects.order_by("-created_at").search(**kwargs) def list_snapshots( - status: str | None = None, - url__icontains: str | None = None, - url__istartswith: str | None = None, - tag: str | None = None, - crawl_id: str | None = None, - limit: int | None = None, - sort: str | None = None, csv: str | None = None, + as_json: bool = False, + as_html: bool = False, with_headers: bool = False, - search: str | None = None, - query: str | None = None, + **kwargs, ) -> int: """ List Snapshots as JSONL with optional filters. @@ -252,90 +243,71 @@ def list_snapshots( 0: Success (even if no results) """ from archivebox.misc.jsonl import write_record - from archivebox.core.models import Snapshot - if with_headers and not csv: - rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr) + output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html)) + if output_formats > 1: + rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr) + return 2 + if with_headers and not output_formats: + rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr) return 2 - is_tty = sys.stdout.isatty() and not csv + is_tty = sys.stdout.isatty() and not output_formats - queryset = build_snapshot_queryset( - status=status, - url__icontains=url__icontains, - url__istartswith=url__istartswith, - tag=tag, - crawl_id=crawl_id, - sort=sort, - search=search, - query=query, - limit=limit, - ) - - if not is_tty: - if limit: - limited_ids = list(queryset.values_list("id", flat=True)[:limit]) - preserved_order = Case( - *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)), - output_field=IntegerField(), - ) - queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order) - queryset = queryset.prefetch_related("tags") - elif limit: - queryset = queryset[:limit] + try: + queryset = build_snapshot_queryset(**kwargs) + except ValueError as err: + rprint(f"[red]{err}[/red]", file=sys.stderr) + return 2 count = 0 + if as_json: + queryset = queryset.prefetch_related("tags") + output = queryset.to_json(with_headers=with_headers) + sys.stdout.write(output) + if output and not output.endswith("\n"): + sys.stdout.write("\n") + rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr) + return 0 + + if as_html: + queryset = queryset.prefetch_related("tags") + output = queryset.to_html(with_headers=with_headers) + sys.stdout.write(output) + if output and not output.endswith("\n"): + sys.stdout.write("\n") + rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr) + return 0 + if csv: cols = [col.strip() for col in csv.split(",") if col.strip()] if not cols: rprint("[red]No CSV columns provided[/red]", file=sys.stderr) return 2 - rows: list[str] = [] if with_headers: - rows.append(",".join(cols)) - simple_cols = { - "id", - "crawl_id", - "url", - "title", - "timestamp", - "depth", - "status", - "fs_version", - "bookmarked_at", - "created_at", - "modified_at", - "retry_at", - "downloaded_at", - } - from archivebox.misc.util import to_json - - for snapshot in queryset.iterator(chunk_size=500): - if set(cols).issubset(simple_cols): - rows.append( - ",".join(to_json(snapshot.serializable_value(col), indent=None) for col in cols), - ) - else: - rows.append(snapshot.to_csv(cols=cols, separator=",")) + sys.stdout.write(",".join(cols)) + sys.stdout.write("\n") + for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500): + sys.stdout.write(snapshot.to_csv(cols=cols, separator=",")) + sys.stdout.write("\n") count += 1 - output = "\n".join(rows) - if output: - sys.stdout.write(output) - if not output.endswith("\n"): - sys.stdout.write("\n") rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr) return 0 - for snapshot in queryset: - if is_tty: - status_color = { - "queued": "yellow", - "started": "blue", - "sealed": "green", - }.get(snapshot.status, "dim") - rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}") - else: + if not is_tty: + for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500): write_record(snapshot.to_json()) + count += 1 + rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr) + return 0 + + for snapshot in queryset.iterator(chunk_size=500): + status_color = { + "queued": "yellow", + "started": "blue", + "sealed": "green", + }.get(snapshot.status, "dim") + rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}") count += 1 rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr) @@ -494,46 +466,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int): @main.command("list") -@click.option("--status", "-s", help="Filter by status (queued, started, sealed)") -@click.option("--url__icontains", help="Filter by URL contains") -@click.option("--url__istartswith", help="Filter by URL starts with") -@click.option("--tag", "-t", help="Filter by tag name") -@click.option("--crawl-id", help="Filter by crawl ID") -@click.option("--limit", "-n", type=int, help="Limit number of results") -@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at") -@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title") -@click.option("--with-headers", is_flag=True, help="Include column headers in structured output") -@click.option("--search", help="Search mode to use for the query") -@click.argument("query", nargs=-1) -def list_cmd( - status: str | None, - url__icontains: str | None, - url__istartswith: str | None, - tag: str | None, - crawl_id: str | None, - limit: int | None, - sort: str | None, - csv: str | None, - with_headers: bool, - search: str | None, - query: tuple[str, ...], -): +@snapshot_output_options +@snapshot_filter_options(default_filter_type="substring") +def list_cmd(**kwargs): """List Snapshots as JSONL.""" - sys.exit( - list_snapshots( - status=status, - url__icontains=url__icontains, - url__istartswith=url__istartswith, - tag=tag, - crawl_id=crawl_id, - limit=limit, - sort=sort, - csv=csv, - with_headers=with_headers, - search=search, - query=" ".join(query), - ), - ) + sys.exit(list_snapshots(**kwargs)) @main.command("update") diff --git a/archivebox/cli/archivebox_snapshot_compat.py b/archivebox/cli/archivebox_snapshot_compat.py deleted file mode 100644 index 62f684e0..00000000 --- a/archivebox/cli/archivebox_snapshot_compat.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -__package__ = "archivebox.cli" -__command__ = "archivebox snapshot" - -import sys - -import rich_click as click - -from archivebox.cli.archivebox_snapshot import create_snapshots - - -@click.command(context_settings={"ignore_unknown_options": True}) -@click.option("--tag", "-t", default="", help="Comma-separated tags to add") -@click.option("--status", "-s", default="queued", help="Initial status (default: queued)") -@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)") -@click.argument("urls", nargs=-1) -def main(tag: str, status: str, depth: int, urls: tuple[str, ...]): - """Backwards-compatible `archivebox snapshot URL...` entrypoint.""" - sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth)) - - -if __name__ == "__main__": - main() diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 3adb935c..75766074 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -15,6 +15,7 @@ from pathlib import Path import rich_click as click from archivebox.misc.util import enforce_types, docstring +from archivebox.cli.archivebox_snapshot import snapshot_filter_options if TYPE_CHECKING: from django.db.models import QuerySet @@ -47,51 +48,19 @@ def _get_search_indexing_plugins() -> list[str]: def _build_filtered_snapshots_queryset( - *, - filter_patterns: Iterable[str], - filter_type: str, - status: str | None = None, - url__icontains: str | None = None, - url__istartswith: str | None = None, - tag: str | None = None, - crawl_id: str | None = None, - limit: int | None = None, - sort: str | None = None, - search: str | None = None, - before: float | None = None, - after: float | None = None, - resume: str | None = None, + **kwargs, ): - from datetime import datetime + from archivebox.core.models import Snapshot from archivebox.cli.archivebox_snapshot import build_snapshot_queryset - filter_patterns = tuple(filter_patterns) - snapshots = build_snapshot_queryset( - status=status, - url__icontains=url__icontains, - url__istartswith=url__istartswith, - tag=tag, - crawl_id=crawl_id, - sort=sort, - search=search, - query=" ".join(filter_patterns) if search else None, - ) - - if filter_patterns and not search: - snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type) - - if before: - snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before)) - if after: - snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after)) - if resume: - snapshots = snapshots.filter(timestamp__lte=resume) - if not sort: - snapshots = snapshots.order_by("-timestamp") + limit = kwargs.pop("limit", None) + snapshots = build_snapshot_queryset(**kwargs) + if kwargs.get("resume"): + snapshots = snapshots.filter(timestamp__lte=kwargs["resume"]) snapshots = snapshots.select_related("crawl") - if limit: - limited_ids = list(snapshots.values_list("id", flat=True)[:limit]) - snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl") + if limit is not None and limit > 0: + snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit]) + snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl") return snapshots @@ -254,27 +223,29 @@ def update( from archivebox.machine.models import Process from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested from archivebox.core.takeover_util import ( - command_owns_runtime_stack, + command_owns_foreground_runner, current_command, ensure_daemon_stack, - standby_until_runtime_stack_needed, + standby_until_foreground_runner_needed, ) - from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process + from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR) def wait_for_turn() -> None: raise_if_shutdown_requested() - standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR) + standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR) raise_if_shutdown_requested() - def run_scoped_runner(*args: str) -> None: + def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None: while True: wait_for_turn() + if ensure_daemon_reason: + ensure_daemon_stack(reason=ensure_daemon_reason) exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}") if exit_code == 0: return - if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR): + if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR): continue raise SystemExit(exit_code) @@ -298,8 +269,6 @@ def update( try: wait_for_turn() - if stop_daemon_stack: - stop_existing_supervisord_process() with foreground_shutdown_signals(), foreground_parent_watchdog(): while True: @@ -466,7 +435,10 @@ def update( for snapshot_id in sorted(touched_snapshot_ids): run_scoped_runner("--snapshot-id", snapshot_id) else: - run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else [])) + run_scoped_runner( + *(["--maintenance-only"] if index_only or migrate_only else []), + ensure_daemon_reason="search indexing" if do_index else None, + ) if not continuous: break @@ -987,22 +959,11 @@ def print_index_stats(stats: dict[str, Any]) -> None: @click.command() @click.option("--resume", type=str, help="Resume from timestamp") -@click.option("--status", "-s", help="Filter by status (queued, started, sealed)") -@click.option("--url__icontains", help="Filter by URL contains") -@click.option("--url__istartswith", help="Filter by URL starts with") -@click.option("--tag", "-t", help="Filter by tag name") -@click.option("--crawl-id", help="Filter by crawl ID") -@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update") -@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at") -@click.option("--search", help="Search mode to use for positional query") -@click.option("--before", type=float, help="Only snapshots before timestamp") -@click.option("--after", type=float, help="Only snapshots after timestamp") -@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact") @click.option("--batch-size", type=int, default=500, help="Commit every N records") @click.option("--continuous", is_flag=True, help="Run continuously as background worker") @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content") @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state") -@click.argument("filter_patterns", nargs=-1) +@snapshot_filter_options(default_filter_type="exact") @docstring(update.__doc__) def main(**kwargs): from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals @@ -1010,6 +971,8 @@ def main(**kwargs): try: with foreground_shutdown_signals(), foreground_parent_watchdog(): update(**kwargs) + except ValueError as err: + raise click.BadParameter(str(err), param_hint="--status") from err except KeyboardInterrupt: raise SystemExit(130) from None diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index de9dbbbb..25b098f9 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -280,7 +280,7 @@ class SnapshotChangeList(SearchResultsChangeList): else: self.full_result_count = self.model_admin.get_paginator( request, - self.model._default_manager.all().order_by(), + self.model._default_manager.all().order_by("-id"), self.list_per_page, ).count self.show_full_result_count = True @@ -465,7 +465,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): ), ) - ordering = ["-timestamp"] + ordering = ["-id"] actions = [ "add_tags", "remove_tags", diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 86f72761..31afba9a 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -267,6 +267,22 @@ class SnapshotQuerySet(models.QuerySet): "tag": lambda pattern: models.Q(tags__name=pattern), "timestamp": lambda pattern: models.Q(timestamp=pattern), } + FILTER_TYPE_CHOICES = tuple(FILTER_TYPES) + FILTER_ARG_KEYS = ( + "after", + "before", + "filter_type", + "filter_patterns", + "status", + "url__icontains", + "url__istartswith", + "tag", + "crawl_id", + "limit", + "sort", + "search", + ) + SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"}) def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> "SnapshotQuerySet": """Filter snapshots by URL patterns using specified filter type""" @@ -283,6 +299,62 @@ class SnapshotQuerySet(models.QuerySet): raise SystemExit(2) return self.filter(q_filter) + def search(self, **kwargs) -> "SnapshotQuerySet": + from datetime import timezone as dt_timezone + + from archivebox.core.snapshot_status import filter_snapshots_by_status + from archivebox.search.query import apply_snapshot_search + + queryset = self + filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ()) + filter_type = kwargs.get("filter_type") or "substring" + query = kwargs.get("query") + if isinstance(query, (list, tuple)): + query = " ".join(str(part) for part in query) + query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip() + + field_names = {field.name for field in self.model._meta.get_fields()} + field_names.update(field.attname for field in self.model._meta.fields) + field_filters = { + key: value + for key, value in kwargs.items() + if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names + } + status = field_filters.pop("status", None) + queryset = filter_snapshots_by_status(queryset, status) + if field_filters: + queryset = queryset.filter(**field_filters) + if kwargs.get("tag"): + queryset = queryset.filter(tags__name__iexact=kwargs["tag"]) + if kwargs.get("before") is not None: + queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=dt_timezone.utc)) + if kwargs.get("after") is not None: + queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=dt_timezone.utc)) + + if query: + queryset = apply_snapshot_search( + queryset, + query, + search_mode=kwargs.get("search"), + ordering=("-created_at",) if not kwargs.get("sort") else None, + max_results=kwargs.get("limit"), + skip_backend_when_metadata_satisfies_limit=True, + include_metadata_for_forced_backend=True, + ) + elif filter_patterns: + queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type) + + if kwargs.get("sort"): + queryset = queryset.order_by(kwargs["sort"]) + elif not queryset.query.order_by: + queryset = queryset.order_by("-created_at") + + limit = kwargs.get("limit") + if limit is not None and limit > 0: + queryset = queryset[:limit] + + return queryset + # ========================================================================= # Export Methods # ========================================================================= @@ -690,6 +762,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW from archivebox.core.routes_util import ( get_admin_host, get_api_host, + get_base_host, get_listen_host, get_public_host, get_web_host, @@ -704,6 +777,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW protected_roots: set[tuple[str, str | None]] = set() for host_value in ( get_listen_host(config=config), + get_base_host(config=config), get_admin_host(config=config), get_web_host(config=config), get_api_host(config=config), @@ -3173,7 +3247,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW """Convert to CSV string""" data = self.to_dict() cols = cols or ["timestamp", "is_archived", "url"] - return separator.join(to_json(data.get(col, ""), indent=None).ljust(ljust) for col in cols) + invalid_cols = [col for col in dict.fromkeys(cols) if col not in data] + if invalid_cols: + supported_cols = ", ".join(sorted(data)) + raise ValueError(f"Invalid CSV field(s): {', '.join(invalid_cols)}\nSupported CSV fields: {supported_cols}") + return separator.join(to_json(data[col], indent=None).ljust(ljust) for col in cols) def write_json_details(self, out_dir: Path | str | None = None) -> None: """Write JSON index file for this snapshot to its output directory""" diff --git a/archivebox/core/recovery_util.py b/archivebox/core/recovery_util.py index 725f5d7f..aeb3b6ce 100644 --- a/archivebox/core/recovery_util.py +++ b/archivebox/core/recovery_util.py @@ -20,6 +20,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int "crawls_queued_without_retry_at": 0, "snapshots_queued_without_retry_at": 0, "archiveresults_backoff": 0, + "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0, "archiveresults_started_without_running_process": 0, "snapshots_started_without_running_results": 0, "crawls_started_with_due_snapshots": 0, @@ -73,6 +74,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int retry_at__isnull=True, ).update(retry_at=now, modified_at=now) cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now) + # Targeted plugin rows on final/paused Snapshots are scheduled through the + # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a + # sealed Snapshot and must not be interpreted as queued work just because + # old/synthetic ArchiveResult rows exist. If takeover kills the runner + # after it leases the Snapshot but before queued ArchiveResult rows finish, + # the rows remain QUEUED while retry_at sits in the future. Recovery runs + # only after this runner has won the single-runner gate, so it can safely + # unlock those stale plugin leases for immediate processing instead of + # waiting out the previous owner's full lock timeout. + queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED) + cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = ( + Snapshot.objects.filter( + id__in=queued_plugin_results.values("snapshot_id"), + status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED], + ) + .filter(retry_at__gt=now) + .update(retry_at=now, modified_at=now) + ) # Impossible state repair: STARTED ArchiveResults without a live Process # have no owner left to emit completion. Requeue only the result row; the # snapshot/crawl schedulers will pick up normal retry processing. diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 4d3f559f..403525d5 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -77,6 +77,8 @@ INSTALLED_APPS = [ "django_extensions", # provides Django Debug Toolbar (and other non-debug helpers) ] +DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST" + MIDDLEWARE = [ "archivebox.core.middleware.TimezoneMiddleware", diff --git a/archivebox/core/snapshot_status.py b/archivebox/core/snapshot_status.py new file mode 100644 index 00000000..f9df3808 --- /dev/null +++ b/archivebox/core/snapshot_status.py @@ -0,0 +1,25 @@ +__package__ = "archivebox.core" + +from django.db.models import QuerySet + + +def snapshot_status_values() -> tuple[str, ...]: + from archivebox.core.models import Snapshot + + return tuple(Snapshot.StatusChoices.values) + + +def normalize_snapshot_status(status: str | None) -> str | None: + value = str(status or "").strip().lower() + if not value: + return None + + valid_statuses = snapshot_status_values() + if value not in valid_statuses: + raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}") + return value + + +def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet: + value = normalize_snapshot_status(status) + return queryset.filter(status=value) if value else queryset diff --git a/archivebox/core/takeover_util.py b/archivebox/core/takeover_util.py index cecd1b94..d5b39eb1 100644 --- a/archivebox/core/takeover_util.py +++ b/archivebox/core/takeover_util.py @@ -18,10 +18,18 @@ def runtime_stack_owner_types(): from archivebox.machine.models import Process return ( - Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR, + ) + + +def foreground_runner_owner_types(): + from archivebox.machine.models import Process + + return ( + Process.TypeChoices.SERVER, Process.TypeChoices.ADD, + Process.TypeChoices.UPDATE, ) @@ -70,15 +78,13 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None): if exclude_id is not None: base_qs = base_qs.exclude(id=exclude_id) - top_level_types = ( - Process.TypeChoices.UPDATE, - Process.TypeChoices.SERVER, - Process.TypeChoices.ADD, - ) for qs in ( - base_qs.filter(process_type__in=top_level_types), + # Only server parents own HTTP runtime leadership. Foreground add/update + # commands can own runner/sonic components, but server startup must never + # wait behind them before binding Daphne. + base_qs.filter(process_type=Process.TypeChoices.SERVER), # A foreground `archivebox run` process is allowed to own the runtime - # stack when no server/update/add parent is alive. A runner launched by + # stack when no server/add parent is alive. A runner launched by # supervisord is only a child worker; after its parent is killed it must # not keep stealing leadership from the next foreground command. base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD), @@ -95,6 +101,30 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool: return bool(owner and owner.id == command.id) +def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None): + from archivebox.machine.models import Machine, Process + + machine = Machine.current() + qs = Process.objects.filter( + machine=machine, + status=Process.StatusChoices.RUNNING, + pwd=str(data_dir), + process_type__in=foreground_runner_owner_types(), + ) + if exclude_id is not None: + qs = qs.exclude(id=exclude_id) + for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50): + if proc.is_running: + return proc + proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0) + return None + + +def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool: + owner = foreground_runner_owner(data_dir=data_dir) + return bool(owner and owner.id == command.id) + + def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str: try: from archivebox.config.common import get_config @@ -108,9 +138,9 @@ def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str: if not names and owner is not None: from archivebox.machine.models import Process - if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}: + if owner.process_type == Process.TypeChoices.SERVER: names = ["orchestrator", "server"] - elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}: + elif owner.process_type == Process.TypeChoices.ORCHESTRATOR: names = ["orchestrator"] return ", ".join(dict.fromkeys(names)) or "runtime stack" @@ -292,3 +322,26 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva command.modified_at = timezone.now() command.save(update_fields=["modified_at"]) return {"resumed": announced, "previous_owner_pid": previous_owner_pid} + + +def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]: + from archivebox.workers.supervisord_util import reap_foreground_supervisord_process + + announced = False + previous_owner_pid = None + while not command_owns_foreground_runner(command, data_dir=data_dir): + reap_foreground_supervisord_process() + if not announced: + owner = foreground_runner_owner(data_dir=data_dir) + owner_pid = owner.pid if owner else "unknown" + previous_owner_pid = owner_pid + rprint( + f"[yellow][*] A newer archivebox process took over the orchestrator, sonic " + f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]", + file=sys.stderr, + ) + announced = True + time.sleep(interval) + command.modified_at = timezone.now() + command.save(update_fields=["modified_at"]) + return {"resumed": announced, "previous_owner_pid": previous_owner_pid} diff --git a/archivebox/crawls/admin.py b/archivebox/crawls/admin.py index c05859ee..150bde3d 100644 --- a/archivebox/crawls/admin.py +++ b/archivebox/crawls/admin.py @@ -943,7 +943,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): updated += len(batch) return updated - @action(label="Recrawl", description="Create a new crawl with the same settings") + @action(label="Recrawl", description="Create a new crawl with the same settings", methods=("POST",)) def recrawl(self, request, obj): """Duplicate this crawl as a new crawl with the same URLs and settings.""" diff --git a/archivebox/machine/admin.py b/archivebox/machine/admin.py index 9fc49093..4119b4c9 100644 --- a/archivebox/machine/admin.py +++ b/archivebox/machine/admin.py @@ -548,6 +548,7 @@ class ProcessAdmin(BaseModelAdmin): label="Kill", description="Kill this process if it is still running", attrs={"class": "deletelink"}, + methods=("POST",), ) def kill_process(self, request, obj): self._terminate_processes(request, [obj]) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 0f8cd81a..5913e8dd 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -56,7 +56,6 @@ from abxbus.event_bus import EventBus, get_current_event, in_handler_context from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError from archivebox.config.common import ArchiveBoxBaseConfig, normalize_runtime_config -from archivebox.core.recovery_util import recover_orchestrator_state from archivebox.misc.db import run_db_analyze_batch from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler @@ -930,15 +929,14 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout), ), ) - # Normal crawl shutdown drives cleanup synchronously so - # ProcessKillEvent handlers get their grace period. During - # OS-signal shutdown, asyncio is already cancelling tasks; - # keep the child event attached for any remaining bus tick, - # but do not call now()/wait() because the bus context may - # disappear before delivery and produce noisy shutdown - # exceptions instead of useful cleanup. - if not self._signal_abort_requested: - await _run_event_now(cleanup_event, crawl_setup_phase_timeout) + # Cleanup owns ProcessKillEvent emission for crawl-scoped + # setup hooks. Even during OS-signal shutdown we must drive + # it synchronously before bus teardown; otherwise daemon/bg + # setup hooks can outlive the foreground runner that + # launched them. _run_event_now() is already bounded by the + # crawl setup timeout and cleanup handlers provide their own + # hook-level grace periods. + await _run_event_now(cleanup_event, crawl_setup_phase_timeout) finally: cancel_watcher.cancel() await asyncio.gather(cancel_watcher, return_exceptions=True) @@ -2134,6 +2132,8 @@ def run_pending_crawls( if daemon: now_monotonic = time.monotonic() if now_monotonic - last_recovery_at >= 30.0: + from archivebox.core.recovery_util import recover_orchestrator_state + recover_orchestrator_state() last_recovery_at = now_monotonic # SQLite query plans degrade as the snapshot/archiveresult tables grow diff --git a/archivebox/templates/admin/crawls/crawl/change_form.html b/archivebox/templates/admin/crawls/crawl/change_form.html index d5a7452e..fb345268 100644 --- a/archivebox/templates/admin/crawls/crawl/change_form.html +++ b/archivebox/templates/admin/crawls/crawl/change_form.html @@ -1,4 +1,5 @@ {% extends "admin/change_form.html" %} +{% load add_preserved_filters from admin_urls %} {% block object-tools-items %} {% if original %} @@ -31,6 +32,12 @@ {% endif %} {% endif %} +{% for tool in objectactions %} +
  • + {% url tools_view_name pk=object_id tool=tool.name as action_url %} + {% include 'django_object_actions/action_trigger.html' %} +
    • +{% endfor %} {{ block.super }} {% endblock %} diff --git a/archivebox/templates/static/admin.css b/archivebox/templates/static/admin.css index 193f4c1e..b01b6f8e 100755 --- a/archivebox/templates/static/admin.css +++ b/archivebox/templates/static/admin.css @@ -134,13 +134,23 @@ body.change-list #content .object-tools { clear: both; } -#content .object-tools a:link, #content .object-tools a:visited { +#content .object-tools a:link, +#content .object-tools a:visited, +#content .object-tools form button { border-radius: 0px; background-color: #f5dd5d; color: #333; font-size: 12px; font-weight: 800; } +#content .object-tools form { + display: inline; + margin: 0; +} +#content .object-tools form button { + border: 0; + cursor: pointer; +} #content .object-tools a.addlink { background-blend-mode: difference; diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index 77fb0e6d..13e4fcde 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -2,7 +2,9 @@ import os import json +import re import secrets +import signal import socket import subprocess import sys @@ -11,7 +13,9 @@ import textwrap import time import shutil from datetime import timedelta +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from threading import Thread from types import SimpleNamespace from typing import Any from collections.abc import Callable @@ -21,8 +25,6 @@ import pytest import requests from django.utils import timezone -pytest_plugins = ["archivebox.tests.fixtures"] - REPO_ROOT = Path(__file__).resolve().parents[2] PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve() SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve() @@ -90,64 +92,168 @@ def _sync_archivebox_test_data_dir(data_dir: Path) -> None: # ============================================================================= +class ArchiveBoxCmdResult: + """Process-like result for completed and live ArchiveBox CLI commands.""" + + def __init__(self, args: list[str], process: subprocess.Popen) -> None: + self.args = args + self._process = process + self._stdout = None + self._stderr = None + + @property + def stdout(self): + if self._stdout is None: + return self._process.stdout + return self._stdout + + @property + def stderr(self): + if self._stderr is None: + return self._process.stderr + return self._stderr + + @property + def stdin(self): + return self._process.stdin + + @property + def returncode(self) -> int | None: + return self._process.returncode + + @property + def pid(self) -> int | None: + return self._process.pid + + def poll(self) -> int | None: + return self._process.poll() + + def wait(self, timeout: float | None = None) -> int | None: + return self._process.wait(timeout=timeout) + + def communicate(self, input=None, timeout: float | None = None): + self._stdout, self._stderr = self._process.communicate(input=input, timeout=timeout) + return self._stdout, self._stderr + + def terminate(self) -> None: + self._process.terminate() + + def kill(self) -> None: + self._process.kill() + + def send_signal(self, sig: int) -> None: + self._process.send_signal(sig) + + def run_archivebox_cmd( args: list[str], - data_dir: Path, - stdin: str | None = None, + *, + cwd: Path | None = None, + input: str | bytes | None = None, timeout: int = 60, env: dict[str, str] | None = None, -) -> tuple[str, str, int]: - """ - Run archivebox command via subprocess, return (stdout, stderr, returncode). + check: bool = False, + text: bool = True, + capture_output: bool = True, + stdout: Any = None, + stderr: Any = None, + stdin: Any = None, + wait: bool = True, + start_new_session: bool = False, + default_cli_env: bool = False, + disable_extractors: bool = False, + replace_env: bool = False, +) -> ArchiveBoxCmdResult: + """Run an ArchiveBox CLI command under test isolation.""" + cwd = cwd or Path.cwd() + cmd = ["archivebox", *args] - Args: - args: Command arguments (e.g., ['crawl', 'create', 'https://example.com']) - data_dir: The DATA_DIR to use - stdin: Optional string to pipe to stdin - timeout: Command timeout in seconds - env: Additional environment variables + _assert_not_repo_path(cwd, label="cwd") - Returns: - Tuple of (stdout, stderr, returncode) - """ - cmd = [sys.executable, "-m", "archivebox"] + args + run_env: dict[str, str] | None = None + if default_cli_env or disable_extractors or env is not None: + run_env = {} if replace_env else os.environ.copy() + if default_cli_env: + run_env["USE_COLOR"] = "False" + run_env["SHOW_PROGRESS"] = "False" + if disable_extractors: + run_env.update( + { + "SAVE_ARCHIVEDOTORG": "False", + "SAVE_TITLE": "False", + "SAVE_FAVICON": "False", + "SAVE_WGET": "False", + "SAVE_WARC": "False", + "SAVE_PDF": "False", + "SAVE_SCREENSHOT": "False", + "SAVE_DOM": "False", + "SAVE_SINGLEFILE": "False", + "SAVE_READABILITY": "False", + "SAVE_MERCURY": "False", + "SAVE_GIT": "False", + "SAVE_YTDLP": "False", + "SAVE_HEADERS": "False", + "SAVE_HTMLTOTEXT": "False", + }, + ) + if env: + run_env.update(env) - _assert_not_repo_path(data_dir, label="cwd") - base_env = os.environ.copy() - base_env["USE_COLOR"] = "False" - base_env["SHOW_PROGRESS"] = "False" - # Disable slow extractors for faster tests - base_env["SAVE_ARCHIVEDOTORG"] = "False" - base_env["SAVE_TITLE"] = "False" - base_env["SAVE_FAVICON"] = "False" - base_env["SAVE_WGET"] = "False" - base_env["SAVE_WARC"] = "False" - base_env["SAVE_PDF"] = "False" - base_env["SAVE_SCREENSHOT"] = "False" - base_env["SAVE_DOM"] = "False" - base_env["SAVE_SINGLEFILE"] = "False" - base_env["SAVE_READABILITY"] = "False" - base_env["SAVE_MERCURY"] = "False" - base_env["SAVE_GIT"] = "False" - base_env["SAVE_YTDLP"] = "False" - base_env["SAVE_HEADERS"] = "False" - base_env["SAVE_HTMLTOTEXT"] = "False" + _assert_safe_runtime_paths(cwd=cwd, env=run_env or os.environ) - if env: - base_env.update(env) + if stdin is not None: + assert input is None, "pass either input or stdin, not both" + if wait: + input = stdin + if isinstance(input, str): + text = True - _assert_safe_runtime_paths(cwd=data_dir, env=base_env) - result = subprocess.run( + if capture_output: + stdout = subprocess.PIPE if stdout is None else stdout + stderr = subprocess.PIPE if stderr is None else stderr + + process = subprocess.Popen( cmd, - input=stdin, - capture_output=True, - text=True, - cwd=data_dir, - env=base_env, - timeout=timeout, + stdin=subprocess.PIPE if wait and input is not None else stdin, + stdout=stdout, + stderr=stderr, + text=text, + cwd=cwd, + env=run_env, + start_new_session=start_new_session, ) + result = ArchiveBoxCmdResult(cmd, process) - return result.stdout, result.stderr, result.returncode + if wait: + try: + result.communicate(input=input, timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + result.communicate() + raise + if check and result.returncode: + raise subprocess.CalledProcessError( + result.returncode, + cmd, + output=result.stdout, + stderr=result.stderr, + ) + + return result + + +def find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: + candidates = {snapshot_id} + if len(snapshot_id) == 32: + candidates.add(f"{snapshot_id[:8]}-{snapshot_id[8:12]}-{snapshot_id[12:16]}-{snapshot_id[16:20]}-{snapshot_id[20:]}") + elif len(snapshot_id) == 36 and "-" in snapshot_id: + candidates.add(snapshot_id.replace("-", "")) + + for needle in candidates: + for path in data_dir.rglob(needle): + if path.is_dir(): + return path + return None # ============================================================================= @@ -251,58 +357,134 @@ def hermetic_lib_dir(tmp_path, monkeypatch): @pytest.fixture -def initialized_archive(isolated_data_dir): +def initialized_archive(tmp_path): """ Initialize ArchiveBox archive in isolated directory. Runs `archivebox init` via subprocess to set up database and directories. """ - stdout, stderr, returncode = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["init", "--quick"], - data_dir=isolated_data_dir, + cwd=tmp_path, timeout=60, + default_cli_env=True, + disable_extractors=True, ) + stderr, returncode = _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox init failed: {stderr}" - return isolated_data_dir + return tmp_path @pytest.fixture -def archivebox_daemon_server(tmp_path, process, free_tcp_port_factory): +def recursive_test_site(): + pages = { + "/": """ + + + Root + + + + About + Blog + Contact + + + """.strip().encode("utf-8"), + "/about": """ + + + Deep About + + + """.strip().encode("utf-8"), + "/blog": """ + + + Deep Blog + + + """.strip().encode("utf-8"), + "/contact": """ + + + Deep Contact + + + """.strip().encode("utf-8"), + "/deep/about": b"

    Deep About

    ", + "/deep/blog": b"

    Deep Blog

    ", + "/deep/contact": b"

    Deep Contact

    ", + "/favicon.ico": b"test-icon", + } + + class RecursiveHandler(BaseHTTPRequestHandler): + def do_GET(self): + body = pages.get(self.path) + if body is None: + self.send_response(404) + self.end_headers() + return + + self.send_response(200) + if self.path.endswith(".ico"): + self.send_header("Content-Type", "image/x-icon") + else: + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), RecursiveHandler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + base_url = f"http://127.0.0.1:{server.server_address[1]}" + yield { + "base_url": base_url, + "root_url": f"{base_url}/", + "child_urls": [f"{base_url}/about", f"{base_url}/blog", f"{base_url}/contact"], + "deep_urls": [f"{base_url}/deep/about", f"{base_url}/deep/blog", f"{base_url}/deep/contact"], + } + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.fixture +def archivebox_daemon_server(initialized_archive, free_tcp_port_factory): """ Start a real daemonized ArchiveBox server in this test's DATA_DIR and always stop its supervisord before the test exits. """ - assert process.returncode == 0, process.stderr started: list[tuple[Path, dict[str, str]]] = [] def start(**env_overrides: str): - env = os.environ.copy() - env.update( - { - "USE_COLOR": "False", - "SHOW_PROGRESS": "False", - "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1", - "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()), - **{key: str(value) for key, value in env_overrides.items()}, - }, + env = cli_env( + live=True, + SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1", + SEARCH_BACKEND_SONIC_PORT=str(free_tcp_port_factory()), + **{key: str(value) for key, value in env_overrides.items()}, ) port = free_tcp_port_factory() - result = subprocess.run( - [sys.executable, "-m", "archivebox", "server", "--daemonize", f"127.0.0.1:{port}"], - cwd=tmp_path, + result = run_archivebox_cmd( + ["server", "--daemonize", f"127.0.0.1:{port}"], + cwd=initialized_archive, env=env, - capture_output=True, - text=True, timeout=90, ) assert result.returncode == 0, result.stderr or result.stdout - started.append((tmp_path, env)) + started.append((initialized_archive, env)) return SimpleNamespace( - data_dir=tmp_path, + data_dir=initialized_archive, env=env, port=port, - worker_state=lambda: _archivebox_worker_state(tmp_path, env), - wait_for_workers=lambda names, timeout=45: _wait_for_archivebox_workers(tmp_path, env, names, timeout=timeout), + worker_state=lambda: _archivebox_worker_state(initialized_archive, env), + wait_for_workers=lambda names, timeout=45: _wait_for_archivebox_workers(initialized_archive, env, names, timeout=timeout), ) try: @@ -330,44 +512,307 @@ def wait_for_process(predicate: Callable[[psutil.Process, str], bool], *, timeou raise AssertionError("No matching live process found. Last seen:\n" + "\n".join(last_seen[-50:])) -# ============================================================================= -# CWD-based CLI Helpers (no DATA_DIR env) -# ============================================================================= +def pid_is_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True -def run_archivebox_cmd_cwd( - args: list[str], +def wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + if not pid_is_alive(pid): + return + time.sleep(0.1) + raise AssertionError(f"PID {pid} is still running") + + +def cleanup_process_group(group_pid: int | None, *child_pids: int | None) -> None: + if group_pid and pid_is_alive(group_pid): + try: + os.killpg(group_pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + try: + os.kill(group_pid, signal.SIGKILL) + except ProcessLookupError: + pass + for pid in child_pids: + if pid and pid_is_alive(pid): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def cli_env( + *, + port: int | None = None, + plugins_root: Path | None = None, + replace: bool = False, + disable_extractors: bool = False, + live: bool = False, + server: bool = False, + wget: bool = False, + **extra: str, +) -> dict[str, str]: + env = {} if replace else os.environ.copy() + env.update({"USE_COLOR": "False", "SHOW_PROGRESS": "False"}) + + if disable_extractors or live or server: + env.update( + { + "SAVE_ARCHIVEDOTORG": "False", + "SAVE_TITLE": "False", + "SAVE_FAVICON": "False", + "SAVE_WARC": "False", + "SAVE_PDF": "False", + "SAVE_SCREENSHOT": "False", + "SAVE_DOM": "False", + "SAVE_SINGLEFILE": "False", + "SAVE_READABILITY": "False", + "SAVE_MERCURY": "False", + "SAVE_GIT": "False", + "SAVE_YTDLP": "False", + "SAVE_HEADERS": "False", + "SAVE_HTMLTOTEXT": "False", + }, + ) + + if live: + env.update( + { + "TIMEOUT": "60", + "WGET_TIMEOUT": "45", + "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1", + "PARSE_HTML_URLS_ENABLED": "True", + "PARSE_DOM_OUTLINKS_ENABLED": "False", + "SEARCH_BACKEND_ENGINE": "sqlite", + }, + ) + + if server: + assert port is not None, "port is required when server=True" + env.update( + { + "PLUGINS": "wget", + "BIND_ADDR": f"127.0.0.1:{port}", + "BASE_URL": f"http://archivebox.localhost:{port}", + "ALLOWED_HOSTS": "*", + "PUBLIC_ADD_VIEW": "True", + "TIMEOUT": "30", + "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*|example\.com", + "SAVE_WGET": "True", + "USE_CHROME": "False", + }, + ) + + if wget: + env.update({"PLUGINS": "wget", "SAVE_WGET": "True"}) + + if plugins_root is not None: + env["ABX_PLUGINS_DIR"] = str(plugins_root) + + env.update(extra) + return env + + +def wait_for_port_open(host: str, port: int, *, timeout: float = 30.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection((host, port), timeout=0.25): + return + except OSError: + time.sleep(0.1) + raise AssertionError(f"server did not listen on {host}:{port}") + + +def wait_for_log(log_path: Path, text: str, *, timeout: float = 30.0) -> str: + deadline = time.time() + timeout + content = "" + while time.time() < deadline: + if log_path.exists(): + content = log_path.read_text(encoding="utf-8", errors="replace") + if text in content: + return content + time.sleep(0.1) + raise AssertionError(f"timed out waiting for {text!r} in {log_path}:\n{content}") + + +def wait_for_log_count(log_path: Path, text: str, count: int, *, timeout: float = 30.0) -> str: + deadline = time.time() + timeout + content = "" + while time.time() < deadline: + if log_path.exists(): + content = log_path.read_text(encoding="utf-8", errors="replace") + if content.count(text) >= count: + return content + time.sleep(0.1) + raise AssertionError(f"timed out waiting for {count} occurrences of {text!r} in {log_path}:\n{content}") + + +def wait_for_log_pattern(log_path: Path, pattern: str, *, timeout: float = 30.0) -> re.Match[str]: + deadline = time.time() + timeout + content = "" + while time.time() < deadline: + if log_path.exists(): + content = log_path.read_text(encoding="utf-8", errors="replace") + match = re.search(pattern, content) + if match: + return match + time.sleep(0.1) + raise AssertionError(f"timed out waiting for pattern {pattern!r} in {log_path}:\n{content}") + + +def supervisor_pid_from_log(log_path: Path) -> int: + content = log_path.read_text(encoding="utf-8", errors="replace") + matches = re.findall(r"Supervisord connected \(pid=(\d+)\)", content) + assert matches, content + return int(matches[-1]) + + +def worker_pid_from_log(log_path: Path, worker_name: str) -> int: + content = log_path.read_text(encoding="utf-8", errors="replace") + matches = re.findall(rf"Worker {re.escape(worker_name)}: started RUNNING \(pid (\d+),", content) + assert matches, content + return int(matches[-1]) + + +def wait_for_worker_pid_from_log(log_path: Path, worker_name: str, *, timeout: float = 45.0) -> int: + deadline = time.time() + timeout + last_error = "" + while time.time() < deadline: + try: + return worker_pid_from_log(log_path, worker_name) + except AssertionError as err: + last_error = str(err) + time.sleep(0.1) + raise AssertionError(last_error or f"timed out waiting for worker {worker_name!r} in {log_path}") + + +def pgrep_data_dir(data_dir: Path) -> list[str]: + result = subprocess.run(["pgrep", "-af", str(data_dir)], capture_output=True, text=True, timeout=5) + lines = [line for line in result.stdout.splitlines() if "pgrep -af" not in line] + + for runtime_root in (Path("/tmp/archivebox"), data_dir / "tmp"): + for config_path in runtime_root.glob("*/supervisord.conf"): + try: + config_text = config_path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + if str(data_dir) not in config_text: + continue + pid_path = config_path.with_name("supervisord.pid") + try: + pid = int(pid_path.read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + continue + if not pid_is_alive(pid): + continue + ps_line = subprocess.run( + ["ps", "-p", str(pid), "-o", "pid=,ppid=,command="], + capture_output=True, + text=True, + timeout=5, + ).stdout.strip() + if ps_line: + lines.append(ps_line) + + return sorted(set(lines)) + + +def assert_no_processes_for_data_dir(data_dir: Path, *, timeout: float = 10.0) -> None: + deadline = time.time() + timeout + remaining: list[str] = [] + while time.time() < deadline: + remaining = pgrep_data_dir(data_dir) + if not remaining: + return + time.sleep(0.25) + raise AssertionError("processes still reference test DATA_DIR:\n" + "\n".join(remaining)) + + +def kill_processes_for_data_dir(data_dir: Path) -> None: + for line in pgrep_data_dir(data_dir): + try: + pid = int(line.split(None, 1)[0]) + except (IndexError, ValueError): + continue + if pid != os.getpid(): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def start_archivebox_server( cwd: Path, - stdin: str | None = None, - timeout: int = 60, + *, + port: int, env: dict[str, str] | None = None, -) -> tuple[str, str, int]: - """ - Run archivebox command via subprocess using cwd as DATA_DIR (no DATA_DIR env). - Returns (stdout, stderr, returncode). - """ - cmd = [sys.executable, "-m", "archivebox"] + args + daemonize: bool | None = None, + log_name: str | None = None, + wait_for_log_text: str | None = "Tailing worker logs", +): + if daemonize is None: + daemonize = log_name is None - _assert_not_repo_path(cwd, label="cwd") - base_env = os.environ.copy() - base_env["USE_COLOR"] = "False" - base_env["SHOW_PROGRESS"] = "False" + args = ["server", f"127.0.0.1:{port}"] + if daemonize: + args.insert(1, "--daemonize") - if env: - base_env.update(env) - - _assert_safe_runtime_paths(cwd=cwd, env=base_env) - result = subprocess.run( - cmd, - input=stdin, - capture_output=True, - text=True, + log_path = cwd / log_name if log_name else None + log = log_path.open("w", encoding="utf-8") if log_path else None + proc = run_archivebox_cmd( + args, cwd=cwd, - env=base_env, - timeout=timeout, + env=env or cli_env(live=True), + stdout=log if log else None, + stderr=subprocess.STDOUT if log else None, + text=daemonize, + start_new_session=not daemonize, + wait=daemonize, ) + if log is not None: + log.close() + proc.log_path = log_path + if daemonize: + assert proc.returncode == 0, proc.stderr or proc.stdout + return proc + wait_for_port_open("127.0.0.1", port) + if log_path is not None and wait_for_log_text is not None: + wait_for_log(log_path, wait_for_log_text, timeout=30.0) + return proc - return result.stdout, result.stderr, result.returncode + +def stop_archivebox_process(proc: subprocess.Popen[str], sig=signal.SIGTERM, *, timeout: float = 15.0) -> str: + if proc.poll() is None: + try: + os.killpg(proc.pid, sig) + except (ProcessLookupError, OSError): + try: + os.kill(proc.pid, sig) + except ProcessLookupError: + pass + try: + stdout, _stderr = proc.communicate(timeout=timeout) + return stdout or "" + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except (ProcessLookupError, OSError): + try: + os.kill(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, _stderr = proc.communicate(timeout=5) + return stdout or "" def run_queued_crawls(cwd: Path, env: dict[str, str] | None = None, timeout: int = 180) -> None: @@ -376,21 +821,21 @@ import json from archivebox.crawls.models import Crawl print(json.dumps([str(crawl_id) for crawl_id in Crawl.objects.order_by("created_at").values_list("id", flat=True)])) """ - stdout, stderr, returncode = run_archivebox_cmd_cwd(["manage", "shell", "-c", script], cwd=cwd, timeout=60, env=env) + _cmd_result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60, env=env) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr or stdout crawl_ids = json.loads(stdout.strip().splitlines()[-1]) for crawl_id in crawl_ids: - stdout, stderr, returncode = run_archivebox_cmd_cwd(["run", f"--crawl-id={crawl_id}"], cwd=cwd, timeout=timeout, env=env) + _cmd_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=cwd, timeout=timeout, env=env) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox run --crawl-id={crawl_id} failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" def _run_archivebox_manage_shell(cwd: Path, env: dict[str, str], script: str, timeout: int = 60) -> str: - result = subprocess.run( - [sys.executable, "-m", "archivebox", "manage", "shell", "-c", script], + result = run_archivebox_cmd( + ["manage", "shell", "-c", script], cwd=cwd, env=env, - capture_output=True, - text=True, timeout=timeout, ) assert result.returncode == 0, result.stderr or result.stdout @@ -469,71 +914,161 @@ def run_python_cwd( # Server/API Integration Helpers # ============================================================================= +API_TEST_HOST = "api.archivebox.localhost:8000" +ADMIN_TEST_HOST = "admin.archivebox.localhost:8000" +PUBLIC_TEST_HOST = "public.archivebox.localhost:8000" +WEB_TEST_HOST = "web.archivebox.localhost:8000" + + +@pytest.fixture +def admin_user(request): + from django.contrib.auth import get_user_model + + username = f"admin_{abs(hash(request.node.nodeid))}" + return get_user_model().objects.create_superuser( + username=username, + email=f"{username}@example.com", + password="testpassword", + ) + + +@pytest.fixture +def admin_client(client, admin_user): + client.force_login(admin_user) + return client + + +@pytest.fixture +def crawl(admin_user, db): + from archivebox.crawls.models import Crawl + + return Crawl.objects.create( + urls="https://example.com\nhttps://example.org", + tags_str="alpha,beta", + created_by=admin_user, + ) + + +@pytest.fixture +def snapshot(crawl, db): + from archivebox.core.models import Snapshot + + return Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + ) + + +@pytest.fixture +def tagged_data(crawl, admin_user): + from archivebox.core.models import Snapshot, Tag + + tag = Tag.objects.create(name="Alpha Research", created_by=admin_user) + first = Snapshot.objects.create( + url="https://example.com/one", + title="Example One", + crawl=crawl, + ) + second = Snapshot.objects.create( + url="https://example.com/two", + title="Example Two", + crawl=crawl, + ) + first.tags.add(tag) + second.tags.add(tag) + return tag, [first, second] + + +@pytest.fixture +def api_admin_user(request): + from django.contrib.auth import get_user_model + + username = f"apiadmin_{abs(hash(request.node.nodeid))}" + return get_user_model().objects.create_superuser( + username=username, + email=f"{username}@example.com", + password="testpass123", + ) + + +@pytest.fixture +def api_token(api_admin_user): + from archivebox.api.auth import get_or_create_api_token + + token = get_or_create_api_token(api_admin_user) + assert token is not None + return token + + +@pytest.fixture +def api_headers(api_token) -> dict[str, str]: + return api_auth_headers(api_token.token, django_client=True) + + +def api_auth_headers(api_token: str, *, django_client: bool = False, port: int | None = None) -> dict[str, str]: + host = f"api.archivebox.localhost:{port}" if port is not None else API_TEST_HOST + if django_client: + return { + "HTTP_HOST": host, + "HTTP_X_ARCHIVEBOX_API_KEY": api_token, + } + return { + "Host": host, + "X-ArchiveBox-API-Key": api_token, + } + + +def wait_for_live_api(port: int, *, path: str = "/api/v1/docs"): + return wait_for_http(port, host=f"api.archivebox.localhost:{port}", path=path) + + +def live_api_request(port: int, method: str, path: str, *, api_token: str, timeout: int = 30, **kwargs): + return requests.request( + method, + f"http://127.0.0.1:{port}{path}", + headers=api_auth_headers(api_token, port=port), + timeout=timeout, + **kwargs, + ) + + +def api_client_request( + client, + method: str, + path: str, + *, + payload: dict[str, Any] | None = None, + api_token: str | None = None, + headers: dict[str, str] | None = None, + **kwargs, +): + request_kwargs = dict(kwargs) + if payload is not None: + request_kwargs["data"] = json.dumps(payload) + request_kwargs["content_type"] = "application/json" + if headers is None: + assert api_token is not None + headers = api_auth_headers(api_token, django_client=True) + request_kwargs.update(headers) + return getattr(client, method.lower())(path, **request_kwargs) + def init_archive(cwd: Path) -> None: - result = subprocess.run( - [sys.executable, "-m", "archivebox", "init", "--quick"], + result = run_archivebox_cmd( + ["init", "--quick"], cwd=cwd, - capture_output=True, - text=True, timeout=60, ) assert result.returncode == 0, result.stderr -def build_test_env(port: int, **extra: str) -> dict[str, str]: - env = os.environ.copy() - env.update( - { - "PLUGINS": "wget", - "BIND_ADDR": f"127.0.0.1:{port}", - "BASE_URL": f"http://archivebox.localhost:{port}", - "ALLOWED_HOSTS": "*", - "PUBLIC_ADD_VIEW": "True", - "USE_COLOR": "False", - "SHOW_PROGRESS": "False", - "TIMEOUT": "30", - "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*|example\.com", - "SAVE_ARCHIVEDOTORG": "False", - "SAVE_TITLE": "False", - "SAVE_FAVICON": "False", - "SAVE_WARC": "False", - "SAVE_PDF": "False", - "SAVE_SCREENSHOT": "False", - "SAVE_DOM": "False", - "SAVE_SINGLEFILE": "False", - "SAVE_READABILITY": "False", - "SAVE_MERCURY": "False", - "SAVE_GIT": "False", - "SAVE_YTDLP": "False", - "SAVE_HEADERS": "False", - "SAVE_HTMLTOTEXT": "False", - "SAVE_WGET": "True", - "USE_CHROME": "False", - }, - ) - env.update(extra) - return env - - def get_free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) return sock.getsockname()[1] -def start_server(cwd: Path, env: dict[str, str], port: int) -> None: - result = subprocess.run( - [sys.executable, "-m", "archivebox", "server", "--daemonize", f"127.0.0.1:{port}"], - cwd=cwd, - capture_output=True, - text=True, - env=env, - timeout=60, - ) - assert result.returncode == 0, result.stderr - - def stop_server(cwd: Path) -> None: script = textwrap.dedent( """ @@ -549,10 +1084,18 @@ def stop_server(cwd: Path) -> None: run_python_cwd(script, cwd=cwd, timeout=30) -def wait_for_http(port: int, host: str, path: str = "/", timeout: int = 30) -> requests.Response: +def wait_for_http( + port: int, + host: str, + path: str = "/", + timeout: float = 30.0, + process: subprocess.Popen[str] | None = None, +) -> requests.Response: deadline = time.time() + timeout last_exc = None while time.time() < deadline: + if process is not None and process.poll() is not None: + raise AssertionError(f"Server exited before becoming ready with code {process.returncode}") try: response = requests.get( f"http://127.0.0.1:{port}{path}", @@ -562,6 +1105,7 @@ def wait_for_http(port: int, host: str, path: str = "/", timeout: int = 30) -> r ) if response.status_code < 500: return response + last_exc = f"HTTP {response.status_code}" except requests.RequestException as exc: last_exc = exc time.sleep(0.5) @@ -886,14 +1430,15 @@ def real_archive_with_example(tmp_path_factory, request): if request.cls is not None: request.cls.data_dir = tmp_path - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["init", "--quick"], cwd=tmp_path, timeout=120, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox init failed: {stderr}" - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( [ "config", "--set", @@ -904,6 +1449,7 @@ def real_archive_with_example(tmp_path_factory, request): ], cwd=tmp_path, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox config failed: {stderr}" add_env = { @@ -915,12 +1461,13 @@ def real_archive_with_example(tmp_path_factory, request): system_browser = _find_system_browser() if system_browser: add_env["CHROME_BINARY"] = str(system_browser) - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["add", "--depth=0", "--plugins=responses", "https://example.com"], cwd=tmp_path, timeout=600, env=add_env, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox add failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" ready = wait_for_archive_outputs(tmp_path, "https://example.com", timeout=60) @@ -941,6 +1488,16 @@ def parse_jsonl_output(stdout: str) -> list[dict[str, Any]]: return Process.parse_records_from_text(stdout or "") +def stdout_lines(stdout: str) -> list[str]: + return [line for line in stdout.splitlines() if line.strip()] + + +def assert_jsonl_only(stdout: str) -> None: + lines = stdout_lines(stdout) + assert lines, "Expected stdout to contain JSONL records" + assert all(line.lstrip().startswith("{") for line in lines), stdout + + def assert_jsonl_contains_type(stdout: str, record_type: str, min_count: int = 1): """Assert output contains at least min_count records of type.""" records = parse_jsonl_output(stdout) diff --git a/archivebox/tests/fixtures.py b/archivebox/tests/fixtures.py deleted file mode 100644 index a674795f..00000000 --- a/archivebox/tests/fixtures.py +++ /dev/null @@ -1,121 +0,0 @@ -import os -import subprocess -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from threading import Thread - -import pytest - - -@pytest.fixture -def process(tmp_path): - process = subprocess.run( - ["archivebox", "init"], - capture_output=True, - cwd=tmp_path, - ) - return process - - -@pytest.fixture -def disable_extractors_dict(): - env = os.environ.copy() - env.update( - { - "SAVE_WGET": "false", - "SAVE_SINGLEFILE": "false", - "SAVE_READABILITY": "false", - "SAVE_MERCURY": "false", - "SAVE_HTMLTOTEXT": "false", - "SAVE_PDF": "false", - "SAVE_SCREENSHOT": "false", - "SAVE_DOM": "false", - "SAVE_HEADERS": "false", - "SAVE_GIT": "false", - "SAVE_YTDLP": "false", - "SAVE_ARCHIVEDOTORG": "false", - "SAVE_TITLE": "false", - "SAVE_FAVICON": "false", - "PLUGINS": "__archivebox_test_no_plugins__", - }, - ) - return env - - -@pytest.fixture -def recursive_test_site(): - pages = { - "/": """ - - - Root - - - - About - Blog - Contact - - - """.strip().encode("utf-8"), - "/about": """ - - - Deep About - - - """.strip().encode("utf-8"), - "/blog": """ - - - Deep Blog - - - """.strip().encode("utf-8"), - "/contact": """ - - - Deep Contact - - - """.strip().encode("utf-8"), - "/deep/about": b"

    Deep About

    ", - "/deep/blog": b"

    Deep Blog

    ", - "/deep/contact": b"

    Deep Contact

    ", - "/favicon.ico": b"test-icon", - } - - class _RecursiveHandler(BaseHTTPRequestHandler): - def do_GET(self): - body = pages.get(self.path) - if body is None: - self.send_response(404) - self.end_headers() - return - - self.send_response(200) - if self.path.endswith(".ico"): - self.send_header("Content-Type", "image/x-icon") - else: - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), _RecursiveHandler) - thread = Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - base_url = f"http://127.0.0.1:{server.server_address[1]}" - yield { - "base_url": base_url, - "root_url": f"{base_url}/", - "child_urls": [f"{base_url}/about", f"{base_url}/blog", f"{base_url}/contact"], - "deep_urls": [f"{base_url}/deep/about", f"{base_url}/deep/blog", f"{base_url}/deep/contact"], - } - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) diff --git a/archivebox/tests/migrations_helpers.py b/archivebox/tests/migrations_helpers.py index 113b6073..dc4a4d63 100644 --- a/archivebox/tests/migrations_helpers.py +++ b/archivebox/tests/migrations_helpers.py @@ -8,14 +8,12 @@ This module provides: - Helper functions to run archivebox commands and verify results """ -import os -import sys import json import sqlite3 -import subprocess from pathlib import Path from datetime import datetime, timezone +from archivebox.tests.conftest import cli_env, run_archivebox_cmd from archivebox.uuid_compat import uuid7 @@ -1020,41 +1018,21 @@ def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]: # ============================================================================= -def run_archivebox(data_dir: Path, args: list, timeout: int = 60, env: dict | None = None) -> subprocess.CompletedProcess: +def run_archivebox_migration_cmd(data_dir: Path, args: list, timeout: int = 60, env: dict | None = None): """Run archivebox command in subprocess with given data directory.""" - base_env = os.environ.copy() - base_env["USE_COLOR"] = "False" - base_env["SHOW_PROGRESS"] = "False" - # Disable ALL extractors for faster tests (can be overridden by env parameter) - base_env["PLUGINS"] = "__archivebox_test_no_plugins__" - base_env["SAVE_ARCHIVEDOTORG"] = "False" - base_env["SAVE_TITLE"] = "False" - base_env["SAVE_FAVICON"] = "False" - base_env["SAVE_WGET"] = "False" - base_env["SAVE_SINGLEFILE"] = "False" - base_env["SAVE_SCREENSHOT"] = "False" - base_env["SAVE_PDF"] = "False" - base_env["SAVE_DOM"] = "False" - base_env["SAVE_READABILITY"] = "False" - base_env["SAVE_MERCURY"] = "False" - base_env["SAVE_GIT"] = "False" - base_env["SAVE_YTDLP"] = "False" - base_env["SAVE_HEADERS"] = "False" - base_env["SAVE_HTMLTOTEXT"] = "False" - - # Override with any custom env vars + base_env = cli_env( + disable_extractors=True, + PLUGINS="__archivebox_test_no_plugins__", + ) if env: base_env.update(env) - cmd = [sys.executable, "-m", "archivebox"] + args - - return subprocess.run( - cmd, - capture_output=True, - text=True, + return run_archivebox_cmd( + args, env=base_env, - cwd=str(data_dir), + cwd=data_dir, timeout=timeout, + replace_env=True, ) diff --git a/archivebox/tests/test_api_archiveresult.py b/archivebox/tests/test_api_archiveresult.py new file mode 100644 index 00000000..06782923 --- /dev/null +++ b/archivebox/tests/test_api_archiveresult.py @@ -0,0 +1 @@ +# Tests moved to test_api_v1_core_archiveresults.py and test_api_v1_core_archiveresult_archiveresult_id.py. diff --git a/archivebox/tests/test_api_cli.py b/archivebox/tests/test_api_cli.py index 0ff51b3d..b4267b1b 100644 --- a/archivebox/tests/test_api_cli.py +++ b/archivebox/tests/test_api_cli.py @@ -1,155 +1 @@ -import os -import time - -import pytest -import requests - -from archivebox.core.models import Snapshot -from archivebox.crawls.models import Crawl -from archivebox.tests.test_orm_helpers import use_archivebox_db -from .conftest import ( - build_test_env, - create_admin_and_token, - get_free_port, - init_archive, - start_server, - stop_server, - wait_for_http, -) - -pytestmark = pytest.mark.django_db(transaction=True) - - -@pytest.mark.timeout(180) -def test_cli_api_add_search_update_remove_over_server(tmp_path): - os.chdir(tmp_path) - init_archive(tmp_path) - - port = get_free_port() - env = build_test_env(port, PUBLIC_INDEX="True") - api_token = create_admin_and_token(tmp_path) - api_headers = { - "Host": f"api.archivebox.localhost:{port}", - "X-ArchiveBox-API-Key": api_token, - } - target_url = "https://example.com/" - - try: - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - add_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/cli/add", - headers=api_headers, - json={ - "urls": [target_url], - "tag": "api-cli", - "depth": 0, - "parser": "url_list", - "plugins": "wget", - "update": True, - "overwrite": False, - "index_only": True, - }, - timeout=10, - ) - assert add_response.status_code == 200, add_response.text - add_payload = add_response.json() - assert add_payload["success"] is True - assert add_payload["result_format"] == "json" - assert add_payload["result"]["num_snapshots"] == 0 - crawl_id = add_payload["result"]["crawl_id"] - assert add_payload["result"]["snapshot_ids"] == [] - stop_server(tmp_path) - from archivebox.services.runner import run_crawl - - with use_archivebox_db(tmp_path): - run_crawl(crawl_id, show_progress=False) - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - deadline = time.time() + 180 - snapshot_id = None - while time.time() < deadline: - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.filter(crawl_id=crawl_id, url=target_url).first() - if snapshot is not None: - snapshot_id = str(snapshot.id) - break - time.sleep(1) - assert snapshot_id is not None - - search_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/cli/search", - headers=api_headers, - json={ - "filter_patterns": [target_url], - "filter_type": "exact", - "status": "indexed", - "sort": "bookmarked_at", - "as_json": True, - "as_html": False, - "as_csv": "", - "with_headers": False, - }, - timeout=10, - ) - assert search_response.status_code == 200, search_response.text - search_payload = search_response.json() - assert search_payload["success"] is True - assert search_payload["result_format"] == "json" - assert any(item["url"] == target_url for item in search_payload["result"]) - - update_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/cli/update", - headers=api_headers, - json={ - "resume": None, - "after": 0, - "before": 4102444800, - "filter_type": "exact", - "filter_patterns": [target_url], - "batch_size": 1, - "continuous": False, - }, - timeout=20, - ) - assert update_response.status_code == 200, update_response.text - assert update_response.json()["success"] is True - stop_server(tmp_path) - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - with use_archivebox_db(tmp_path): - crawl_obj = Crawl.objects.filter(pk=crawl_id).first() - crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None - - assert crawl is not None - assert crawl[0] == 0 - assert crawl[1] == "api-cli" - assert crawl[2]["INDEX_ONLY"] is True - - remove_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/cli/remove", - headers=api_headers, - json={ - "delete": True, - "after": 0, - "before": 4102444800, - "filter_type": "exact", - "filter_patterns": [target_url], - }, - timeout=20, - ) - assert remove_response.status_code == 200, remove_response.text - remove_payload = remove_response.json() - assert remove_payload["success"] is True - assert remove_payload["result"]["removed_count"] == 1 - assert snapshot_id in remove_payload["result"]["removed_snapshot_ids"] - - with use_archivebox_db(tmp_path): - snapshot_count = Snapshot.objects.filter(pk=snapshot_id).count() - - assert snapshot_count == 0 - finally: - stop_server(tmp_path) +# CLI endpoint tests moved to test_api_v1_cli_add.py and test_api_v1_cli_update.py. diff --git a/archivebox/tests/test_api_cli_schedule.py b/archivebox/tests/test_api_cli_schedule.py index 7aa6eee7..def03802 100644 --- a/archivebox/tests/test_api_cli_schedule.py +++ b/archivebox/tests/test_api_cli_schedule.py @@ -1,83 +1 @@ -import os -from io import StringIO - -import pytest -import requests -from django.contrib.auth import get_user_model -from django.test import RequestFactory - -from archivebox.api.v1_cli import ScheduleCommandSchema, cli_schedule -from archivebox.crawls.models import CrawlSchedule -from .conftest import ( - build_test_env, - create_admin_and_token, - get_free_port, - init_archive, - start_server, - stop_server, - wait_for_http, -) - -User = get_user_model() - - -@pytest.mark.django_db -def test_schedule_api_creates_schedule_via_view_request(): - user = User.objects.create_user( - username="api-user", - password="testpass123", - email="api@example.com", - ) - request = RequestFactory().post("/api/v1/cli/schedule") - request.user = user - setattr(request, "stdout", StringIO()) - setattr(request, "stderr", StringIO()) - args = ScheduleCommandSchema( - every="daily", - import_path="https://example.com/feed.xml", - quiet=True, - ) - - response = cli_schedule(request, args) - - assert response["success"] is True - assert response["result_format"] == "json" - assert CrawlSchedule.objects.count() == 1 - assert len(response["result"]["created_schedule_ids"]) == 1 - - -@pytest.mark.django_db(transaction=True) -@pytest.mark.timeout(180) -def test_api_v1_cli_schedule_creates_schedule_over_server(tmp_path, recursive_test_site): - os.chdir(tmp_path) - init_archive(tmp_path) - - port = get_free_port() - env = build_test_env(port) - api_token = create_admin_and_token(tmp_path) - - try: - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - response = requests.post( - f"http://127.0.0.1:{port}/api/v1/cli/schedule", - headers={ - "Host": f"api.archivebox.localhost:{port}", - "X-ArchiveBox-API-Key": api_token, - }, - json={ - "every": "daily", - "import_path": recursive_test_site["root_url"], - "quiet": True, - }, - timeout=10, - ) - - assert response.status_code == 200, response.text - payload = response.json() - assert payload["success"] is True - assert payload["result_format"] == "json" - assert len(payload["result"]["created_schedule_ids"]) == 1 - finally: - stop_server(tmp_path) +# CLI schedule endpoint tests moved to test_api_v1_cli_schedule.py. diff --git a/archivebox/tests/test_api_core.py b/archivebox/tests/test_api_core.py deleted file mode 100644 index 8843519c..00000000 --- a/archivebox/tests/test_api_core.py +++ /dev/null @@ -1,374 +0,0 @@ -import os -from datetime import timedelta - -import pytest -import requests -from django.core.files.uploadedfile import SimpleUploadedFile -from django.contrib.auth import get_user_model -from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart -from django.utils import timezone - -from archivebox.core.models import ArchiveResult, Snapshot, Tag -from archivebox.crawls.models import Crawl -from archivebox.tests.test_orm_helpers import use_archivebox_db -from .conftest import ( - build_test_env, - create_admin_and_token, - get_free_port, - init_archive, - start_server, - stop_server, - wait_for_http, -) - -pytestmark = pytest.mark.django_db(transaction=True) - - -def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client): - from archivebox.api.auth import get_or_create_api_token - - user = get_user_model().objects.create_superuser( - username="uploadapiadmin", - email="uploadapiadmin@example.com", - password="testpass123", - ) - api_token = get_or_create_api_token(user) - assert api_token is not None - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by=user, - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - active_retry_at = timezone.now() + timedelta(minutes=5) - active_snapshot = Snapshot.objects.create( - url="https://example.com/active", - crawl=crawl, - status=Snapshot.StatusChoices.STARTED, - retry_at=active_retry_at, - ) - sealed_snapshot = Snapshot.objects.create( - url="https://example.com/sealed", - crawl=crawl, - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - ) - - active_response = client.post( - "/api/v1/core/archiveresults", - { - "snapshot_id": str(active_snapshot.id), - "plugin": "chrome_extension_dom", - "hook_name": "on_Snapshot__archivebox_browser_extension_upload", - "status": ArchiveResult.StatusChoices.SUCCEEDED, - "output_str": "uploaded active snapshot output", - }, - HTTP_HOST="api.archivebox.localhost:8000", - HTTP_X_ARCHIVEBOX_API_KEY=api_token.token, - ) - assert active_response.status_code == 200, active_response.content - active_snapshot.refresh_from_db() - assert active_snapshot.status == Snapshot.StatusChoices.STARTED - assert active_snapshot.retry_at == active_retry_at - assert active_snapshot.downloaded_at is not None - - sealed_response = client.post( - "/api/v1/core/archiveresults", - { - "snapshot_id": str(sealed_snapshot.id), - "plugin": "chrome_extension_mhtml", - "hook_name": "on_Snapshot__archivebox_browser_extension_upload", - "status": ArchiveResult.StatusChoices.SUCCEEDED, - "output_str": "uploaded sealed snapshot output", - }, - HTTP_HOST="api.archivebox.localhost:8000", - HTTP_X_ARCHIVEBOX_API_KEY=api_token.token, - ) - assert sealed_response.status_code == 200, sealed_response.content - sealed_snapshot.refresh_from_db() - assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED - assert sealed_snapshot.retry_at is not None - assert sealed_snapshot.downloaded_at is not None - - -def test_archiveresult_patch_upload_finalizes_queued_result(client): - from archivebox.api.auth import get_or_create_api_token - - user = get_user_model().objects.create_superuser( - username="patchuploadapiadmin", - email="patchuploadapiadmin@example.com", - password="testpass123", - ) - api_token = get_or_create_api_token(user) - assert api_token is not None - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by=user, - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - snapshot = Snapshot.objects.create( - url="https://example.com/upload-patch", - crawl=crawl, - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="dom", - hook_name="on_Snapshot__archivebox_browser_extension_upload", - status=ArchiveResult.StatusChoices.QUEUED, - ) - - response = client.generic( - "PATCH", - f"/api/v1/core/archiveresult/{result.id}", - encode_multipart( - BOUNDARY, - { - "files": SimpleUploadedFile("output.html", b"uploaded", content_type="text/html"), - "output_paths": "output.html", - "output_str": "output.html", - }, - ), - content_type=MULTIPART_CONTENT, - HTTP_HOST="api.archivebox.localhost:8000", - HTTP_X_ARCHIVEBOX_API_KEY=api_token.token, - ) - assert response.status_code == 200, response.content - - result.refresh_from_db() - snapshot.refresh_from_db() - assert result.status == ArchiveResult.StatusChoices.SUCCEEDED - assert result.output_str == "output.html" - assert snapshot.status == Snapshot.StatusChoices.SEALED - assert snapshot.retry_at is not None - - -def test_crawl_cancel_api_defers_cleanup_to_runner(client): - from archivebox.api.auth import get_or_create_api_token - from archivebox.services.runner import run_due_crawl - - user = get_user_model().objects.create_superuser( - username="cancelapiadmin", - email="cancelapiadmin@example.com", - password="testpass123", - ) - api_token = get_or_create_api_token(user) - assert api_token is not None - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by=user, - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now() + timedelta(minutes=5), - ) - child = Snapshot.objects.create( - url="https://example.com/cancel-child", - crawl=crawl, - status=Snapshot.StatusChoices.STARTED, - retry_at=timezone.now() + timedelta(minutes=5), - ) - crawl.output_dir.mkdir(parents=True, exist_ok=True) - pid_file = crawl.output_dir / "cleanup-test.pid" - pid_file.write_text("12345") - - response = client.patch( - f"/api/v1/crawls/crawl/{crawl.id}", - {"action": "cancel"}, - content_type="application/json", - HTTP_HOST="api.archivebox.localhost:8000", - HTTP_X_ARCHIVEBOX_API_KEY=api_token.token, - ) - assert response.status_code == 200, response.content - - crawl.refresh_from_db() - child.refresh_from_db() - assert crawl.status == Crawl.StatusChoices.SEALED - assert crawl.retry_at is not None - assert crawl.retry_at <= timezone.now() - assert child.status == Snapshot.StatusChoices.STARTED - assert child.retry_at is not None - assert child.retry_at <= timezone.now() - assert pid_file.exists() - - assert run_due_crawl(crawl, lock_seconds=60) is True - crawl.refresh_from_db() - assert crawl.retry_at is None - assert not pid_file.exists() - - -@pytest.mark.timeout(180) -def test_core_api_crud_uses_token_auth_and_persists_side_effects_over_server(tmp_path, recursive_test_site): - os.chdir(tmp_path) - init_archive(tmp_path) - - port = get_free_port() - env = build_test_env(port, PUBLIC_INDEX="True") - api_token = create_admin_and_token(tmp_path) - api_headers = { - "Host": f"api.archivebox.localhost:{port}", - "X-ArchiveBox-API-Key": api_token, - } - - try: - start_server(tmp_path, env=env, port=port) - docs = wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - assert docs.status_code == 200 - openapi = wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/openapi.json") - assert openapi.status_code == 200 - paths = openapi.json()["paths"] - assert "/api/v1/core/snapshots" in paths - assert "/api/v1/crawls/crawls" in paths - - unauth = requests.get( - f"http://127.0.0.1:{port}/api/v1/crawls/crawls", - headers={"Host": f"api.archivebox.localhost:{port}"}, - timeout=10, - ) - assert unauth.status_code in (401, 403) - bad_auth = requests.get( - f"http://127.0.0.1:{port}/api/v1/crawls/crawls", - headers={"Host": f"api.archivebox.localhost:{port}", "X-ArchiveBox-API-Key": "bad-token"}, - timeout=10, - ) - assert bad_auth.status_code in (401, 403) - - crawl_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/crawls/crawls", - headers=api_headers, - json={ - "urls": [recursive_test_site["root_url"]], - "max_depth": 2, - "tags": ["api-depth-two"], - "label": "api crawl", - "notes": "created through REST API", - "config": { - "PLUGINS": "wget,parse_html_urls", - "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*", - "CRAWL_MAX_URLS": 7, - "CRAWL_MAX_SIZE": "0", - "SNAPSHOT_MAX_SIZE": "0", - }, - }, - timeout=10, - ) - assert crawl_response.status_code == 200, crawl_response.text - crawl_payload = crawl_response.json() - crawl_id = crawl_payload["id"] - assert crawl_payload["max_depth"] == 2 - assert crawl_payload["tags_str"] == "api-depth-two" - assert crawl_payload["config"]["PLUGINS"] == "wget,parse_html_urls" - assert crawl_payload["config"]["CRAWL_MAX_URLS"] == 7 - - snapshot_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/core/snapshots", - headers=api_headers, - json={ - "url": recursive_test_site["child_urls"][0], - "crawl_id": crawl_id, - "depth": 1, - "title": "API child snapshot", - "tags": ["api-child"], - "status": "queued", - }, - timeout=10, - ) - assert snapshot_response.status_code == 200, snapshot_response.text - snapshot_payload = snapshot_response.json() - snapshot_id = snapshot_payload["id"] - assert snapshot_payload["url"] == recursive_test_site["child_urls"][0] - assert snapshot_payload["tags"] == ["api-child"] - - patch_snapshot = requests.patch( - f"http://127.0.0.1:{port}/api/v1/core/snapshot/{snapshot_id}", - headers=api_headers, - json={"status": "sealed", "tags": ["api-child", "api-patched"]}, - timeout=10, - ) - assert patch_snapshot.status_code == 200, patch_snapshot.text - assert patch_snapshot.json()["status"] == "sealed" - assert set(patch_snapshot.json()["tags"]) == {"api-child", "api-patched"} - - tag_create = requests.post( - f"http://127.0.0.1:{port}/api/v1/core/tags/create/", - headers=api_headers, - json={"name": "api-extra"}, - timeout=10, - ) - assert tag_create.status_code == 200, tag_create.text - tag_id = tag_create.json()["tag_id"] - - add_tag = requests.post( - f"http://127.0.0.1:{port}/api/v1/core/tags/add-to-snapshot/", - headers=api_headers, - json={"snapshot_id": snapshot_id, "tag_id": tag_id}, - timeout=10, - ) - assert add_tag.status_code == 200, add_tag.text - remove_tag = requests.post( - f"http://127.0.0.1:{port}/api/v1/core/tags/remove-from-snapshot/", - headers=api_headers, - json={"snapshot_id": snapshot_id, "tag_name": "api-extra"}, - timeout=10, - ) - assert remove_tag.status_code == 200, remove_tag.text - - crawl_patch = requests.patch( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - json={"status": "sealed", "tags": ["api-sealed"]}, - timeout=10, - ) - assert crawl_patch.status_code == 200, crawl_patch.text - assert crawl_patch.json()["status"] == "sealed" - assert crawl_patch.json()["tags_str"] == "api-sealed" - - snapshots_list = requests.get( - f"http://127.0.0.1:{port}/api/v1/core/snapshots?tag=api-patched&with_archiveresults=true", - headers=api_headers, - timeout=10, - ) - assert snapshots_list.status_code == 200, snapshots_list.text - snapshot_items = snapshots_list.json()["items"] - assert len(snapshot_items) == 1 - assert snapshot_items[0]["id"] == snapshot_id - assert snapshot_items[0]["archiveresults"] == [] - - bearer_response = requests.get( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers={"Host": f"api.archivebox.localhost:{port}", "Authorization": f"Bearer {api_token}"}, - timeout=10, - ) - assert bearer_response.status_code == 200, bearer_response.text - query_response = requests.get( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}?api_key={api_token}", - headers={"Host": f"api.archivebox.localhost:{port}"}, - timeout=10, - ) - assert query_response.status_code == 200, query_response.text - - delete_snapshot = requests.delete( - f"http://127.0.0.1:{port}/api/v1/core/snapshot/{snapshot_id}", - headers=api_headers, - timeout=10, - ) - assert delete_snapshot.status_code == 200, delete_snapshot.text - assert delete_snapshot.json()["success"] is True - - delete_crawl = requests.delete( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - timeout=10, - ) - assert delete_crawl.status_code == 200, delete_crawl.text - assert delete_crawl.json()["success"] is True - - with use_archivebox_db(tmp_path): - assert Crawl.objects.filter(pk=crawl_id).count() == 0 - assert Snapshot.objects.filter(pk=snapshot_id).count() == 0 - assert Tag.objects.filter(name="api-extra").count() == 1 - finally: - stop_server(tmp_path) diff --git a/archivebox/tests/test_api_crawl.py b/archivebox/tests/test_api_crawl.py new file mode 100644 index 00000000..49248675 --- /dev/null +++ b/archivebox/tests/test_api_crawl.py @@ -0,0 +1 @@ +# Tests moved to test_api_v1_crawls_crawl_crawl_id.py. diff --git a/archivebox/tests/test_api_crud.py b/archivebox/tests/test_api_crud.py new file mode 100644 index 00000000..98b61d35 --- /dev/null +++ b/archivebox/tests/test_api_crud.py @@ -0,0 +1 @@ +# Tests moved to test_api_v1_workflow_core_token_auth_side_effects.py and exact endpoint files under test_api_v1_core_*. diff --git a/archivebox/tests/test_api_delete_paths.py b/archivebox/tests/test_api_delete_paths.py index cfb5f9e7..46113b20 100644 --- a/archivebox/tests/test_api_delete_paths.py +++ b/archivebox/tests/test_api_delete_paths.py @@ -1,88 +1 @@ -import json -from pathlib import Path - -import pytest -from django.contrib.auth import get_user_model - -from archivebox.api.auth import get_or_create_api_token -from archivebox.cli.archivebox_add import add -from archivebox.cli.archivebox_run import run_runner -from archivebox.core.models import Snapshot -from archivebox.crawls.models import Crawl - - -pytestmark = pytest.mark.django_db(transaction=True) - -API_HOST = "api.archivebox.localhost:8000" - - -def _api_headers(token): - return { - "HTTP_HOST": API_HOST, - "HTTP_X_ARCHIVEBOX_API_KEY": token.token, - } - - -def _admin_token(): - user = get_user_model().objects.create_superuser( - username="deletepathadmin", - email="deletepathadmin@test.com", - password="testpassword", - ) - token = get_or_create_api_token(user) - assert token is not None - return token - - -def test_rest_snapshot_delete_removes_output_dir(client): - token = _admin_token() - url = "https://example.com/delete-path-snapshot" - - response = client.post( - "/api/v1/core/snapshots", - data=json.dumps({"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED}), - content_type="application/json", - **_api_headers(token), - ) - assert response.status_code == 200, response.content.decode() - - snapshot = Snapshot.objects.get(url=url) - snapshot_dir = Path(snapshot.output_dir) - snapshot_dir.mkdir(parents=True, exist_ok=True) - (snapshot_dir / "delete-path-test.txt").write_text("snapshot output") - assert snapshot_dir.exists() - - response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **_api_headers(token)) - assert response.status_code == 200, response.content.decode() - assert not Snapshot.objects.filter(pk=snapshot.pk).exists() - assert not snapshot_dir.exists() - - -def test_rest_crawl_delete_removes_crawl_and_snapshot_output_dirs(client): - token = _admin_token() - url = "https://example.com/delete-path-crawl" - - crawl, _snapshots = add( - urls=[url], - depth=0, - max_urls=1, - plugins="__archivebox_test_no_plugins__", - bg=True, - ) - assert run_runner(daemon=False, crawl_id=str(crawl.id)) == 0 - snapshot = Snapshot.objects.get(crawl=crawl, url=url) - crawl_dir = Path(crawl.output_dir) - snapshot_dir = Path(snapshot.output_dir) - crawl_dir.mkdir(parents=True, exist_ok=True) - snapshot_dir.mkdir(parents=True, exist_ok=True) - (crawl_dir / "delete-path-crawl.txt").write_text("crawl output") - (snapshot_dir / "delete-path-snapshot.txt").write_text("snapshot output") - assert crawl_dir.exists() - assert snapshot_dir.exists() - - response = client.delete(f"/api/v1/crawls/crawl/{crawl.id}", **_api_headers(token)) - assert response.status_code == 200, response.content.decode() - assert not Crawl.objects.filter(pk=crawl.pk).exists() - assert not Snapshot.objects.filter(pk=snapshot.pk).exists() - assert not crawl_dir.exists() - assert not snapshot_dir.exists() +# Tests moved to test_api_v1_core_snapshot_snapshot_id.py and test_api_v1_crawls_crawl_crawl_id.py. diff --git a/archivebox/tests/test_api_personas.py b/archivebox/tests/test_api_personas.py new file mode 100644 index 00000000..227efc93 --- /dev/null +++ b/archivebox/tests/test_api_personas.py @@ -0,0 +1 @@ +# Tests moved to test_api_v1_personas_sync.py and test_api_v1_personas_personas.py. diff --git a/archivebox/tests/test_api_remove.py b/archivebox/tests/test_api_remove.py new file mode 100644 index 00000000..80a2395f --- /dev/null +++ b/archivebox/tests/test_api_remove.py @@ -0,0 +1 @@ +# CLI remove endpoint tests moved to test_api_v1_cli_remove.py. diff --git a/archivebox/tests/test_api_rss.py b/archivebox/tests/test_api_rss.py index e5d892c8..dd3acbbc 100644 --- a/archivebox/tests/test_api_rss.py +++ b/archivebox/tests/test_api_rss.py @@ -1,166 +1 @@ -from datetime import datetime -from typing import cast - -import pytest -from django.contrib.auth import get_user_model -from django.contrib.auth.models import UserManager -from django.utils import timezone - - -pytestmark = pytest.mark.django_db - - -User = get_user_model() -ADMIN_HOST = "admin.archivebox.localhost:8000" - - -@pytest.fixture -def admin_user(db): - return cast(UserManager, User.objects).create_superuser( - username="rssadmin", - email="rssadmin@test.com", - password="testpassword", - ) - - -@pytest.fixture -def other_user(db): - return cast(UserManager, User.objects).create_user( - username="rssother", - email="rssother@test.com", - password="testpassword", - ) - - -@pytest.fixture -def api_token(admin_user): - from archivebox.api.auth import get_or_create_api_token - - token = get_or_create_api_token(admin_user) - assert token is not None - return token.token - - -def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime): - from archivebox.core.models import Snapshot - from archivebox.crawls.models import Crawl - - crawl = Crawl.objects.create(urls=url, created_by=user) - snapshot = Snapshot.objects.create( - url=url, - title=title, - crawl=crawl, - bookmarked_at=bookmarked_at, - ) - return crawl, snapshot - - -def test_snapshots_rss_filters_by_user_and_orders_newest_first(client, api_token, admin_user, other_user): - from archivebox.core.models import Tag - - older_at = timezone.make_aware(datetime(2026, 5, 22, 8, 0, 0)) - newer_at = timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0)) - _crawl, older_snapshot = make_snapshot( - user=admin_user, - url="https://example.com/rss-older", - title="Older & Escaped", - bookmarked_at=older_at, - ) - make_snapshot( - user=admin_user, - url="https://example.com/rss-newer", - title="Newer Snapshot", - bookmarked_at=newer_at, - ) - make_snapshot( - user=other_user, - url="https://example.com/rss-other-user", - title="Other User", - bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)), - ) - older_snapshot.tags.add(Tag.objects.create(name="rss-tag", created_by=admin_user)) - - response = client.get( - "/api/v1/core/snapshots.rss", - {"created_by": admin_user.username, "limit": 50, "api_key": api_token}, - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - assert response["Content-Type"].startswith("application/rss+xml") - body = response.content.decode() - assert 'rss-tag" in body - assert "rss-other-user" not in body - assert body.index("rss-newer") < body.index("rss-older") - - -def test_snapshots_rss_supports_before_yyyymmdd_and_limit(client, api_token, admin_user): - make_snapshot( - user=admin_user, - url="https://example.com/rss-before-too-new", - title="Too New", - bookmarked_at=timezone.make_aware(datetime(2026, 5, 24, 8, 0, 0)), - ) - make_snapshot( - user=admin_user, - url="https://example.com/rss-before-keep-one", - title="Keep One", - bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 12, 0, 0)), - ) - make_snapshot( - user=admin_user, - url="https://example.com/rss-before-keep-two", - title="Keep Two", - bookmarked_at=timezone.make_aware(datetime(2026, 5, 22, 12, 0, 0)), - ) - - response = client.get( - "/api/v1/core/snapshots.rss", - {"created_by": str(admin_user.pk), "before": "20260523", "limit": 1, "api_key": api_token}, - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - body = response.content.decode() - assert "rss-before-too-new" not in body - assert "rss-before-keep-one" in body - assert "rss-before-keep-two" not in body - - -def test_crawl_as_rss_redirects_to_canonical_snapshots_feed(client, api_token, admin_user, other_user): - crawl, _snapshot = make_snapshot( - user=admin_user, - url="https://example.com/rss-crawl-feed", - title="Crawl Feed Snapshot", - bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0)), - ) - make_snapshot( - user=other_user, - url="https://example.com/rss-crawl-other", - title="Other Crawl Snapshot", - bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)), - ) - - response = client.get( - f"/api/v1/crawls/crawl/{crawl.id}", - {"as_rss": "true", "limit": 50, "api_key": api_token}, - HTTP_HOST=ADMIN_HOST, - follow=True, - ) - - assert response.status_code == 200 - assert response.redirect_chain - redirect_url = response.redirect_chain[0][0] - assert redirect_url.startswith("/api/v1/core/snapshots.rss?") - assert f"crawl_id={crawl.id}" in redirect_url - assert "as_rss" not in redirect_url - assert response["Content-Type"].startswith("application/rss+xml") - body = response.content.decode() - assert "rss-crawl-feed" in body - assert "rss-crawl-other" not in body +# Tests moved to test_api_v1_core_snapshots_rss.py and test_api_v1_crawls_crawl_crawl_id.py. diff --git a/archivebox/tests/test_api_search.py b/archivebox/tests/test_api_search.py new file mode 100644 index 00000000..df3443be --- /dev/null +++ b/archivebox/tests/test_api_search.py @@ -0,0 +1 @@ +# Tests moved to test_api_v1_core_snapshots.py. diff --git a/archivebox/tests/test_api_v1_auth_check_api_token.py b/archivebox/tests/test_api_v1_auth_check_api_token.py new file mode 100644 index 00000000..d03e73d4 --- /dev/null +++ b/archivebox/tests/test_api_v1_auth_check_api_token.py @@ -0,0 +1,19 @@ +import pytest + +from archivebox.tests.conftest import API_TEST_HOST, api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_token): + response = api_client_request( + client, + "post", + "/api/v1/auth/check_api_token", + payload={"token": api_token.token}, + headers={"HTTP_HOST": API_TEST_HOST}, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_auth_get_api_token.py b/archivebox/tests/test_api_v1_auth_get_api_token.py new file mode 100644 index 00000000..6a6f4027 --- /dev/null +++ b/archivebox/tests/test_api_v1_auth_get_api_token.py @@ -0,0 +1,22 @@ +import pytest + +from archivebox.tests.conftest import API_TEST_HOST, api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user): + response = api_client_request( + client, + "post", + "/api/v1/auth/get_api_token", + payload={ + "username": api_admin_user.username, + "password": "testpass123", + }, + headers={"HTTP_HOST": API_TEST_HOST}, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_cli_add.py b/archivebox/tests/test_api_v1_cli_add.py new file mode 100644 index 00000000..17a01957 --- /dev/null +++ b/archivebox/tests/test_api_v1_cli_add.py @@ -0,0 +1,29 @@ +import pytest + +from .conftest import ( + api_client_request, + init_archive, +) + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + init_archive(tmp_path) + + response = api_client_request( + client, + "post", + "/api/v1/cli/add", + payload={ + "urls": ["https://example.com/api-cli-add-basic"], + "depth": 0, + "parser": "url_list", + "plugins": "__archivebox_test_no_plugins__", + "index_only": True, + }, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_cli_remove.py b/archivebox/tests/test_api_v1_cli_remove.py new file mode 100644 index 00000000..f1eabc31 --- /dev/null +++ b/archivebox/tests/test_api_v1_cli_remove.py @@ -0,0 +1,209 @@ +from datetime import datetime, timedelta +from pathlib import Path + +import pytest +from django.utils import timezone + +from archivebox.core.models import Snapshot, Tag +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import api_client_request, init_archive + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def _crawl(user, urls: str, label: str) -> Crawl: + return Crawl.objects.create( + urls=urls, + created_by=user, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + tags_str=label, + ) + + +def _snapshot(crawl: Crawl, url: str, *, status: str = Snapshot.StatusChoices.SEALED, tag: str = "", bookmarked_at=None) -> Snapshot: + snapshot = Snapshot.objects.create( + url=url, + crawl=crawl, + status=status, + retry_at=None, + bookmarked_at=bookmarked_at or timezone.now(), + ) + if tag: + tag_obj, _ = Tag.objects.get_or_create(name=tag) + snapshot.tags.add(tag_obj) + return snapshot + + +def _touch_output(snapshot: Snapshot) -> Path: + output_dir = Path(snapshot.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "api-remove-test.txt").write_text(str(snapshot.id)) + return output_dir + + +def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 30000) -> tuple[list[Snapshot], dict[str, Path]]: + base = timezone.make_aware(datetime(2026, 2, 1, 12, 0, 0)) + snapshots = [ + Snapshot( + url=f"https://example.com/remove-timeout-{idx}", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + timestamp=f"88{idx:030d}", + bookmarked_at=base + timedelta(seconds=idx), + created_at=base + timedelta(seconds=idx), + ) + for idx in range(count) + ] + Snapshot.objects.bulk_create(snapshots, batch_size=1000) + + sample = [*snapshots[-200:], *snapshots[:200]] + return snapshots, {str(snapshot.id): _touch_output(snapshot) for snapshot in sample} + + +def _post_remove(client, api_headers, body: dict): + return api_client_request( + client, + "post", + "/api/v1/cli/remove", + payload=body, + headers=api_headers, + ) + + +def test_cli_remove_api_removes_rows_and_respects_snapshot_filters(client, tmp_path, api_admin_user, api_headers): + init_archive(tmp_path) + base = timezone.make_aware(datetime(2026, 1, 1, 12, 0, 0)) + + crawl_a = _crawl(api_admin_user, "https://alpha.example.com/articles/needle", "crawl-a") + crawl_b = _crawl(api_admin_user, "https://beta.example.org/posts/needle", "crawl-b") + exact = _snapshot(crawl_a, "https://alpha.example.com/articles/exact", tag="api-keep", bookmarked_at=base) + keep = _snapshot(crawl_a, "https://alpha.example.com/articles/needle", tag="api-keep", bookmarked_at=base + timedelta(hours=1)) + wrong_status = _snapshot( + crawl_a, + "https://alpha.example.com/articles/needle-queued", + status=Snapshot.StatusChoices.QUEUED, + tag="api-keep", + bookmarked_at=base + timedelta(hours=2), + ) + other_crawl = _snapshot(crawl_b, "https://beta.example.org/posts/needle", tag="api-other", bookmarked_at=base + timedelta(hours=3)) + exact_dir = _touch_output(exact) + keep_dir = _touch_output(keep) + wrong_status_dir = _touch_output(wrong_status) + other_crawl_dir = _touch_output(other_crawl) + + exact_response = _post_remove( + client, + api_headers, + { + "filter_type": "exact", + "filter_patterns": [exact.url], + "timeout": 60, + }, + ) + assert exact_response.status_code == 200, exact_response.content + exact_payload = exact_response.json() + assert exact_payload["success"] is True + assert exact_payload["result"]["removed_count"] == 1 + assert exact_payload["result"]["removed_snapshot_ids"] == [str(exact.id)] + assert exact_payload["result"]["not_removed_count"] == 0 + assert not Snapshot.objects.filter(pk=exact.pk).exists() + assert not exact_dir.exists() + + filtered_response = _post_remove( + client, + api_headers, + { + "filter_type": "substring", + "filter_patterns": ["needle"], + "status": Snapshot.StatusChoices.SEALED, + "tag": "api-keep", + "url__istartswith": "https://alpha.example.com", + "crawl_id": str(crawl_a.id), + "after": (base + timedelta(minutes=30)).timestamp(), + "before": (base + timedelta(hours=2)).timestamp(), + "timeout": 60, + }, + ) + assert filtered_response.status_code == 200, filtered_response.content + filtered_payload = filtered_response.json() + assert filtered_payload["success"] is True + assert filtered_payload["result"]["removed_count"] == 1 + assert filtered_payload["result"]["removed_snapshot_ids"] == [str(keep.id)] + assert filtered_payload["result"]["not_removed_count"] == 0 + assert not Snapshot.objects.filter(pk=keep.pk).exists() + assert not keep_dir.exists() + assert Snapshot.objects.filter(pk=wrong_status.pk).exists() + assert Snapshot.objects.filter(pk=other_crawl.pk).exists() + assert wrong_status_dir.exists() + assert other_crawl_dir.exists() + + +def test_cli_remove_api_reports_timeout_and_clamps_timeout_to_sixty_seconds(client, tmp_path, api_admin_user, api_headers): + init_archive(tmp_path) + crawl = _crawl(api_admin_user, "https://example.com/remove-timeout-0", "timeout") + snapshots, output_dirs_by_id = _bulk_timeout_snapshots(crawl) + + timeout_response = _post_remove( + client, + api_headers, + { + "filter_type": "substring", + "filter_patterns": ["remove-timeout-"], + "timeout": 3, + }, + ) + assert timeout_response.status_code == 200, timeout_response.content + timeout_payload = timeout_response.json() + assert timeout_payload["success"] is False + assert timeout_payload["errors"] + assert set(timeout_payload["result"]) == { + "removed_count", + "removed_snapshot_ids", + "not_removed_count", + "not_removed_snapshot_ids", + "success", + "error", + "timeout", + } + assert timeout_payload["result"]["success"] is False + assert timeout_payload["result"]["timeout"] == 3.0 + assert timeout_payload["result"]["error"] + assert timeout_payload["result"]["removed_count"] == len(timeout_payload["result"]["removed_snapshot_ids"]) + assert timeout_payload["result"]["not_removed_count"] == len(timeout_payload["result"]["not_removed_snapshot_ids"]) + assert timeout_payload["result"]["removed_count"] > 0 + assert timeout_payload["result"]["not_removed_count"] > 0 + assert timeout_payload["result"]["removed_count"] + timeout_payload["result"]["not_removed_count"] == len(snapshots) + + removed_ids = set(timeout_payload["result"]["removed_snapshot_ids"]) + not_removed_ids = set(timeout_payload["result"]["not_removed_snapshot_ids"]) + assert Snapshot.objects.filter(url__icontains="remove-timeout-").count() == len(not_removed_ids) + assert removed_ids & set(output_dirs_by_id) + assert not_removed_ids & set(output_dirs_by_id) + for snapshot_id in removed_ids & set(output_dirs_by_id): + assert not Snapshot.objects.filter(pk=snapshot_id).exists() + assert not output_dirs_by_id[snapshot_id].exists() + for snapshot_id in not_removed_ids & set(output_dirs_by_id): + assert Snapshot.objects.filter(pk=snapshot_id).exists() + assert output_dirs_by_id[snapshot_id].exists() + + clamp_snapshot = _snapshot(crawl, "https://example.com/remove-timeout-clamp") + clamp_dir = _touch_output(clamp_snapshot) + clamp_response = _post_remove( + client, + api_headers, + { + "filter_type": "exact", + "filter_patterns": [clamp_snapshot.url], + "timeout": 999, + }, + ) + assert clamp_response.status_code == 200, clamp_response.content + clamp_payload = clamp_response.json() + assert clamp_payload["success"] is True + assert clamp_payload["result"]["timeout"] == 60.0 + assert clamp_payload["result"]["removed_snapshot_ids"] == [str(clamp_snapshot.id)] + assert not Snapshot.objects.filter(pk=clamp_snapshot.pk).exists() + assert not clamp_dir.exists() diff --git a/archivebox/tests/test_api_v1_cli_schedule.py b/archivebox/tests/test_api_v1_cli_schedule.py new file mode 100644 index 00000000..97e91961 --- /dev/null +++ b/archivebox/tests/test_api_v1_cli_schedule.py @@ -0,0 +1,71 @@ +from io import StringIO + +import pytest +import requests +from django.test import RequestFactory + +from archivebox.api.v1_cli import ScheduleCommandSchema, cli_schedule +from archivebox.crawls.models import CrawlSchedule +from .conftest import ( + api_auth_headers, + cli_env, + create_admin_and_token, + get_free_port, + init_archive, + start_archivebox_server, + stop_server, + wait_for_http, +) + + +@pytest.mark.django_db +def test_schedule_api_creates_schedule_via_view_request(api_admin_user): + request = RequestFactory().post("/api/v1/cli/schedule") + request.user = api_admin_user + setattr(request, "stdout", StringIO()) + setattr(request, "stderr", StringIO()) + args = ScheduleCommandSchema( + every="daily", + import_path="https://example.com/feed.xml", + quiet=True, + ) + + response = cli_schedule(request, args) + + assert response["success"] is True + assert response["result_format"] == "json" + assert CrawlSchedule.objects.count() == 1 + assert len(response["result"]["created_schedule_ids"]) == 1 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.timeout(180) +def test_api_v1_cli_schedule_creates_schedule_over_server(tmp_path, recursive_test_site): + init_archive(tmp_path) + + port = get_free_port() + env = cli_env(port=port, server=True) + api_token = create_admin_and_token(tmp_path) + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + + response = requests.post( + f"http://127.0.0.1:{port}/api/v1/cli/schedule", + headers=api_auth_headers(api_token, port=port), + json={ + "every": "daily", + "import_path": recursive_test_site["root_url"], + "quiet": True, + }, + timeout=10, + ) + + assert response.status_code == 200, response.text + payload = response.json() + assert payload["success"] is True + assert payload["result_format"] == "json" + assert len(payload["result"]["created_schedule_ids"]) == 1 + finally: + stop_server(tmp_path) diff --git a/archivebox/tests/test_api_v1_cli_search.py b/archivebox/tests/test_api_v1_cli_search.py new file mode 100644 index 00000000..7fa60b74 --- /dev/null +++ b/archivebox/tests/test_api_v1_cli_search.py @@ -0,0 +1,31 @@ +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from .conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/api-cli-search-basic", created_by=api_admin_user) + Snapshot.objects.create(url="https://example.com/api-cli-search-basic", crawl=crawl) + + response = api_client_request( + client, + "post", + "/api/v1/cli/search", + payload={ + "filter_patterns": ["https://example.com/api-cli-search-basic"], + "filter_type": "exact", + "as_json": True, + "as_html": False, + "as_csv": "", + "with_headers": False, + }, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_cli_update.py b/archivebox/tests/test_api_v1_cli_update.py new file mode 100644 index 00000000..0892e51a --- /dev/null +++ b/archivebox/tests/test_api_v1_cli_update.py @@ -0,0 +1,137 @@ +import json + +import pytest + +from .conftest import ( + api_client_request, + cli_env, + create_admin_and_token, + get_free_port, + init_archive, + live_api_request, + parse_jsonl_output, + run_archivebox_cmd, + start_archivebox_server, + stop_server, + wait_for_live_api, +) + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_cli_update_api_accepts_empty_json_without_traceback(client, tmp_path, api_headers): + init_archive(tmp_path) + + try: + response = api_client_request( + client, + "post", + "/api/v1/cli/update", + payload={}, + headers=api_headers, + ) + finally: + stop_server(tmp_path) + + assert response.status_code == 200, response.content + payload = response.json() + assert payload["success"] is True + assert "Traceback" not in response.content.decode() + + +@pytest.mark.timeout(180) +def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_path): + env = cli_env(disable_extractors=True) + init_archive(tmp_path) + + records = [ + { + "type": "Snapshot", + "url": "https://alpha.example.com/articles/needle", + "title": "Needle Alpha", + "tags": "api-keep", + "timestamp": "1700000000", + "bookmarked_at": "2023-11-14T22:13:20+00:00", + }, + { + "type": "Snapshot", + "url": "https://beta.example.org/posts/haystack", + "title": "Haystack Beta", + "tags": "api-other", + "timestamp": "1710000000", + "bookmarked_at": "2024-03-09T16:00:00+00:00", + }, + { + "type": "Snapshot", + "url": "https://docs.archivebox.io/manual", + "title": "Manual Gamma", + "tags": "api-docs", + "timestamp": "1720000000", + "bookmarked_at": "2024-07-03T09:46:40+00:00", + }, + ] + stdin = "\n".join(json.dumps(record) for record in records) + "\n" + run_archivebox_cmd(["snapshot", "create"], cwd=tmp_path, stdin=stdin, env=env, check=True) + + port = get_free_port() + env = { + **cli_env(port=port, server=True, PUBLIC_INDEX="True"), + **env, + } + api_token = create_admin_and_token(tmp_path) + + def assert_update_filter(label, body, expected_records): + response = live_api_request( + port, + "post", + "/api/v1/cli/update", + api_token=api_token, + json={**body, "batch_size": 100}, + timeout=30, + ) + assert response.status_code == 200, f"{label}: {response.text}" + assert "Traceback" not in response.text + payload = response.json() + assert payload["success"] is True, label + expected_ids = {record["id"] for record in expected_records} + assert set(payload["result"]["snapshot_ids"]) == expected_ids, label + assert payload["result"]["matched_count"] == len(expected_ids), label + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + list_result = run_archivebox_cmd(["snapshot", "list", "--sort", "timestamp"], cwd=tmp_path, env=env, check=True) + snapshots = {record["url"]: record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot"} + alpha = snapshots["https://alpha.example.com/articles/needle"] + beta = snapshots["https://beta.example.org/posts/haystack"] + gamma = snapshots["https://docs.archivebox.io/manual"] + status_result = run_archivebox_cmd( + ["snapshot", "list", "--status", alpha["status"]], + cwd=tmp_path, + env=env, + check=True, + ) + status_records = [record for record in parse_jsonl_output(status_result.stdout) if record.get("type") == "Snapshot"] + + cases = [ + ("status", {"status": alpha["status"]}, status_records), + ("filter_type exact", {"filter_type": "exact", "filter_patterns": [alpha["url"]]}, [alpha]), + ("filter_type substring", {"filter_type": "substring", "filter_patterns": ["needle"]}, [alpha]), + ("filter_type regex", {"filter_type": "regex", "filter_patterns": [r"alpha\.example\.com/.+needle"]}, [alpha]), + ("filter_type domain", {"filter_type": "domain", "filter_patterns": ["alpha.example.com"]}, [alpha]), + ("filter_type tag", {"filter_type": "tag", "filter_patterns": ["api-keep"]}, [alpha]), + ("filter_type timestamp", {"filter_type": "timestamp", "filter_patterns": [alpha["timestamp"]]}, [alpha]), + ("url__icontains", {"url__icontains": "needle"}, [alpha]), + ("url__istartswith", {"url__istartswith": "https://alpha.example.com"}, [alpha]), + ("tag", {"tag": "api-keep"}, [alpha]), + ("crawl_id", {"crawl_id": alpha["crawl_id"]}, [alpha]), + ("limit and sort", {"limit": 1, "sort": "timestamp"}, [alpha]), + ("search", {"search": "meta", "filter_patterns": ["Needle Alpha"]}, [alpha]), + ("before", {"before": 1715000000}, [alpha, beta]), + ("after", {"after": 1715000000}, [gamma]), + ("resume", {"resume": beta["timestamp"]}, [alpha, beta]), + ] + for label, body, expected_records in cases: + assert_update_filter(label, body, expected_records) + finally: + stop_server(tmp_path) diff --git a/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py b/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py new file mode 100644 index 00000000..1fd7eec3 --- /dev/null +++ b/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py @@ -0,0 +1,160 @@ +import time + +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.test_orm_helpers import use_archivebox_db +from .conftest import ( + cli_env, + create_admin_and_token, + get_free_port, + init_archive, + live_api_request, + start_archivebox_server, + stop_server, + wait_for_live_api, +) + +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.mark.timeout(180) +def test_cli_api_add_search_update_remove_over_server(tmp_path): + init_archive(tmp_path) + + port = get_free_port() + env = cli_env(port=port, server=True, PUBLIC_INDEX="True") + api_token = create_admin_and_token(tmp_path) + target_url = "https://example.com/" + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + add_response = live_api_request( + port, + "post", + "/api/v1/cli/add", + api_token=api_token, + json={ + "urls": [target_url], + "tag": "api-cli", + "depth": 0, + "parser": "url_list", + "plugins": "wget", + "update": True, + "overwrite": False, + "index_only": True, + }, + timeout=10, + ) + assert add_response.status_code == 200, add_response.text + add_payload = add_response.json() + assert add_payload["success"] is True + assert add_payload["result_format"] == "json" + assert add_payload["result"]["num_snapshots"] == 0 + crawl_id = add_payload["result"]["crawl_id"] + assert add_payload["result"]["snapshot_ids"] == [] + stop_server(tmp_path) + from archivebox.services.runner import run_crawl + + with use_archivebox_db(tmp_path): + run_crawl(crawl_id, show_progress=False) + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + deadline = time.time() + 180 + snapshot_id = None + snapshot_status = None + while time.time() < deadline: + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.filter(crawl_id=crawl_id, url=target_url).first() + if snapshot is not None: + snapshot_id = str(snapshot.id) + snapshot_status = snapshot.status + break + time.sleep(1) + assert snapshot_id is not None + assert snapshot_status is not None + + search_response = live_api_request( + port, + "post", + "/api/v1/cli/search", + api_token=api_token, + json={ + "filter_patterns": [target_url], + "filter_type": "exact", + "status": snapshot_status, + "sort": "bookmarked_at", + "as_json": True, + "as_html": False, + "as_csv": "", + "with_headers": False, + }, + timeout=10, + ) + assert search_response.status_code == 200, search_response.text + search_payload = search_response.json() + assert search_payload["success"] is True + assert search_payload["result_format"] == "json" + assert any(item["url"] == target_url for item in search_payload["result"]) + + update_response = live_api_request( + port, + "post", + "/api/v1/cli/update", + api_token=api_token, + json={ + "resume": None, + "after": 0, + "before": 4102444800, + "filter_type": "exact", + "filter_patterns": [target_url], + "batch_size": 1, + "continuous": False, + }, + timeout=20, + ) + assert update_response.status_code == 200, update_response.text + assert update_response.json()["success"] is True + stop_server(tmp_path) + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + with use_archivebox_db(tmp_path): + crawl_obj = Crawl.objects.filter(pk=crawl_id).first() + crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None + + assert crawl is not None + assert crawl[0] == 0 + assert crawl[1] == "api-cli" + assert crawl[2]["INDEX_ONLY"] is True + + remove_response = live_api_request( + port, + "post", + "/api/v1/cli/remove", + api_token=api_token, + json={ + "delete": True, + "after": 0, + "before": 4102444800, + "filter_type": "exact", + "filter_patterns": [target_url], + }, + timeout=20, + ) + assert remove_response.status_code == 200, remove_response.text + remove_payload = remove_response.json() + assert remove_payload["success"] is True + assert remove_payload["result"]["removed_count"] == 1 + assert snapshot_id in remove_payload["result"]["removed_snapshot_ids"] + + with use_archivebox_db(tmp_path): + snapshot_count = Snapshot.objects.filter(pk=snapshot_id).count() + + assert snapshot_count == 0 + finally: + stop_server(tmp_path) diff --git a/archivebox/tests/test_api_v1_core_any_id.py b/archivebox/tests/test_api_v1_core_any_id.py new file mode 100644 index 00000000..da4efe39 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_any_id.py @@ -0,0 +1,16 @@ +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/any", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/any", crawl=crawl) + + response = client.get(f"/api/v1/core/any/{snapshot.id}", follow=True, **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py b/archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py new file mode 100644 index 00000000..43285a2c --- /dev/null +++ b/archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py @@ -0,0 +1,69 @@ +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart + +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/archiveresult-detail", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/archiveresult-detail", crawl=crawl) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="api-basic", + hook_name="on_Snapshot__api_basic", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_str="ok", + ) + + response = client.get(f"/api/v1/core/archiveresult/{result.id}", **api_headers) + + assert response.status_code == 200, response.content + + +def test_archiveresult_patch_upload_finalizes_queued_result(client, api_admin_user, api_headers): + crawl = Crawl.objects.create( + urls="https://example.com", + created_by=api_admin_user, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com/upload-patch", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="dom", + hook_name="on_Snapshot__archivebox_browser_extension_upload", + status=ArchiveResult.StatusChoices.QUEUED, + ) + + response = client.generic( + "PATCH", + f"/api/v1/core/archiveresult/{result.id}", + encode_multipart( + BOUNDARY, + { + "files": SimpleUploadedFile("output.html", b"uploaded", content_type="text/html"), + "output_paths": "output.html", + "output_str": "output.html", + }, + ), + content_type=MULTIPART_CONTENT, + **api_headers, + ) + assert response.status_code == 200, response.content + + result.refresh_from_db() + snapshot.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.SUCCEEDED + assert result.output_str == "output.html" + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert snapshot.retry_at is not None diff --git a/archivebox/tests/test_api_v1_core_archiveresults.py b/archivebox/tests/test_api_v1_core_archiveresults.py new file mode 100644 index 00000000..09c3af9e --- /dev/null +++ b/archivebox/tests/test_api_v1_core_archiveresults.py @@ -0,0 +1,170 @@ +from datetime import timedelta + +import pytest +from django.db import connection +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client, api_admin_user, api_headers): + crawl = Crawl.objects.create( + urls="https://example.com", + created_by=api_admin_user, + status=Crawl.StatusChoices.STARTED, + retry_at=timezone.now(), + ) + active_retry_at = timezone.now() + timedelta(minutes=5) + active_snapshot = Snapshot.objects.create( + url="https://example.com/active", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + retry_at=active_retry_at, + ) + sealed_snapshot = Snapshot.objects.create( + url="https://example.com/sealed", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + + active_response = client.post( + "/api/v1/core/archiveresults", + { + "snapshot_id": str(active_snapshot.id), + "plugin": "chrome_extension_dom", + "hook_name": "on_Snapshot__archivebox_browser_extension_upload", + "status": ArchiveResult.StatusChoices.SUCCEEDED, + "output_str": "uploaded active snapshot output", + }, + **api_headers, + ) + assert active_response.status_code == 200, active_response.content + active_snapshot.refresh_from_db() + assert active_snapshot.status == Snapshot.StatusChoices.STARTED + assert active_snapshot.retry_at == active_retry_at + assert active_snapshot.downloaded_at is not None + + sealed_response = client.post( + "/api/v1/core/archiveresults", + { + "snapshot_id": str(sealed_snapshot.id), + "plugin": "chrome_extension_mhtml", + "hook_name": "on_Snapshot__archivebox_browser_extension_upload", + "status": ArchiveResult.StatusChoices.SUCCEEDED, + "output_str": "uploaded sealed snapshot output", + }, + **api_headers, + ) + assert sealed_response.status_code == 200, sealed_response.content + sealed_snapshot.refresh_from_db() + assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED + assert sealed_snapshot.retry_at is not None + assert sealed_snapshot.downloaded_at is not None + + +def test_archiveresults_api_limit_uses_exact_count_without_full_row_distinct(client, api_headers): + snapshot_response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + payload={ + "url": "https://example.com/archive-result-pagination", + "title": "ArchiveResult pagination", + "status": Snapshot.StatusChoices.QUEUED, + }, + headers=api_headers, + ) + assert snapshot_response.status_code == 200, snapshot_response.content + snapshot_id = snapshot_response.json()["id"] + + for plugin_name in ("dom", "screenshot"): + result_response = client.post( + "/api/v1/core/archiveresults", + { + "snapshot_id": snapshot_id, + "plugin": plugin_name, + "hook_name": f"on_Snapshot__test_{plugin_name}", + "status": ArchiveResult.StatusChoices.SUCCEEDED, + "output_str": f"{plugin_name} output", + }, + **api_headers, + ) + assert result_response.status_code == 200, result_response.content + + total_archiveresults = ArchiveResult.objects.count() + with CaptureQueriesContext(connection) as captured_queries: + response = client.get( + "/api/v1/core/archiveresults?limit=1", + **api_headers, + ) + + assert response.status_code == 200, response.content + payload = response.json() + assert payload["count"] == total_archiveresults + assert payload["total_items"] == total_archiveresults + assert payload["limit"] == 1 + assert payload["num_items"] == 1 + + count_queries = [ + query["sql"] for query in captured_queries if "COUNT" in query["sql"].upper() and '"core_archiveresult"' in query["sql"] + ] + assert count_queries + assert not any("SELECT DISTINCT" in query.upper() for query in count_queries), count_queries + + +def test_archiveresults_api_join_filters_count_distinct_primary_keys(client, api_headers): + snapshot_response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + payload={ + "url": "https://example.com/archive-result-tag-pagination", + "title": "ArchiveResult tag pagination", + "tags": ["api-tag-pagination-one", "api-tag-pagination-two"], + "status": Snapshot.StatusChoices.QUEUED, + }, + headers=api_headers, + ) + assert snapshot_response.status_code == 200, snapshot_response.content + snapshot_id = snapshot_response.json()["id"] + + result_response = client.post( + "/api/v1/core/archiveresults", + { + "snapshot_id": snapshot_id, + "plugin": "dom", + "hook_name": "on_Snapshot__test_tag_pagination", + "status": ArchiveResult.StatusChoices.SUCCEEDED, + "output_str": "tag pagination output", + }, + **api_headers, + ) + assert result_response.status_code == 200, result_response.content + + with CaptureQueriesContext(connection) as captured_queries: + response = client.get( + "/api/v1/core/archiveresults?search=api-tag-pagination&limit=1", + **api_headers, + ) + + assert response.status_code == 200, response.content + payload = response.json() + assert payload["count"] == 1 + assert payload["total_items"] == 1 + assert payload["num_items"] == 1 + assert [item["id"] for item in payload["items"]] == [result_response.json()["id"]] + + count_queries = [ + query["sql"] for query in captured_queries if "COUNT" in query["sql"].upper() and '"core_archiveresult"' in query["sql"] + ] + assert count_queries + assert any("SELECT DISTINCT" in query.upper() for query in count_queries), count_queries + assert not any('"core_archiveresult"."output_files" AS' in query for query in count_queries), count_queries + assert not any('"core_archiveresult"."notes" AS' in query for query in count_queries), count_queries diff --git a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py new file mode 100644 index 00000000..4d3e75a2 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py @@ -0,0 +1,497 @@ +import json +import time +from pathlib import Path + +import pytest +from django.utils import timezone + +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import run_archivebox_cmd +from archivebox.tests.test_orm_helpers import use_archivebox_db +from archivebox.workers.models import RETRY_AT_MAX + +from .conftest import ( + api_client_request, + cli_env, + create_admin_and_token, + get_crawl_runtime_state, + get_free_port, + init_archive, + live_api_request, + start_archivebox_server, + stop_server, + wait_for_live_api, + wait_for_snapshot_capture, +) + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def _seed_archiveresult( + snapshot: Snapshot, + *, + plugin: str, + hook_name: str, + status: str, + output_text: str = "", + output_path: str | None = None, +) -> ArchiveResult: + output_files = {} + output_size = 0 + output_mimetypes = "" + if output_path is not None: + output_bytes = output_text.encode() + absolute_path = Path(snapshot.output_dir) / output_path + absolute_path.parent.mkdir(parents=True, exist_ok=True) + absolute_path.write_bytes(output_bytes) + output_size = len(output_bytes) + output_mimetypes = "text/plain" + output_files[output_path] = { + "extension": Path(output_path).suffix.lstrip("."), + "mimetype": "text/plain", + "size": output_size, + } + + now = timezone.now() + return ArchiveResult.objects.create( + snapshot=snapshot, + plugin=plugin, + hook_name=hook_name, + status=status, + output_str=output_path or output_text, + output_files=output_files, + output_size=output_size, + output_mimetypes=output_mimetypes, + start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None, + end_ts=now if status in ArchiveResult.FINAL_STATES else None, + ) + + +def _snapshot_hook_name(plugin_name: str) -> str: + from abx_dl.models import discover_plugins + + plugin = discover_plugins().get(plugin_name) + assert plugin is not None, f"missing test plugin {plugin_name}" + hooks = plugin.filter_hooks("Snapshot") + assert hooks, f"missing Snapshot hooks for {plugin_name}" + return hooks[0].name + + +def _snapshot_state(cwd: Path, url: str) -> dict[str, object]: + with use_archivebox_db(cwd): + snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").get(url=url) + snapshot_dir = Path(snapshot.output_dir) + crawl_dir = Path(snapshot.crawl.output_dir) + crawl_link = crawl_dir / "snapshots" / Snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id) + results = list( + ArchiveResult.objects.filter(snapshot=snapshot) + .order_by("plugin", "hook_name") + .values("plugin", "hook_name", "status", "output_files", "output_size"), + ) + return { + "id": str(snapshot.id), + "crawl_id": str(snapshot.crawl_id), + "status": snapshot.status, + "retry_at": snapshot.retry_at, + "downloaded_at": snapshot.downloaded_at, + "output_size": snapshot.output_size, + "snapshot_dir": snapshot_dir, + "crawl_dir": crawl_dir, + "crawl_link": crawl_link, + "results": results, + } + + +def _paused_snapshot_state(cwd: Path, snapshot_id: str) -> dict[str, object]: + with use_archivebox_db(cwd): + snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) + succeeded_results = ArchiveResult.objects.filter(snapshot=snapshot, status=ArchiveResult.StatusChoices.SUCCEEDED).count() + return { + "status": snapshot.status, + "retry_at": snapshot.retry_at, + "crawl_status": snapshot.crawl.status, + "succeeded_results": succeeded_results, + "snapshot_dir": Path(snapshot.output_dir), + } + + +def _wait_for_paused_scheduler_marker(cwd: Path, snapshot_id: str, timeout: int = 60) -> dict[str, object]: + deadline = time.time() + timeout + last_state: dict[str, object] = {} + while time.time() < deadline: + last_state = _paused_snapshot_state(cwd, snapshot_id) + if last_state["status"] == Snapshot.StatusChoices.PAUSED and last_state["retry_at"] == RETRY_AT_MAX: + return last_state + if last_state["status"] == Snapshot.StatusChoices.SEALED: + return last_state + time.sleep(1) + raise AssertionError(f"paused snapshot did not settle back to retry_at=MAX: {last_state}") + + +def _wait_for_crawl_snapshot_rows(cwd: Path, crawl_id: str, timeout: int = 45) -> dict[str, object]: + deadline = time.time() + timeout + latest_state: dict[str, object] | None = None + while time.time() < deadline: + latest_state = get_crawl_runtime_state(cwd, crawl_id) + if latest_state["snapshots"]: + return latest_state + time.sleep(0.2) + raise AssertionError(f"timed out waiting for snapshot rows for crawl {crawl_id}: {latest_state}") + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/snapshot-detail", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/snapshot-detail", crawl=crawl) + + response = client.get(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers) + + assert response.status_code == 200, response.content + + +def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows( + tmp_path, + client, + recursive_test_site, +): + init_archive(tmp_path) + api_token = create_admin_and_token(tmp_path) + + with use_archivebox_db(tmp_path): + create_response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + api_token=api_token, + payload={ + "url": recursive_test_site["root_url"], + "depth": 0, + "title": "Snapshot pause target", + "tags": ["snapshot-pause-e2e"], + "status": "queued", + }, + ) + assert create_response.status_code == 200, create_response.content.decode() + snapshot_id = json.loads(create_response.content.decode())["id"] + snapshot = Snapshot.objects.get(id=snapshot_id) + + queued_result = _seed_archiveresult( + snapshot, + plugin="manualqueue", + hook_name="on_Snapshot__manual_queue", + status=ArchiveResult.StatusChoices.QUEUED, + ) + started_result = _seed_archiveresult( + snapshot, + plugin="manualstart", + hook_name="on_Snapshot__manual_start", + status=ArchiveResult.StatusChoices.STARTED, + ) + succeeded_result = _seed_archiveresult( + snapshot, + plugin="manualdone", + hook_name="on_Snapshot__manual_done", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_text="finished result should stay finished", + output_path="manualdone/final.txt", + ) + failed_result = _seed_archiveresult( + snapshot, + plugin="manualfail", + hook_name="on_Snapshot__manual_fail", + status=ArchiveResult.StatusChoices.FAILED, + output_text="failed result should stay failed", + ) + + invalid_response = api_client_request( + client, + "patch", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + payload={"action": "hold"}, + ) + assert invalid_response.status_code == 400 + snapshot = Snapshot.objects.get(id=snapshot_id) + assert snapshot.status == Snapshot.StatusChoices.QUEUED + + pause_response = api_client_request( + client, + "patch", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + payload={"action": "pause"}, + ) + assert pause_response.status_code == 200, pause_response.content.decode() + assert json.loads(pause_response.content.decode())["status"] == Snapshot.StatusChoices.PAUSED + + snapshot.refresh_from_db() + crawl = Crawl.objects.get(id=snapshot.crawl_id) + assert snapshot.status == Snapshot.StatusChoices.PAUSED + assert snapshot.retry_at == RETRY_AT_MAX + assert crawl.status == Crawl.StatusChoices.QUEUED + + active_rows = { + row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) + } + assert active_rows == { + "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + } + + finished_rows = { + row.plugin: (row.status, row.retry_at, row.output_size) + for row in ArchiveResult.objects.filter(id__in=[succeeded_result.id, failed_result.id]) + } + assert finished_rows["manualdone"][0] == ArchiveResult.StatusChoices.SUCCEEDED + assert finished_rows["manualdone"][1] is None + assert finished_rows["manualdone"][2] == len("finished result should stay finished") + assert finished_rows["manualfail"] == (ArchiveResult.StatusChoices.FAILED, None, 0) + + succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id) + output_path = Path(snapshot.output_dir) / next(iter(succeeded_row.output_files)) + assert output_path.read_text() == "finished result should stay finished" + + resume_response = api_client_request( + client, + "patch", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + payload={"action": "resume"}, + ) + assert resume_response.status_code == 200, resume_response.content.decode() + assert json.loads(resume_response.content.decode())["status"] == Snapshot.StatusChoices.QUEUED + + snapshot.refresh_from_db() + crawl.refresh_from_db() + assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert snapshot.retry_at is not None + assert snapshot.retry_at != RETRY_AT_MAX + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at is not None + assert crawl.retry_at != RETRY_AT_MAX + + resumed_rows = { + row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) + } + assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["manualqueue"][1] is not None + assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX + assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["manualstart"][1] is not None + assert resumed_rows["manualstart"][1] != RETRY_AT_MAX + + assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED + assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED + assert output_path.read_text() == "finished result should stay finished" + + +def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused( + tmp_path, + client, + recursive_test_site, +): + init_archive(tmp_path) + api_token = create_admin_and_token(tmp_path) + + with use_archivebox_db(tmp_path): + snapshot_response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + api_token=api_token, + payload={ + "url": recursive_test_site["root_url"], + "depth": 0, + "title": "Paused targeted retry", + "tags": ["targeted-extract-pause"], + "status": "queued", + }, + ) + assert snapshot_response.status_code == 200, snapshot_response.content.decode() + snapshot_id = json.loads(snapshot_response.content.decode())["id"] + snapshot = Snapshot.objects.get(id=snapshot_id) + + wget_result = _seed_archiveresult( + snapshot, + plugin="wget", + hook_name=_snapshot_hook_name("wget"), + status=ArchiveResult.StatusChoices.FAILED, + output_text="initial failure before targeted retry", + ) + unrelated_result = _seed_archiveresult( + snapshot, + plugin="manualqueue", + hook_name="on_Snapshot__manual_queue", + status=ArchiveResult.StatusChoices.QUEUED, + ) + finished_result = _seed_archiveresult( + snapshot, + plugin="manualdone", + hook_name="on_Snapshot__manual_done", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_text="finished row must survive targeted retry", + output_path="manualdone/targeted.txt", + ) + + pause_response = api_client_request( + client, + "patch", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + payload={"action": "pause"}, + ) + assert pause_response.status_code == 200, pause_response.content.decode() + assert json.loads(pause_response.content.decode())["status"] == Snapshot.StatusChoices.PAUSED + + snapshot = Snapshot.objects.get(id=snapshot_id) + assert snapshot.status == Snapshot.StatusChoices.PAUSED + assert snapshot.retry_at == RETRY_AT_MAX + assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED + assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED + finished_row = ArchiveResult.objects.get(id=finished_result.id) + finished_output_path = Path(snapshot.output_dir) / next(iter(finished_row.output_files)) + assert finished_output_path.read_text() == "finished row must survive targeted retry" + + env = cli_env( + port=get_free_port(), + PLUGINS="wget", + SAVE_WGET="True", + WGET_WARC_ENABLED="False", + URL_ALLOWLIST=r"127\.0\.0\.1[:/].*", + ) + extract = run_archivebox_cmd( + ["extract", str(wget_result.id)], + cwd=tmp_path, + env=env, + timeout=150, + ) + assert extract.returncode == 0, f"STDOUT:\n{extract.stdout}\nSTDERR:\n{extract.stderr}" + + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.get(id=snapshot_id) + assert snapshot.status == Snapshot.StatusChoices.PAUSED + assert snapshot.retry_at == RETRY_AT_MAX + + retried_wget = ArchiveResult.objects.get(id=wget_result.id) + assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED + assert retried_wget.output_size > 0 + assert retried_wget.output_files + + unrelated = ArchiveResult.objects.get(id=unrelated_result.id) + assert unrelated.status == ArchiveResult.StatusChoices.PAUSED + assert unrelated.retry_at == RETRY_AT_MAX + + finished = ArchiveResult.objects.get(id=finished_result.id) + assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED + assert finished.retry_at is None + assert finished_output_path.read_text() == "finished row must survive targeted retry" + + +@pytest.mark.timeout(240) +def test_paused_snapshot_survives_server_restart_and_resumes_via_api(tmp_path, recursive_test_site): + init_archive(tmp_path) + + port = get_free_port() + env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True") + api_token = create_admin_and_token(tmp_path) + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + crawl_response = live_api_request( + port, + "post", + "/api/v1/crawls/crawls", + api_token=api_token, + json={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 0, + "tags": ["snapshot-pause-restart-e2e"], + "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + }, + timeout=10, + ) + assert crawl_response.status_code == 200, crawl_response.text + crawl_id = crawl_response.json()["id"] + crawl_state = _wait_for_crawl_snapshot_rows(tmp_path, crawl_id) + snapshot_id = crawl_state["snapshots"][0]["id"] + + pause_response = live_api_request( + port, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + json={"action": "pause"}, + timeout=10, + ) + assert pause_response.status_code == 200, pause_response.text + + current_state = _paused_snapshot_state(tmp_path, snapshot_id) + if current_state["status"] == Snapshot.StatusChoices.SEALED: + assert current_state["succeeded_results"] > 0 + return + + paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id) + if paused_state["status"] == Snapshot.StatusChoices.SEALED: + assert paused_state["succeeded_results"] > 0 + return + assert paused_state["succeeded_results"] == 0 + assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html")) + + stop_server(tmp_path) + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + restarted_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id) + assert restarted_state["status"] == Snapshot.StatusChoices.PAUSED + assert restarted_state["succeeded_results"] == 0 + + resume_response = live_api_request( + port, + "patch", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + json={"action": "resume"}, + timeout=10, + ) + assert resume_response.status_code == 200, resume_response.text + assert resume_response.json()["status"] == Snapshot.StatusChoices.QUEUED + + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + assert "About" in captured_text + + final_state = _snapshot_state(tmp_path, recursive_test_site["root_url"]) + assert final_state["status"] == Snapshot.StatusChoices.SEALED + assert final_state["downloaded_at"] is not None + assert any( + result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in final_state["results"] + ) + finally: + stop_server(tmp_path) + + +def test_rest_snapshot_delete_removes_output_dir(client, api_headers): + url = "https://example.com/delete-path-snapshot" + + response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED}, + headers=api_headers, + ) + assert response.status_code == 200, response.content.decode() + + snapshot = Snapshot.objects.get(url=url) + snapshot_dir = Path(snapshot.output_dir) + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "delete-path-test.txt").write_text("snapshot output") + assert snapshot_dir.exists() + + response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers) + assert response.status_code == 200, response.content.decode() + assert not Snapshot.objects.filter(pk=snapshot.pk).exists() + assert not snapshot_dir.exists() diff --git a/archivebox/tests/test_api_v1_core_snapshots.py b/archivebox/tests/test_api_v1_core_snapshots.py new file mode 100644 index 00000000..54edfa92 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_snapshots.py @@ -0,0 +1,46 @@ +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_snapshots_api_filters_status_column_and_rejects_legacy_status(client, api_admin_user, api_headers): + crawl = Crawl.objects.create( + urls="https://example.com", + created_by=api_admin_user, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + Snapshot.objects.create( + url="https://example.com/api-status-queued", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + ) + sealed_snapshot = Snapshot.objects.create( + url="https://example.com/api-status-sealed", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + + response = client.get( + "/api/v1/core/snapshots", + {"status": "sealed"}, + **api_headers, + ) + assert response.status_code == 200, response.content + payload = response.json() + items = payload["items"] if isinstance(payload, dict) and "items" in payload else payload + assert [item["id"] for item in items] == [str(sealed_snapshot.id)] + assert [item["status"] for item in items] == ["sealed"] + + legacy_response = client.get( + "/api/v1/core/snapshots", + {"status": "unarchived"}, + **api_headers, + ) + assert legacy_response.status_code == 400 + assert "Invalid snapshot status" in legacy_response.content.decode() diff --git a/archivebox/tests/test_api_v1_core_snapshots_rss.py b/archivebox/tests/test_api_v1_core_snapshots_rss.py new file mode 100644 index 00000000..c5f2d1a5 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_snapshots_rss.py @@ -0,0 +1,115 @@ +from datetime import datetime +from typing import cast + +import pytest +from django.contrib.auth import get_user_model +from django.contrib.auth.models import UserManager +from django.utils import timezone + + +pytestmark = pytest.mark.django_db + + +User = get_user_model() +ADMIN_HOST = "admin.archivebox.localhost:8000" + + +@pytest.fixture +def other_user(db): + return cast(UserManager, User.objects).create_user( + username="rssother", + email="rssother@test.com", + password="testpassword", + ) + + +def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime): + from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl + + crawl = Crawl.objects.create(urls=url, created_by=user) + snapshot = Snapshot.objects.create( + url=url, + title=title, + crawl=crawl, + bookmarked_at=bookmarked_at, + ) + return crawl, snapshot + + +def test_snapshots_rss_filters_by_user_and_orders_newest_first(client, api_token, api_admin_user, other_user): + from archivebox.core.models import Tag + + older_at = timezone.make_aware(datetime(2026, 5, 22, 8, 0, 0)) + newer_at = timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0)) + _crawl, older_snapshot = make_snapshot( + user=api_admin_user, + url="https://example.com/rss-older", + title="Older & Escaped", + bookmarked_at=older_at, + ) + make_snapshot( + user=api_admin_user, + url="https://example.com/rss-newer", + title="Newer Snapshot", + bookmarked_at=newer_at, + ) + make_snapshot( + user=other_user, + url="https://example.com/rss-other-user", + title="Other User", + bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)), + ) + older_snapshot.tags.add(Tag.objects.create(name="rss-tag", created_by=api_admin_user)) + + response = client.get( + "/api/v1/core/snapshots.rss", + {"created_by": api_admin_user.username, "limit": 50, "api_key": api_token.token}, + HTTP_HOST=ADMIN_HOST, + ) + + assert response.status_code == 200 + assert response["Content-Type"].startswith("application/rss+xml") + body = response.content.decode() + assert 'rss-tag" in body + assert "rss-other-user" not in body + assert body.index("rss-newer") < body.index("rss-older") + + +def test_snapshots_rss_supports_before_yyyymmdd_and_limit(client, api_token, api_admin_user): + make_snapshot( + user=api_admin_user, + url="https://example.com/rss-before-too-new", + title="Too New", + bookmarked_at=timezone.make_aware(datetime(2026, 5, 24, 8, 0, 0)), + ) + make_snapshot( + user=api_admin_user, + url="https://example.com/rss-before-keep-one", + title="Keep One", + bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 12, 0, 0)), + ) + make_snapshot( + user=api_admin_user, + url="https://example.com/rss-before-keep-two", + title="Keep Two", + bookmarked_at=timezone.make_aware(datetime(2026, 5, 22, 12, 0, 0)), + ) + + response = client.get( + "/api/v1/core/snapshots.rss", + {"created_by": str(api_admin_user.pk), "before": "20260523", "limit": 1, "api_key": api_token.token}, + HTTP_HOST=ADMIN_HOST, + ) + + assert response.status_code == 200 + body = response.content.decode() + assert "rss-before-too-new" not in body + assert "rss-before-keep-one" in body + assert "rss-before-keep-two" not in body diff --git a/archivebox/tests/test_api_v1_core_tag_tag_id.py b/archivebox/tests/test_api_v1_core_tag_tag_id.py new file mode 100644 index 00000000..2da14fdb --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tag_tag_id.py @@ -0,0 +1,15 @@ +import pytest + +from archivebox.core.models import Tag + + +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.mark.parametrize("request_method", ("get", "delete")) +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers, request_method): + tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + + response = getattr(client, request_method)(f"/api/v1/core/tag/{tag.id}", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_core_tag_tag_id_rename.py b/archivebox/tests/test_api_v1_core_tag_tag_id_rename.py new file mode 100644 index 00000000..c5c65426 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tag_tag_id_rename.py @@ -0,0 +1,38 @@ +import pytest + +from archivebox.core.models import Tag +from archivebox.tests.conftest import ADMIN_TEST_HOST, api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + + response = api_client_request( + client, + "post", + f"/api/v1/core/tag/{tag.id}/rename", + payload={"name": "api-basic-renamed"}, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + + +def test_tag_rename_api_updates_name(client, api_token, tagged_data): + tag, _ = tagged_data + + response = api_client_request( + client, + "post", + f"/api/v1/core/tag/{tag.id}/rename?api_key={api_token.token}", + payload={"name": "Alpha Archive"}, + headers={"HTTP_HOST": ADMIN_TEST_HOST}, + ) + + assert response.status_code == 200 + + tag.refresh_from_db() + assert tag.name == "Alpha Archive" diff --git a/archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py b/archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py new file mode 100644 index 00000000..31a9d07e --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py @@ -0,0 +1,36 @@ +import pytest + +from archivebox.core.models import Snapshot, Tag +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import ADMIN_TEST_HOST + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + crawl = Crawl.objects.create(urls="https://example.com/tag-jsonl-export", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/tag-jsonl-export", crawl=crawl) + snapshot.tags.add(tag) + + response = client.get(f"/api/v1/core/tag/{tag.id}/snapshots.jsonl", **api_headers) + + assert response.status_code == 200, response.content + + +def test_tag_snapshots_export_returns_jsonl(client, api_token, tagged_data): + tag, _ = tagged_data + + response = client.get( + f"/api/v1/core/tag/{tag.id}/snapshots.jsonl", + {"api_key": api_token.token}, + HTTP_HOST=ADMIN_TEST_HOST, + ) + + assert response.status_code == 200 + assert response["Content-Type"].startswith("application/x-ndjson") + assert f"tag-{tag.slug}-snapshots.jsonl" in response["Content-Disposition"] + body = response.content.decode() + assert '"type": "Snapshot"' in body + assert '"tags": "Alpha Research"' in body diff --git a/archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py b/archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py new file mode 100644 index 00000000..88aaf861 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py @@ -0,0 +1,35 @@ +import pytest + +from archivebox.core.models import Snapshot, Tag +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import ADMIN_TEST_HOST + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + crawl = Crawl.objects.create(urls="https://example.com/tag-url-export", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/tag-url-export", crawl=crawl) + snapshot.tags.add(tag) + + response = client.get(f"/api/v1/core/tag/{tag.id}/urls.txt", **api_headers) + + assert response.status_code == 200, response.content + + +def test_tag_urls_export_returns_plain_text_urls(client, api_token, tagged_data): + tag, snapshots = tagged_data + + response = client.get( + f"/api/v1/core/tag/{tag.id}/urls.txt", + {"api_key": api_token.token}, + HTTP_HOST=ADMIN_TEST_HOST, + ) + + assert response.status_code == 200 + assert response["Content-Type"].startswith("text/plain") + assert f"tag-{tag.slug}-urls.txt" in response["Content-Disposition"] + exported_urls = set(filter(None, response.content.decode().splitlines())) + assert exported_urls == {snapshot.url for snapshot in snapshots} diff --git a/archivebox/tests/test_api_v1_core_tags.py b/archivebox/tests/test_api_v1_core_tags.py new file mode 100644 index 00000000..fcf7464a --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tags.py @@ -0,0 +1,14 @@ +import pytest + +from archivebox.core.models import Tag + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + + response = client.get("/api/v1/core/tags", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py b/archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py new file mode 100644 index 00000000..c732ca39 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py @@ -0,0 +1,25 @@ +import pytest + +from archivebox.core.models import Snapshot, Tag +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/tag-add", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/tag-add", crawl=crawl) + tag = Tag.objects.create(name="api-basic-add-tag", created_by=api_admin_user) + + response = api_client_request( + client, + "post", + "/api/v1/core/tags/add-to-snapshot/", + payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id}, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_core_tags_autocomplete.py b/archivebox/tests/test_api_v1_core_tags_autocomplete.py new file mode 100644 index 00000000..71f242eb --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tags_autocomplete.py @@ -0,0 +1,18 @@ +import pytest + +from archivebox.core.models import Tag + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_token): + Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + + response = client.get( + "/api/v1/core/tags/autocomplete/", + {"q": "api-basic", "api_key": api_token.token}, + HTTP_HOST="api.archivebox.localhost:8000", + ) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_core_tags_create.py b/archivebox/tests/test_api_v1_core_tags_create.py new file mode 100644 index 00000000..c069c515 --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tags_create.py @@ -0,0 +1,19 @@ +import pytest + +from archivebox.tests.conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, api_headers): + response = api_client_request( + client, + "post", + "/api/v1/core/tags/create/", + payload={"name": "api-basic-created-tag"}, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py b/archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py new file mode 100644 index 00000000..2b56770c --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py @@ -0,0 +1,26 @@ +import pytest + +from archivebox.core.models import Snapshot, Tag +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/tag-remove", created_by=api_admin_user) + snapshot = Snapshot.objects.create(url="https://example.com/tag-remove", crawl=crawl) + tag = Tag.objects.create(name="api-basic-remove-tag", created_by=api_admin_user) + snapshot.tags.add(tag) + + response = api_client_request( + client, + "post", + "/api/v1/core/tags/remove-from-snapshot/", + payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id}, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_core_tags_search.py b/archivebox/tests/test_api_v1_core_tags_search.py new file mode 100644 index 00000000..b489e90b --- /dev/null +++ b/archivebox/tests/test_api_v1_core_tags_search.py @@ -0,0 +1,83 @@ +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone + +from archivebox.core.models import Snapshot, Tag +from archivebox.tests.conftest import ADMIN_TEST_HOST + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + Tag.objects.create(name="api-basic-tag", created_by=api_admin_user) + + response = client.get("/api/v1/core/tags/search/", {"q": "api-basic"}, **api_headers) + + assert response.status_code == 200, response.content + + +def test_tag_search_api_returns_card_payload(client, api_token, tagged_data): + tag, snapshots = tagged_data + + response = client.get( + "/api/v1/core/tags/search/", + {"q": "Alpha", "api_key": api_token.token}, + HTTP_HOST=ADMIN_TEST_HOST, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["sort"] == "created_desc" + assert payload["created_by"] == "" + assert payload["year"] == "" + assert payload["has_snapshots"] == "all" + assert payload["tags"][0]["id"] == tag.id + assert payload["tags"][0]["name"] == "Alpha Research" + assert payload["tags"][0]["num_snapshots"] == 2 + assert payload["tags"][0]["snapshots"] == [] + assert payload["tags"][0]["export_jsonl_url"].endswith(f"/api/v1/core/tag/{tag.id}/snapshots.jsonl") + assert payload["tags"][0]["filter_url"].endswith(f"/admin/core/snapshot/?tags__id__exact={tag.id}") + assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"} + + +def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data): + from datetime import datetime + + other_user = get_user_model().objects.create_user( + username="tagother", + email="tagother@test.com", + password="unused", + ) + tag_with_snapshots = tagged_data[0] + empty_tag = Tag.objects.create(name="Zulu Empty", created_by=other_user) + alpha_tag = Tag.objects.create(name="Alpha Empty", created_by=other_user) + Snapshot.objects.create( + url="https://example.com/three", + title="Example Three", + crawl=crawl, + ).tags.add(alpha_tag) + + Tag.objects.filter(pk=empty_tag.pk).update(created_at=timezone.make_aware(datetime(2024, 1, 1, 12, 0, 0))) + Tag.objects.filter(pk=alpha_tag.pk).update(created_at=timezone.make_aware(datetime(2025, 1, 1, 12, 0, 0))) + Tag.objects.filter(pk=tag_with_snapshots.pk).update(created_at=timezone.make_aware(datetime(2026, 1, 1, 12, 0, 0))) + + response = client.get( + "/api/v1/core/tags/search/", + { + "sort": "name_desc", + "created_by": str(other_user.pk), + "year": "2024", + "has_snapshots": "no", + "api_key": api_token.token, + }, + HTTP_HOST=ADMIN_TEST_HOST, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["sort"] == "name_desc" + assert payload["created_by"] == str(other_user.pk) + assert payload["year"] == "2024" + assert payload["has_snapshots"] == "no" + assert [tag["name"] for tag in payload["tags"]] == ["Zulu Empty"] diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py new file mode 100644 index 00000000..2e391dce --- /dev/null +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py @@ -0,0 +1,614 @@ +import json +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import cast + +import pytest +from django.contrib.auth import get_user_model +from django.contrib.auth.models import UserManager +from django.utils import timezone + +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.test_orm_helpers import use_archivebox_db +from archivebox.workers.models import RETRY_AT_MAX + +from .conftest import ( + api_client_request, + cli_env, + create_admin_and_token, + get_crawl_runtime_state, + get_free_port, + init_archive, + live_api_request, + run_archivebox_cmd, + start_archivebox_server, + stop_server, + wait_for_live_api, + wait_for_snapshot_capture, +) + + +pytestmark = pytest.mark.django_db(transaction=True) +User = get_user_model() +ADMIN_HOST = "admin.archivebox.localhost:8000" + + +@pytest.fixture +def other_user(db): + return cast(UserManager, User.objects).create_user( + username="rssother", + email="rssother@test.com", + password="testpassword", + ) + + +def _seed_archiveresult( + snapshot: Snapshot, + *, + plugin: str, + hook_name: str, + status: str, + output_text: str = "", + output_path: str | None = None, +) -> ArchiveResult: + output_files = {} + output_size = 0 + output_mimetypes = "" + if output_path is not None: + output_bytes = output_text.encode() + absolute_path = Path(snapshot.output_dir) / output_path + absolute_path.parent.mkdir(parents=True, exist_ok=True) + absolute_path.write_bytes(output_bytes) + output_size = len(output_bytes) + output_mimetypes = "text/plain" + output_files[output_path] = { + "extension": Path(output_path).suffix.lstrip("."), + "mimetype": "text/plain", + "size": output_size, + } + + now = timezone.now() + return ArchiveResult.objects.create( + snapshot=snapshot, + plugin=plugin, + hook_name=hook_name, + status=status, + output_str=output_path or output_text, + output_files=output_files, + output_size=output_size, + output_mimetypes=output_mimetypes, + start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None, + end_ts=now if status in ArchiveResult.FINAL_STATES else None, + ) + + +def wait_for_crawl_snapshot_rows(cwd, crawl_id, timeout=45): + deadline = time.time() + timeout + latest_state = None + while time.time() < deadline: + latest_state = get_crawl_runtime_state(cwd, crawl_id) + if latest_state["snapshots"]: + return latest_state + time.sleep(0.2) + raise AssertionError(f"timed out waiting for runner to create snapshots for crawl {crawl_id}: {latest_state}") + + +def wait_for_crawl_child_snapshots_paused_or_sealed(cwd, crawl_id, timeout=45): + deadline = time.time() + timeout + latest_state = None + while time.time() < deadline: + latest_state = get_crawl_runtime_state(cwd, crawl_id) + snapshots = latest_state["snapshots"] + if snapshots and all(snapshot["status"] in {"paused", "sealed"} for snapshot in snapshots): + return latest_state + time.sleep(0.2) + raise AssertionError(f"timed out waiting for runner to pause or seal snapshots for crawl {crawl_id}: {latest_state}") + + +def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime): + crawl = Crawl.objects.create(urls=url, created_by=user) + snapshot = Snapshot.objects.create( + url=url, + title=title, + crawl=crawl, + bookmarked_at=bookmarked_at, + ) + return crawl, snapshot + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/crawl-detail", created_by=api_admin_user) + + response = client.get(f"/api/v1/crawls/crawl/{crawl.id}", **api_headers) + + assert response.status_code == 200, response.content + + +def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone( + tmp_path, + client, + recursive_test_site, +): + init_archive(tmp_path) + api_token = create_admin_and_token(tmp_path) + + with use_archivebox_db(tmp_path): + crawl_response = api_client_request( + client, + "post", + "/api/v1/crawls/crawls", + api_token=api_token, + payload={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 0, + "tags": ["crawl-archiveresult-pause"], + "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + }, + ) + assert crawl_response.status_code == 200, crawl_response.content.decode() + crawl_id = json.loads(crawl_response.content.decode())["id"] + from archivebox.services.runner import run_due_snapshot + + active_response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + api_token=api_token, + payload={ + "url": recursive_test_site["root_url"], + "crawl_id": crawl_id, + "depth": 0, + "title": "Active child", + "status": "queued", + }, + ) + assert active_response.status_code == 200, active_response.content.decode() + active_snapshot = Snapshot.objects.get(id=json.loads(active_response.content.decode())["id"]) + + sealed_response = api_client_request( + client, + "post", + "/api/v1/core/snapshots", + api_token=api_token, + payload={ + "url": recursive_test_site["child_urls"][0], + "crawl_id": crawl_id, + "depth": 0, + "title": "Already sealed child", + "status": "queued", + }, + ) + assert sealed_response.status_code == 200, sealed_response.content.decode() + sealed_snapshot_id = json.loads(sealed_response.content.decode())["id"] + sealed_snapshot = Snapshot.objects.get(id=sealed_snapshot_id) + sealed_done = _seed_archiveresult( + sealed_snapshot, + plugin="sealedone", + hook_name="on_Snapshot__sealed_done", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_text="sealed snapshot result remains finished", + output_path="sealedone/final.txt", + ) + sealed_snapshot.sm.seal() + sealed_snapshot.refresh_from_db() + assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED + assert sealed_snapshot.retry_at is None + + active_queued = _seed_archiveresult( + active_snapshot, + plugin="manualqueue", + hook_name="on_Snapshot__manual_queue", + status=ArchiveResult.StatusChoices.QUEUED, + ) + active_started = _seed_archiveresult( + active_snapshot, + plugin="manualstart", + hook_name="on_Snapshot__manual_start", + status=ArchiveResult.StatusChoices.STARTED, + ) + active_done = _seed_archiveresult( + active_snapshot, + plugin="manualdone", + hook_name="on_Snapshot__manual_done", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_text="parent cascade should not rewrite finished rows", + output_path="manualdone/cascade.txt", + ) + pause_response = api_client_request( + client, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + payload={"action": "pause"}, + ) + assert pause_response.status_code == 200, pause_response.content.decode() + assert json.loads(pause_response.content.decode())["status"] == Crawl.StatusChoices.PAUSED + + active_snapshot.refresh_from_db() + sealed_snapshot.refresh_from_db() + crawl = Crawl.objects.get(id=crawl_id) + assert crawl.status == Crawl.StatusChoices.PAUSED + assert crawl.retry_at == RETRY_AT_MAX + assert active_snapshot.status == Snapshot.StatusChoices.QUEUED + assert active_snapshot.retry_at is not None + assert active_snapshot.retry_at <= timezone.now() + assert ArchiveResult.objects.get(id=active_queued.id).status == ArchiveResult.StatusChoices.QUEUED + assert ArchiveResult.objects.get(id=active_started.id).status == ArchiveResult.StatusChoices.STARTED + + assert run_due_snapshot(active_snapshot, lock_seconds=60) is True + active_snapshot.refresh_from_db() + sealed_snapshot.refresh_from_db() + assert active_snapshot.status == Snapshot.StatusChoices.PAUSED + assert active_snapshot.retry_at == RETRY_AT_MAX + assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED + assert sealed_snapshot.retry_at is None + + paused_rows = { + row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) + } + assert paused_rows == { + "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + } + + active_done_row = ArchiveResult.objects.get(id=active_done.id) + sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id) + active_done_path = Path(active_snapshot.output_dir) / next(iter(active_done_row.output_files)) + sealed_done_path = Path(sealed_snapshot.output_dir) / next(iter(sealed_done_row.output_files)) + assert active_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED + assert active_done_row.retry_at is None + assert active_done_path.read_text() == "parent cascade should not rewrite finished rows" + assert sealed_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED + assert sealed_done_row.retry_at is None + assert sealed_done_path.read_text() == "sealed snapshot result remains finished" + + resume_response = api_client_request( + client, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + payload={"action": "resume"}, + ) + assert resume_response.status_code == 200, resume_response.content.decode() + assert json.loads(resume_response.content.decode())["status"] == Crawl.StatusChoices.QUEUED + + active_snapshot.refresh_from_db() + sealed_snapshot.refresh_from_db() + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at is not None + assert crawl.retry_at != RETRY_AT_MAX + assert active_snapshot.status == Snapshot.StatusChoices.QUEUED + assert active_snapshot.retry_at is not None + assert active_snapshot.retry_at != RETRY_AT_MAX + assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED + assert sealed_snapshot.retry_at is None + + resumed_rows = { + row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) + } + assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["manualqueue"][1] is not None + assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX + assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["manualstart"][1] is not None + assert resumed_rows["manualstart"][1] != RETRY_AT_MAX + assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED + assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED + assert active_done_path.read_text() == "parent cascade should not rewrite finished rows" + assert sealed_done_path.read_text() == "sealed snapshot result remains finished" + + +@pytest.mark.timeout(240) +def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(tmp_path, recursive_test_site): + init_archive(tmp_path) + + port = get_free_port() + env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True") + api_token = create_admin_and_token(tmp_path) + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + crawl_response = live_api_request( + port, + "post", + "/api/v1/crawls/crawls", + api_token=api_token, + json={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 0, + "tags": ["pause-resume-e2e"], + "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + }, + timeout=10, + ) + assert crawl_response.status_code == 200, crawl_response.text + crawl_id = crawl_response.json()["id"] + wait_for_crawl_snapshot_rows(tmp_path, crawl_id) + + pause_response = live_api_request( + port, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + json={"action": "pause"}, + timeout=10, + ) + assert pause_response.status_code == 200, pause_response.text + assert pause_response.json()["status"] == "paused" + + paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id) + assert paused_state["crawl_status"] == "paused" + assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"] + assert len(paused_state["snapshots"]) == 1 + snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed" + if snapshot_finished_before_pause: + assert any(result["status"] == "succeeded" for result in paused_state["results"]) + else: + assert paused_state["snapshots"][0]["status"] == "paused" + assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"] + + stop_server(tmp_path) + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + restarted_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert restarted_state["crawl_status"] == "paused" + assert restarted_state["crawl_retry_at"] == restarted_state["retry_at_max"] + if snapshot_finished_before_pause: + assert restarted_state["snapshots"][0]["status"] == "sealed" + assert any(result["status"] == "succeeded" for result in restarted_state["results"]) + return + assert restarted_state["snapshots"][0]["status"] == "paused" + assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"] + assert not any(result["status"] == "succeeded" for result in restarted_state["results"]) + + resume_response = live_api_request( + port, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + json={"action": "resume"}, + timeout=10, + ) + assert resume_response.status_code == 200, resume_response.text + assert resume_response.json()["status"] == "queued" + + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + assert "About" in captured_text + + final_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert final_state["snapshots"][0]["status"] == "sealed" + wget_results = [result for result in final_state["results"] if result["plugin"] == "wget"] + assert wget_results + assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) + finally: + stop_server(tmp_path) + + +@pytest.mark.timeout(180) +def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(tmp_path, recursive_test_site): + init_archive(tmp_path) + + port = get_free_port() + env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True") + api_token = create_admin_and_token(tmp_path) + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + crawl_response = live_api_request( + port, + "post", + "/api/v1/crawls/crawls", + api_token=api_token, + json={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 0, + "tags": ["paused-index-e2e"], + "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + }, + timeout=10, + ) + assert crawl_response.status_code == 200, crawl_response.text + crawl_id = crawl_response.json()["id"] + wait_for_crawl_snapshot_rows(tmp_path, crawl_id) + + pause_response = live_api_request( + port, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + json={"action": "pause"}, + timeout=10, + ) + assert pause_response.status_code == 200, pause_response.text + assert pause_response.json()["status"] == "paused" + paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id) + snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed" + finally: + stop_server(tmp_path) + + if snapshot_finished_before_pause: + indexed_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert indexed_state["crawl_status"] == "paused" + assert indexed_state["snapshots"][0]["status"] == "sealed" + return + + update_env = cli_env( + port=port, + PLUGINS="search_backend_sqlite", + SEARCH_BACKEND_ENGINE="sqlite", + ) + update_process = run_archivebox_cmd( + [ + "update", + "--index-only", + "--crawl-id", + crawl_id, + "--limit", + "1", + "--batch-size", + "1", + ], + cwd=tmp_path, + env=update_env, + timeout=120, + ) + assert update_process.returncode == 0, update_process.stderr + + indexed_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert indexed_state["crawl_status"] == "paused" + assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"] + assert indexed_state["snapshots"][0]["status"] == "paused" + assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"] + search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"] + assert search_results + assert all(result["status"] not in {"queued", "started", "paused"} for result in search_results) + + try: + start_archivebox_server(tmp_path, env=env, port=port) + wait_for_live_api(port) + + still_paused_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert still_paused_state["crawl_status"] == "paused" + assert still_paused_state["snapshots"][0]["status"] == "paused" + assert not any(result["plugin"] == "wget" and result["status"] == "succeeded" for result in still_paused_state["results"]) + + resume_response = live_api_request( + port, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + json={"action": "resume"}, + timeout=10, + ) + assert resume_response.status_code == 200, resume_response.text + assert resume_response.json()["status"] == "queued" + + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + assert "About" in captured_text + + resumed_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert resumed_state["snapshots"][0]["status"] == "sealed" + wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"] + assert wget_results + assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) + finally: + stop_server(tmp_path) + + +def test_crawl_cancel_api_defers_cleanup_to_runner(client, api_admin_user, api_headers): + from archivebox.services.runner import run_due_crawl + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by=api_admin_user, + status=Crawl.StatusChoices.STARTED, + retry_at=timezone.now() + timedelta(minutes=5), + ) + child = Snapshot.objects.create( + url="https://example.com/cancel-child", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + retry_at=timezone.now() + timedelta(minutes=5), + ) + crawl.output_dir.mkdir(parents=True, exist_ok=True) + pid_file = crawl.output_dir / "cleanup-test.pid" + pid_file.write_text("12345") + + response = api_client_request( + client, + "patch", + f"/api/v1/crawls/crawl/{crawl.id}", + payload={"action": "cancel"}, + headers=api_headers, + ) + assert response.status_code == 200, response.content + + crawl.refresh_from_db() + child.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.SEALED + assert crawl.retry_at is not None + assert crawl.retry_at <= timezone.now() + assert child.status == Snapshot.StatusChoices.STARTED + assert child.retry_at is not None + assert child.retry_at <= timezone.now() + assert pid_file.exists() + + assert run_due_crawl(crawl, lock_seconds=60) is True + crawl.refresh_from_db() + assert crawl.retry_at is None + assert not pid_file.exists() + + +def test_rest_crawl_delete_removes_crawl_and_snapshot_output_dirs(client, api_admin_user, api_headers): + url = "https://example.com/delete-path-crawl" + + crawl = Crawl.objects.create( + urls=url, + max_depth=0, + created_by=api_admin_user, + status=Crawl.StatusChoices.SEALED, + ) + snapshot = Snapshot.objects.create( + crawl=crawl, + url=url, + depth=0, + status=Snapshot.StatusChoices.SEALED, + ) + crawl_dir = Path(crawl.output_dir) + snapshot_dir = Path(snapshot.output_dir) + crawl_dir.mkdir(parents=True, exist_ok=True) + snapshot_dir.mkdir(parents=True, exist_ok=True) + (crawl_dir / "delete-path-crawl.txt").write_text("crawl output") + (snapshot_dir / "delete-path-snapshot.txt").write_text("snapshot output") + assert crawl_dir.exists() + assert snapshot_dir.exists() + + response = client.delete(f"/api/v1/crawls/crawl/{crawl.id}", **api_headers) + assert response.status_code == 200, response.content.decode() + assert not Crawl.objects.filter(pk=crawl.pk).exists() + assert not Snapshot.objects.filter(pk=snapshot.pk).exists() + assert not crawl_dir.exists() + assert not snapshot_dir.exists() + + +def test_crawl_as_rss_redirects_to_canonical_snapshots_feed(client, api_token, api_admin_user, other_user): + crawl, _snapshot = make_snapshot( + user=api_admin_user, + url="https://example.com/rss-crawl-feed", + title="Crawl Feed Snapshot", + bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0)), + ) + make_snapshot( + user=other_user, + url="https://example.com/rss-crawl-other", + title="Other Crawl Snapshot", + bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)), + ) + + response = client.get( + f"/api/v1/crawls/crawl/{crawl.id}", + {"as_rss": "true", "limit": 50, "api_key": api_token.token}, + HTTP_HOST=ADMIN_HOST, + follow=True, + ) + + assert response.status_code == 200 + assert response.redirect_chain + redirect_url = response.redirect_chain[0][0] + assert redirect_url.startswith("/api/v1/core/snapshots.rss?") + assert f"crawl_id={crawl.id}" in redirect_url + assert "as_rss" not in redirect_url + assert response["Content-Type"].startswith("application/rss+xml") + body = response.content.decode() + assert "rss-crawl-feed" in body + assert "rss-crawl-other" not in body diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py new file mode 100644 index 00000000..b8b14317 --- /dev/null +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py @@ -0,0 +1,16 @@ +import pytest + +from archivebox.crawls.models import Crawl + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/crawl-file-root", created_by=api_admin_user) + crawl.output_dir.mkdir(parents=True, exist_ok=True) + (crawl.output_dir / "basic.txt").write_text("ok") + + response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/basic.txt", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py new file mode 100644 index 00000000..14885982 --- /dev/null +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py @@ -0,0 +1,17 @@ +import pytest + +from archivebox.crawls.models import Crawl + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/crawl-file-nested", created_by=api_admin_user) + nested_dir = crawl.output_dir / "folder" + nested_dir.mkdir(parents=True, exist_ok=True) + (nested_dir / "basic.txt").write_text("ok") + + response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/basic.txt", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py new file mode 100644 index 00000000..3a3fbca3 --- /dev/null +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py @@ -0,0 +1,17 @@ +import pytest + +from archivebox.crawls.models import Crawl + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): + crawl = Crawl.objects.create(urls="https://example.com/crawl-file-deep", created_by=api_admin_user) + nested_dir = crawl.output_dir / "folder" / "subfolder" + nested_dir.mkdir(parents=True, exist_ok=True) + (nested_dir / "basic.txt").write_text("ok") + + response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/subfolder/basic.txt", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_crawls_crawls.py b/archivebox/tests/test_api_v1_crawls_crawls.py new file mode 100644 index 00000000..8a18d2cb --- /dev/null +++ b/archivebox/tests/test_api_v1_crawls_crawls.py @@ -0,0 +1,38 @@ +from django.test import RequestFactory + +import pytest + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_create_crawl_api_queues_crawl_without_spawning_runner(): + from django.contrib.auth import get_user_model + + from archivebox.api.v1_crawls import CrawlCreateSchema, create_crawl + + user = get_user_model().objects.create_superuser( + username="runner-api-admin", + email="runner-api-admin@example.com", + password="testpassword", + ) + request = RequestFactory().post("/api/v1/crawls") + request.user = user + + crawl = create_crawl( + request, + CrawlCreateSchema( + urls=["https://example.com"], + max_depth=0, + tags=[], + tags_str="", + label="", + notes="", + config={}, + ), + ) + + assert str(crawl.id) + assert crawl.status == "queued" + assert crawl.retry_at is not None + assert crawl.snapshot_set.count() == 0 diff --git a/archivebox/tests/test_api_v1_machine_binaries.py b/archivebox/tests/test_api_v1_machine_binaries.py new file mode 100644 index 00000000..9ae80e67 --- /dev/null +++ b/archivebox/tests/test_api_v1_machine_binaries.py @@ -0,0 +1,22 @@ +import pytest + +from archivebox.machine.models import Binary, Machine + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + machine = Machine.current(refresh=True) + Binary.objects.create( + machine=machine, + name="api-basic-bin", + binprovider="env", + abspath="/usr/bin/env", + version="1.0", + status=Binary.StatusChoices.INSTALLED, + ) + + response = client.get("/api/v1/machine/binaries", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_machine_binary_binary_id.py b/archivebox/tests/test_api_v1_machine_binary_binary_id.py new file mode 100644 index 00000000..a8c1d541 --- /dev/null +++ b/archivebox/tests/test_api_v1_machine_binary_binary_id.py @@ -0,0 +1,22 @@ +import pytest + +from archivebox.machine.models import Binary, Machine + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + machine = Machine.current(refresh=True) + binary = Binary.objects.create( + machine=machine, + name="api-basic-bin", + binprovider="env", + abspath="/usr/bin/env", + version="1.0", + status=Binary.StatusChoices.INSTALLED, + ) + + response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_machine_binary_by_name_name.py b/archivebox/tests/test_api_v1_machine_binary_by_name_name.py new file mode 100644 index 00000000..45744fa2 --- /dev/null +++ b/archivebox/tests/test_api_v1_machine_binary_by_name_name.py @@ -0,0 +1,22 @@ +import pytest + +from archivebox.machine.models import Binary, Machine + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + machine = Machine.current(refresh=True) + Binary.objects.create( + machine=machine, + name="api-basic-bin", + binprovider="env", + abspath="/usr/bin/env", + version="1.0", + status=Binary.StatusChoices.INSTALLED, + ) + + response = client.get("/api/v1/machine/binary/by-name/api-basic-bin", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_machine_machine_current.py b/archivebox/tests/test_api_v1_machine_machine_current.py new file mode 100644 index 00000000..ec303476 --- /dev/null +++ b/archivebox/tests/test_api_v1_machine_machine_current.py @@ -0,0 +1,10 @@ +import pytest + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + response = client.get("/api/v1/machine/machine/current", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_machine_machine_machine_id.py b/archivebox/tests/test_api_v1_machine_machine_machine_id.py new file mode 100644 index 00000000..db5a5ea7 --- /dev/null +++ b/archivebox/tests/test_api_v1_machine_machine_machine_id.py @@ -0,0 +1,14 @@ +import pytest + +from archivebox.machine.models import Machine + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + machine = Machine.current(refresh=True) + + response = client.get(f"/api/v1/machine/machine/{machine.id}", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_machine_machines.py b/archivebox/tests/test_api_v1_machine_machines.py new file mode 100644 index 00000000..3101c2d1 --- /dev/null +++ b/archivebox/tests/test_api_v1_machine_machines.py @@ -0,0 +1,14 @@ +import pytest + +from archivebox.machine.models import Machine + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, tmp_path, api_headers): + Machine.current(refresh=True) + + response = client.get("/api/v1/machine/machines", **api_headers) + + assert response.status_code == 200, response.content diff --git a/archivebox/tests/test_api_v1_personas_personas.py b/archivebox/tests/test_api_v1_personas_personas.py new file mode 100644 index 00000000..04cb74a7 --- /dev/null +++ b/archivebox/tests/test_api_v1_personas_personas.py @@ -0,0 +1,57 @@ +import pytest + +from archivebox.personas.models import Persona +from archivebox.tests.conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_personas_api_returns_paginated_envelope(client, api_headers): + for name in ("api-persona-alpha", "api-persona-bravo", "api-persona-charlie"): + create_response = api_client_request( + client, + "post", + "/api/v1/personas/sync", + payload={ + "extension_persona_id": f"extension-{name}", + "name": name, + "settings": {}, + "cookies_txt": "", + "auth_json": {}, + }, + headers=api_headers, + ) + assert create_response.status_code == 200, create_response.content + assert create_response.json()["created"] is True + + total_personas = Persona.objects.count() + response = client.get( + "/api/v1/personas/personas?limit=1", + **api_headers, + ) + + assert response.status_code == 200, response.content + payload = response.json() + assert isinstance(payload, dict) + assert set(payload) >= {"items", "count", "total_items", "total_pages", "page", "limit", "offset", "num_items"} + assert payload["count"] == total_personas + assert payload["total_items"] == total_personas + assert payload["limit"] == 1 + assert payload["offset"] == 0 + assert payload["num_items"] == 1 + assert len(payload["items"]) == 1 + + limit_two_response = client.get( + "/api/v1/personas/personas?limit=2", + **api_headers, + ) + assert limit_two_response.status_code == 200, limit_two_response.content + limit_two_payload = limit_two_response.json() + assert isinstance(limit_two_payload, dict) + assert limit_two_payload["count"] == total_personas + assert limit_two_payload["total_items"] == total_personas + assert limit_two_payload["limit"] == 2 + assert limit_two_payload["offset"] == 0 + assert limit_two_payload["num_items"] == 2 + assert len(limit_two_payload["items"]) == 2 diff --git a/archivebox/tests/test_api_v1_personas_sync.py b/archivebox/tests/test_api_v1_personas_sync.py new file mode 100644 index 00000000..cea4d5d2 --- /dev/null +++ b/archivebox/tests/test_api_v1_personas_sync.py @@ -0,0 +1,25 @@ +import pytest + +from archivebox.tests.conftest import api_client_request + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_basic_success_case_request(client, api_headers): + response = api_client_request( + client, + "post", + "/api/v1/personas/sync", + payload={ + "extension_persona_id": "extension-api-persona-basic", + "name": "api-persona-basic", + "settings": {}, + "cookies_txt": "", + "auth_json": {}, + }, + headers=api_headers, + ) + + assert response.status_code == 200, response.content + assert response.json()["success"] is True diff --git a/archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py b/archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py new file mode 100644 index 00000000..d7917289 --- /dev/null +++ b/archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py @@ -0,0 +1,207 @@ +import pytest +import requests + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import ( + cli_env, + create_admin_and_token, + get_free_port, + init_archive, + live_api_request, + start_archivebox_server, + stop_server, + wait_for_live_api, +) +from archivebox.tests.test_orm_helpers import use_archivebox_db + + +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.mark.timeout(180) +def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server(tmp_path, recursive_test_site): + init_archive(tmp_path) + + port = get_free_port() + env = cli_env(port=port, server=True, PUBLIC_INDEX="True") + api_token = create_admin_and_token(tmp_path) + + try: + start_archivebox_server(tmp_path, env=env, port=port) + docs = wait_for_live_api(port) + assert docs.status_code == 200 + openapi = wait_for_live_api(port, path="/api/v1/openapi.json") + assert openapi.status_code == 200 + paths = openapi.json()["paths"] + assert "/api/v1/core/snapshots" in paths + assert "/api/v1/crawls/crawls" in paths + + unauth = requests.get( + f"http://127.0.0.1:{port}/api/v1/crawls/crawls", + headers={"Host": f"api.archivebox.localhost:{port}"}, + timeout=10, + ) + assert unauth.status_code in (401, 403) + bad_auth = requests.get( + f"http://127.0.0.1:{port}/api/v1/crawls/crawls", + headers={"Host": f"api.archivebox.localhost:{port}", "X-ArchiveBox-API-Key": "bad-token"}, + timeout=10, + ) + assert bad_auth.status_code in (401, 403) + + crawl_response = live_api_request( + port, + "post", + "/api/v1/crawls/crawls", + api_token=api_token, + json={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 2, + "tags": ["api-depth-two"], + "label": "api crawl", + "notes": "created through REST API", + "config": { + "PLUGINS": "wget,parse_html_urls", + "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*", + "CRAWL_MAX_URLS": 7, + "CRAWL_MAX_SIZE": "0", + "SNAPSHOT_MAX_SIZE": "0", + }, + }, + timeout=10, + ) + assert crawl_response.status_code == 200, crawl_response.text + crawl_payload = crawl_response.json() + crawl_id = crawl_payload["id"] + assert crawl_payload["max_depth"] == 2 + assert crawl_payload["tags_str"] == "api-depth-two" + assert crawl_payload["config"]["PLUGINS"] == "wget,parse_html_urls" + assert crawl_payload["config"]["CRAWL_MAX_URLS"] == 7 + + snapshot_response = live_api_request( + port, + "post", + "/api/v1/core/snapshots", + api_token=api_token, + json={ + "url": recursive_test_site["child_urls"][0], + "crawl_id": crawl_id, + "depth": 1, + "title": "API child snapshot", + "tags": ["api-child"], + "status": "queued", + }, + timeout=10, + ) + assert snapshot_response.status_code == 200, snapshot_response.text + snapshot_payload = snapshot_response.json() + snapshot_id = snapshot_payload["id"] + assert snapshot_payload["url"] == recursive_test_site["child_urls"][0] + assert snapshot_payload["tags"] == ["api-child"] + + patch_snapshot = live_api_request( + port, + "patch", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + json={"status": "sealed", "tags": ["api-child", "api-patched"]}, + timeout=10, + ) + assert patch_snapshot.status_code == 200, patch_snapshot.text + assert patch_snapshot.json()["status"] == "sealed" + assert set(patch_snapshot.json()["tags"]) == {"api-child", "api-patched"} + + tag_create = live_api_request( + port, + "post", + "/api/v1/core/tags/create/", + api_token=api_token, + json={"name": "api-extra"}, + timeout=10, + ) + assert tag_create.status_code == 200, tag_create.text + tag_id = tag_create.json()["tag_id"] + + add_tag = live_api_request( + port, + "post", + "/api/v1/core/tags/add-to-snapshot/", + api_token=api_token, + json={"snapshot_id": snapshot_id, "tag_id": tag_id}, + timeout=10, + ) + assert add_tag.status_code == 200, add_tag.text + remove_tag = live_api_request( + port, + "post", + "/api/v1/core/tags/remove-from-snapshot/", + api_token=api_token, + json={"snapshot_id": snapshot_id, "tag_name": "api-extra"}, + timeout=10, + ) + assert remove_tag.status_code == 200, remove_tag.text + + crawl_patch = live_api_request( + port, + "patch", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + json={"status": "sealed", "tags": ["api-sealed"]}, + timeout=10, + ) + assert crawl_patch.status_code == 200, crawl_patch.text + assert crawl_patch.json()["status"] == "sealed" + assert crawl_patch.json()["tags_str"] == "api-sealed" + + snapshots_list = live_api_request( + port, + "get", + "/api/v1/core/snapshots?tag=api-patched&with_archiveresults=true", + api_token=api_token, + timeout=10, + ) + assert snapshots_list.status_code == 200, snapshots_list.text + snapshot_items = snapshots_list.json()["items"] + assert len(snapshot_items) == 1 + assert snapshot_items[0]["id"] == snapshot_id + assert snapshot_items[0]["archiveresults"] == [] + + bearer_response = requests.get( + f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", + headers={"Host": f"api.archivebox.localhost:{port}", "Authorization": f"Bearer {api_token}"}, + timeout=10, + ) + assert bearer_response.status_code == 200, bearer_response.text + query_response = requests.get( + f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}?api_key={api_token}", + headers={"Host": f"api.archivebox.localhost:{port}"}, + timeout=10, + ) + assert query_response.status_code == 200, query_response.text + + delete_snapshot = live_api_request( + port, + "delete", + f"/api/v1/core/snapshot/{snapshot_id}", + api_token=api_token, + timeout=10, + ) + assert delete_snapshot.status_code == 200, delete_snapshot.text + assert delete_snapshot.json()["success"] is True + + delete_crawl = live_api_request( + port, + "delete", + f"/api/v1/crawls/crawl/{crawl_id}", + api_token=api_token, + timeout=10, + ) + assert delete_crawl.status_code == 200, delete_crawl.text + assert delete_crawl.json()["success"] is True + + with use_archivebox_db(tmp_path): + assert Crawl.objects.filter(pk=crawl_id).count() == 0 + assert Snapshot.objects.filter(pk=snapshot_id).count() == 0 + finally: + stop_server(tmp_path) diff --git a/archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py b/archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py new file mode 100644 index 00000000..9f68f3af --- /dev/null +++ b/archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py @@ -0,0 +1,66 @@ +from django.test import RequestFactory + +import pytest + + +pytestmark = pytest.mark.django_db(transaction=True) + + +SENSITIVE_SECRET = "raw-twocaptcha-secret-for-frozen-crawl-test" + + +@pytest.fixture +def archivebox_db(initialized_archive): + from archivebox.tests.test_orm_helpers import use_archivebox_db + + with use_archivebox_db(initialized_archive): + yield initialized_archive + + +def _user(username="frozen-config-admin"): + from django.contrib.auth import get_user_model + + return get_user_model().objects.create_superuser( + username=username, + email=f"{username}@example.com", + password="testpassword", + ) + + +def test_api_create_and_cli_add_store_full_frozen_config(archivebox_db): + from archivebox.api.v1_crawls import CrawlCreateSchema, CrawlSchema, create_crawl + from archivebox.cli.archivebox_add import add + from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED + + user = _user("frozen-config-api-admin") + request = RequestFactory().post("/api/v1/crawls") + request.user = user + + api_crawl = create_crawl( + request, + CrawlCreateSchema( + urls=["https://example.com/api"], + max_depth=0, + tags=[], + tags_str="", + label="API frozen config", + notes="", + config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 33, "SECRET_KEY": "must-not-freeze", "PUBLIC_ADD_VIEW": True}, + ), + ) + assert "CHECK_SSL_VALIDITY" in api_crawl.config + assert api_crawl.config["TIMEOUT"] == 33 + assert api_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET + assert "SECRET_KEY" not in api_crawl.config + assert "PUBLIC_ADD_VIEW" not in api_crawl.config + assert CrawlSchema.resolve_config(api_crawl)["TWOCAPTCHA_API_KEY"] == SENSITIVE_CONFIG_VALUE_REDACTED + + cli_crawl, _snapshots = add( + "https://example.com/cli", + bg=True, + created_by_id=user.pk, + config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 44}, + ) + assert "CHECK_SSL_VALIDITY" in cli_crawl.config + assert cli_crawl.config["TIMEOUT"] == 44 + assert cli_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET diff --git a/archivebox/tests/test_archiveresult_pause.py b/archivebox/tests/test_archiveresult_pause.py index c19fd6ef..d1974060 100644 --- a/archivebox/tests/test_archiveresult_pause.py +++ b/archivebox/tests/test_archiveresult_pause.py @@ -1,372 +1,2 @@ -import json -import os -import subprocess -import sys -from pathlib import Path - -import pytest -from django.utils import timezone - -from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.crawls.models import Crawl -from archivebox.tests.test_orm_helpers import use_archivebox_db -from archivebox.workers.models import RETRY_AT_MAX - -from .conftest import build_test_env, create_admin_and_token, get_free_port, init_archive - -pytestmark = pytest.mark.django_db(transaction=True) - -API_HOST = "api.archivebox.localhost:8000" - - -def _api_headers(token: str) -> dict[str, str]: - return { - "HTTP_HOST": API_HOST, - "HTTP_X_ARCHIVEBOX_API_KEY": token, - } - - -def _json_response(response): - return json.loads(response.content.decode()) - - -def _post_json(client, path: str, token: str, payload: dict): - return client.post( - path, - data=json.dumps(payload), - content_type="application/json", - **_api_headers(token), - ) - - -def _patch_json(client, path: str, token: str, payload: dict): - return client.patch( - path, - data=json.dumps(payload), - content_type="application/json", - **_api_headers(token), - ) - - -def _seed_archiveresult( - snapshot: Snapshot, - *, - plugin: str, - hook_name: str, - status: str, - output_text: str = "", - output_path: str | None = None, -) -> ArchiveResult: - output_files = {} - output_size = 0 - output_mimetypes = "" - if output_path is not None: - output_bytes = output_text.encode() - absolute_path = Path(snapshot.output_dir) / output_path - absolute_path.parent.mkdir(parents=True, exist_ok=True) - absolute_path.write_bytes(output_bytes) - output_size = len(output_bytes) - output_mimetypes = "text/plain" - output_files[output_path] = { - "extension": Path(output_path).suffix.lstrip("."), - "mimetype": "text/plain", - "size": output_size, - } - - now = timezone.now() - return ArchiveResult.objects.create( - snapshot=snapshot, - plugin=plugin, - hook_name=hook_name, - status=status, - output_str=output_path or output_text, - output_files=output_files, - output_size=output_size, - output_mimetypes=output_mimetypes, - start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None, - end_ts=now if status in ArchiveResult.FINAL_STATES else None, - ) - - -def _snapshot_hook_name(plugin_name: str) -> str: - from abx_dl.models import discover_plugins - - plugin = discover_plugins().get(plugin_name) - assert plugin is not None, f"missing test plugin {plugin_name}" - hooks = plugin.filter_hooks("Snapshot") - assert hooks, f"missing Snapshot hooks for {plugin_name}" - return hooks[0].name - - -def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone( - tmp_path, - client, - recursive_test_site, -): - os.chdir(tmp_path) - init_archive(tmp_path) - api_token = create_admin_and_token(tmp_path) - - with use_archivebox_db(tmp_path): - crawl_response = _post_json( - client, - "/api/v1/crawls/crawls", - api_token, - { - "urls": [recursive_test_site["root_url"]], - "max_depth": 0, - "tags": ["crawl-archiveresult-pause"], - "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, - }, - ) - assert crawl_response.status_code == 200, crawl_response.content.decode() - crawl_id = _json_response(crawl_response)["id"] - from archivebox.services.runner import run_due_snapshot - - active_response = _post_json( - client, - "/api/v1/core/snapshots", - api_token, - { - "url": recursive_test_site["root_url"], - "crawl_id": crawl_id, - "depth": 0, - "title": "Active child", - "status": "queued", - }, - ) - assert active_response.status_code == 200, active_response.content.decode() - active_snapshot = Snapshot.objects.get(id=_json_response(active_response)["id"]) - - sealed_response = _post_json( - client, - "/api/v1/core/snapshots", - api_token, - { - "url": recursive_test_site["child_urls"][0], - "crawl_id": crawl_id, - "depth": 0, - "title": "Already sealed child", - "status": "queued", - }, - ) - assert sealed_response.status_code == 200, sealed_response.content.decode() - sealed_snapshot_id = _json_response(sealed_response)["id"] - sealed_snapshot = Snapshot.objects.get(id=sealed_snapshot_id) - sealed_done = _seed_archiveresult( - sealed_snapshot, - plugin="sealedone", - hook_name="on_Snapshot__sealed_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="sealed snapshot result remains finished", - output_path="sealedone/final.txt", - ) - sealed_snapshot.sm.seal() - sealed_snapshot.refresh_from_db() - assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED - assert sealed_snapshot.retry_at is None - - active_queued = _seed_archiveresult( - active_snapshot, - plugin="manualqueue", - hook_name="on_Snapshot__manual_queue", - status=ArchiveResult.StatusChoices.QUEUED, - ) - active_started = _seed_archiveresult( - active_snapshot, - plugin="manualstart", - hook_name="on_Snapshot__manual_start", - status=ArchiveResult.StatusChoices.STARTED, - ) - active_done = _seed_archiveresult( - active_snapshot, - plugin="manualdone", - hook_name="on_Snapshot__manual_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="parent cascade should not rewrite finished rows", - output_path="manualdone/cascade.txt", - ) - pause_response = _patch_json( - client, - f"/api/v1/crawls/crawl/{crawl_id}", - api_token, - {"action": "pause"}, - ) - assert pause_response.status_code == 200, pause_response.content.decode() - assert _json_response(pause_response)["status"] == Crawl.StatusChoices.PAUSED - - active_snapshot.refresh_from_db() - sealed_snapshot.refresh_from_db() - crawl = Crawl.objects.get(id=crawl_id) - assert crawl.status == Crawl.StatusChoices.PAUSED - assert crawl.retry_at == RETRY_AT_MAX - assert active_snapshot.status == Snapshot.StatusChoices.QUEUED - assert active_snapshot.retry_at is not None - assert active_snapshot.retry_at <= timezone.now() - assert ArchiveResult.objects.get(id=active_queued.id).status == ArchiveResult.StatusChoices.QUEUED - assert ArchiveResult.objects.get(id=active_started.id).status == ArchiveResult.StatusChoices.STARTED - - assert run_due_snapshot(active_snapshot, lock_seconds=60) is True - active_snapshot.refresh_from_db() - sealed_snapshot.refresh_from_db() - assert active_snapshot.status == Snapshot.StatusChoices.PAUSED - assert active_snapshot.retry_at == RETRY_AT_MAX - assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED - assert sealed_snapshot.retry_at is None - - paused_rows = { - row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) - } - assert paused_rows == { - "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - } - - active_done_row = ArchiveResult.objects.get(id=active_done.id) - sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id) - active_done_path = Path(active_snapshot.output_dir) / next(iter(active_done_row.output_files)) - sealed_done_path = Path(sealed_snapshot.output_dir) / next(iter(sealed_done_row.output_files)) - assert active_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED - assert active_done_row.retry_at is None - assert active_done_path.read_text() == "parent cascade should not rewrite finished rows" - assert sealed_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED - assert sealed_done_row.retry_at is None - assert sealed_done_path.read_text() == "sealed snapshot result remains finished" - - resume_response = _patch_json( - client, - f"/api/v1/crawls/crawl/{crawl_id}", - api_token, - {"action": "resume"}, - ) - assert resume_response.status_code == 200, resume_response.content.decode() - assert _json_response(resume_response)["status"] == Crawl.StatusChoices.QUEUED - - active_snapshot.refresh_from_db() - sealed_snapshot.refresh_from_db() - crawl.refresh_from_db() - assert crawl.status == Crawl.StatusChoices.QUEUED - assert crawl.retry_at is not None - assert crawl.retry_at != RETRY_AT_MAX - assert active_snapshot.status == Snapshot.StatusChoices.QUEUED - assert active_snapshot.retry_at is not None - assert active_snapshot.retry_at != RETRY_AT_MAX - assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED - assert sealed_snapshot.retry_at is None - - resumed_rows = { - row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) - } - assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualqueue"][1] is not None - assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX - assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualstart"][1] is not None - assert resumed_rows["manualstart"][1] != RETRY_AT_MAX - assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED - assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED - assert active_done_path.read_text() == "parent cascade should not rewrite finished rows" - assert sealed_done_path.read_text() == "sealed snapshot result remains finished" - - -def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused( - tmp_path, - client, - recursive_test_site, -): - os.chdir(tmp_path) - init_archive(tmp_path) - api_token = create_admin_and_token(tmp_path) - - with use_archivebox_db(tmp_path): - snapshot_response = _post_json( - client, - "/api/v1/core/snapshots", - api_token, - { - "url": recursive_test_site["root_url"], - "depth": 0, - "title": "Paused targeted retry", - "tags": ["targeted-extract-pause"], - "status": "queued", - }, - ) - assert snapshot_response.status_code == 200, snapshot_response.content.decode() - snapshot_id = _json_response(snapshot_response)["id"] - snapshot = Snapshot.objects.get(id=snapshot_id) - - wget_result = _seed_archiveresult( - snapshot, - plugin="wget", - hook_name=_snapshot_hook_name("wget"), - status=ArchiveResult.StatusChoices.FAILED, - output_text="initial failure before targeted retry", - ) - unrelated_result = _seed_archiveresult( - snapshot, - plugin="manualqueue", - hook_name="on_Snapshot__manual_queue", - status=ArchiveResult.StatusChoices.QUEUED, - ) - finished_result = _seed_archiveresult( - snapshot, - plugin="manualdone", - hook_name="on_Snapshot__manual_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="finished row must survive targeted retry", - output_path="manualdone/targeted.txt", - ) - - pause_response = _patch_json( - client, - f"/api/v1/core/snapshot/{snapshot_id}", - api_token, - {"action": "pause"}, - ) - assert pause_response.status_code == 200, pause_response.content.decode() - assert _json_response(pause_response)["status"] == Snapshot.StatusChoices.PAUSED - - snapshot = Snapshot.objects.get(id=snapshot_id) - assert snapshot.status == Snapshot.StatusChoices.PAUSED - assert snapshot.retry_at == RETRY_AT_MAX - assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED - assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED - finished_row = ArchiveResult.objects.get(id=finished_result.id) - finished_output_path = Path(snapshot.output_dir) / next(iter(finished_row.output_files)) - assert finished_output_path.read_text() == "finished row must survive targeted retry" - - env = build_test_env( - get_free_port(), - PLUGINS="wget", - SAVE_WGET="True", - WGET_WARC_ENABLED="False", - URL_ALLOWLIST=r"127\.0\.0\.1[:/].*", - ) - extract = subprocess.run( - [sys.executable, "-m", "archivebox", "extract", str(wget_result.id)], - cwd=tmp_path, - capture_output=True, - text=True, - env=env, - timeout=150, - ) - assert extract.returncode == 0, f"STDOUT:\n{extract.stdout}\nSTDERR:\n{extract.stderr}" - - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.get(id=snapshot_id) - assert snapshot.status == Snapshot.StatusChoices.PAUSED - assert snapshot.retry_at == RETRY_AT_MAX - - retried_wget = ArchiveResult.objects.get(id=wget_result.id) - assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED - assert retried_wget.output_size > 0 - assert retried_wget.output_files - - unrelated = ArchiveResult.objects.get(id=unrelated_result.id) - assert unrelated.status == ArchiveResult.StatusChoices.PAUSED - assert unrelated.retry_at == RETRY_AT_MAX - - finished = ArchiveResult.objects.get(id=finished_result.id) - assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED - assert finished.retry_at is None - assert finished_output_path.read_text() == "finished row must survive targeted retry" +# test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone moved to test_api_v1_crawls_crawl_crawl_id.py. +# test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused moved to test_api_v1_core_snapshot_snapshot_id.py. diff --git a/archivebox/tests/test_auth_ldap.py b/archivebox/tests/test_auth_ldap.py index c0c92513..dc143e6d 100644 --- a/archivebox/tests/test_auth_ldap.py +++ b/archivebox/tests/test_auth_ldap.py @@ -132,27 +132,33 @@ class TestArchiveBoxWithLDAP: def test_archivebox_init_without_ldap(self, tmp_path): """Test that archivebox init works without LDAP enabled.""" - _, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["init"], - data_dir=tmp_path, + cwd=tmp_path, timeout=45, env={"LDAP_ENABLED": "False"}, + default_cli_env=True, + disable_extractors=True, ) + _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode # Should succeed assert code == 0, f"archivebox init failed: {stderr}" def test_archivebox_version_with_ldap_config(self, tmp_path): """Test that archivebox version works with LDAP config set.""" - _, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["version"], - data_dir=tmp_path, + cwd=tmp_path, timeout=10, env={ "LDAP_ENABLED": "False", "LDAP_SERVER_URI": "ldap://ldap-test.localhost:389", }, + default_cli_env=True, + disable_extractors=True, ) + _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode # Should succeed assert code == 0, f"archivebox version failed: {stderr}" @@ -163,15 +169,18 @@ class TestLDAPConfigValidationInArchiveBox: def test_archivebox_init_with_incomplete_ldap_config(self, tmp_path): """Test that archivebox init fails with helpful error when LDAP config is incomplete.""" - _, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["init"], - data_dir=tmp_path, + cwd=tmp_path, timeout=45, env={ "LDAP_ENABLED": "True", # Missing: LDAP_SERVER_URI, LDAP_BIND_DN, etc. }, + default_cli_env=True, + disable_extractors=True, ) + _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode # Should fail with validation error assert code != 0, "Should fail with incomplete LDAP config" diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py index 9fe1e277..aa73f654 100644 --- a/archivebox/tests/test_binary_service.py +++ b/archivebox/tests/test_binary_service.py @@ -24,10 +24,6 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) -> return link -def _binary_request(name: str, *, binproviders: str = "env") -> str: - return json.dumps({"type": "BinaryRequest", "name": name, "binproviders": binproviders}) + "\n" - - def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]: return { "LIB_DIR": str(data_dir / "lib"), @@ -100,13 +96,16 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized _link_real_binary(bootstrap_bin_dir, "uv") _link_real_binary(provider_bin_dir, name, source="rg") - stdout, stderr, returncode = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], - data_dir=initialized_archive, - stdin=_binary_request(name), + cwd=initialized_archive, + stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n", timeout=120, env=_runtime_env(initialized_archive, bootstrap_bin_dir), + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr output_records = parse_jsonl_output(stdout) @@ -135,12 +134,15 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized assert binary_processes[-1].exit_code == 0 assert any(f"--name={name}" in arg for arg in binary_processes[-1].cmd) - version_stdout, version_stderr, version_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["version"], - data_dir=initialized_archive, + cwd=initialized_archive, timeout=60, env=_runtime_env(initialized_archive, bootstrap_bin_dir), + default_cli_env=True, + disable_extractors=True, ) + version_stdout, version_stderr, version_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert version_code == 0, version_stderr assert name in version_stdout assert binary.version in version_stdout @@ -149,12 +151,15 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized (initialized_archive / "lib" / "bin" / name).unlink(missing_ok=True) _link_real_binary(bootstrap_bin_dir, name, source="rg") - rerun_stdout, rerun_stderr, rerun_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run", f"--binary-id={first_binary_id}"], - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=_runtime_env(initialized_archive, bootstrap_bin_dir), + default_cli_env=True, + disable_extractors=True, ) + rerun_stdout, rerun_stderr, rerun_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert rerun_code == 0, rerun_stdout + rerun_stderr with use_archivebox_db(initialized_archive): @@ -174,13 +179,16 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res provider_bin_dir = initialized_archive / "lib" / "env" / "bin" _link_real_binary(bootstrap_bin_dir, "uv") - stdout, stderr, returncode = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], - data_dir=initialized_archive, - stdin=_binary_request(name), + cwd=initialized_archive, + stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n", timeout=120, env=_runtime_env(initialized_archive, bootstrap_bin_dir), + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in parse_jsonl_output(stdout)) @@ -201,12 +209,15 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res _link_real_binary(provider_bin_dir, name, source="rg") - recover_stdout, recover_stderr, recover_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run", f"--binary-id={queued_id}"], - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=_runtime_env(initialized_archive, bootstrap_bin_dir), + default_cli_env=True, + disable_extractors=True, ) + recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert recover_code == 0, recover_stdout + recover_stderr with use_archivebox_db(initialized_archive): diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py index 3a661494..df95c552 100644 --- a/archivebox/tests/test_cli_add.py +++ b/archivebox/tests/test_cli_add.py @@ -5,64 +5,49 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie """ import os -import subprocess -from pathlib import Path +import json import pytest -from archivebox.core.models import Snapshot +from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl -from archivebox.tests.conftest import run_queued_crawls +from archivebox.machine.models import Process +from archivebox.tests.conftest import _find_system_browser, find_snapshot_dir, run_archivebox_cmd, run_queued_crawls, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def _find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: - candidates = {snapshot_id} - if len(snapshot_id) == 32: - candidates.add(f"{snapshot_id[:8]}-{snapshot_id[8:12]}-{snapshot_id[12:16]}-{snapshot_id[16:20]}-{snapshot_id[20:]}") - elif len(snapshot_id) == 36 and "-" in snapshot_id: - candidates.add(snapshot_id.replace("-", "")) - - for needle in candidates: - for path in data_dir.rglob(needle): - if path.is_dir(): - return path - return None - - -def test_add_single_url_creates_snapshot_in_db(tmp_path, process, disable_extractors_dict): +def test_add_single_url_creates_snapshot_in_db(initialized_archive): """Test that adding a single URL queues a crawl whose runner creates the snapshot.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) assert result.returncode == 0 - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshots = list(Snapshot.objects.values_list("url", flat=True)) assert len(snapshots) == 1 assert snapshots[0] == "https://example.com" -def test_add_bg_queues_crawl_without_creating_snapshots(tmp_path, process, disable_extractors_dict): +def test_add_bg_queues_crawl_without_creating_snapshots(initialized_archive): """Background add should leave root snapshot creation to the runner.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--bg", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--bg", "--depth=0", "https://example.com"], + env=env, ) assert result.returncode == 0 - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() snapshot_count = Snapshot.objects.count() @@ -71,25 +56,23 @@ def test_add_bg_queues_crawl_without_creating_snapshots(tmp_path, process, disab assert snapshot_count == 0 -def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(tmp_path, process, disable_extractors_dict): +def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initialized_archive): """Index-only add only creates the crawl; rejected URLs are sealed by the runner.""" - os.chdir(tmp_path) - result = subprocess.run( + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( [ - "archivebox", "add", "--index-only", "--depth=0", "--url-denylist=example.com", "https://example.com", ], - capture_output=True, - env=disable_extractors_dict, + env=env, ) assert result.returncode == 0 - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() snapshot_count = Snapshot.objects.count() @@ -97,9 +80,9 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(tmp_p assert crawl.retry_at is None assert snapshot_count == 0 - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() snapshot_count = Snapshot.objects.count() @@ -108,31 +91,55 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(tmp_p assert snapshot_count == 0 -def test_add_creates_crawl_record(tmp_path, process, disable_extractors_dict): - """Test that add command creates a Crawl record in the database.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, +def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive): + """Index-only add must apply the same internal URL guard as snapshot creation.""" + env = cli_env(disable_extractors=True) + internal_urls = [ + "http://archivebox.localhost:9292/admin/", + "http://web.archivebox.localhost:9292/", + "http://api.archivebox.localhost:9292/api/v1/docs", + "http://snap-2fb8e923c58c.archivebox.localhost:9292/index.html", + ] + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", *internal_urls], + env={**env, "BASE_URL": "http://archivebox.localhost:9292"}, ) - with use_archivebox_db(tmp_path): + assert result.returncode == 0 + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.get() + snapshot_count = Snapshot.objects.count() + + assert crawl.get_urls_list() == [] + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at is None + assert snapshot_count == 0 + + +def test_add_creates_crawl_record(initialized_archive): + """Test that add command creates a Crawl record in the database.""" + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, + ) + + with use_archivebox_db(initialized_archive): crawl_count = Crawl.objects.count() assert crawl_count == 1 -def test_add_creates_source_file(tmp_path, process, disable_extractors_dict): +def test_add_creates_source_file(initialized_archive): """Test that add creates a source file with the URL.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - sources_dir = tmp_path / "sources" + sources_dir = initialized_archive / "sources" assert sources_dir.exists() source_files = list(sources_dir.glob("*cli_add.txt")) @@ -142,19 +149,18 @@ def test_add_creates_source_file(tmp_path, process, disable_extractors_dict): assert "https://example.com" in source_content -def test_add_multiple_urls_single_command(tmp_path, process, disable_extractors_dict): +def test_add_multiple_urls_single_command(initialized_archive): """Test adding multiple URLs in a single command.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com", "https://example.org"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com", "https://example.org"], + env=env, ) assert result.returncode == 0 - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshot_count = Snapshot.objects.count() urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) @@ -163,28 +169,27 @@ def test_add_multiple_urls_single_command(tmp_path, process, disable_extractors_ assert urls[1] == "https://example.org" -def test_add_from_file(tmp_path, process, disable_extractors_dict): +def test_add_from_file(initialized_archive): """Test adding URLs from a file. The add command should treat a file argument as URL input and create snapshots for each URL it contains. """ - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Create a file with URLs - urls_file = tmp_path / "urls.txt" + urls_file = initialized_archive / "urls.txt" urls_file.write_text("https://example.com\nhttps://example.org\n") - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", str(urls_file)], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", str(urls_file)], + env=env, ) assert result.returncode == 0 - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl_count = Crawl.objects.count() snapshot_count = Snapshot.objects.count() @@ -193,91 +198,85 @@ def test_add_from_file(tmp_path, process, disable_extractors_dict): assert snapshot_count == 2 -def test_add_with_depth_0_flag(tmp_path, process, disable_extractors_dict): +def test_add_with_depth_0_flag(initialized_archive): """Test that --depth=0 flag is accepted and works.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) assert result.returncode == 0 - assert "unrecognized arguments: --depth" not in result.stderr.decode("utf-8") + assert "unrecognized arguments: --depth" not in result.stderr -def test_add_with_depth_1_flag(tmp_path, process, disable_extractors_dict): +def test_add_with_depth_1_flag(initialized_archive): """Test that --depth=1 flag is accepted.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=1", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=1", "https://example.com"], + env=env, ) assert result.returncode == 0 - assert "unrecognized arguments: --depth" not in result.stderr.decode("utf-8") + assert "unrecognized arguments: --depth" not in result.stderr -def test_add_rejects_invalid_depth_values(tmp_path, process, disable_extractors_dict): +def test_add_rejects_invalid_depth_values(initialized_archive): """Test that add rejects depth values outside the supported range.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for depth in ("5", "-1"): - result = subprocess.run( - ["archivebox", "add", "--index-only", f"--depth={depth}", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["add", "--index-only", f"--depth={depth}", "https://example.com"], + env=env, ) - stderr = result.stderr.decode("utf-8").lower() + stderr = result.stderr.lower() assert result.returncode != 0 assert "invalid" in stderr or "not one of" in stderr -def test_add_with_tags(tmp_path, process, disable_extractors_dict): +def test_add_with_tags(initialized_archive): """Test adding URL with tags stores tags_str in crawl. With --index-only, Tag objects are not created until archiving happens. Tags are stored as a string in the Crawl.tags_str field. """ - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "--tag=test,example", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "--tag=test,example", "https://example.com"], + env=env, ) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): tags_str = Crawl.objects.values_list("tags_str", flat=True).get() # Tags are stored as a comma-separated string in crawl assert "test" in tags_str or "example" in tags_str -def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extractors_dict): +def test_add_records_selected_persona_on_crawl(initialized_archive): """Test add persists the selected persona so browser config derives from it later.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "--persona=Default", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "--persona=Default", "https://example.com"], + env=env, ) assert result.returncode == 0 - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() assert crawl.persona_id assert "ACTIVE_PERSONA" not in crawl.config - assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir() + assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir() -def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_extractors_dict): - os.chdir(tmp_path) - result = subprocess.run( +def test_add_records_url_filter_overrides_on_crawl(initialized_archive): + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( [ - "archivebox", "add", "--index-only", "--depth=0", @@ -285,45 +284,42 @@ def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_ex "--domain-denylist=static.example.com", "https://example.com", ], - capture_output=True, - env=disable_extractors_dict, + env=env, ) assert result.returncode == 0 - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com" assert crawl.config["URL_DENYLIST"] == "static.example.com" - assert not (tmp_path / "personas" / "Default" / "chrome_extensions").exists() + assert not (initialized_archive / "personas" / "Default" / "chrome_extensions").exists() -def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_extractors_dict): +def test_add_duplicate_url_creates_separate_crawls(initialized_archive): """Test that adding the same URL twice creates separate crawls and snapshots. Each 'add' command creates a new Crawl. Multiple crawls can archive the same URL. This allows re-archiving URLs at different times. """ - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add URL first time - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Add same URL second time with --update to opt out of ONLY_NEW. - subprocess.run( - ["archivebox", "add", "--index-only", "--update", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--update", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshot_count = Snapshot.objects.filter(url="https://example.com").count() crawl_count = Crawl.objects.count() @@ -332,54 +328,48 @@ def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_ex assert snapshot_count == 2 -def test_add_with_overwrite_flag(tmp_path, process, disable_extractors_dict): +def test_add_with_overwrite_flag(initialized_archive): """Test that --overwrite flag forces re-archiving.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add URL first time - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) # Add with overwrite - result = subprocess.run( - ["archivebox", "add", "--index-only", "--overwrite", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["add", "--index-only", "--overwrite", "https://example.com"], + env=env, ) assert result.returncode == 0 - assert "unrecognized arguments: --overwrite" not in result.stderr.decode("utf-8") + assert "unrecognized arguments: --overwrite" not in result.stderr -def test_add_creates_snapshot_output_directory(tmp_path, process, disable_extractors_dict): +def test_add_creates_snapshot_output_directory(initialized_archive): """Test that add creates the current snapshot output directory on disk.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get()) - snapshot_dir = _find_snapshot_dir(tmp_path, snapshot_id) + snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id) assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}" assert snapshot_dir.is_dir() -def test_add_help_shows_depth_and_tag_options(tmp_path, process): +def test_add_help_shows_depth_and_tag_options(initialized_archive): """Test that add --help documents the main filter and crawl options.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["add", "--help"], ) assert result.returncode == 0 @@ -391,11 +381,10 @@ def test_add_help_shows_depth_and_tag_options(tmp_path, process): assert "--tag" in result.stdout -def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable_extractors_dict): - os.chdir(tmp_path) - result = subprocess.run( +def test_add_records_max_url_and_size_limits_on_crawl(initialized_archive): + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( [ - "archivebox", "add", "--index-only", "--depth=1", @@ -405,14 +394,13 @@ def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable "--snapshot-max-size=5mb", "https://example.com", ], - capture_output=True, - env=disable_extractors_dict, + env=env, ) assert result.returncode == 0 columns = {field.name for field in Crawl._meta.local_fields} - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): config = Crawl.objects.values_list("config", flat=True).get() or {} assert {"max_urls", "crawl_max_size", "crawl_timeout", "snapshot_max_size"}.isdisjoint(columns) @@ -422,14 +410,11 @@ def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable assert config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024 -def test_add_without_args_shows_usage(tmp_path, process): +def test_add_without_args_shows_usage(initialized_archive): """Test that add without URLs fails with a usage hint instead of crashing.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["add"], ) combined = result.stdout + result.stderr @@ -437,19 +422,18 @@ def test_add_without_args_shows_usage(tmp_path, process): assert "usage" in combined.lower() or "url" in combined.lower() -def test_add_index_only_queues_crawl_without_starting_runner(tmp_path, process, disable_extractors_dict): +def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive): """Test that --index-only creates only a queued crawl and returns fast.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, timeout=30, # Should be fast ) assert result.returncode == 0 - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() snapshot_count = Snapshot.objects.count() @@ -458,35 +442,260 @@ def test_add_index_only_queues_crawl_without_starting_runner(tmp_path, process, assert snapshot_count == 0 -def test_add_links_snapshot_to_crawl(tmp_path, process, disable_extractors_dict): +def test_add_links_snapshot_to_crawl(initialized_archive): """Test that add links the snapshot to the crawl via crawl_id.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl_id = Crawl.objects.values_list("id", flat=True).get() snapshot_crawl = Snapshot.objects.values_list("crawl_id", flat=True).get() assert snapshot_crawl == crawl_id -def test_add_sets_snapshot_timestamp(tmp_path, process, disable_extractors_dict): +def test_add_sets_snapshot_timestamp(initialized_archive): """Test that add sets a timestamp on the snapshot.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): timestamp = Snapshot.objects.values_list("timestamp", flat=True).get() assert timestamp is not None assert len(str(timestamp)) > 0 + + +@pytest.mark.timeout(180) +def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_archive): + + wget_urls = [ + "https://example.com", + "https://pirate.github.io/stress-tests/challenge.html", + ] + chrome_url = "https://example.com/?archivebox-chrome-flow=1" + env = os.environ.copy() + env.pop("CHROME_BINARY", None) + env.update( + { + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + "TIMEOUT": "60", + "SAVE_WGET": "true", + "SAVE_HEADERS": "false", + "SAVE_TITLE": "false", + "SAVE_READABILITY": "false", + "SAVE_SINGLEFILE": "false", + "SAVE_MERCURY": "false", + "SAVE_SCREENSHOT": "false", + "SAVE_PDF": "false", + "SAVE_DOM": "false", + "SAVE_ARCHIVEDOTORG": "false", + "SAVE_GIT": "false", + "SAVE_YTDLP": "false", + "SAVE_FAVICON": "false", + }, + ) + _cmd_result = run_archivebox_cmd( + [ + "add", + "--depth=0", + "--max-urls=2", + "--crawl-max-size=10mb", + "--tag=real-flow,challenge", + "--parser=url_list", + "--plugins=wget", + *wget_urls, + ], + cwd=initialized_archive, + env=env, + timeout=180, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + + chrome_env = env | { + "SAVE_WGET": "false", + "SAVE_HEADERS": "true", + "SAVE_TITLE": "true", + "CHROME_HEADLESS": "true", + "CHROME_SANDBOX": "false", + "CHROME_ISOLATION": "snapshot", + } + system_browser = _find_system_browser() + if system_browser: + chrome_env["CHROME_BINARY"] = str(system_browser) + else: + _cmd_result = run_archivebox_cmd( + ["install", "chrome"], + cwd=initialized_archive, + env=chrome_env, + timeout=600, + ) + install_stdout, install_stderr, install_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert install_returncode == 0, install_stderr or install_stdout + _cmd_result = run_archivebox_cmd( + [ + "add", + "--depth=0", + "--max-urls=1", + "--crawl-max-size=10mb", + "--tag=chrome-flow", + "--parser=url_list", + "--plugins=chrome,wget,headers,title", + chrome_url, + ], + cwd=initialized_archive, + env=chrome_env, + timeout=180, + ) + chrome_stdout, chrome_stderr, chrome_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert chrome_returncode == 0, chrome_stderr or chrome_stdout + + _cmd_result = run_archivebox_cmd( + ["list", "--tag=real-flow"], + cwd=initialized_archive, + env=env, + timeout=60, + ) + list_stdout, list_stderr, list_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert list_returncode == 0, list_stderr or list_stdout + listed = [json.loads(line) for line in list_stdout.splitlines() if line.strip()] + assert {item["url"] for item in listed} >= set(wget_urls) + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first() + real_flow_crawl = Crawl.objects.filter(tags_str="real-flow,challenge").values_list("max_depth", "tags_str", "config").first() + snapshots = list(Snapshot.objects.order_by("url").values_list("id", "url", "depth", "status", "title")) + archive_results = list( + ArchiveResult.objects.select_related("snapshot") + .order_by("snapshot__url", "plugin") + .values_list("snapshot__url", "plugin", "status", "output_files", "output_size", "output_str"), + ) + processes = list(Process.objects.filter(process_type="hook").values_list("process_type", "status", "exit_code", "pwd", "cmd")) + + assert real_flow_crawl is not None + assert real_flow_crawl[0] == 0 + assert real_flow_crawl[1] == "real-flow,challenge" + real_flow_config = real_flow_crawl[2] or {} + assert real_flow_config["CRAWL_MAX_URLS"] == 2 + assert real_flow_config["CRAWL_MAX_SIZE"] == 10 * 1024 * 1024 + assert real_flow_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 + assert "wget" in real_flow_config["PLUGINS"] + assert crawl is not None + assert crawl[1] == "chrome-flow" + assert "wget,headers,title" in json.dumps(crawl[2] or {}) + + snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots} + assert snapshot_urls >= {*wget_urls, chrome_url} + assert all(depth == 0 for _id, _url, depth, _status, _title in snapshots) + + by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size, _output in archive_results} + assert by_url_plugin[("https://example.com", "wget")] == "succeeded" + assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded" + assert (chrome_url, "headers") in by_url_plugin + assert (chrome_url, "title") in by_url_plugin + failed_results = [(url, plugin, output) for url, plugin, status, _files, _size, output in archive_results if status == "failed"] + assert len(failed_results) <= 2, failed_results + + snapshot_root = initialized_archive / "archive/users/system/snapshots" + html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()] + header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0] + title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0] + index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()] + assert html_outputs + if by_url_plugin[(chrome_url, "headers")] == "succeeded": + assert header_outputs + if by_url_plugin[(chrome_url, "title")] == "succeeded": + assert title_outputs + assert any("Example Domain" in path.read_text(errors="ignore") for path in title_outputs) + assert len(index_outputs) >= len(wget_urls) + 1 + + combined_html = "\n".join(path.read_text(errors="ignore") for path in html_outputs) + assert "Example Domain" in combined_html + assert "Browser Agent Challenge for AI Browser Drivers" in combined_html + + assert processes + assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes) + assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes) + + +@pytest.mark.timeout(180) +def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive): + + env = os.environ.copy() + env.update( + { + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + "TIMEOUT": "60", + "SAVE_WGET": "true", + "SAVE_HEADERS": "false", + "SAVE_TITLE": "false", + "SAVE_READABILITY": "false", + "SAVE_SINGLEFILE": "false", + "SAVE_MERCURY": "false", + "SAVE_SCREENSHOT": "false", + "SAVE_PDF": "false", + "SAVE_DOM": "false", + "SAVE_ARCHIVEDOTORG": "false", + "SAVE_GIT": "false", + "SAVE_YTDLP": "false", + "SAVE_FAVICON": "false", + "PARSE_HTML_URLS_ENABLED": "true", + "PARSE_DOM_OUTLINKS_ENABLED": "false", + }, + ) + + _cmd_result = run_archivebox_cmd( + [ + "add", + "--depth=2", + "--max-urls=2", + "--crawl-max-size=50mb", + "--tag=recursive-flow", + "--parser=url_list", + "--plugins=wget,parse_html_urls", + "https://example.com", + ], + cwd=initialized_archive, + env=env, + timeout=180, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first() + snapshots = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status")) + archive_results = list( + ArchiveResult.objects.select_related("snapshot") + .order_by("snapshot__depth", "snapshot__url", "plugin") + .values_list("snapshot__url", "plugin", "status", "output_files"), + ) + + assert crawl[0] == 2 + assert crawl[1] == "recursive-flow" + crawl_config = crawl[2] or {} + assert crawl_config["CRAWL_MAX_URLS"] == 2 + assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024 + assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 + assert ("https://example.com", 0, "sealed") in snapshots + assert any(url == "https://iana.org/domains/example" and depth == 1 and status == "sealed" for url, depth, status in snapshots) + + by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results} + assert by_url_plugin[("https://example.com", "wget")] == "succeeded" + assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded" + assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded" + + urls_outputs = list((initialized_archive / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl")) + assert urls_outputs + assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs) diff --git a/archivebox/tests/test_cli_archiveresult.py b/archivebox/tests/test_cli_archiveresult.py index 3a4174b6..681a8a1f 100644 --- a/archivebox/tests/test_cli_archiveresult.py +++ b/archivebox/tests/test_cli_archiveresult.py @@ -32,15 +32,24 @@ class TestArchiveResultCreate: url = create_test_url() # Create a snapshot first - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] # Pipe snapshot to archiveresult create - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=title"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" @@ -57,14 +66,23 @@ class TestArchiveResultCreate: def test_create_with_specific_plugin(self, initialized_archive): """Create archive result for specific plugin.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=screenshot"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout2) @@ -77,21 +95,28 @@ class TestArchiveResultCreate: url = create_test_url() # Create crawl and snapshot - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode # Now pipe all to archiveresult create - stdout3, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=title"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout3) @@ -105,11 +130,14 @@ class TestArchiveResultCreate: """Only pass-through records but no new snapshots returns success.""" crawl_record = {"type": "Crawl", "id": "fake-id", "urls": "https://example.com"} - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create"], stdin=json.dumps(crawl_record), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Passed through" in stderr @@ -120,10 +148,13 @@ class TestArchiveResultList: def test_list_empty(self, initialized_archive): """List with no archive results returns empty.""" - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "list"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Listed 0 archive results" in stderr @@ -132,36 +163,53 @@ class TestArchiveResultList: """Filter archive results by status.""" # Create snapshot and materialize an archive result via the runner url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode run_archivebox_cmd( ["run"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PROJECTOR_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - created = parse_jsonl_output( - run_archivebox_cmd( - ["archiveresult", "list", "--plugin=favicon"], - data_dir=initialized_archive, - )[0], - )[0] + _cmd_result = run_archivebox_cmd( + ["archiveresult", "list", "--plugin=favicon"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + created = parse_jsonl_output(_cmd_result.stdout)[0] run_archivebox_cmd( ["archiveresult", "update", "--status=queued"], stdin=json.dumps(created), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--status=queued"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -171,25 +219,39 @@ class TestArchiveResultList: def test_list_filter_by_plugin(self, initialized_archive): """Filter archive results by plugin.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode run_archivebox_cmd( ["run"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PROJECTOR_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--plugin=favicon"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -201,25 +263,39 @@ class TestArchiveResultList: # Create multiple archive results for _ in range(3): url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode run_archivebox_cmd( ["run"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PROJECTOR_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--limit=2"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -232,32 +308,50 @@ class TestArchiveResultUpdate: def test_update_status(self, initialized_archive): """Update archive result status.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) - stdout_run, _, _ = run_archivebox_cmd( + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( ["run"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PROJECTOR_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - stdout_list, _, _ = run_archivebox_cmd( + _stdout_run, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--plugin=favicon"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout_list, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode ar = parse_jsonl_output(stdout_list)[0] - stdout3, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "update", "--status=failed"], stdin=json.dumps(ar), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout3, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Updated 1 archive results" in stderr @@ -272,32 +366,50 @@ class TestArchiveResultDelete: def test_delete_requires_yes(self, initialized_archive): """Delete requires --yes flag.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) - stdout_run, _, _ = run_archivebox_cmd( + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( ["run"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PROJECTOR_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - stdout_list, _, _ = run_archivebox_cmd( + _stdout_run, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--plugin=favicon"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout_list, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode ar = parse_jsonl_output(stdout_list)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "delete"], stdin=json.dumps(ar), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 1 assert "--yes" in stderr @@ -305,32 +417,50 @@ class TestArchiveResultDelete: def test_delete_with_yes(self, initialized_archive): """Delete with --yes flag works.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) - stdout_run, _, _ = run_archivebox_cmd( + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( ["run"], stdin=stdout2, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PROJECTOR_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - stdout_list, _, _ = run_archivebox_cmd( + _stdout_run, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--plugin=favicon"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout_list, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode ar = parse_jsonl_output(stdout_list)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "delete", "--yes"], stdin=json.dumps(ar), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Deleted 1 archive results" in stderr diff --git a/archivebox/tests/test_cli_binary.py b/archivebox/tests/test_cli_binary.py new file mode 100644 index 00000000..480728ea --- /dev/null +++ b/archivebox/tests/test_cli_binary.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox binary command. + +TODO: expand beyond command discovery into create/list/update/delete behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_binary_help_runs_successfully(tmp_path): + """The binary command should be registered and expose help.""" + + result = run_archivebox_cmd(["binary", "--help"]) + + assert result.returncode == 0 + assert "binary" in result.stdout.lower() + assert "list" in result.stdout diff --git a/archivebox/tests/test_cli_config.py b/archivebox/tests/test_cli_config.py index 5cb28a48..ba017bba 100644 --- a/archivebox/tests/test_cli_config.py +++ b/archivebox/tests/test_cli_config.py @@ -4,14 +4,12 @@ Comprehensive tests for archivebox config command. Verify config reads/writes ArchiveBox.conf file correctly. """ -import os -import subprocess +from archivebox.tests.conftest import run_archivebox_cmd -def test_config_displays_all_config(tmp_path, process): +def test_config_displays_all_config(initialized_archive): """Test that config without args displays all configuration.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "config"], capture_output=True, text=True) + result = run_archivebox_cmd(["config"]) assert result.returncode == 0 output = result.stdout @@ -21,180 +19,232 @@ def test_config_displays_all_config(tmp_path, process): assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output -def test_config_get_specific_key(tmp_path, process): +def test_config_shows_derived_collection_paths_but_not_runtime_dirs(initialized_archive): + """The CLI should expose collection paths, not per-crawl/per-snapshot runtime dirs.""" + run_archivebox_cmd(["init"], cwd=initialized_archive, check=True) + + result = run_archivebox_cmd(["config"], cwd=initialized_archive) + + assert result.returncode == 0, result.stderr + unwrapped_output = result.stdout.replace("\n", "") + assert "DATA_DIR" in result.stdout + assert result.stdout.count("\nDATA_DIR =") == 1 + assert str(initialized_archive) in unwrapped_output + assert "PERSONAS_DIR" in result.stdout + assert result.stdout.count("\nPERSONAS_DIR =") == 1 + assert "SNAP_DIR" not in result.stdout + assert "CRAWL_DIR" not in result.stdout + + +def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive): + run_archivebox_cmd(["init"], cwd=initialized_archive, check=True) + + data_dir = run_archivebox_cmd(["config", "--get", "DATA_DIR"], cwd=initialized_archive) + snap_dir = run_archivebox_cmd(["config", "--get", "SNAP_DIR"], cwd=initialized_archive) + + assert data_dir.returncode == 0, data_dir.stderr + assert "DATA_DIR" in data_dir.stdout + assert str(initialized_archive) in data_dir.stdout.replace("\n", "") + assert snap_dir.returncode != 0 + assert "SNAP_DIR =" not in snap_dir.stdout + + +def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive): + run_archivebox_cmd(["init"], cwd=initialized_archive, check=True) + + data_dir = run_archivebox_cmd( + ["config", "--set", f"DATA_DIR={initialized_archive / 'other'}"], + cwd=initialized_archive, + ) + crawl_dir = run_archivebox_cmd( + ["config", "--set", f"CRAWL_DIR={initialized_archive / 'crawl'}"], + cwd=initialized_archive, + ) + + assert data_dir.returncode != 0 + assert crawl_dir.returncode != 0 + content = (initialized_archive / "ArchiveBox.conf").read_text() + assert "DATA_DIR" not in content + assert "CRAWL_DIR" not in content + + +def test_config_get_specific_key(initialized_archive): """Test that config --get KEY retrieves specific value.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "config", "--get", "TIMEOUT"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--get", "TIMEOUT"], ) assert result.returncode == 0 assert "TIMEOUT" in result.stdout -def test_config_set_writes_to_file(tmp_path, process): +def test_config_set_writes_to_file(initialized_archive): """Test that config --set KEY=VALUE writes to ArchiveBox.conf.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=120"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--set", "TIMEOUT=120"], ) assert result.returncode == 0 # Verify config file was updated - config_file = tmp_path / "ArchiveBox.conf" + config_file = initialized_archive / "ArchiveBox.conf" assert config_file.exists() content = config_file.read_text() assert "TIMEOUT" in content or "120" in content -def test_config_set_and_get_roundtrip(tmp_path, process): +def test_config_set_and_get_roundtrip(initialized_archive): """Test that set value can be retrieved with get.""" - os.chdir(tmp_path) # Set a unique value - subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=987"], - capture_output=True, - text=True, + run_archivebox_cmd( + ["config", "--set", "TIMEOUT=987"], ) # Get the value back - result = subprocess.run( - ["archivebox", "config", "--get", "TIMEOUT"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--get", "TIMEOUT"], ) assert "987" in result.stdout -def test_config_set_multiple_values(tmp_path, process): +def test_config_set_multiple_values(initialized_archive): """Test setting multiple config values at once.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"], ) assert result.returncode == 0 # Verify both were written - config_file = tmp_path / "ArchiveBox.conf" + config_file = initialized_archive / "ArchiveBox.conf" content = config_file.read_text() assert "111" in content assert "222" in content -def test_config_set_invalid_key_fails(tmp_path, process): +def test_config_set_invalid_key_fails(initialized_archive): """Test that setting invalid config key fails.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"], ) assert result.returncode != 0 -def test_config_set_requires_equals_sign(tmp_path, process): +def test_config_set_requires_equals_sign(initialized_archive): """Test that set requires KEY=VALUE format.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--set", "TIMEOUT"], ) assert result.returncode != 0 -def test_config_search_finds_keys(tmp_path, process): +def test_config_search_finds_keys(initialized_archive): """Test that config --search finds matching keys.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "config", "--search", "TIMEOUT"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--search", "TIMEOUT"], ) # Should find timeout-related config assert "TIMEOUT" in result.stdout -def test_config_preserves_existing_values(tmp_path, process): +def test_config_preserves_existing_values(initialized_archive): """Test that setting new values preserves existing ones.""" - os.chdir(tmp_path) # Set first value - subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=100"], - capture_output=True, + run_archivebox_cmd( + ["config", "--set", "TIMEOUT=100"], ) # Set second value - subprocess.run( - ["archivebox", "config", "--set", "YTDLP_TIMEOUT=200"], - capture_output=True, + run_archivebox_cmd( + ["config", "--set", "YTDLP_TIMEOUT=200"], ) # Verify both are in config file - config_file = tmp_path / "ArchiveBox.conf" + config_file = initialized_archive / "ArchiveBox.conf" content = config_file.read_text() assert "TIMEOUT" in content assert "YTDLP_TIMEOUT" in content -def test_config_file_is_valid_toml(tmp_path, process): +def test_config_file_is_valid_toml(initialized_archive): """Test that config file remains valid TOML after set.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=150"], - capture_output=True, + run_archivebox_cmd( + ["config", "--set", "TIMEOUT=150"], ) - config_file = tmp_path / "ArchiveBox.conf" + config_file = initialized_archive / "ArchiveBox.conf" content = config_file.read_text() # Basic TOML validation - should have sections and key=value pairs assert "[" in content or "=" in content -def test_config_updates_existing_value(tmp_path, process): +def test_config_updates_existing_value(initialized_archive): """Test that setting same key twice updates the value.""" - os.chdir(tmp_path) # Set initial value - subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=100"], - capture_output=True, + run_archivebox_cmd( + ["config", "--set", "TIMEOUT=100"], ) # Update to new value - subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=200"], - capture_output=True, + run_archivebox_cmd( + ["config", "--set", "TIMEOUT=200"], ) # Get current value - result = subprocess.run( - ["archivebox", "config", "--get", "TIMEOUT"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["config", "--get", "TIMEOUT"], ) # Should show updated value assert "200" in result.stdout + + +def test_config_ignores_legacy_unknown_keys(tmp_path, initialized_archive): + """Old ArchiveBox.conf keys should not prevent startup during upgrades.""" + (tmp_path / "ArchiveBox.conf").write_text( + """ +[ARCHIVING_CONFIG] +MAX_MEDIA_SIZE = "750m" + +[SEARCH_BACKEND_CONFIG] +SEARCH_BACKEND_HOST_NAME = "sonic" +SEARCH_BACKEND_PASSWORD = "SecretPassword" +""", + ) + + result = run_archivebox_cmd( + ["version"], + ) + + assert result.returncode == 0, result.stderr + assert "Extra inputs are not permitted" not in result.stderr + + +class TestConfigCLI: + """Test the CLI interface for config command.""" + + def test_cli_help(self, tmp_path, initialized_archive): + """Test that --help works for config command.""" + + result = run_archivebox_cmd( + ["config", "--help"], + ) + + assert result.returncode == 0 + assert "--get" in result.stdout + assert "--set" in result.stdout diff --git a/archivebox/tests/test_cli_crawl.py b/archivebox/tests/test_cli_crawl.py index 62482b10..a4ba957f 100644 --- a/archivebox/tests/test_cli_crawl.py +++ b/archivebox/tests/test_cli_crawl.py @@ -10,11 +10,20 @@ Tests cover: import json +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl from archivebox.tests.conftest import ( - run_archivebox_cmd, - parse_jsonl_output, + cli_env, create_test_url, + parse_jsonl_output, + run_archivebox_cmd, + run_queued_crawls, ) +from archivebox.tests.test_orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) class TestCrawlCreate: @@ -24,10 +33,13 @@ class TestCrawlCreate: """Create crawl from URL arguments.""" url = create_test_url() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" assert "Created crawl" in stderr @@ -43,11 +55,14 @@ class TestCrawlCreate: urls = [create_test_url() for _ in range(3)] stdin = "\n".join(urls) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create"], stdin=stdin, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" @@ -63,10 +78,13 @@ class TestCrawlCreate: """Create crawl with --depth flag.""" url = create_test_url() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create", "--depth=2", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -76,10 +94,13 @@ class TestCrawlCreate: """Create crawl with --tag flag.""" url = create_test_url() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create", "--tag=test-tag", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -91,11 +112,14 @@ class TestCrawlCreate: url = create_test_url() stdin = json.dumps(tag_record) + "\n" + json.dumps({"url": url}) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create"], stdin=stdin, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -109,15 +133,19 @@ class TestCrawlCreate: """Existing Crawl records (with id) are passed through.""" # First create a crawl url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] # Now pipe it back - should pass through - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout2) @@ -130,10 +158,13 @@ class TestCrawlList: def test_list_empty(self, initialized_archive): """List with no crawls returns empty.""" - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "list"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Listed 0 crawls" in stderr @@ -141,12 +172,15 @@ class TestCrawlList: def test_list_returns_created(self, initialized_archive): """List returns previously created crawls.""" url = create_test_url() - run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "list"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -156,12 +190,15 @@ class TestCrawlList: def test_list_filter_by_status(self, initialized_archive): """Filter crawls by status.""" url = create_test_url() - run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "list", "--status=queued"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -172,12 +209,20 @@ class TestCrawlList: """Limit number of results.""" # Create multiple crawls for _ in range(3): - run_archivebox_cmd(["crawl", "create", create_test_url()], data_dir=initialized_archive) + run_archivebox_cmd( + ["crawl", "create", create_test_url()], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "list", "--limit=2"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -191,15 +236,19 @@ class TestCrawlUpdate: """Update crawl status.""" # Create a crawl url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] # Update it - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "update", "--status=started"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Updated 1 crawls" in stderr @@ -214,14 +263,18 @@ class TestCrawlDelete: def test_delete_requires_yes(self, initialized_archive): """Delete requires --yes flag.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "delete"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 1 assert "--yes" in stderr @@ -229,14 +282,18 @@ class TestCrawlDelete: def test_delete_with_yes(self, initialized_archive): """Delete with --yes flag works.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "delete", "--yes"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Deleted 1 crawls" in stderr @@ -244,15 +301,172 @@ class TestCrawlDelete: def test_delete_dry_run(self, initialized_archive): """Dry run shows what would be deleted.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "delete", "--dry-run"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Would delete" in stderr assert "dry run" in stderr.lower() + + +def test_crawl_creates_crawl_object(initialized_archive): + """Test that crawl command creates a Crawl object.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + ["crawl", "create", "https://example.com"], + cwd=initialized_archive, + env=env, + check=True, + ) + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.order_by("-created_at").first() + + assert crawl is not None, "Crawl object should be created" + + +def test_crawl_depth_sets_max_depth_in_crawl(initialized_archive): + """Test that --depth option sets max_depth in the Crawl object.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + ["crawl", "create", "--depth=2", "https://example.com"], + cwd=initialized_archive, + env=env, + check=True, + ) + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.order_by("-created_at").first() + + assert crawl is not None + assert crawl.max_depth == 2, "Crawl max_depth should match --depth=2" + + +def test_crawl_creates_snapshot_for_url(initialized_archive): + """Test that crawl creates a Snapshot for the input URL.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + ["crawl", "create", "https://example.com"], + cwd=initialized_archive, + env=env, + check=True, + ) + run_queued_crawls(initialized_archive, env) + + with use_archivebox_db(initialized_archive): + snapshot = Snapshot.objects.filter(url="https://example.com").first() + + assert snapshot is not None, "Snapshot should be created for input URL" + + +def test_crawl_links_snapshot_to_crawl(initialized_archive): + """Test that Snapshot is linked to Crawl via crawl_id.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + ["crawl", "create", "https://example.com"], + cwd=initialized_archive, + env=env, + check=True, + ) + run_queued_crawls(initialized_archive, env) + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.order_by("-created_at").first() + assert crawl is not None + snapshot = Snapshot.objects.filter(url="https://example.com").first() + + assert snapshot is not None + assert snapshot.crawl_id == crawl.id, "Snapshot should be linked to Crawl" + + +def test_crawl_multiple_urls_creates_multiple_snapshots(initialized_archive): + """Test that crawling multiple URLs creates multiple snapshots.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + [ + "crawl", + "create", + "https://example.com", + "https://iana.org", + ], + cwd=initialized_archive, + env=env, + check=True, + ) + run_queued_crawls(initialized_archive, env) + + with use_archivebox_db(initialized_archive): + urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) + + assert "https://example.com" in urls + assert "https://iana.org" in urls + + +def test_crawl_from_file_creates_snapshot(initialized_archive): + """Test that crawl can create snapshots from a file of URLs.""" + env = cli_env(disable_extractors=True) + + # Write URLs to a file + urls_file = initialized_archive / "urls.txt" + urls_file.write_text("https://example.com\n") + + run_archivebox_cmd( + ["crawl", "create", str(urls_file)], + cwd=initialized_archive, + env=env, + check=True, + ) + run_queued_crawls(initialized_archive, env) + + with use_archivebox_db(initialized_archive): + snapshot = Snapshot.objects.first() + + # Should create at least one snapshot (the source file or the URL) + assert snapshot is not None, "Should create at least one snapshot" + + +def test_crawl_persists_input_urls_on_crawl(initialized_archive): + """Test that crawl input URLs are stored on the Crawl record.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + ["crawl", "create", "https://example.com"], + cwd=initialized_archive, + env=env, + check=True, + ) + + with use_archivebox_db(initialized_archive): + crawl = Crawl.objects.order_by("-created_at").first() + + assert crawl is not None, "Crawl should be created for crawl input" + assert "https://example.com" in crawl.urls, "Crawl should persist input URLs" + + +class TestCrawlCLI: + """Test the CLI interface for crawl command.""" + + def test_cli_help(self, tmp_path, initialized_archive): + """Test that --help works for crawl command.""" + + result = run_archivebox_cmd( + ["crawl", "--help"], + ) + + assert result.returncode == 0 + assert "create" in result.stdout diff --git a/archivebox/tests/test_cli_extract.py b/archivebox/tests/test_cli_extract.py index f754fa06..b271f180 100644 --- a/archivebox/tests/test_cli_extract.py +++ b/archivebox/tests/test_cli_extract.py @@ -4,35 +4,31 @@ Tests for archivebox extract command. Verify extract re-runs extractors on existing snapshots. """ -import os -import subprocess - import pytest from archivebox.core.models import Snapshot -from archivebox.tests.conftest import run_queued_crawls +from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def test_extract_runs_on_existing_snapshots(tmp_path, process, disable_extractors_dict): +def test_extract_runs_on_existing_snapshots(initialized_archive): """Test that extract command runs on existing snapshots.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add a snapshot first - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Run extract - result = subprocess.run( - ["archivebox", "extract"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["extract"], + env=env, timeout=30, ) @@ -40,30 +36,28 @@ def test_extract_runs_on_existing_snapshots(tmp_path, process, disable_extractor assert result.returncode in [0, 1] -def test_extract_preserves_snapshot_count(tmp_path, process, disable_extractors_dict): +def test_extract_preserves_snapshot_count(initialized_archive): """Test that extract doesn't change snapshot count.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count_before = Snapshot.objects.count() # Run extract - subprocess.run( - ["archivebox", "extract", "--overwrite"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["extract", "--overwrite"], + env=env, timeout=30, ) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count_after = Snapshot.objects.count() assert count_after == count_before diff --git a/archivebox/tests/test_cli_extract_input.py b/archivebox/tests/test_cli_extract_input.py index 9138ee63..19427cdb 100644 --- a/archivebox/tests/test_cli_extract_input.py +++ b/archivebox/tests/test_cli_extract_input.py @@ -1,221 +1,210 @@ """Tests for archivebox extract input handling and pipelines.""" -import os import subprocess import json import pytest from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.tests.conftest import run_queued_crawls +from archivebox.tests.conftest import run_archivebox_cmd, run_queued_crawls, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def _snapshot_id(data_dir): - with use_archivebox_db(data_dir): - return Snapshot.objects.values_list("id", flat=True).first() - - -def test_extract_runs_on_snapshot_id(tmp_path, process, disable_extractors_dict): +def test_extract_runs_on_snapshot_id(initialized_archive): """Test that extract command accepts a snapshot ID.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # First create a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - snapshot_id = _snapshot_id(tmp_path) + with use_archivebox_db(initialized_archive): + snapshot_id = Snapshot.objects.values_list("id", flat=True).first() # Run extract on the snapshot - result = subprocess.run( - ["archivebox", "extract", "--no-wait", str(snapshot_id)], - capture_output=True, - text=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["extract", "--no-wait", str(snapshot_id)], + env=env, ) # Should not error about invalid snapshot ID assert "not found" not in result.stderr.lower() -def test_extract_with_enabled_extractor_creates_archiveresult(tmp_path, process, disable_extractors_dict): +def test_extract_with_enabled_extractor_creates_archiveresult(initialized_archive): """Test that extract creates ArchiveResult when extractor is enabled.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # First create a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - snapshot_id = _snapshot_id(tmp_path) + with use_archivebox_db(initialized_archive): + snapshot_id = Snapshot.objects.values_list("id", flat=True).first() # Run extract with title extractor enabled - env = disable_extractors_dict.copy() + env = env.copy() env["SAVE_TITLE"] = "true" - subprocess.run( - ["archivebox", "extract", "--no-wait", str(snapshot_id)], - capture_output=True, - text=True, + run_archivebox_cmd( + ["extract", "--no-wait", str(snapshot_id)], env=env, ) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count = ArchiveResult.objects.filter(snapshot_id=snapshot_id).count() # May or may not have results depending on timing assert count >= 0 -def test_extract_plugin_option_accepted(tmp_path, process, disable_extractors_dict): +def test_extract_plugin_option_accepted(initialized_archive): """Test that --plugin option is accepted.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # First create a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - snapshot_id = _snapshot_id(tmp_path) + with use_archivebox_db(initialized_archive): + snapshot_id = Snapshot.objects.values_list("id", flat=True).first() - result = subprocess.run( - ["archivebox", "extract", "--plugin=title", "--no-wait", str(snapshot_id)], - capture_output=True, - text=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["extract", "--plugin=title", "--no-wait", str(snapshot_id)], + env=env, ) assert "unrecognized arguments: --plugin" not in result.stderr -def test_extract_stdin_snapshot_id(tmp_path, process, disable_extractors_dict): +def test_extract_stdin_snapshot_id(initialized_archive): """Test that extract reads snapshot IDs from stdin.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # First create a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - snapshot_id = _snapshot_id(tmp_path) + with use_archivebox_db(initialized_archive): + snapshot_id = Snapshot.objects.values_list("id", flat=True).first() - result = subprocess.run( - ["archivebox", "extract", "--no-wait"], + result = run_archivebox_cmd( + ["extract", "--no-wait"], input=f"{snapshot_id}\n", - capture_output=True, - text=True, - env=disable_extractors_dict, + env=env, ) # Should not show "not found" error assert "not found" not in result.stderr.lower() or result.returncode == 0 -def test_extract_stdin_jsonl_input(tmp_path, process, disable_extractors_dict): +def test_extract_stdin_jsonl_input(initialized_archive): """Test that extract reads JSONL records from stdin.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # First create a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - snapshot_id = _snapshot_id(tmp_path) + with use_archivebox_db(initialized_archive): + snapshot_id = Snapshot.objects.values_list("id", flat=True).first() jsonl_input = json.dumps({"type": "Snapshot", "id": str(snapshot_id)}) + "\n" - result = subprocess.run( - ["archivebox", "extract", "--no-wait"], + result = run_archivebox_cmd( + ["extract", "--no-wait"], input=jsonl_input, - capture_output=True, - text=True, - env=disable_extractors_dict, + env=env, ) # Should not show "not found" error assert "not found" not in result.stderr.lower() or result.returncode == 0 -def test_extract_pipeline_from_snapshot(tmp_path, process, disable_extractors_dict): +def test_extract_pipeline_from_snapshot(initialized_archive): """Test piping snapshot output to extract.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Create snapshot and pipe to extract - snapshot_proc = subprocess.Popen( - ["archivebox", "snapshot", "https://example.com"], + snapshot_proc = run_archivebox_cmd( + ["snapshot", "https://example.com"], + cwd=initialized_archive, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=disable_extractors_dict, + env=env, + wait=False, ) - subprocess.run( - ["archivebox", "extract", "--no-wait"], + extract_proc = run_archivebox_cmd( + ["extract", "--no-wait"], + cwd=initialized_archive, stdin=snapshot_proc.stdout, - capture_output=True, - text=True, - env=disable_extractors_dict, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + wait=False, ) + if snapshot_proc.stdout is not None: + snapshot_proc.stdout.close() - snapshot_proc.wait() + extract_stdout, extract_stderr = extract_proc.communicate(timeout=60) + snapshot_stdout, snapshot_stderr = snapshot_proc.communicate(timeout=60) + assert snapshot_proc.returncode == 0, (snapshot_stdout or "") + (snapshot_stderr or "") - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshot = Snapshot.objects.filter(url="https://example.com").first() assert snapshot is not None, "Snapshot should be created by pipeline" -def test_extract_multiple_snapshots(tmp_path, process, disable_extractors_dict): +def test_extract_multiple_snapshots(initialized_archive): """Test extracting from multiple snapshots.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Create multiple snapshots one at a time to avoid deduplication issues - subprocess.run( - ["archivebox", "add", "--index-only", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://example.com"], + env=env, ) - subprocess.run( - ["archivebox", "add", "--index-only", "https://iana.org"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "https://iana.org"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshot_ids = list(Snapshot.objects.values_list("id", flat=True)) assert len(snapshot_ids) >= 2, "Should have at least 2 snapshots" # Extract from all snapshots ids_input = "\n".join(str(snapshot_id) for snapshot_id in snapshot_ids) + "\n" - result = subprocess.run( - ["archivebox", "extract", "--no-wait"], + result = run_archivebox_cmd( + ["extract", "--no-wait"], input=ids_input, - capture_output=True, - text=True, - env=disable_extractors_dict, + env=env, ) assert result.returncode == 0, result.stderr - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count = Snapshot.objects.count() assert count >= 2, "Both snapshots should still exist after extraction" @@ -224,29 +213,23 @@ def test_extract_multiple_snapshots(tmp_path, process, disable_extractors_dict): class TestExtractCLI: """Test the CLI interface for extract command.""" - def test_cli_help(self, tmp_path, process): + def test_cli_help(self, initialized_archive): """Test that --help works for extract command.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "extract", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["extract", "--help"], ) assert result.returncode == 0 assert "--plugin" in result.stdout or "-p" in result.stdout assert "--wait" in result.stdout or "--no-wait" in result.stdout - def test_cli_no_snapshots_shows_warning(self, tmp_path, process): + def test_cli_no_snapshots_shows_warning(self, initialized_archive): """Test that running without snapshots shows a warning.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "extract", "--no-wait"], + result = run_archivebox_cmd( + ["extract", "--no-wait"], input="", - capture_output=True, - text=True, ) # Should show warning about no snapshots or exit normally (empty input) diff --git a/archivebox/tests/test_cli_help.py b/archivebox/tests/test_cli_help.py index 772e2a08..bf2ee195 100644 --- a/archivebox/tests/test_cli_help.py +++ b/archivebox/tests/test_cli_help.py @@ -4,14 +4,12 @@ Tests for archivebox help command. Verify command runs successfully and produces output. """ -import os -import subprocess +from archivebox.tests.conftest import run_archivebox_cmd def test_help_runs_successfully(tmp_path): """Test that help command runs and produces output.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "help"], capture_output=True, text=True) + result = run_archivebox_cmd(["help"]) assert result.returncode == 0 combined = result.stdout + result.stderr @@ -19,10 +17,9 @@ def test_help_runs_successfully(tmp_path): assert "archivebox" in combined.lower() -def test_help_in_initialized_dir(tmp_path, process): +def test_help_in_initialized_dir(initialized_archive): """Test help command in initialized data directory.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "help"], capture_output=True, text=True) + result = run_archivebox_cmd(["help"]) assert result.returncode == 0 combined = result.stdout + result.stderr diff --git a/archivebox/tests/test_cli_init.py b/archivebox/tests/test_cli_init.py index 89f9db6c..e43cb2c1 100644 --- a/archivebox/tests/test_cli_init.py +++ b/archivebox/tests/test_cli_init.py @@ -4,9 +4,6 @@ Comprehensive tests for archivebox init command. Verify init creates correct database schema, filesystem structure, and config. """ -import os -import subprocess - import pytest from django.utils import timezone from django.db import connections @@ -16,7 +13,8 @@ from archivebox.config.common import get_config from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl from archivebox.machine.models import Machine -from archivebox.tests.conftest import run_queued_crawls +from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) @@ -27,8 +25,7 @@ DIR_PERMISSIONS = get_config().OUTPUT_PERMISSIONS.replace("6", "7").replace("4", def test_init_creates_database_file(tmp_path): """Test that init creates index.sqlite3 database file.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "init"], capture_output=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 0 db_path = tmp_path / "index.sqlite3" @@ -38,8 +35,7 @@ def test_init_creates_database_file(tmp_path): def test_init_creates_archive_directory(tmp_path): """Test that init creates archive directory.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) archive_dir = tmp_path / "archive" assert archive_dir.exists() @@ -48,9 +44,8 @@ def test_init_creates_archive_directory(tmp_path): def test_init_uses_cwd_archive_and_users_dirs(tmp_path): """Test that init creates archive/users storage roots under cwd.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "init"], capture_output=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 0 assert (tmp_path / "archive").is_dir() @@ -59,8 +54,7 @@ def test_init_uses_cwd_archive_and_users_dirs(tmp_path): def test_init_creates_sources_directory(tmp_path): """Test that init creates sources directory.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) sources_dir = tmp_path / "sources" assert sources_dir.exists() @@ -69,8 +63,7 @@ def test_init_creates_sources_directory(tmp_path): def test_init_creates_logs_directory(tmp_path): """Test that init creates logs directory.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) logs_dir = tmp_path / "logs" assert logs_dir.exists() @@ -79,8 +72,7 @@ def test_init_creates_logs_directory(tmp_path): def test_init_creates_config_file(tmp_path): """Test that init creates ArchiveBox.conf config file.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) config_file = tmp_path / "ArchiveBox.conf" assert config_file.exists() @@ -89,8 +81,7 @@ def test_init_creates_config_file(tmp_path): def test_init_runs_migrations(tmp_path): """Test that init runs Django migrations and creates core tables.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) with use_archivebox_db(tmp_path): migration_count = MigrationRecorder.Migration.objects.count() @@ -100,8 +91,7 @@ def test_init_runs_migrations(tmp_path): def test_init_creates_core_snapshot_table(tmp_path): """Test that init creates core_snapshot table.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) assert Snapshot._meta.db_table == "core_snapshot" with use_archivebox_db(tmp_path): @@ -110,8 +100,7 @@ def test_init_creates_core_snapshot_table(tmp_path): def test_init_creates_crawls_crawl_table(tmp_path): """Test that init creates crawls_crawl table.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) assert Crawl._meta.db_table == "crawls_crawl" with use_archivebox_db(tmp_path): @@ -120,8 +109,7 @@ def test_init_creates_crawls_crawl_table(tmp_path): def test_init_creates_core_archiveresult_table(tmp_path): """Test that init creates core_archiveresult table.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) assert ArchiveResult._meta.db_table == "core_archiveresult" with use_archivebox_db(tmp_path): @@ -130,8 +118,7 @@ def test_init_creates_core_archiveresult_table(tmp_path): def test_init_sets_correct_file_permissions(tmp_path): """Test that init sets correct permissions on created files.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) # Check database permissions db_path = tmp_path / "index.sqlite3" @@ -144,15 +131,14 @@ def test_init_sets_correct_file_permissions(tmp_path): def test_init_is_idempotent(tmp_path): """Test that running init multiple times is safe (idempotent).""" - os.chdir(tmp_path) # First init - result1 = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result1 = run_archivebox_cmd(["init"]) assert result1.returncode == 0 assert "Initializing a new ArchiveBox" in result1.stdout # Second init should update, not fail - result2 = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result2 = run_archivebox_cmd(["init"]) assert result2.returncode == 0 assert "updating existing ArchiveBox" in result2.stdout or "up-to-date" in result2.stdout.lower() @@ -164,15 +150,14 @@ def test_init_is_idempotent(tmp_path): def test_init_refuses_database_migrated_by_newer_code(tmp_path): """A downgraded ArchiveBox build must fail before serving a newer DB schema.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 0 with use_archivebox_db(tmp_path): MigrationRecorder.Migration.objects.create(app="crawls", name="9999_future_test", applied=timezone.now()) connections["default"].commit() - result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 3 assert "migrated by a newer version of ArchiveBox" in result.stderr assert "crawls.9999_future_test" in result.stderr @@ -183,8 +168,7 @@ def test_init_recovers_from_pre_squash_dev_history(tmp_path): """Pre-squash dev DBs (rows for migrations now absorbed by ``replaces=``) must NOT trip the newer-DB guard — every historical squash would otherwise brick beta-tester collections that pre-date the squash commit.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 0 # Sampling — one name per affected app, all listed in the ``replaces=`` @@ -205,43 +189,41 @@ def test_init_recovers_from_pre_squash_dev_history(tmp_path): MigrationRecorder.Migration.objects.create(app=app, name=name, applied=timezone.now()) connections["default"].commit() - result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 0, f"init refused to recover pre-squash dev DB.\nstdout={result.stdout}\nstderr={result.stderr}" assert "migrated by a newer version of ArchiveBox" not in result.stderr -def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_extractors_dict): +def test_init_with_existing_data_preserves_snapshots(initialized_archive): """Test that re-running init preserves existing snapshot data.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Check snapshot was created - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count_before = Snapshot.objects.count() assert count_before == 1 # Run init again - result = subprocess.run(["archivebox", "init"], capture_output=True) + result = run_archivebox_cmd(["init"]) assert result.returncode == 0 # Snapshot should still exist - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count_after = Snapshot.objects.count() assert count_after == count_before def test_init_quick_flag_skips_checks(tmp_path): """Test that init --quick runs faster by skipping some checks.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "init", "--quick"], capture_output=True, text=True) + result = run_archivebox_cmd(["init", "--quick"]) assert result.returncode == 0 # Database should still be created @@ -251,8 +233,7 @@ def test_init_quick_flag_skips_checks(tmp_path): def test_init_creates_machine_table(tmp_path): """Test that init creates the machine_machine table.""" - os.chdir(tmp_path) - subprocess.run(["archivebox", "init"], capture_output=True) + run_archivebox_cmd(["init"]) assert Machine._meta.db_table == "machine_machine" with use_archivebox_db(tmp_path): @@ -261,31 +242,27 @@ def test_init_creates_machine_table(tmp_path): def test_init_output_shows_collection_info(tmp_path): """Test that init output shows helpful collection information.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + result = run_archivebox_cmd(["init"]) output = result.stdout # Should show some helpful info about the collection assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output -def test_init_ignores_unrecognized_archive_directories(tmp_path, process, disable_extractors_dict): +def test_init_ignores_unrecognized_archive_directories(initialized_archive): """Test that init upgrades existing dirs without choking on extra folders.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) - (tmp_path / "archive" / "some_random_folder").mkdir(parents=True, exist_ok=True) + run_queued_crawls(initialized_archive, env) + (initialized_archive / "archive" / "some_random_folder").mkdir(parents=True, exist_ok=True) - result = subprocess.run( - ["archivebox", "init"], - capture_output=True, - text=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["init"], + env=env, ) assert result.returncode == 0, result.stdout + result.stderr diff --git a/archivebox/tests/test_cli_install.py b/archivebox/tests/test_cli_install.py index cbc2db10..59161cc5 100644 --- a/archivebox/tests/test_cli_install.py +++ b/archivebox/tests/test_cli_install.py @@ -5,8 +5,8 @@ Verify install detects and records binary dependencies in DB. """ import os -import subprocess from pathlib import Path +from archivebox.tests.conftest import run_archivebox_cmd import pytest @@ -18,13 +18,10 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def test_install_runs_successfully(tmp_path, process): +def test_install_runs_successfully(initialized_archive): """Test that install command runs without error.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--dry-run"], timeout=60, ) @@ -32,28 +29,23 @@ def test_install_runs_successfully(tmp_path, process): assert result.returncode in [0, 1] # May return 1 if binaries missing -def test_install_creates_binary_records_in_db(tmp_path, process): +def test_install_creates_binary_records_in_db(initialized_archive): """Test that install creates Binary records in database.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, + run_archivebox_cmd( + ["install", "--dry-run"], timeout=60, ) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): Binary.objects.count() -def test_install_dry_run_does_not_install(tmp_path, process): +def test_install_dry_run_does_not_install(initialized_archive): """Test that --dry-run doesn't actually install anything.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--dry-run"], timeout=60, ) @@ -61,14 +53,11 @@ def test_install_dry_run_does_not_install(tmp_path, process): assert "dry" in result.stdout.lower() or result.returncode in [0, 1] -def test_install_detects_system_binaries(tmp_path, process): +def test_install_detects_system_binaries(initialized_archive): """Test that install detects existing system binaries.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--dry-run"], timeout=60, ) @@ -76,14 +65,11 @@ def test_install_detects_system_binaries(tmp_path, process): assert result.returncode in [0, 1] -def test_install_shows_binary_status(tmp_path, process): +def test_install_shows_binary_status(initialized_archive): """Test that install shows status of binaries.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--dry-run"], timeout=60, ) @@ -92,13 +78,10 @@ def test_install_shows_binary_status(tmp_path, process): assert len(output) > 50 -def test_install_dry_run_prints_dry_run_message(tmp_path, process): +def test_install_dry_run_prints_dry_run_message(initialized_archive): """Test that install --dry-run clearly reports that no changes will be made.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--dry-run"], timeout=60, ) @@ -108,11 +91,8 @@ def test_install_dry_run_prints_dry_run_message(tmp_path, process): def test_install_help_lists_dry_run_flag(tmp_path): """Test that install --help documents the dry-run option.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--help"], ) assert result.returncode == 0 @@ -121,11 +101,8 @@ def test_install_help_lists_dry_run_flag(tmp_path): def test_install_invalid_option_fails(tmp_path): """Test that invalid install options fail cleanly.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--invalid-option"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--invalid-option"], ) assert result.returncode != 0 @@ -133,11 +110,8 @@ def test_install_invalid_option_fails(tmp_path): def test_install_from_empty_dir_initializes_collection(tmp_path): """Test that install bootstraps an empty dir before performing work.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "install", "--dry-run"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "--dry-run"], ) output = result.stdout + result.stderr @@ -145,11 +119,10 @@ def test_install_from_empty_dir_initializes_collection(tmp_path): assert "Initializing" in output or "Dry run" in output or "init" in output.lower() -def test_install_updates_binary_table(tmp_path, process): +def test_install_updates_binary_table(initialized_archive): """Test that install completes and only mutates dependency state.""" - os.chdir(tmp_path) env = os.environ.copy() - tmp_short = Path("/tmp") / f"abx-install-{tmp_path.name}" + tmp_short = Path("/tmp") / f"abx-install-{initialized_archive.name}" tmp_short.mkdir(parents=True, exist_ok=True) env.update( { @@ -158,10 +131,8 @@ def test_install_updates_binary_table(tmp_path, process): }, ) - result = subprocess.run( - ["archivebox", "install", "git"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["install", "git"], timeout=120, env=env, ) @@ -169,7 +140,7 @@ def test_install_updates_binary_table(tmp_path, process): output = result.stdout + result.stderr assert result.returncode == 0, output - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): binary_counts = { status: Binary.objects.filter(status=status).count() for status in Binary.objects.values_list("status", flat=True).distinct() } diff --git a/archivebox/tests/test_cli_list.py b/archivebox/tests/test_cli_list.py index eb33612a..bb791cef 100644 --- a/archivebox/tests/test_cli_list.py +++ b/archivebox/tests/test_cli_list.py @@ -5,42 +5,135 @@ Verify list emits snapshot JSONL and applies the documented filters. """ import json -import os -import subprocess +import sys import pytest +from django.db import connection +from django.utils import timezone from archivebox.core.models import Snapshot -from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls +from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def _parse_jsonl(stdout: str) -> list[dict]: - return [json.loads(line) for line in stdout.splitlines() if line.strip().startswith("{")] +class CountingStdout: + encoding = "utf-8" + + def __init__(self): + self.rows = 0 + self._pending = "" + + def isatty(self): + return False + + def write(self, text): + self._pending += text + lines = self._pending.split("\n") + self._pending = lines.pop() + self.rows += sum(1 for line in lines if line.startswith("{")) + return len(text) + + def flush(self): + return None -def test_list_outputs_existing_snapshots_as_jsonl(tmp_path, process, disable_extractors_dict): +def test_list_limit_zero_streams_one_million_snapshots_without_materializing(admin_user, monkeypatch): + """Regression: archivebox list --limit=0 must stream unbounded result sets.""" + from archivebox.cli.archivebox_snapshot import list_snapshots + from archivebox.crawls.models import Crawl + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by=admin_user, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + now = timezone.now().isoformat() + with connection.cursor() as cursor: + cursor.execute( + """ + WITH RECURSIVE seq(n) AS ( + SELECT 1 + UNION ALL + SELECT n + 1 FROM seq WHERE n < 1000000 + ) + INSERT INTO core_snapshot ( + id, + url, + timestamp, + title, + bookmarked_at, + created_at, + modified_at, + downloaded_at, + fs_version, + crawl_id, + config, + current_step, + depth, + notes, + num_uses_failed, + num_uses_succeeded, + retry_at, + status, + delete_at, + output_size, + parent_snapshot_id + ) + SELECT + lower(hex(randomblob(16))), + 'https://example.com/page-' || n, + printf('9%031d', n), + '', + %s, + %s, + %s, + NULL, + '0.9.0', + %s, + '{}', + 0, + 0, + '', + 0, + 0, + NULL, + 'sealed', + NULL, + 0, + NULL + FROM seq + """, + [now, now, now, str(crawl.id).replace("-", "")], + ) + + stdout = CountingStdout() + monkeypatch.setattr(sys, "stdout", stdout) + + assert list_snapshots(limit=0) == 0 + assert stdout.rows == 1000000 + + +def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive): """Test that list prints one JSON object per stored snapshot.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for url in ["https://example.com", "https://iana.org"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run( - ["archivebox", "list"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["list"], timeout=30, ) - rows = _parse_jsonl(result.stdout) + rows = parse_jsonl_output(result.stdout) urls = {row["url"] for row in rows} assert result.returncode == 0, result.stderr @@ -48,95 +141,83 @@ def test_list_outputs_existing_snapshots_as_jsonl(tmp_path, process, disable_ext assert "https://iana.org" in urls -def test_list_filters_by_url_icontains(tmp_path, process, disable_extractors_dict): +def test_list_filters_by_url_icontains(initialized_archive): """Test that list --url__icontains returns only matching snapshots.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for url in ["https://example.com", "https://iana.org"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run( - ["archivebox", "list", "--url__icontains", "example.com"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["list", "--url__icontains", "example.com"], timeout=30, ) - rows = _parse_jsonl(result.stdout) + rows = parse_jsonl_output(result.stdout) assert result.returncode == 0, result.stderr assert len(rows) == 1 assert rows[0]["url"] == "https://example.com" -def test_list_filters_by_crawl_id_and_limit(tmp_path, process, disable_extractors_dict): +def test_list_filters_by_crawl_id_and_limit(initialized_archive): """Test that crawl-id and limit filters constrain the result set.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for url in ["https://example.com", "https://iana.org"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl_id = str(Snapshot.objects.values_list("crawl_id", flat=True).get(url="https://example.com")) - result = subprocess.run( - ["archivebox", "list", "--crawl-id", crawl_id, "--limit", "1"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["list", "--crawl-id", crawl_id, "--limit", "1"], timeout=30, ) - rows = _parse_jsonl(result.stdout) + rows = parse_jsonl_output(result.stdout) assert result.returncode == 0, result.stderr assert len(rows) == 1 assert rows[0]["crawl_id"].replace("-", "") == crawl_id.replace("-", "") assert rows[0]["url"] == "https://example.com" -def test_list_filters_by_status(tmp_path, process, disable_extractors_dict): +def test_list_filters_by_status(initialized_archive): """Test that list can filter using the current snapshot status.""" - os.chdir(tmp_path) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): status = Snapshot.objects.values_list("status", flat=True).get() - result = subprocess.run( - ["archivebox", "list", "--status", status], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["list", "--status", status], timeout=30, ) - rows = _parse_jsonl(result.stdout) + rows = parse_jsonl_output(result.stdout) assert result.returncode == 0, result.stderr assert len(rows) == 1 assert rows[0]["status"] == status -def test_list_help_lists_filter_options(tmp_path, process): +def test_list_help_lists_filter_options(initialized_archive): """Test that list --help documents the supported filter flags.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "list", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["list", "--help"], timeout=30, ) @@ -145,28 +226,28 @@ def test_list_help_lists_filter_options(tmp_path, process): assert "--crawl-id" in result.stdout assert "--limit" in result.stdout assert "--search" in result.stdout + assert "--json" in result.stdout + assert "--html" in result.stdout + assert "--with-headers" in result.stdout -def test_list_allows_sort_with_limit(tmp_path, process, disable_extractors_dict): +def test_list_allows_sort_with_limit(initialized_archive): """Test that list can sort and then apply limit without queryset slicing errors.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for url in ["https://example.com", "https://iana.org", "https://example.net"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run( - ["archivebox", "list", "--limit", "2", "--sort", "-created_at"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["list", "--limit", "2", "--sort", "-created_at"], timeout=30, ) - rows = _parse_jsonl(result.stdout) + rows = parse_jsonl_output(result.stdout) assert result.returncode == 0, result.stderr assert len(rows) == 2 @@ -174,12 +255,15 @@ def test_list_allows_sort_with_limit(tmp_path, process, disable_extractors_dict) def test_snapshot_list_search_meta(initialized_archive): """snapshot list should support metadata search mode.""" url = create_test_url(domain="meta-search-example.com") - run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", "--search=meta", "meta-search-example.com"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" records = parse_jsonl_output(stdout) @@ -190,12 +274,15 @@ def test_snapshot_list_search_meta(initialized_archive): def test_list_search_meta_matches_metadata(initialized_archive): """top-level list --search=meta should apply metadata search to the queryset.""" url = create_test_url(domain="top-level-meta-search-example.com") - run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["list", "--search=meta", "top-level-meta-search-example.com"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" records = parse_jsonl_output(stdout) @@ -204,62 +291,128 @@ def test_list_search_meta_matches_metadata(initialized_archive): def test_search_command_finds_snapshots(initialized_archive): - run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd(["search", "example"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, stderr assert "example" in stdout def test_search_command_returns_no_results_for_missing_term(initialized_archive): - run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - _stdout, _stderr, code = run_archivebox_cmd(["search", "nonexistentterm12345"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["search", "nonexistentterm12345"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code in [0, 1] def test_search_command_on_empty_archive(initialized_archive): - _stdout, _stderr, code = run_archivebox_cmd(["search", "anything"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["search", "anything"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code in [0, 1] -def test_search_command_json_outputs_matching_snapshots(initialized_archive): - run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive) +def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive): + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd(["search", "--json"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, stderr - payload = json.loads(stdout) + records = parse_jsonl_output(stdout) + assert any("example.com" in row.get("url", "") for row in records) + + +def test_search_command_json_outputs_matching_snapshots(initialized_archive): + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + + result = run_archivebox_cmd(["search", "--json"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) assert any("example.com" in row.get("url", "") for row in payload) def test_search_command_json_with_headers_wraps_links_payload(initialized_archive): - run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd(["search", "--json", "--with-headers"], data_dir=initialized_archive) + result = run_archivebox_cmd( + ["search", "--json", "--with-headers"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - assert code == 0, stderr - payload = json.loads(stdout) - links = payload.get("links", payload) - assert any("example.com" in row.get("url", "") for row in links) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert "links" in payload + assert any("example.com" in row.get("url", "") for row in payload["links"]) def test_search_command_html_outputs_markup(initialized_archive): - run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd(["search", "--html"], data_dir=initialized_archive) + result = run_archivebox_cmd(["search", "--html"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - assert code == 0, stderr - assert "<" in stdout + assert result.returncode == 0, result.stderr + assert "<" in result.stdout + assert "example.com" in result.stdout def test_search_command_csv_outputs_requested_column(initialized_archive): - run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd(["search", "--csv", "url", "--with-headers"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["search", "--csv", "url", "--with-headers"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, stderr assert "url" in stdout @@ -267,26 +420,38 @@ def test_search_command_csv_outputs_requested_column(initialized_archive): def test_search_command_with_headers_requires_structured_output_format(initialized_archive): - _stdout, stderr, code = run_archivebox_cmd(["search", "--with-headers"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["search", "--with-headers"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code != 0 - assert "requires" in stderr.lower() or "json" in stderr.lower() + assert "requires" in stderr.lower() + assert "json" in stderr.lower() def test_search_command_sort_option_runs_successfully(initialized_archive): for url in ["https://iana.org", "https://example.com"]: - run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd(["search", "--csv", "url", "--sort=url"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["search", "--csv", "url", "--sort=url"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, stderr assert "example.com" in stdout or "iana.org" in stdout def test_search_command_help_lists_supported_filters(initialized_archive): - stdout, _stderr, code = run_archivebox_cmd(["search", "--help"], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["search", "--help"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 - assert "--filter-type" in stdout or "-f" in stdout + assert "--url__icontains" in stdout + assert "--crawl-id" in stdout assert "--status" in stdout assert "--sort" in stdout + assert "--json" in stdout + assert "--html" in stdout diff --git a/archivebox/tests/test_cli_machine.py b/archivebox/tests/test_cli_machine.py new file mode 100644 index 00000000..602ea12d --- /dev/null +++ b/archivebox/tests/test_cli_machine.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox machine command. + +TODO: expand beyond command discovery into list/filter behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_machine_help_runs_successfully(tmp_path): + """The machine command should be registered and expose help.""" + + result = run_archivebox_cmd(["machine", "--help"]) + + assert result.returncode == 0 + assert "machine" in result.stdout.lower() + assert "list" in result.stdout diff --git a/archivebox/tests/test_cli_manage.py b/archivebox/tests/test_cli_manage.py index 9634b632..4f0f73e7 100644 --- a/archivebox/tests/test_cli_manage.py +++ b/archivebox/tests/test_cli_manage.py @@ -4,18 +4,14 @@ Tests for archivebox manage command. Verify manage command runs Django management commands. """ -import os -import subprocess +from archivebox.tests.conftest import run_archivebox_cmd -def test_manage_help_works(tmp_path, process): +def test_manage_help_works(initialized_archive): """Test that manage help command works.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "manage", "help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["manage", "help"], timeout=30, ) @@ -23,14 +19,11 @@ def test_manage_help_works(tmp_path, process): assert len(result.stdout) > 100 -def test_manage_showmigrations_works(tmp_path, process): +def test_manage_showmigrations_works(initialized_archive): """Test that manage showmigrations works.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "manage", "showmigrations"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["manage", "showmigrations"], timeout=30, ) @@ -39,14 +32,11 @@ def test_manage_showmigrations_works(tmp_path, process): assert "core" in result.stdout or "[" in result.stdout -def test_manage_dbshell_command_exists(tmp_path, process): +def test_manage_dbshell_command_exists(initialized_archive): """Test that manage dbshell command is recognized.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "manage", "help", "dbshell"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["manage", "help", "dbshell"], timeout=30, ) @@ -55,14 +45,11 @@ def test_manage_dbshell_command_exists(tmp_path, process): assert "dbshell" in result.stdout or "database" in result.stdout.lower() -def test_manage_check_works(tmp_path, process): +def test_manage_check_works(initialized_archive): """Test that manage check works.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "manage", "check"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["manage", "check"], timeout=30, ) diff --git a/archivebox/tests/test_cli_mcp.py b/archivebox/tests/test_cli_mcp.py new file mode 100644 index 00000000..82b2c97b --- /dev/null +++ b/archivebox/tests/test_cli_mcp.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox mcp command. + +TODO: expand beyond command discovery into JSON-RPC stdio behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_mcp_help_runs_successfully(tmp_path): + """The mcp command should be registered and expose help.""" + + result = run_archivebox_cmd(["mcp", "--help"]) + + assert result.returncode == 0 + assert "mcp" in result.stdout.lower() diff --git a/archivebox/tests/test_cli_persona.py b/archivebox/tests/test_cli_persona.py new file mode 100644 index 00000000..6f45cc54 --- /dev/null +++ b/archivebox/tests/test_cli_persona.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox persona command. + +TODO: expand beyond command discovery into create/list/update/delete behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_persona_help_runs_successfully(tmp_path): + """The persona command should be registered and expose help.""" + + result = run_archivebox_cmd(["persona", "--help"]) + + assert result.returncode == 0 + assert "persona" in result.stdout.lower() + assert "list" in result.stdout diff --git a/archivebox/tests/test_cli_piping.py b/archivebox/tests/test_cli_piping.py index a58f24ef..34d1e0bc 100644 --- a/archivebox/tests/test_cli_piping.py +++ b/archivebox/tests/test_cli_piping.py @@ -15,6 +15,7 @@ import pytest from archivebox.core.models import Snapshot from archivebox.machine.models import Binary from archivebox.tests.conftest import ( + assert_jsonl_only, create_test_url, parse_jsonl_output, run_archivebox_cmd, @@ -41,20 +42,6 @@ class MockTTYStringIO(StringIO): return self._is_tty -def _stdout_lines(stdout: str) -> list[str]: - return [line for line in stdout.splitlines() if line.strip()] - - -def _assert_stdout_is_jsonl_only(stdout: str) -> None: - lines = _stdout_lines(stdout) - assert lines, "Expected stdout to contain JSONL records" - assert all(line.lstrip().startswith("{") for line in lines), stdout - - -def _uuid(value: str) -> uuid.UUID: - return uuid.UUID(value) - - def test_parse_line_accepts_supported_piping_inputs(): """The JSONL parser should normalize the input forms CLI pipes accept.""" from archivebox.misc.jsonl import TYPE_CRAWL, TYPE_SNAPSHOT, parse_line @@ -196,30 +183,36 @@ def test_crawl_create_stdout_pipes_into_run(initialized_archive): """`archivebox crawl create | archivebox run` should queue and materialize snapshots.""" url = create_test_url() - create_stdout, create_stderr, create_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + create_stdout, create_stderr, create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert create_code == 0, create_stderr - _assert_stdout_is_jsonl_only(create_stdout) + assert_jsonl_only(create_stdout) crawl = next(record for record in parse_jsonl_output(create_stdout) if record.get("type") == "Crawl") - run_stdout, run_stderr, run_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=create_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert run_code == 0, run_stderr - _assert_stdout_is_jsonl_only(run_stdout) + assert_jsonl_only(run_stdout) run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") == "Crawl" and record.get("id") == crawl["id"] for record in run_records) with use_archivebox_db(initialized_archive): - snapshot_count = Snapshot.objects.filter(crawl_id=_uuid(crawl["id"])).count() + snapshot_count = Snapshot.objects.filter(crawl_id=uuid.UUID(crawl["id"])).count() assert isinstance(snapshot_count, int) assert snapshot_count >= 1 @@ -228,40 +221,52 @@ def test_snapshot_list_stdout_pipes_into_run(initialized_archive): """`archivebox snapshot list | archivebox run` should requeue listed snapshots.""" url = create_test_url() - create_stdout, create_stderr, create_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + create_stdout, create_stderr, create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert create_code == 0, create_stderr snapshot = next(record for record in parse_jsonl_output(create_stdout) if record.get("type") == "Snapshot") - list_stdout, list_stderr, list_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", "--status=queued", f"--url__icontains={snapshot['id']}"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + list_stdout, list_stderr, list_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode if list_code != 0 or not parse_jsonl_output(list_stdout): - list_stdout, list_stderr, list_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", f"--url__icontains={url}"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + list_stdout, list_stderr, list_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert list_code == 0, list_stderr - _assert_stdout_is_jsonl_only(list_stdout) + assert_jsonl_only(list_stdout) - run_stdout, run_stderr, run_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=list_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert run_code == 0, run_stderr - _assert_stdout_is_jsonl_only(run_stdout) + assert_jsonl_only(run_stdout) run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") == "Snapshot" and record.get("id") == snapshot["id"] for record in run_records) with use_archivebox_db(initialized_archive): - snapshot_status = Snapshot.objects.values_list("status", flat=True).get(pk=_uuid(snapshot["id"])) + snapshot_status = Snapshot.objects.values_list("status", flat=True).get(pk=uuid.UUID(snapshot["id"])) assert snapshot_status == "sealed" @@ -269,48 +274,62 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive): """`archivebox archiveresult list | archivebox run` should preserve clean JSONL stdout.""" url = create_test_url() - snapshot_stdout, snapshot_stderr, snapshot_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create", url], - data_dir=initialized_archive, + cwd=initialized_archive, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + snapshot_stdout, snapshot_stderr, snapshot_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert snapshot_code == 0, snapshot_stderr - ar_create_stdout, ar_create_stderr, ar_create_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=snapshot_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + ar_create_stdout, ar_create_stderr, ar_create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert ar_create_code == 0, ar_create_stderr run_archivebox_cmd( ["run"], stdin=ar_create_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) - list_stdout, list_stderr, list_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "list", "--plugin=favicon"], - data_dir=initialized_archive, + cwd=initialized_archive, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + list_stdout, list_stderr, list_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert list_code == 0, list_stderr - _assert_stdout_is_jsonl_only(list_stdout) + assert_jsonl_only(list_stdout) listed_records = parse_jsonl_output(list_stdout) archiveresult = next(record for record in listed_records if record.get("type") == "ArchiveResult") - run_stdout, run_stderr, run_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=list_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert run_code == 0, run_stderr - _assert_stdout_is_jsonl_only(run_stdout) + assert_jsonl_only(run_stdout) run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") == "ArchiveResult" and record.get("id") == archiveresult["id"] for record in run_records) @@ -318,29 +337,35 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive): def test_binary_create_stdout_pipes_into_run(initialized_archive): """`archivebox binary create | archivebox run` should queue the binary record for processing.""" - create_stdout, create_stderr, create_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["binary", "create", "--name=python3", f"--abspath={sys.executable}", "--version=test"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + create_stdout, create_stderr, create_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert create_code == 0, create_stderr - _assert_stdout_is_jsonl_only(create_stdout) + assert_jsonl_only(create_stdout) binary = next(record for record in parse_jsonl_output(create_stdout) if record.get("type") in {"BinaryRequest", "Binary"}) - run_stdout, run_stderr, run_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=create_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, + default_cli_env=True, + disable_extractors=True, ) + run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert run_code == 0, run_stderr - _assert_stdout_is_jsonl_only(run_stdout) + assert_jsonl_only(run_stdout) run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") in {"BinaryRequest", "Binary"} and record.get("id") == binary["id"] for record in run_records) with use_archivebox_db(initialized_archive): - status = Binary.objects.values_list("status", flat=True).get(pk=_uuid(binary["id"])) + status = Binary.objects.values_list("status", flat=True).get(pk=uuid.UUID(binary["id"])) assert status in {"queued", "installed"} @@ -348,43 +373,55 @@ def test_multi_stage_pipeline_into_run(initialized_archive): """`crawl create | snapshot create | archiveresult create | run` should preserve JSONL and finish work.""" url = create_test_url() - crawl_stdout, crawl_stderr, crawl_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["crawl", "create", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + crawl_stdout, crawl_stderr, crawl_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert crawl_code == 0, crawl_stderr - _assert_stdout_is_jsonl_only(crawl_stdout) + assert_jsonl_only(crawl_stdout) - snapshot_stdout, snapshot_stderr, snapshot_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create"], stdin=crawl_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + snapshot_stdout, snapshot_stderr, snapshot_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert snapshot_code == 0, snapshot_stderr - _assert_stdout_is_jsonl_only(snapshot_stdout) + assert_jsonl_only(snapshot_stdout) - archiveresult_stdout, archiveresult_stderr, archiveresult_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=snapshot_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + archiveresult_stdout, archiveresult_stderr, archiveresult_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert archiveresult_code == 0, archiveresult_stderr - _assert_stdout_is_jsonl_only(archiveresult_stdout) + assert_jsonl_only(archiveresult_stdout) - run_stdout, run_stderr, run_code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=archiveresult_stdout, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=PIPE_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert run_code == 0, run_stderr - _assert_stdout_is_jsonl_only(run_stdout) + assert_jsonl_only(run_stdout) run_records = parse_jsonl_output(run_stdout) snapshot = next(record for record in run_records if record.get("type") == "Snapshot") assert any(record.get("type") == "ArchiveResult" for record in run_records) with use_archivebox_db(initialized_archive): - snapshot_status = Snapshot.objects.values_list("status", flat=True).get(pk=_uuid(snapshot["id"])) + snapshot_status = Snapshot.objects.values_list("status", flat=True).get(pk=uuid.UUID(snapshot["id"])) assert snapshot_status == "sealed" diff --git a/archivebox/tests/test_cli_pluginmap.py b/archivebox/tests/test_cli_pluginmap.py new file mode 100644 index 00000000..d7266ec3 --- /dev/null +++ b/archivebox/tests/test_cli_pluginmap.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox pluginmap command. + +TODO: expand beyond command discovery into quiet/event output behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_pluginmap_help_runs_successfully(tmp_path): + """The pluginmap command should be registered and expose help.""" + + result = run_archivebox_cmd(["pluginmap", "--help"]) + + assert result.returncode == 0 + assert "pluginmap" in result.stdout.lower() + assert "--event" in result.stdout diff --git a/archivebox/tests/test_cli_process.py b/archivebox/tests/test_cli_process.py new file mode 100644 index 00000000..1968fc50 --- /dev/null +++ b/archivebox/tests/test_cli_process.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox process command. + +TODO: expand beyond command discovery into list/filter behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_process_help_runs_successfully(tmp_path): + """The process command should be registered and expose help.""" + + result = run_archivebox_cmd(["process", "--help"]) + + assert result.returncode == 0 + assert "process" in result.stdout.lower() + assert "list" in result.stdout diff --git a/archivebox/tests/test_cli_real_flows.py b/archivebox/tests/test_cli_real_flows.py deleted file mode 100644 index 54f08e5f..00000000 --- a/archivebox/tests/test_cli_real_flows.py +++ /dev/null @@ -1,1091 +0,0 @@ -#!/usr/bin/env python3 -"""Real user-facing archive flows against live URLs.""" - -import json -import os -import re -import signal -import socket -import subprocess -import sys -import time -from pathlib import Path - -import pytest - -from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.crawls.models import Crawl -from archivebox.machine.models import Process -from archivebox.tests.test_orm_helpers import use_archivebox_db - -from .conftest import _find_system_browser, wait_for_process - -pytestmark = pytest.mark.django_db(transaction=True) - - -def _pid_is_alive(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - return True - - -def _wait_for_pid_exit(pid: int, *, timeout: float = 5.0) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - if not _pid_is_alive(pid): - return - time.sleep(0.05) - raise AssertionError(f"PID {pid} is still alive") - - -def _cleanup_process_group(group_pid: int | None, *child_pids: int | None) -> None: - if group_pid and _pid_is_alive(group_pid): - try: - os.killpg(group_pid, signal.SIGKILL) - except ProcessLookupError: - pass - except OSError: - try: - os.kill(group_pid, signal.SIGKILL) - except ProcessLookupError: - pass - for pid in child_pids: - if pid and _pid_is_alive(pid): - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - - -def _free_port() -> int: - with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - - -def _live_exit_env(data_dir, *, plugins_root=None, extra=None): - env = os.environ.copy() - env.update( - { - "DATA_DIR": str(data_dir), - "USE_COLOR": "false", - "SHOW_PROGRESS": "false", - "SAVE_ARCHIVEDOTORG": "false", - "SAVE_FAVICON": "false", - "SAVE_HEADERS": "false", - "SAVE_TITLE": "false", - "SAVE_READABILITY": "false", - "SAVE_SINGLEFILE": "false", - "SAVE_MERCURY": "false", - "SAVE_SCREENSHOT": "false", - "SAVE_PDF": "false", - "SAVE_DOM": "false", - "SAVE_GIT": "false", - "SAVE_YTDLP": "false", - "TIMEOUT": "60", - "WGET_TIMEOUT": "45", - "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1", - "PARSE_HTML_URLS_ENABLED": "true", - "PARSE_DOM_OUTLINKS_ENABLED": "false", - "SEARCH_BACKEND_ENGINE": "sqlite", - }, - ) - if plugins_root is not None: - env["ABX_PLUGINS_DIR"] = str(plugins_root) - if extra: - env.update(extra) - return env - - -def _wait_for_port(host: str, port: int, *, timeout: float = 30.0) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - try: - with socket.create_connection((host, port), timeout=0.25): - return - except OSError: - time.sleep(0.1) - raise AssertionError(f"server did not listen on {host}:{port}") - - -def _wait_for_log(log_path: Path, text: str, *, timeout: float = 30.0) -> str: - deadline = time.time() + timeout - content = "" - while time.time() < deadline: - if log_path.exists(): - content = log_path.read_text(encoding="utf-8", errors="replace") - if text in content: - return content - time.sleep(0.1) - raise AssertionError(f"timed out waiting for {text!r} in {log_path}:\n{content}") - - -def _wait_for_log_count(log_path: Path, text: str, count: int, *, timeout: float = 30.0) -> str: - deadline = time.time() + timeout - content = "" - while time.time() < deadline: - if log_path.exists(): - content = log_path.read_text(encoding="utf-8", errors="replace") - if content.count(text) >= count: - return content - time.sleep(0.1) - raise AssertionError(f"timed out waiting for {count} occurrences of {text!r} in {log_path}:\n{content}") - - -def _wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - if not _pid_is_alive(pid): - return - time.sleep(0.1) - raise AssertionError(f"PID {pid} is still running") - - -def _supervisor_pid_from_log(log_path: Path) -> int: - content = log_path.read_text(encoding="utf-8", errors="replace") - matches = re.findall(r"Supervisord connected \(pid=(\d+)\)", content) - assert matches, content - return int(matches[-1]) - - -def _worker_pid_from_log(log_path: Path, worker_name: str) -> int: - content = log_path.read_text(encoding="utf-8", errors="replace") - matches = re.findall(rf"Worker {re.escape(worker_name)}: started RUNNING \(pid (\d+),", content) - assert matches, content - return int(matches[-1]) - - -def _pgrep_data_dir(data_dir) -> list[str]: - result = subprocess.run(["pgrep", "-af", str(data_dir)], capture_output=True, text=True, timeout=5) - lines = [line for line in result.stdout.splitlines() if "pgrep -af" not in line] - - # A foreground ArchiveBox process can be killed with SIGKILL before Python - # cleanup runs. Supervisord's command line only points at its generated - # config file, so catch orphaned supervisors by resolving pidfiles whose - # configs still reference this real test DATA_DIR. - for runtime_root in (Path("/tmp/archivebox"), Path(data_dir) / "tmp"): - for config_path in runtime_root.glob("*/supervisord.conf"): - try: - config_text = config_path.read_text(encoding="utf-8", errors="replace") - except OSError: - continue - if str(data_dir) not in config_text: - continue - pid_path = config_path.with_name("supervisord.pid") - try: - pid = int(pid_path.read_text(encoding="utf-8").strip()) - except (OSError, ValueError): - continue - if not _pid_is_alive(pid): - continue - ps_line = subprocess.run( - ["ps", "-p", str(pid), "-o", "pid=,ppid=,command="], - capture_output=True, - text=True, - timeout=5, - ).stdout.strip() - if ps_line: - lines.append(ps_line) - - return sorted(set(lines)) - - -def _assert_no_processes_for_data_dir(data_dir, *, timeout: float = 10.0) -> None: - deadline = time.time() + timeout - remaining: list[str] = [] - while time.time() < deadline: - remaining = _pgrep_data_dir(data_dir) - if not remaining: - return - time.sleep(0.25) - raise AssertionError("processes still reference test DATA_DIR:\n" + "\n".join(remaining)) - - -def _kill_processes_for_data_dir(data_dir) -> None: - for line in _pgrep_data_dir(data_dir): - try: - pid = int(line.split(None, 1)[0]) - except (IndexError, ValueError): - continue - if pid != os.getpid(): - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - - -def _start_server(data_dir, *, port: int, log_name: str, env: dict[str, str] | None = None) -> tuple[subprocess.Popen[str], Path]: - log_path = data_dir / log_name - log = log_path.open("w", encoding="utf-8") - proc = subprocess.Popen( - [sys.executable, "-m", "archivebox", "server", f"127.0.0.1:{port}"], - cwd=data_dir, - env=env or _live_exit_env(data_dir), - stdout=log, - stderr=subprocess.STDOUT, - text=True, - start_new_session=True, - ) - log.close() - _wait_for_port("127.0.0.1", port) - _wait_for_log(log_path, "Tailing worker logs", timeout=30.0) - return proc, log_path - - -def _stop_process(proc: subprocess.Popen[str], sig=signal.SIGTERM, *, timeout: float = 15.0) -> str: - if proc.poll() is None: - try: - os.killpg(proc.pid, sig) - except (ProcessLookupError, OSError): - try: - os.kill(proc.pid, sig) - except ProcessLookupError: - pass - try: - stdout, _stderr = proc.communicate(timeout=timeout) - return stdout or "" - except subprocess.TimeoutExpired: - try: - os.killpg(proc.pid, signal.SIGKILL) - except (ProcessLookupError, OSError): - try: - os.kill(proc.pid, signal.SIGKILL) - except ProcessLookupError: - pass - stdout, _stderr = proc.communicate(timeout=5) - return stdout or "" - - -def _write_slow_snapshot_plugin(plugins_root, marker_dir): - plugin_dir = plugins_root / "slow_exit" - plugin_dir.mkdir(parents=True, exist_ok=True) - hook = plugin_dir / "on_Snapshot__09_slow_exit.finite.bg.sh" - hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - f"marker_dir={str(marker_dir)!r}", - 'mkdir -p "$marker_dir"', - 'echo $$ >> "$marker_dir/hook-pids.txt"', - 'touch "$marker_dir/hook-started"', - "trap 'touch \"$marker_dir/hook-stopped\"; exit 0' TERM INT HUP", - "while true; do sleep 1; done", - "", - ], - ), - encoding="utf-8", - ) - hook.chmod(0o755) - return plugin_dir - - -def _wait_for_crawl_state(data_dir, predicate, *, timeout: float = 30.0): - deadline = time.time() + timeout - last = None - while time.time() < deadline: - with use_archivebox_db(data_dir): - last = { - "crawls": list(Crawl.objects.order_by("created_at").values("id", "status", "retry_at")), - "snapshots": list(Snapshot.objects.order_by("created_at").values("id", "url", "status", "retry_at")), - "results": list(ArchiveResult.objects.order_by("created_at").values("id", "plugin", "status")), - "processes": list(Process.objects.order_by("created_at").values("id", "process_type", "status", "pid", "cmd")), - } - if predicate(last): - return last - time.sleep(0.25) - raise AssertionError(f"timed out waiting for crawl state, last={last}") - - -def _wait_for_hook_runs(marker_dir: Path, count: int, *, timeout: float = 45.0) -> list[int]: - pid_file = marker_dir / "hook-pids.txt" - deadline = time.time() + timeout - pids: list[int] = [] - while time.time() < deadline: - if pid_file.exists(): - pids = [int(line.strip()) for line in pid_file.read_text().splitlines() if line.strip()] - if len(pids) >= count: - return pids - time.sleep(0.25) - raise AssertionError(f"timed out waiting for {count} slow hook runs, got {pids}") - - -def _start_live_add( - data_dir, - env, - *, - url="https://example.com", - max_urls="2", - log_name="archivebox-add.log", -) -> tuple[subprocess.Popen[str], Path]: - log_path = data_dir / log_name - log = log_path.open("w", encoding="utf-8") - urls = [url] if isinstance(url, str) else url - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "archivebox", - "add", - "--depth=1", - f"--max-urls={max_urls}", - "--crawl-max-size=50mb", - "--plugins=wget,parse_html_urls,slow_exit", - *urls, - ], - cwd=data_dir, - env=env, - stdout=log, - stderr=subprocess.STDOUT, - text=True, - start_new_session=True, - ) - log.close() - return proc, log_path - - -@pytest.mark.timeout(90) -def test_cli_run_signal_cleans_background_hook_process_group(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - plugins_root = tmp_path / "runtime_plugins" - plugin_dir = plugins_root / "cancel_group" - plugin_dir.mkdir(parents=True) - daemon_hook = plugin_dir / "on_CrawlSetup__10_daemon.daemon.bg.sh" - foreground_hook = plugin_dir / "on_CrawlSetup__20_foreground.sh" - daemon_hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'test_dir="${LEAK_TEST_DIR:?}"', - "sleep 600 &", - 'echo $$ > "$test_dir/daemon.pid"', - 'echo $! > "$test_dir/daemon-child.pid"', - 'echo ready > "$test_dir/daemon.ready"', - "trap 'echo cleaned > \"$test_dir/daemon.cleaned\"; exit 0' TERM INT", - "wait", - "", - ], - ), - ) - foreground_hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'test_dir="${LEAK_TEST_DIR:?}"', - 'echo $$ > "$test_dir/foreground.pid"', - 'echo ready > "$test_dir/foreground.ready"', - "trap 'echo cleaned > \"$test_dir/foreground.cleaned\"; exit 0' TERM INT", - "while true; do sleep 1; done", - "", - ], - ), - ) - daemon_hook.chmod(0o755) - foreground_hook.chmod(0o755) - - leak_test_dir = tmp_path / "leak-check" - leak_test_dir.mkdir() - env = os.environ.copy() - env.update( - { - "ABX_PLUGINS_DIR": str(plugins_root), - "LEAK_TEST_DIR": str(leak_test_dir), - "PLUGINS": "cancel_group", - "TIMEOUT": "30", - "USE_COLOR": "false", - "SHOW_PROGRESS": "false", - }, - ) - - create_result = subprocess.run( - [sys.executable, "-m", "archivebox", "crawl", "create", "https://example.com"], - cwd=tmp_path, - capture_output=True, - text=True, - env=env, - timeout=60, - ) - assert create_result.returncode == 0, create_result.stderr or create_result.stdout - crawl_records = [json.loads(line) for line in create_result.stdout.splitlines() if line.strip().startswith("{")] - crawl_id = next(record["id"] for record in crawl_records if record.get("type") == "Crawl") - - daemon_pid: int | None = None - daemon_child_pid: int | None = None - foreground_pid: int | None = None - run_process = subprocess.Popen( - [sys.executable, "-m", "archivebox", "run", f"--crawl-id={crawl_id}"], - cwd=tmp_path, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - start_new_session=True, - ) - try: - deadline = time.time() + 20 - while time.time() < deadline: - if (leak_test_dir / "daemon.ready").exists() and (leak_test_dir / "foreground.ready").exists(): - break - if run_process.poll() is not None: - output = run_process.communicate(timeout=1)[0] - raise AssertionError(f"archivebox run exited before hooks were ready:\n{output}") - time.sleep(0.05) - assert (leak_test_dir / "daemon.ready").exists() - assert (leak_test_dir / "foreground.ready").exists() - - daemon_pid = int((leak_test_dir / "daemon.pid").read_text().strip()) - daemon_child_pid = int((leak_test_dir / "daemon-child.pid").read_text().strip()) - foreground_pid = int((leak_test_dir / "foreground.pid").read_text().strip()) - assert _pid_is_alive(daemon_pid) - assert _pid_is_alive(daemon_child_pid) - assert _pid_is_alive(foreground_pid) - - run_process.send_signal(signal.SIGTERM) - time.sleep(0.1) - if run_process.poll() is None: - run_process.send_signal(signal.SIGTERM) - output = run_process.communicate(timeout=20)[0] - assert "Runner error" not in output - - _wait_for_pid_exit(daemon_pid) - _wait_for_pid_exit(daemon_child_pid) - _wait_for_pid_exit(foreground_pid) - assert (leak_test_dir / "daemon.cleaned").read_text().strip() == "cleaned" - assert (leak_test_dir / "foreground.cleaned").read_text().strip() == "cleaned" - finally: - if run_process.poll() is None: - try: - os.killpg(run_process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - run_process.communicate(timeout=5) - _cleanup_process_group(daemon_pid, daemon_child_pid) - _cleanup_process_group(foreground_pid) - - -@pytest.mark.timeout(300) -@pytest.mark.parametrize( - ("stop_signal", "expected_notice"), - [ - (signal.SIGHUP, "Got SIGHUP"), - (signal.SIGINT, "Got SIGINT"), - (signal.SIGTERM, "Got SIGTERM"), - (signal.SIGKILL, None), - ], -) -def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(tmp_path, process, stop_signal, expected_notice): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - env = _live_exit_env(tmp_path) - port = _free_port() - server = None - resumed = None - try: - server, server_log = _start_server(tmp_path, port=port, log_name=f"server-{stop_signal.name}.log", env=env) - - os.kill(server.pid, stop_signal) - try: - server.wait(timeout=20 if stop_signal != signal.SIGKILL else 5) - except subprocess.TimeoutExpired: - os.kill(server.pid, signal.SIGKILL) - server.wait(timeout=5) - - if expected_notice: - log_text = server_log.read_text(encoding="utf-8", errors="replace") - assert expected_notice in log_text - assert "ArchiveBox server shut down gracefully" in log_text - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - - resumed, resumed_log = _start_server(tmp_path, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env) - status = subprocess.run( - [sys.executable, "-m", "archivebox", "status"], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - timeout=60, - ) - assert status.returncode == 0, status.stderr or status.stdout - - os.kill(resumed.pid, signal.SIGTERM) - resumed.wait(timeout=20) - resumed_text = resumed_log.read_text(encoding="utf-8", errors="replace") - assert "Got SIGTERM" in resumed_text - assert "ArchiveBox server shut down gracefully" in resumed_text - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - finally: - for proc in (server, resumed): - if proc is not None and proc.poll() is None: - _stop_process(proc, signal.SIGKILL) - _kill_processes_for_data_dir(tmp_path) - - -@pytest.mark.timeout(180) -def test_live_daemonized_server_keeps_supervisord_owned_by_archivebox_parent(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - env = _live_exit_env(tmp_path) - port = _free_port() - bind_url = f"http://127.0.0.1:{port}" - try: - result = subprocess.run( - [sys.executable, "-m", "archivebox", "server", "--daemonize", f"127.0.0.1:{port}"], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - timeout=90, - ) - assert result.returncode == 0, result.stderr or result.stdout - _wait_for_port("127.0.0.1", port, timeout=30) - - server_process = wait_for_process( - lambda _proc, command: "archivebox" in command and " server " in f" {command} " and bind_url.replace("http://", "") in command, - ) - supervisord = wait_for_process( - lambda proc, command: proc.ppid() == server_process.pid and "supervisord" in command, - ) - wait_for_process( - lambda proc, command: proc.ppid() == supervisord.pid and "supervisord_watchdog" in command, - ) - - os.kill(server_process.pid, signal.SIGKILL) - _wait_for_pid_to_disappear(server_process.pid, timeout=10) - _wait_for_pid_to_disappear(supervisord.pid, timeout=20) - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - finally: - _kill_processes_for_data_dir(tmp_path) - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - - -@pytest.mark.timeout(240) -def test_live_second_server_takes_over_existing_server_process(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - env = _live_exit_env(tmp_path) - port = _free_port() - first = None - second = None - try: - first, first_log = _start_server(tmp_path, port=port, log_name="server-first.log", env=env) - second, second_log = _start_server(tmp_path, port=port, log_name="server-second.log", env=env) - - assert first.poll() is None - first_text = first_log.read_text(encoding="utf-8", errors="replace") - second_text = second_log.read_text(encoding="utf-8", errors="replace") - assert "A newer archivebox process took over the orchestrator, server" in first_text - assert "Starting orchestrator, server" in second_text - - status = subprocess.run( - [sys.executable, "-m", "archivebox", "status"], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - timeout=60, - ) - assert status.returncode == 0, status.stderr or status.stdout - - first_resumes = first_log.read_text(encoding="utf-8", errors="replace").count("Other newer archivebox process") - _stop_process(second, signal.SIGTERM) - second = None - _wait_for_log_count(first_log, "Other newer archivebox process", first_resumes + 1, timeout=35) - assert first.poll() is None - finally: - if second is not None and second.poll() is None: - _stop_process(second, signal.SIGTERM) - if first is not None and first.poll() is None: - _stop_process(first, signal.SIGKILL) - _kill_processes_for_data_dir(tmp_path) - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - - -@pytest.mark.timeout(420) -def test_live_repeated_server_startups_take_over_cleanly(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - env = _live_exit_env(tmp_path) - port = _free_port() - servers: list[subprocess.Popen[str]] = [] - server_pids: list[int] = [] - daphne_pids: list[int] = [] - runner_pids: list[int] = [] - try: - for index in range(5): - server, log_path = _start_server(tmp_path, port=port, log_name=f"server-chaos-{index}.log", env=env) - servers.append(server) - server_pids.append(server.pid) - daphne_pids.append(_worker_pid_from_log(log_path, "worker_daphne")) - runner_pids.append(_worker_pid_from_log(log_path, "worker_runner")) - - if index > 0: - previous_server = servers[index - 1] - previous_log = (tmp_path / f"server-chaos-{index - 1}.log").read_text(encoding="utf-8", errors="replace") - current_log = log_path.read_text(encoding="utf-8", errors="replace") - assert previous_server.poll() is None - assert _pid_is_alive(server_pids[index - 1]) - assert "A newer archivebox process took over the orchestrator, server" in previous_log - assert "Starting orchestrator, server" in current_log - _wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15) - _wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15) - - status = subprocess.run( - [sys.executable, "-m", "archivebox", "status"], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - timeout=60, - ) - assert status.returncode == 0, status.stderr or status.stdout - time.sleep(5) - - assert servers[-1].poll() is None - assert all(server.poll() is None for server in servers) - listener = subprocess.run( - ["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"], - capture_output=True, - text=True, - timeout=10, - ) - assert listener.returncode == 0, listener.stderr or listener.stdout - assert listener.stdout.count(f":{port} (LISTEN)") == 1 - - previous_log_path = tmp_path / "server-chaos-3.log" - previous_takeovers = previous_log_path.read_text(encoding="utf-8", errors="replace").count( - "Other newer archivebox process", - ) - _stop_process(servers[-1], signal.SIGTERM) - _wait_for_log_count(previous_log_path, "Other newer archivebox process", previous_takeovers + 1, timeout=35) - assert servers[3].poll() is None - finally: - for server in reversed(servers): - if server.poll() is None: - _stop_process(server, signal.SIGTERM) - _kill_processes_for_data_dir(tmp_path) - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - - -@pytest.mark.timeout(240) -def test_live_servers_in_different_data_dirs_do_not_interfere(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - first_data_dir = tmp_path - second_data_dir = tmp_path.parent / f"{tmp_path.name}-second" - second_data_dir.mkdir() - second_env = _live_exit_env(second_data_dir) - second_init = subprocess.run( - [sys.executable, "-m", "archivebox", "init"], - cwd=second_data_dir, - env=second_env, - capture_output=True, - text=True, - timeout=90, - ) - assert second_init.returncode == 0, second_init.stderr or second_init.stdout - - first_port = _free_port() - second_port = _free_port() - first = None - second = None - first_resumed = None - try: - first = _start_server(first_data_dir, port=first_port, log_name="server-first-data-dir.log", env=_live_exit_env(first_data_dir))[0] - second = _start_server(second_data_dir, port=second_port, log_name="server-second-data-dir.log", env=second_env)[0] - - first_status = subprocess.run( - [sys.executable, "-m", "archivebox", "status"], - cwd=first_data_dir, - env=_live_exit_env(first_data_dir), - capture_output=True, - text=True, - timeout=60, - ) - second_status = subprocess.run( - [sys.executable, "-m", "archivebox", "status"], - cwd=second_data_dir, - env=second_env, - capture_output=True, - text=True, - timeout=60, - ) - assert first_status.returncode == 0, first_status.stderr or first_status.stdout - assert second_status.returncode == 0, second_status.stderr or second_status.stdout - - _stop_process(first, signal.SIGTERM) - first = None - assert second.poll() is None, "stopping one DATA_DIR server must not stop another DATA_DIR server" - - first_resumed = _start_server( - first_data_dir, - port=first_port, - log_name="server-first-data-dir-resumed.log", - env=_live_exit_env(first_data_dir), - )[0] - assert second.poll() is None, "restarting one DATA_DIR server must not take over another DATA_DIR supervisor" - finally: - for proc in (first, first_resumed, second): - if proc is not None and proc.poll() is None: - _stop_process(proc, signal.SIGTERM) - _kill_processes_for_data_dir(first_data_dir) - _kill_processes_for_data_dir(second_data_dir) - _assert_no_processes_for_data_dir(first_data_dir, timeout=12) - _assert_no_processes_for_data_dir(second_data_dir, timeout=12) - - -@pytest.mark.timeout(420) -def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - plugins_root = tmp_path / "runtime_plugins" - marker_dir = tmp_path / "slow-plugin-markers" - _write_slow_snapshot_plugin(plugins_root, marker_dir) - env = _live_exit_env(tmp_path, plugins_root=plugins_root) - port = _free_port() - server = None - server2 = None - server3 = None - add_proc = None - add_proc2 = None - try: - server, server_log = _start_server(tmp_path, port=port, log_name="server-add-owner-1.log", env=env) - supervisor_pid_before = _supervisor_pid_from_log(server_log) - - update_result = subprocess.run( - [sys.executable, "-m", "archivebox", "update", "--index-only", "--batch-size=10"], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - timeout=90, - ) - assert update_result.returncode == 0, update_result.stderr or update_result.stdout - assert server.poll() is None - assert _pid_is_alive(supervisor_pid_before) - assert _supervisor_pid_from_log(server_log) == supervisor_pid_before - - add_proc, add_log = _start_live_add( - tmp_path, - env, - url=["https://example.com", "https://blog.sweeting.me"], - log_name="archivebox-add-1.log", - ) - _wait_for_hook_runs(marker_dir, 1) - _wait_for_crawl_state( - tmp_path, - lambda state: any(snapshot["status"] == Snapshot.StatusChoices.STARTED for snapshot in state["snapshots"]), - timeout=30, - ) - - os.kill(server.pid, signal.SIGTERM) - server.wait(timeout=20) - assert add_proc.poll() is None, "foreground add should keep owning its crawl after the server exits" - assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace") - - server2, _server2_log = _start_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env) - os.killpg(add_proc.pid, signal.SIGTERM) - add_proc.wait(timeout=30) - add_output = add_log.read_text(encoding="utf-8", errors="replace") - assert "Runner error" not in add_output - _wait_for_hook_runs(marker_dir, 2, timeout=60) - _wait_for_crawl_state( - tmp_path, - lambda state: ( - any(crawl["status"] in (Crawl.StatusChoices.STARTED, Crawl.StatusChoices.QUEUED) for crawl in state["crawls"]) - and any(result["plugin"] == "slow_exit" for result in state["results"]) - ), - timeout=30, - ) - - add_proc2, add_log2 = _start_live_add( - tmp_path, - env, - url="https://example.com/?exit-resume=2", - max_urls="1", - log_name="archivebox-add-2.log", - ) - _wait_for_hook_runs(marker_dir, 3, timeout=60) - os.killpg(add_proc2.pid, signal.SIGTERM) - os.kill(server2.pid, signal.SIGTERM) - add_proc2.wait(timeout=30) - add_output2 = add_log2.read_text(encoding="utf-8", errors="replace") - server2.wait(timeout=20) - assert "Runner error" not in add_output2 - - server3, _server3_log = _start_server(tmp_path, port=port, log_name="server-add-owner-3.log", env=env) - _wait_for_hook_runs(marker_dir, 4, timeout=70) - - with use_archivebox_db(tmp_path): - crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at")) - snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at")) - failed_results = list( - ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).values_list("plugin", "output_str"), - ) - assert crawls - assert snapshots - assert not failed_results - finally: - for proc in (add_proc, add_proc2, server, server2, server3): - if proc is not None and proc.poll() is None: - _stop_process(proc, signal.SIGTERM, timeout=10) - _kill_processes_for_data_dir(tmp_path) - _assert_no_processes_for_data_dir(tmp_path, timeout=12) - - -@pytest.mark.timeout(180) -def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - wget_urls = [ - "https://example.com", - "https://pirate.github.io/stress-tests/challenge.html", - ] - chrome_url = "https://example.com/?archivebox-chrome-flow=1" - env = os.environ.copy() - env.pop("CHROME_BINARY", None) - env.update( - { - "USE_COLOR": "false", - "SHOW_PROGRESS": "false", - "TIMEOUT": "60", - "SAVE_WGET": "true", - "SAVE_HEADERS": "false", - "SAVE_TITLE": "false", - "SAVE_READABILITY": "false", - "SAVE_SINGLEFILE": "false", - "SAVE_MERCURY": "false", - "SAVE_SCREENSHOT": "false", - "SAVE_PDF": "false", - "SAVE_DOM": "false", - "SAVE_ARCHIVEDOTORG": "false", - "SAVE_GIT": "false", - "SAVE_YTDLP": "false", - "SAVE_FAVICON": "false", - }, - ) - result = subprocess.run( - [ - sys.executable, - "-m", - "archivebox", - "add", - "--depth=0", - "--max-urls=2", - "--crawl-max-size=10mb", - "--tag=real-flow,challenge", - "--parser=url_list", - "--plugins=wget", - *wget_urls, - ], - cwd=tmp_path, - capture_output=True, - text=True, - env=env, - timeout=180, - ) - assert result.returncode == 0, result.stderr or result.stdout - - chrome_env = env | { - "SAVE_WGET": "false", - "SAVE_HEADERS": "true", - "SAVE_TITLE": "true", - "CHROME_HEADLESS": "true", - "CHROME_SANDBOX": "false", - "CHROME_ISOLATION": "snapshot", - } - system_browser = _find_system_browser() - if system_browser: - chrome_env["CHROME_BINARY"] = str(system_browser) - else: - install_result = subprocess.run( - [sys.executable, "-m", "archivebox", "install", "chrome"], - cwd=tmp_path, - capture_output=True, - text=True, - env=chrome_env, - timeout=600, - ) - assert install_result.returncode == 0, install_result.stderr or install_result.stdout - chrome_result = subprocess.run( - [ - sys.executable, - "-m", - "archivebox", - "add", - "--depth=0", - "--max-urls=1", - "--crawl-max-size=10mb", - "--tag=chrome-flow", - "--parser=url_list", - "--plugins=chrome,wget,headers,title", - chrome_url, - ], - cwd=tmp_path, - capture_output=True, - text=True, - env=chrome_env, - timeout=180, - ) - assert chrome_result.returncode == 0, chrome_result.stderr or chrome_result.stdout - - list_result = subprocess.run( - [sys.executable, "-m", "archivebox", "list", "--tag=real-flow"], - cwd=tmp_path, - capture_output=True, - text=True, - env=env, - timeout=60, - ) - assert list_result.returncode == 0, list_result.stderr or list_result.stdout - listed = [json.loads(line) for line in list_result.stdout.splitlines() if line.strip()] - assert {item["url"] for item in listed} >= set(wget_urls) - - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first() - real_flow_crawl = Crawl.objects.filter(tags_str="real-flow,challenge").values_list("max_depth", "tags_str", "config").first() - snapshots = list(Snapshot.objects.order_by("url").values_list("id", "url", "depth", "status", "title")) - archive_results = list( - ArchiveResult.objects.select_related("snapshot") - .order_by("snapshot__url", "plugin") - .values_list("snapshot__url", "plugin", "status", "output_files", "output_size", "output_str"), - ) - processes = list(Process.objects.filter(process_type="hook").values_list("process_type", "status", "exit_code", "pwd", "cmd")) - - assert real_flow_crawl is not None - assert real_flow_crawl[0] == 0 - assert real_flow_crawl[1] == "real-flow,challenge" - real_flow_config = real_flow_crawl[2] or {} - assert real_flow_config["CRAWL_MAX_URLS"] == 2 - assert real_flow_config["CRAWL_MAX_SIZE"] == 10 * 1024 * 1024 - assert real_flow_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 - assert "wget" in real_flow_config["PLUGINS"] - assert crawl is not None - assert crawl[1] == "chrome-flow" - assert "wget,headers,title" in json.dumps(crawl[2] or {}) - - snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots} - assert snapshot_urls >= {*wget_urls, chrome_url} - assert all(depth == 0 for _id, _url, depth, _status, _title in snapshots) - - by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size, _output in archive_results} - assert by_url_plugin[("https://example.com", "wget")] == "succeeded" - assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded" - assert (chrome_url, "headers") in by_url_plugin - assert (chrome_url, "title") in by_url_plugin - failed_results = [(url, plugin, output) for url, plugin, status, _files, _size, output in archive_results if status == "failed"] - assert len(failed_results) <= 2, failed_results - - snapshot_root = tmp_path / "archive/users/system/snapshots" - html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()] - header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0] - title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0] - index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()] - assert html_outputs - if by_url_plugin[(chrome_url, "headers")] == "succeeded": - assert header_outputs - if by_url_plugin[(chrome_url, "title")] == "succeeded": - assert title_outputs - assert any("Example Domain" in path.read_text(errors="ignore") for path in title_outputs) - assert len(index_outputs) >= len(wget_urls) + 1 - - combined_html = "\n".join(path.read_text(errors="ignore") for path in html_outputs) - assert "Example Domain" in combined_html - assert "Browser Agent Challenge for AI Browser Drivers" in combined_html - - assert processes - assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes) - assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes) - - -@pytest.mark.timeout(180) -def test_cli_recursive_crawl_processes_discovered_html_urls(tmp_path, process): - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - - env = os.environ.copy() - env.update( - { - "USE_COLOR": "false", - "SHOW_PROGRESS": "false", - "TIMEOUT": "60", - "SAVE_WGET": "true", - "SAVE_HEADERS": "false", - "SAVE_TITLE": "false", - "SAVE_READABILITY": "false", - "SAVE_SINGLEFILE": "false", - "SAVE_MERCURY": "false", - "SAVE_SCREENSHOT": "false", - "SAVE_PDF": "false", - "SAVE_DOM": "false", - "SAVE_ARCHIVEDOTORG": "false", - "SAVE_GIT": "false", - "SAVE_YTDLP": "false", - "SAVE_FAVICON": "false", - "PARSE_HTML_URLS_ENABLED": "true", - "PARSE_DOM_OUTLINKS_ENABLED": "false", - }, - ) - - result = subprocess.run( - [ - sys.executable, - "-m", - "archivebox", - "add", - "--depth=2", - "--max-urls=2", - "--crawl-max-size=50mb", - "--tag=recursive-flow", - "--parser=url_list", - "--plugins=wget,parse_html_urls", - "https://example.com", - ], - cwd=tmp_path, - capture_output=True, - text=True, - env=env, - timeout=180, - ) - assert result.returncode == 0, result.stderr or result.stdout - - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first() - snapshots = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status")) - archive_results = list( - ArchiveResult.objects.select_related("snapshot") - .order_by("snapshot__depth", "snapshot__url", "plugin") - .values_list("snapshot__url", "plugin", "status", "output_files"), - ) - - assert crawl[0] == 2 - assert crawl[1] == "recursive-flow" - crawl_config = crawl[2] or {} - assert crawl_config["CRAWL_MAX_URLS"] == 2 - assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024 - assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 - assert ("https://example.com", 0, "sealed") in snapshots - assert any(url == "https://iana.org/domains/example" and depth == 1 and status == "sealed" for url, depth, status in snapshots) - - by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results} - assert by_url_plugin[("https://example.com", "wget")] == "succeeded" - assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded" - assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded" - - urls_outputs = list((tmp_path / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl")) - assert urls_outputs - assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs) diff --git a/archivebox/tests/test_cli_remove.py b/archivebox/tests/test_cli_remove.py index 6e071980..9e3c3435 100644 --- a/archivebox/tests/test_cli_remove.py +++ b/archivebox/tests/test_cli_remove.py @@ -4,26 +4,10 @@ Comprehensive tests for archivebox remove command. Verify remove deletes snapshots from DB and filesystem. """ -import os import json -import subprocess from pathlib import Path -from archivebox.tests.conftest import run_queued_crawls - - -def _find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: - candidates = {snapshot_id} - if len(snapshot_id) == 32: - candidates.add(f"{snapshot_id[:8]}-{snapshot_id[8:12]}-{snapshot_id[12:16]}-{snapshot_id[16:20]}-{snapshot_id[20:]}") - elif len(snapshot_id) == 36 and "-" in snapshot_id: - candidates.add(snapshot_id.replace("-", "")) - - for needle in candidates: - for path in data_dir.rglob(needle): - if path.is_dir(): - return path - return None +from archivebox.tests.conftest import find_snapshot_dir, run_archivebox_cmd, run_queued_crawls, cli_env def _snapshot_rows(data_dir: Path, env: dict) -> list[dict]: @@ -35,260 +19,218 @@ print(json.dumps([ for snapshot in Snapshot.objects.order_by("url") ])) """ - result = subprocess.run( - ["archivebox", "manage", "shell", "-c", script], + result = run_archivebox_cmd( + ["manage", "shell", "-c", script], cwd=data_dir, - capture_output=True, env=env, timeout=30, check=True, ) - return json.loads(result.stdout.decode("utf-8").strip().splitlines()[-1]) + return json.loads(result.stdout.strip().splitlines()[-1]) -def test_remove_deletes_snapshot_from_db(tmp_path, process, disable_extractors_dict): +def test_remove_deletes_snapshot_from_db(initialized_archive): """Test that remove command deletes snapshot from database.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - rows = _snapshot_rows(tmp_path, disable_extractors_dict) + rows = _snapshot_rows(initialized_archive, env) assert len(rows) == 1 snapshot_id = rows[0]["id"] - snapshot_dir = _find_snapshot_dir(tmp_path, snapshot_id) + snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id) assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}" # Remove it - subprocess.run( - ["archivebox", "remove", "https://example.com", "--yes"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["remove", "https://example.com", "--yes"], + env=env, ) - assert len(_snapshot_rows(tmp_path, disable_extractors_dict)) == 0 + assert len(_snapshot_rows(initialized_archive, env)) == 0 assert not snapshot_dir.exists() -def test_remove_deletes_archive_directory(tmp_path, process, disable_extractors_dict): +def test_remove_deletes_archive_directory(initialized_archive): """Test that remove --yes removes the current snapshot output directory.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - rows = _snapshot_rows(tmp_path, disable_extractors_dict) + rows = _snapshot_rows(initialized_archive, env) assert len(rows) == 1 snapshot_id = rows[0]["id"] - snapshot_dir = _find_snapshot_dir(tmp_path, snapshot_id) + snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id) assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}" - subprocess.run( - ["archivebox", "remove", "https://example.com", "--yes"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["remove", "https://example.com", "--yes"], + env=env, ) assert not snapshot_dir.exists() -def test_remove_yes_flag_skips_confirmation(tmp_path, process, disable_extractors_dict): +def test_remove_yes_flag_skips_confirmation(initialized_archive): """Test that --yes flag skips confirmation prompt.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Remove with --yes should complete without interaction - result = subprocess.run( - ["archivebox", "remove", "https://example.com", "--yes"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["remove", "https://example.com", "--yes"], + env=env, timeout=30, ) assert result.returncode == 0 - output = result.stdout.decode("utf-8") + result.stderr.decode("utf-8") + output = result.stdout + result.stderr assert "Index now contains 0 links." in output -def test_remove_without_yes_prompts_and_keeps_snapshot(tmp_path, process, disable_extractors_dict): +def test_remove_without_yes_prompts_and_keeps_snapshot(initialized_archive): """Test that omitting --yes prompts for confirmation and keeps data when declined.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - rows = _snapshot_rows(tmp_path, disable_extractors_dict) + rows = _snapshot_rows(initialized_archive, env) assert len(rows) == 1 - snapshot_dir = _find_snapshot_dir(tmp_path, rows[0]["id"]) + snapshot_dir = find_snapshot_dir(initialized_archive, rows[0]["id"]) assert snapshot_dir is not None - result = subprocess.run( - ["archivebox", "remove", "https://example.com"], - input=b"n\n", - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["remove", "https://example.com"], + input="n\n", + env=env, timeout=30, ) - output = result.stdout.decode("utf-8") + result.stderr.decode("utf-8") + output = result.stdout + result.stderr assert result.returncode == 0 assert "Do you want to proceed" in output or "y/[n]" in output - assert len(_snapshot_rows(tmp_path, disable_extractors_dict)) == 1 + assert len(_snapshot_rows(initialized_archive, env)) == 1 assert snapshot_dir.exists() -def test_remove_multiple_snapshots(tmp_path, process, disable_extractors_dict): +def test_remove_multiple_snapshots(initialized_archive): """Test removing multiple snapshots at once.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add multiple snapshots for url in ["https://example.com", "https://example.org"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - assert len(_snapshot_rows(tmp_path, disable_extractors_dict)) == 2 + assert len(_snapshot_rows(initialized_archive, env)) == 2 # Remove both - subprocess.run( - ["archivebox", "remove", "https://example.com", "https://example.org", "--yes"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["remove", "https://example.com", "https://example.org", "--yes"], + env=env, ) - assert len(_snapshot_rows(tmp_path, disable_extractors_dict)) == 0 + assert len(_snapshot_rows(initialized_archive, env)) == 0 -def test_remove_with_filter(tmp_path, process, disable_extractors_dict): - """Test removing snapshots using filter.""" - os.chdir(tmp_path) - - # Add snapshots - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, - ) - run_queued_crawls(tmp_path, disable_extractors_dict) - - # Remove using filter - result = subprocess.run( - ["archivebox", "remove", "--filter-type=search", "--filter=example.com", "--yes"], - capture_output=True, - env=disable_extractors_dict, - timeout=30, - ) - - # Should complete (exit code depends on implementation) - assert result.returncode in [0, 1, 2] - - -def test_remove_with_regex_filter_deletes_all_matches(tmp_path, process, disable_extractors_dict): +def test_remove_with_regex_filter_deletes_all_matches(initialized_archive): """Test regex filters remove every matching snapshot.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for url in ["https://example.com", "https://iana.org"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run( - ["archivebox", "remove", "--filter-type=regex", ".*", "--yes"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["remove", "--filter-type=regex", ".*", "--yes"], + env=env, check=True, ) - output = result.stdout.decode("utf-8") + result.stderr.decode("utf-8") - assert len(_snapshot_rows(tmp_path, disable_extractors_dict)) == 0 + output = result.stdout + result.stderr + assert len(_snapshot_rows(initialized_archive, env)) == 0 assert "Removed" in output or "Found" in output -def test_remove_nonexistent_url_fails_gracefully(tmp_path, process, disable_extractors_dict): +def test_remove_nonexistent_url_fails_gracefully(initialized_archive): """Test that removing non-existent URL fails gracefully.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - result = subprocess.run( - ["archivebox", "remove", "https://nonexistent-url-12345.com", "--yes"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["remove", "https://nonexistent-url-12345.com", "--yes"], + env=env, ) # Should fail or show error - stdout_text = result.stdout.decode("utf-8", errors="replace").lower() + stdout_text = result.stdout.lower() assert result.returncode != 0 or "not found" in stdout_text or "no matches" in stdout_text -def test_remove_reports_remaining_link_count_correctly(tmp_path, process, disable_extractors_dict): +def test_remove_reports_remaining_link_count_correctly(initialized_archive): """Test remove reports the remaining snapshot count after deletion.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) for url in ["https://example.com", "https://example.org"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, check=True, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run( - ["archivebox", "remove", "https://example.org", "--yes"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["remove", "https://example.org", "--yes"], + env=env, check=True, ) - output = result.stdout.decode("utf-8") + result.stderr.decode("utf-8") + output = result.stdout + result.stderr assert "Removed 1 out of 2 links" in output assert "Index now contains 1 links." in output -def test_remove_after_flag(tmp_path, process, disable_extractors_dict): +def test_remove_after_flag(initialized_archive): """Test remove --after flag removes snapshots after date.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Try remove with --after flag (should work or show usage) - result = subprocess.run( - ["archivebox", "remove", "--after=2020-01-01", "--yes"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["remove", "--after=2020-01-01", "--yes"], + env=env, timeout=30, ) diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index c15b9047..4eb5686d 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -17,11 +17,15 @@ import time import pytest from archivebox.tests.conftest import ( + cleanup_process_group, + cli_env, run_archivebox_cmd, parse_jsonl_output, create_test_url, create_test_crawl_json, create_test_snapshot_json, + pid_is_alive, + wait_for_pid_to_disappear, ) RUN_TEST_ENV = { @@ -30,6 +34,124 @@ RUN_TEST_ENV = { } +@pytest.mark.django_db(transaction=True) +@pytest.mark.timeout(90) +def test_cli_run_signal_cleans_background_hook_process_group(initialized_archive): + + plugins_root = initialized_archive / "runtime_plugins" + plugin_dir = plugins_root / "cancel_group" + plugin_dir.mkdir(parents=True) + daemon_hook = plugin_dir / "on_CrawlSetup__10_daemon.daemon.bg.sh" + foreground_hook = plugin_dir / "on_CrawlSetup__20_foreground.sh" + daemon_hook.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'test_dir="${LEAK_TEST_DIR:?}"', + "sleep 600 &", + 'echo $$ > "$test_dir/daemon.pid"', + 'echo $! > "$test_dir/daemon-child.pid"', + 'echo ready > "$test_dir/daemon.ready"', + "trap 'echo cleaned > \"$test_dir/daemon.cleaned\"; exit 0' TERM INT", + "wait", + "", + ], + ), + ) + foreground_hook.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'test_dir="${LEAK_TEST_DIR:?}"', + 'echo $$ > "$test_dir/foreground.pid"', + 'echo ready > "$test_dir/foreground.ready"', + "trap 'echo cleaned > \"$test_dir/foreground.cleaned\"; exit 0' TERM INT", + "while true; do sleep 1; done", + "", + ], + ), + ) + daemon_hook.chmod(0o755) + foreground_hook.chmod(0o755) + + leak_test_dir = initialized_archive / "leak-check" + leak_test_dir.mkdir() + env = os.environ.copy() + env.update( + { + "ABX_PLUGINS_DIR": str(plugins_root), + "LEAK_TEST_DIR": str(leak_test_dir), + "PLUGINS": "cancel_group", + "TIMEOUT": "30", + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + }, + ) + + _cmd_result = run_archivebox_cmd( + ["crawl", "create", "https://example.com"], + cwd=initialized_archive, + env=env, + timeout=60, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + crawl_records = [json.loads(line) for line in stdout.splitlines() if line.strip().startswith("{")] + crawl_id = next(record["id"] for record in crawl_records if record.get("type") == "Crawl") + + daemon_pid: int | None = None + daemon_child_pid: int | None = None + foreground_pid: int | None = None + run_process = run_archivebox_cmd( + ["run", f"--crawl-id={crawl_id}"], + cwd=initialized_archive, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + wait=False, + ) + try: + deadline = time.time() + 20 + while time.time() < deadline: + if (leak_test_dir / "daemon.ready").exists() and (leak_test_dir / "foreground.ready").exists(): + break + if run_process.poll() is not None: + output = run_process.communicate(timeout=1)[0] + raise AssertionError(f"archivebox run exited before hooks were ready:\n{output}") + time.sleep(0.05) + assert (leak_test_dir / "daemon.ready").exists() + assert (leak_test_dir / "foreground.ready").exists() + + daemon_pid = int((leak_test_dir / "daemon.pid").read_text().strip()) + daemon_child_pid = int((leak_test_dir / "daemon-child.pid").read_text().strip()) + foreground_pid = int((leak_test_dir / "foreground.pid").read_text().strip()) + assert pid_is_alive(daemon_pid) + assert pid_is_alive(daemon_child_pid) + assert pid_is_alive(foreground_pid) + + run_process.send_signal(signal.SIGTERM) + output = run_process.communicate(timeout=20)[0] + assert "Runner error" not in output + + wait_for_pid_to_disappear(daemon_pid, timeout=5) + wait_for_pid_to_disappear(daemon_child_pid, timeout=5) + wait_for_pid_to_disappear(foreground_pid, timeout=5) + assert (leak_test_dir / "daemon.cleaned").read_text().strip() == "cleaned" + assert (leak_test_dir / "foreground.cleaned").read_text().strip() == "cleaned" + finally: + if run_process.poll() is None: + try: + os.killpg(run_process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + run_process.communicate(timeout=5) + cleanup_process_group(daemon_pid, daemon_child_pid) + cleanup_process_group(foreground_pid) + + class TestRunWithCrawl: """Tests for `archivebox run` with Crawl input.""" @@ -37,13 +159,16 @@ class TestRunWithCrawl: """Run creates and processes a new Crawl (no id).""" crawl_record = create_test_crawl_json() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(crawl_record), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" @@ -58,17 +183,27 @@ class TestRunWithCrawl: url = create_test_url() # First create a crawl - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive, env=RUN_TEST_ENV) + _cmd_result = run_archivebox_cmd( + ["crawl", "create", url], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] # Run with the existing crawl - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout2) @@ -82,13 +217,16 @@ class TestRunWithSnapshot: """Run creates and processes a new Snapshot (no id, just url).""" snapshot_record = create_test_snapshot_json() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(snapshot_record), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" @@ -102,17 +240,27 @@ class TestRunWithSnapshot: url = create_test_url() # First create a snapshot - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive, env=RUN_TEST_ENV) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] # Run with the existing snapshot - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout2) @@ -123,13 +271,16 @@ class TestRunWithSnapshot: url = create_test_url() url_record = {"url": url} - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(url_record), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -144,15 +295,25 @@ class TestRunWithArchiveResult: url = create_test_url() # Create snapshot and archive result - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive, env=RUN_TEST_ENV) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, _, _ = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode ar = next(r for r in parse_jsonl_output(stdout2) if r.get("type") == "ArchiveResult") # Update to failed @@ -160,18 +321,23 @@ class TestRunWithArchiveResult: run_archivebox_cmd( ["archiveresult", "update", "--status=failed"], stdin=json.dumps(ar), - data_dir=initialized_archive, + cwd=initialized_archive, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) # Now run should re-queue it - stdout3, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(ar), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout3) @@ -210,12 +376,15 @@ class TestRunRecovery: crawl_id = crawl.id snapshot_id = snapshot.id - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run", "--maintenance-only"], - data_dir=initialized_archive, + cwd=initialized_archive, timeout=90, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, stdout + stderr assert "Repairing" in stderr @@ -239,11 +408,14 @@ class TestRunPassThrough: """Run passes through records with unknown types.""" unknown_record = {"type": "Unknown", "id": "fake-id", "data": "test"} - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(unknown_record), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -256,13 +428,16 @@ class TestRunPassThrough: url = create_test_url() crawl_record = create_test_crawl_json(urls=[url]) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(crawl_record), - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -287,13 +462,16 @@ class TestRunMixedInput: ], ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=stdin, - data_dir=initialized_archive, + cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -308,11 +486,14 @@ class TestRunEmpty: def test_run_empty_stdin(self, initialized_archive): """Run with empty stdin returns success.""" - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin="", - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 @@ -320,11 +501,14 @@ class TestRunEmpty: """Run with only pass-through records shows message.""" unknown = {"type": "Unknown", "id": "fake"} - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["run"], stdin=json.dumps(unknown), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "No records to process" in stderr @@ -344,23 +528,16 @@ class TestRunDaemonMode: else: piped_stdin = "{this is not jsonl}\n" - env = os.environ.copy() - env.update( - { - "DATA_DIR": str(initialized_archive), - "USE_COLOR": "False", - "SHOW_PROGRESS": "False", - }, - ) - proc = subprocess.Popen( - [sys.executable, "-m", "archivebox", "run", "--daemon"], + env = cli_env() + proc = run_archivebox_cmd( + ["run", "--daemon"], cwd=initialized_archive, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, start_new_session=True, + wait=False, ) assert proc.stdin is not None assert proc.stdout is not None @@ -410,24 +587,17 @@ class TestRunDaemonMode: from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE from archivebox.tests.test_orm_helpers import use_archivebox_db - env = os.environ.copy() - env.update( - { - "DATA_DIR": str(initialized_archive), - "USE_COLOR": "False", - "SHOW_PROGRESS": "False", - }, - ) + env = cli_env() procs = [ - subprocess.Popen( - [sys.executable, "-m", "archivebox", "run", "--daemon"], + run_archivebox_cmd( + ["run", "--daemon"], cwd=initialized_archive, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, start_new_session=True, + wait=False, ) for _ in range(2) ] @@ -468,15 +638,15 @@ class TestRunDaemonMode: assert len(active) == 1 os.killpg(active_pid, signal.SIGKILL) - replacement = subprocess.Popen( - [sys.executable, "-m", "archivebox", "run", "--daemon"], + replacement = run_archivebox_cmd( + ["run", "--daemon"], cwd=initialized_archive, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, start_new_session=True, + wait=False, ) procs.append(replacement) deadline = time.monotonic() + 30 @@ -518,7 +688,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -544,7 +714,8 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state, run_due_crawl + from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.services.runner import run_due_crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -579,7 +750,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state user_id = get_or_create_system_user_pk() queued_crawl = Crawl.objects.create( @@ -629,7 +800,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -666,7 +837,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -708,7 +879,8 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state, run_due_crawl, run_due_snapshot + from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.services.runner import run_due_crawl, run_due_snapshot old = timezone.now() - timedelta(hours=13) crawl = Crawl.objects.create( @@ -944,7 +1116,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state old = timezone.now() - timedelta(hours=13) crawl = Crawl.objects.create( @@ -970,7 +1142,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state future = timezone.now() + timedelta(seconds=45) crawl = Crawl.objects.create( @@ -1007,7 +1179,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state future = timezone.now() + timedelta(seconds=45) crawl = Crawl.objects.create( @@ -1046,7 +1218,7 @@ class TestRecoverOrchestratorState: from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state worker = subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(60)"], @@ -1109,7 +1281,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state from archivebox.workers.models import RETRY_AT_MAX crawl = Crawl.objects.create( @@ -1143,7 +1315,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -1319,7 +1491,7 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -1350,7 +1522,8 @@ class TestRecoverOrchestratorState: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state, run_due_snapshot + from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( urls="https://example.com", @@ -1719,7 +1892,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -1755,7 +1928,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -1780,7 +1953,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -1801,7 +1974,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://www.mathjax.org/", @@ -1834,7 +2007,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://revealjs.com/", @@ -1875,7 +2048,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://pdfobject.com/pdf/sample-3pp.pdf", @@ -1921,7 +2094,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://mermaid-js.github.io/mermaid/", @@ -2227,7 +2400,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state old = timezone.now() - timedelta(hours=13) crawl = Crawl.objects.create( @@ -2263,7 +2436,7 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.services.runner import recover_orchestrator_state + from archivebox.core.recovery_util import recover_orchestrator_state runtime_dir = tmp_path / "https_example_com" / ".hooks" / "on_Snapshot__01_title.py" runtime_dir.mkdir(parents=True) diff --git a/archivebox/tests/test_cli_schedule.py b/archivebox/tests/test_cli_schedule.py index 7075b844..81bae29e 100644 --- a/archivebox/tests/test_cli_schedule.py +++ b/archivebox/tests/test_cli_schedule.py @@ -1,21 +1,19 @@ #!/usr/bin/env python3 """CLI-specific tests for archivebox schedule.""" -import os -import subprocess -import sys +from archivebox.tests.conftest import run_archivebox_cmd import pytest from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.tests.test_orm_helpers import use_archivebox_db from .conftest import ( - build_test_env, + cli_env, get_counts, get_free_port, init_archive, make_latest_schedule_due, - start_server, + start_archivebox_server, stop_server, wait_for_http, wait_for_snapshot_capture, @@ -24,27 +22,23 @@ from .conftest import ( pytestmark = pytest.mark.django_db(transaction=True) -def test_schedule_run_all_enqueues_scheduled_crawl(tmp_path, process, disable_extractors_dict): - os.chdir(tmp_path) +def test_schedule_run_all_enqueues_scheduled_crawl(initialized_archive): - subprocess.run( - ["archivebox", "schedule", "--every=daily", "--depth=0", "https://example.com"], - capture_output=True, - text=True, + env = cli_env(disable_extractors=True) + run_archivebox_cmd( + ["schedule", "--every=daily", "--depth=0", "https://example.com"], check=True, ) - result = subprocess.run( - ["archivebox", "schedule", "--run-all"], - capture_output=True, - text=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["schedule", "--run-all"], + env=env, ) assert result.returncode == 0 assert "Enqueued 1 scheduled crawl" in result.stdout - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): crawl_count = Crawl.objects.count() queued_count = Crawl.objects.filter(status="queued").count() @@ -52,36 +46,30 @@ def test_schedule_run_all_enqueues_scheduled_crawl(tmp_path, process, disable_ex assert queued_count >= 1 -def test_schedule_without_import_path_creates_maintenance_schedule(tmp_path, process): - os.chdir(tmp_path) +def test_schedule_without_import_path_creates_maintenance_schedule(initialized_archive): - result = subprocess.run( - ["archivebox", "schedule", "--every=day"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["schedule", "--every=day"], ) assert result.returncode == 0 assert "Created scheduled maintenance update" in result.stdout - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): row = Crawl.objects.order_by("-created_at").values_list("urls", "status").first() assert row == ("archivebox://update", "sealed") -def test_schedule_creates_enabled_db_schedule(tmp_path, process): - os.chdir(tmp_path) +def test_schedule_creates_enabled_db_schedule(initialized_archive): - result = subprocess.run( - ["archivebox", "schedule", "--every=daily", "--depth=1", "https://example.com/feed.xml"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["schedule", "--every=daily", "--depth=1", "https://example.com/feed.xml"], ) assert result.returncode == 0 - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): schedule_row = CrawlSchedule.objects.order_by("-created_at").values_list("schedule", "is_enabled", "label").first() crawl = Crawl.objects.order_by("-created_at").first() @@ -92,20 +80,15 @@ def test_schedule_creates_enabled_db_schedule(tmp_path, process): assert crawl.max_depth == 1 -def test_schedule_show_lists_enabled_schedules(tmp_path, process): - os.chdir(tmp_path) +def test_schedule_show_lists_enabled_schedules(initialized_archive): - subprocess.run( - ["archivebox", "schedule", "--every=weekly", "https://example.com/feed.xml"], - capture_output=True, - text=True, + run_archivebox_cmd( + ["schedule", "--every=weekly", "https://example.com/feed.xml"], check=True, ) - result = subprocess.run( - ["archivebox", "schedule", "--show"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["schedule", "--show"], ) assert result.returncode == 0 @@ -114,26 +97,21 @@ def test_schedule_show_lists_enabled_schedules(tmp_path, process): assert "weekly" in result.stdout -def test_schedule_clear_disables_existing_schedules(tmp_path, process): - os.chdir(tmp_path) +def test_schedule_clear_disables_existing_schedules(initialized_archive): - subprocess.run( - ["archivebox", "schedule", "--every=daily", "https://example.com/feed.xml"], - capture_output=True, - text=True, + run_archivebox_cmd( + ["schedule", "--every=daily", "https://example.com/feed.xml"], check=True, ) - result = subprocess.run( - ["archivebox", "schedule", "--clear"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["schedule", "--clear"], ) assert result.returncode == 0 assert "Disabled 1 scheduled crawl" in result.stdout - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): disabled_count = CrawlSchedule.objects.filter(is_enabled=False).count() enabled_count = CrawlSchedule.objects.filter(is_enabled=True).count() @@ -141,26 +119,20 @@ def test_schedule_clear_disables_existing_schedules(tmp_path, process): assert enabled_count == 0 -def test_schedule_every_requires_valid_period(tmp_path, process): - os.chdir(tmp_path) +def test_schedule_every_requires_valid_period(initialized_archive): - result = subprocess.run( - ["archivebox", "schedule", "--every=invalid_period", "https://example.com/feed.xml"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["schedule", "--every=invalid_period", "https://example.com/feed.xml"], ) assert result.returncode != 0 assert "Invalid schedule" in result.stderr or "Invalid schedule" in result.stdout -def test_schedule_help_lists_schedule_options(tmp_path, process): - os.chdir(tmp_path) +def test_schedule_help_lists_schedule_options(initialized_archive): - result = subprocess.run( - ["archivebox", "schedule", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["schedule", "--help"], ) assert result.returncode == 0 @@ -172,17 +144,14 @@ def test_schedule_help_lists_schedule_options(tmp_path, process): @pytest.mark.timeout(180) def test_schedule_due_crawl_runs_over_server_and_saves_real_content(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env(port) + env = cli_env(port=port, server=True) - schedule_result = subprocess.run( - [sys.executable, "-m", "archivebox", "schedule", "--every=daily", "--depth=0", recursive_test_site["root_url"]], + schedule_result = run_archivebox_cmd( + ["schedule", "--every=daily", "--depth=0", recursive_test_site["root_url"]], cwd=tmp_path, - capture_output=True, - text=True, env=env, timeout=60, ) @@ -192,7 +161,7 @@ def test_schedule_due_crawl_runs_over_server_and_saves_real_content(tmp_path, re make_latest_schedule_due(tmp_path) try: - start_server(tmp_path, env=env, port=port) + start_archivebox_server(tmp_path, env=env, port=port) wait_for_http(port, host=f"web.archivebox.localhost:{port}") captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) assert "Root" in captured_text @@ -203,19 +172,16 @@ def test_schedule_due_crawl_runs_over_server_and_saves_real_content(tmp_path, re @pytest.mark.timeout(180) def test_add_remains_one_shot_when_schedule_is_due(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env(port) + env = cli_env(port=port, server=True) scheduled_url = recursive_test_site["root_url"] one_shot_url = recursive_test_site["child_urls"][0] - schedule_result = subprocess.run( - [sys.executable, "-m", "archivebox", "schedule", "--every=daily", "--depth=0", scheduled_url], + schedule_result = run_archivebox_cmd( + ["schedule", "--every=daily", "--depth=0", scheduled_url], cwd=tmp_path, - capture_output=True, - text=True, env=env, timeout=60, ) @@ -223,11 +189,9 @@ def test_add_remains_one_shot_when_schedule_is_due(tmp_path, recursive_test_site make_latest_schedule_due(tmp_path) - add_result = subprocess.run( - [sys.executable, "-m", "archivebox", "add", "--depth=0", "--plugins=wget", one_shot_url], + add_result = run_archivebox_cmd( + ["add", "--depth=0", "--plugins=wget", one_shot_url], cwd=tmp_path, - capture_output=True, - text=True, env=env, timeout=120, ) diff --git a/archivebox/tests/test_cli_search.py b/archivebox/tests/test_cli_search.py new file mode 100644 index 00000000..f3780796 --- /dev/null +++ b/archivebox/tests/test_cli_search.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox search command. + +TODO: keep search-specific tests here instead of folding all coverage into test_cli_list.py. +""" + +import json + +from archivebox.tests.conftest import cli_env, run_archivebox_cmd + + +def test_search_help_runs_successfully(tmp_path): + """The search alias should be registered and expose list/search filters.""" + + result = run_archivebox_cmd(["search", "--help"]) + + assert result.returncode == 0 + assert "search" in result.stdout.lower() + assert "--csv" in result.stdout + + +def test_cli_search_status_filters_snapshot_status_column(tmp_path, initialized_archive): + env = cli_env(disable_extractors=True) + for url in ( + "https://example.com/search-status-queued", + "https://example.com/search-status-paused", + "https://example.com/search-status-sealed", + ): + result = run_archivebox_cmd( + ["snapshot", "create", url], + env=env, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + for status, needle in ( + ("paused", "search-status-paused"), + ("sealed", "search-status-sealed"), + ): + listed = run_archivebox_cmd( + ["snapshot", "list", "--url__icontains", needle], + env=env, + timeout=30, + ) + assert listed.returncode == 0, listed.stderr + updated = run_archivebox_cmd( + ["snapshot", "update", "--status", status], + input=listed.stdout, + env=env, + timeout=30, + ) + assert updated.returncode == 0, updated.stderr + + result = run_archivebox_cmd( + ["search", "--status", "sealed", "search-status"], + env=env, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + rows = [json.loads(line) for line in result.stdout.splitlines() if line.strip().startswith("{")] + assert [row["status"] for row in rows] == ["sealed"] + assert [row["url"] for row in rows] == ["https://example.com/search-status-sealed"] + + legacy_result = run_archivebox_cmd( + ["search", "--status", "unarchived", "search-status"], + env=env, + timeout=30, + ) + + assert legacy_result.returncode != 0 + assert "Invalid snapshot status" in legacy_result.stderr diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index f31650b5..0d349d74 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -15,12 +15,26 @@ import time from datetime import datetime from types import SimpleNamespace +import pytest + +from archivebox.tests.conftest import ( + assert_no_processes_for_data_dir, + get_free_port, + kill_processes_for_data_dir, + cli_env, + start_archivebox_server, + stop_archivebox_process, + wait_for_pid_to_disappear, + wait_for_port_open, + wait_for_process, + run_archivebox_cmd, +) + def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, monkeypatch): """Admin sessions must survive `archivebox server` restarts for a collection.""" from archivebox.config.collection import write_config_file - os.chdir(tmp_path) (tmp_path / ".archivebox_id").write_text("testcoll") monkeypatch.setenv("BASE_URL", "http://archivebox.localhost:9292") @@ -91,16 +105,13 @@ def test_sqlite_connections_use_explicit_busy_timeout(): assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"] -def test_server_shows_usage_info(tmp_path, process): +def test_server_shows_usage_info(initialized_archive): """Test that server command shows usage or starts.""" - os.chdir(tmp_path) # Just check that the command is recognized # We won't actually start a full server in tests - result = subprocess.run( - ["archivebox", "server", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["server", "--help"], timeout=10, ) @@ -108,15 +119,12 @@ def test_server_shows_usage_info(tmp_path, process): assert "server" in result.stdout.lower() or "http" in result.stdout.lower() -def test_server_help_lists_runtime_options(tmp_path, process): +def test_server_help_lists_runtime_options(initialized_archive): """Test that server help exposes the current runtime options.""" - os.chdir(tmp_path) # Check init flag is recognized - result = subprocess.run( - ["archivebox", "server", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["server", "--help"], timeout=10, ) @@ -158,14 +166,6 @@ def test_reload_workers_use_current_interpreter_and_supervisord_managed_runner() assert watcher["command"] == f"{sys.executable} -m archivebox manage runner_watch --bind-url=http://127.0.0.1:8000" -def _free_port(): - import socket - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - - def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_server): server = archivebox_daemon_server( SEARCH_BACKEND_ENGINE="sqlite", @@ -279,7 +279,7 @@ def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path): DATA_DIR=str(tmp_path), SEARCH_BACKEND_SONIC_ENABLED=False, SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1", - SEARCH_BACKEND_SONIC_PORT=_free_port(), + SEARCH_BACKEND_SONIC_PORT=get_free_port(), SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword", SONIC_BINARY="sonic", ), @@ -294,7 +294,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler from abx_plugins.plugins.search_backend_sonic.daemon import prepare_sonic_daemon - sonic_port = _free_port() + sonic_port = get_free_port() server = archivebox_daemon_server( SEARCH_BACKEND_ENGINE="sonic", SEARCH_BACKEND_SONIC_PORT=str(sonic_port), @@ -329,7 +329,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo asyncio.run(run_test()) -def test_supervisord_sync_does_not_start_duplicate_sonic_listener(tmp_path, process, db): +def test_supervisord_sync_does_not_start_duplicate_sonic_listener(initialized_archive, db): from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker from archivebox.tests.test_orm_helpers import use_archivebox_db from archivebox.workers.supervisord_util import ( @@ -339,9 +339,6 @@ def test_supervisord_sync_does_not_start_duplicate_sonic_listener(tmp_path, proc sync_supervisord_workers, ) - os.chdir(tmp_path) - assert process.returncode == 0, process.stderr - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(("127.0.0.1", 0)) @@ -349,7 +346,7 @@ def test_supervisord_sync_does_not_start_duplicate_sonic_listener(tmp_path, proc sonic_port = listener.getsockname()[1] worker = get_sonic_supervisord_worker( SimpleNamespace( - DATA_DIR=str(tmp_path), + DATA_DIR=str(initialized_archive), SEARCH_BACKEND_ENGINE="sonic", SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1", SEARCH_BACKEND_SONIC_PORT=sonic_port, @@ -360,7 +357,7 @@ def test_supervisord_sync_does_not_start_duplicate_sonic_listener(tmp_path, proc assert worker is not None try: - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): supervisor = get_or_create_supervisord_process(daemonize=False) state = sync_supervisord_workers(supervisor, [(worker, False)], prune=True) sonic_state = state["worker_sonic"] @@ -368,11 +365,11 @@ def test_supervisord_sync_does_not_start_duplicate_sonic_listener(tmp_path, proc assert get_worker(supervisor, "worker_sonic")["statename"] != "RUNNING" finally: listener.close() - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): stop_existing_supervisord_process() -def test_supervisord_takeover_stops_all_live_process_rows(tmp_path, process, db): +def test_supervisord_takeover_stops_all_live_process_rows(initialized_archive, db): import psutil from django.utils import timezone @@ -380,23 +377,22 @@ def test_supervisord_takeover_stops_all_live_process_rows(tmp_path, process, db) from archivebox.machine.models import Machine, Process from archivebox.tests.test_orm_helpers import use_archivebox_db - assert process.returncode == 0, process.stderr - env = os.environ.copy() - env.update({"DATA_DIR": str(tmp_path), "USE_COLOR": "False", "SHOW_PROGRESS": "False"}) + env = cli_env() procs = [] try: for _index in range(2): - proc = subprocess.Popen( - [sys.executable, "-m", "archivebox", "run", "--daemon"], - cwd=tmp_path, + proc = run_archivebox_cmd( + ["run", "--daemon"], + cwd=initialized_archive, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True, + wait=False, ) procs.append(proc) started_at = datetime.fromtimestamp(psutil.Process(proc.pid).create_time(), tz=timezone.get_current_timezone()) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): Process.objects.create( machine=Machine.current(), process_type=Process.TypeChoices.SUPERVISORD, @@ -408,14 +404,14 @@ def test_supervisord_takeover_stops_all_live_process_rows(tmp_path, process, db) status=Process.StatusChoices.RUNNING, ) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): from archivebox.workers.supervisord_util import stop_existing_supervisord_process stop_existing_supervisord_process() for proc in procs: proc.wait(timeout=10) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): assert not Process.objects.filter( process_type=Process.TypeChoices.SUPERVISORD, status=Process.StatusChoices.RUNNING, @@ -425,3 +421,154 @@ def test_supervisord_takeover_stops_all_live_process_rows(tmp_path, process, db) for proc in procs: if proc.poll() is None: os.killpg(proc.pid, signal.SIGKILL) + + +@pytest.mark.timeout(300) +@pytest.mark.parametrize( + ("stop_signal", "expected_notice"), + [ + (signal.SIGHUP, "Got SIGHUP"), + (signal.SIGINT, "Got SIGINT"), + (signal.SIGTERM, "Got SIGTERM"), + (signal.SIGKILL, None), + ], +) +def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initialized_archive, stop_signal, expected_notice): + + env = cli_env(live=True) + port = get_free_port() + server = None + resumed = None + try: + server = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}.log", env=env) + server_log = server.log_path + + os.kill(server.pid, stop_signal) + try: + server.wait(timeout=20 if stop_signal != signal.SIGKILL else 5) + except subprocess.TimeoutExpired: + os.kill(server.pid, signal.SIGKILL) + server.wait(timeout=5) + + if expected_notice: + log_text = server_log.read_text(encoding="utf-8", errors="replace") + assert expected_notice in log_text + assert "ArchiveBox server shut down gracefully" in log_text + assert_no_processes_for_data_dir(initialized_archive, timeout=12) + + resumed = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env) + resumed_log = resumed.log_path + _cmd_result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + + os.kill(resumed.pid, signal.SIGTERM) + resumed.wait(timeout=20) + resumed_text = resumed_log.read_text(encoding="utf-8", errors="replace") + assert "Got SIGTERM" in resumed_text + assert "ArchiveBox server shut down gracefully" in resumed_text + assert_no_processes_for_data_dir(initialized_archive, timeout=12) + finally: + for proc in (server, resumed): + if proc is not None and proc.poll() is None: + stop_archivebox_process(proc, signal.SIGKILL) + kill_processes_for_data_dir(initialized_archive) + + +@pytest.mark.timeout(180) +def test_live_daemonized_server_keeps_supervisord_owned_by_archivebox_parent(initialized_archive): + + env = cli_env(live=True) + port = get_free_port() + bind_url = f"http://127.0.0.1:{port}" + try: + _cmd_result = run_archivebox_cmd( + ["server", "--daemonize", f"127.0.0.1:{port}"], + cwd=initialized_archive, + env=env, + timeout=90, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + wait_for_port_open("127.0.0.1", port, timeout=30) + + server_process = wait_for_process( + lambda _proc, command: "archivebox" in command and " server " in f" {command} " and bind_url.replace("http://", "") in command, + ) + supervisord = wait_for_process( + lambda proc, command: proc.ppid() == server_process.pid and "supervisord" in command, + ) + wait_for_process( + lambda proc, command: proc.ppid() == supervisord.pid and "supervisord_watchdog" in command, + ) + + os.kill(server_process.pid, signal.SIGKILL) + wait_for_pid_to_disappear(server_process.pid, timeout=10) + wait_for_pid_to_disappear(supervisord.pid, timeout=20) + assert_no_processes_for_data_dir(initialized_archive, timeout=12) + finally: + kill_processes_for_data_dir(initialized_archive) + assert_no_processes_for_data_dir(initialized_archive, timeout=12) + + +@pytest.mark.timeout(240) +def test_live_servers_in_different_data_dirs_do_not_interfere(initialized_archive): + + first_data_dir = initialized_archive + second_data_dir = initialized_archive.parent / f"{initialized_archive.name}-second" + second_data_dir.mkdir() + second_env = cli_env(live=True) + _cmd_result = run_archivebox_cmd(["init"], cwd=second_data_dir, env=second_env, timeout=90) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + + first_port = get_free_port() + second_port = get_free_port() + first = None + second = None + first_resumed = None + try: + first = start_archivebox_server( + first_data_dir, + port=first_port, + log_name="server-first-data-dir.log", + env=cli_env(live=True), + ) + second = start_archivebox_server(second_data_dir, port=second_port, log_name="server-second-data-dir.log", env=second_env) + + _cmd_result = run_archivebox_cmd( + ["status"], + cwd=first_data_dir, + env=cli_env(live=True), + timeout=60, + ) + first_stdout, first_stderr, first_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + _cmd_result = run_archivebox_cmd( + ["status"], + cwd=second_data_dir, + env=second_env, + timeout=60, + ) + second_stdout, second_stderr, second_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert first_returncode == 0, first_stderr or first_stdout + assert second_returncode == 0, second_stderr or second_stdout + + stop_archivebox_process(first, signal.SIGTERM) + first = None + assert second.poll() is None, "stopping one DATA_DIR server must not stop another DATA_DIR server" + + first_resumed = start_archivebox_server( + first_data_dir, + port=first_port, + log_name="server-first-data-dir-resumed.log", + env=cli_env(live=True), + ) + assert second.poll() is None, "restarting one DATA_DIR server must not take over another DATA_DIR supervisor" + finally: + for proc in (first, first_resumed, second): + if proc is not None and proc.poll() is None: + stop_archivebox_process(proc, signal.SIGTERM) + kill_processes_for_data_dir(first_data_dir) + kill_processes_for_data_dir(second_data_dir) + assert_no_processes_for_data_dir(first_data_dir, timeout=12) + assert_no_processes_for_data_dir(second_data_dir, timeout=12) diff --git a/archivebox/tests/test_cli_shell.py b/archivebox/tests/test_cli_shell.py index c2a8142c..c9aaeb80 100644 --- a/archivebox/tests/test_cli_shell.py +++ b/archivebox/tests/test_cli_shell.py @@ -4,19 +4,15 @@ Tests for archivebox shell command. Verify shell command starts Django shell (basic smoke tests only). """ -import os -import subprocess +from archivebox.tests.conftest import run_archivebox_cmd -def test_shell_command_exists(tmp_path, process): +def test_shell_command_exists(initialized_archive): """Test that shell command is recognized.""" - os.chdir(tmp_path) # Test that the command exists (will fail without input but should recognize command) - result = subprocess.run( - ["archivebox", "shell", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["shell", "--help"], timeout=10, ) @@ -24,14 +20,11 @@ def test_shell_command_exists(tmp_path, process): assert result.returncode in [0, 1, 2] -def test_shell_c_executes_python(tmp_path, process): +def test_shell_c_executes_python(initialized_archive): """shell -c should fully initialize Django and run the provided command.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "shell", "-c", 'print("shell-ok")'], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["shell", "-c", 'print("shell-ok")'], timeout=30, ) diff --git a/archivebox/tests/test_cli_snapshot.py b/archivebox/tests/test_cli_snapshot.py index 23202929..21a12078 100644 --- a/archivebox/tests/test_cli_snapshot.py +++ b/archivebox/tests/test_cli_snapshot.py @@ -9,12 +9,21 @@ Tests cover: """ import json +import os +import pytest + +from archivebox.core.models import Snapshot, Tag +from archivebox.machine.models import Process from archivebox.tests.conftest import ( - run_archivebox_cmd, - parse_jsonl_output, + cli_env, create_test_url, + parse_jsonl_output, + run_archivebox_cmd, ) +from archivebox.tests.test_orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) class TestSnapshotCreate: @@ -24,10 +33,13 @@ class TestSnapshotCreate: """Create snapshot from URL arguments.""" url = create_test_url() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" assert "Created" in stderr @@ -42,15 +54,19 @@ class TestSnapshotCreate: url = create_test_url() # First create a crawl - stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode crawl = parse_jsonl_output(stdout1)[0] # Pipe crawl to snapshot create - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create"], stdin=json.dumps(crawl), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" @@ -67,10 +83,13 @@ class TestSnapshotCreate: """Create snapshot with --tag flag.""" url = create_test_url() - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create", "--tag=test-tag", url], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -82,11 +101,14 @@ class TestSnapshotCreate: url = create_test_url() stdin = json.dumps(tag_record) + "\n" + json.dumps({"url": url}) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create"], stdin=stdin, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -99,10 +121,13 @@ class TestSnapshotCreate: """Create snapshots from multiple URLs.""" urls = [create_test_url() for _ in range(3)] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "create"] + urls, - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -118,10 +143,13 @@ class TestSnapshotList: def test_list_empty(self, initialized_archive): """List with no snapshots returns empty.""" - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Listed 0 snapshots" in stderr @@ -129,12 +157,15 @@ class TestSnapshotList: def test_list_returns_created(self, initialized_archive): """List returns previously created snapshots.""" url = create_test_url() - run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -144,12 +175,15 @@ class TestSnapshotList: def test_list_filter_by_status(self, initialized_archive): """Filter snapshots by status.""" url = create_test_url() - run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", "--status=queued"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -159,12 +193,15 @@ class TestSnapshotList: def test_list_filter_by_url_contains(self, initialized_archive): """Filter snapshots by URL contains.""" url = create_test_url(domain="unique-domain-12345.com") - run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", "--url__icontains=unique-domain-12345"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -174,12 +211,20 @@ class TestSnapshotList: def test_list_with_limit(self, initialized_archive): """Limit number of results.""" for _ in range(3): - run_archivebox_cmd(["snapshot", "create", create_test_url()], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", create_test_url()], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", "--limit=2"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 records = parse_jsonl_output(stdout) @@ -188,12 +233,20 @@ class TestSnapshotList: def test_list_with_sort_and_limit(self, initialized_archive): """Sorting should be applied before limiting.""" for _ in range(3): - run_archivebox_cmd(["snapshot", "create", create_test_url()], data_dir=initialized_archive) + run_archivebox_cmd( + ["snapshot", "create", create_test_url()], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "list", "--limit=2", "--sort=-created_at"], - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0, f"Command failed: {stderr}" records = parse_jsonl_output(stdout) @@ -206,14 +259,23 @@ class TestSnapshotUpdate: def test_update_status(self, initialized_archive): """Update snapshot status.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "update", "--status=started"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Updated 1 snapshots" in stderr @@ -224,14 +286,23 @@ class TestSnapshotUpdate: def test_update_add_tag(self, initialized_archive): """Update snapshot by adding tag.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout2, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "update", "--tag=new-tag"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Updated 1 snapshots" in stderr @@ -243,14 +314,23 @@ class TestSnapshotDelete: def test_delete_requires_yes(self, initialized_archive): """Delete requires --yes flag.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "delete"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 1 assert "--yes" in stderr @@ -258,14 +338,23 @@ class TestSnapshotDelete: def test_delete_with_yes(self, initialized_archive): """Delete with --yes flag works.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "delete", "--yes"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Deleted 1 snapshots" in stderr @@ -273,14 +362,181 @@ class TestSnapshotDelete: def test_delete_dry_run(self, initialized_archive): """Dry run shows what would be deleted.""" url = create_test_url() - stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive) + _cmd_result = run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode snapshot = parse_jsonl_output(stdout1)[0] - stdout, stderr, code = run_archivebox_cmd( + _cmd_result = run_archivebox_cmd( ["snapshot", "delete", "--dry-run"], stdin=json.dumps(snapshot), - data_dir=initialized_archive, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Would delete" in stderr + + +def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive): + """Test that snapshot stores the exact URL in the database.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=tmp_path, + env=env, + ) + + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.select_related("crawl__created_by").get(url="https://example.com") + username = snapshot.crawl.created_by.username + + # Verify the crawl tree contains a relative symlink to the user-scoped snapshot output. + snapshots_root = tmp_path / "archive" / "users" / username / "snapshots" + crawl_root = tmp_path / "archive" / "users" / username / "crawls" + symlinks = [p for p in crawl_root.rglob("*") if p.is_symlink() and p.resolve().is_dir() and p.resolve().is_relative_to(snapshots_root)] + assert symlinks, "Snapshot symlink should exist under crawl dir" + link_path = symlinks[0] + + assert link_path.is_symlink(), "Snapshot symlink should exist under crawl dir" + link_target = os.readlink(link_path) + assert not os.path.isabs(link_target), "Symlink should be relative" + + +def test_snapshot_multiple_urls_creates_multiple_records(tmp_path, initialized_archive): + """Test that multiple URLs each get their own snapshot record.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + [ + "snapshot", + "create", + "https://example.com", + "https://iana.org", + ], + cwd=tmp_path, + env=env, + ) + + with use_archivebox_db(tmp_path): + urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) + + assert "https://example.com" in urls + assert "https://iana.org" in urls + assert len(urls) >= 2 + + +def test_snapshot_tag_creates_tag_and_links_to_snapshot(tmp_path, initialized_archive): + """Test that --tag creates tag record and links it to the snapshot.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + [ + "snapshot", + "create", + "--tag=mytesttag", + "https://example.com", + ], + cwd=tmp_path, + env=env, + ) + + with use_archivebox_db(tmp_path): + tag = Tag.objects.filter(name="mytesttag").first() + assert tag is not None, "Tag 'mytesttag' should exist in core_tag" + snapshot = Snapshot.objects.filter(url="https://example.com").first() + assert snapshot is not None + assert snapshot.tags.filter(pk=tag.pk).exists(), "Tag should be linked to snapshot via core_snapshot_tags" + + +def test_snapshot_jsonl_output_has_correct_structure(tmp_path, initialized_archive): + """Test that JSONL output contains required fields with correct types.""" + env = cli_env(disable_extractors=True) + + # Pass URL as argument instead of stdin for more reliable behavior + result = run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=tmp_path, + env=env, + ) + + # Parse JSONL output lines + records = Process.parse_records_from_text(result.stdout) + snapshot_records = [r for r in records if r.get("type") == "Snapshot"] + + assert len(snapshot_records) >= 1, "Should output at least one Snapshot JSONL record" + + record = snapshot_records[0] + assert record.get("type") == "Snapshot" + assert "id" in record, "Snapshot record should have 'id' field" + assert "url" in record, "Snapshot record should have 'url' field" + assert record["url"] == "https://example.com" + + +def test_snapshot_with_tag_stores_tag_name(tmp_path, initialized_archive): + """Test that title is stored when provided via tag option.""" + env = cli_env(disable_extractors=True) + + # Use command line args instead of stdin + run_archivebox_cmd( + ["snapshot", "create", "--tag=customtag", "https://example.com"], + cwd=tmp_path, + env=env, + ) + + with use_archivebox_db(tmp_path): + tag = Tag.objects.filter(name="customtag").first() + + assert tag is not None + assert tag.name == "customtag" + + +def test_snapshot_with_depth_sets_snapshot_depth(tmp_path, initialized_archive): + """Test that --depth sets snapshot depth when creating snapshots.""" + env = cli_env(disable_extractors=True) + + run_archivebox_cmd( + [ + "snapshot", + "create", + "--depth=1", + "https://example.com", + ], + cwd=tmp_path, + env=env, + ) + + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.order_by("-created_at").first() + + assert snapshot is not None, "Snapshot should be created when depth is provided" + assert snapshot.depth == 1, "Snapshot depth should match --depth value" + + +def test_snapshot_allows_duplicate_urls_across_crawls(tmp_path, initialized_archive): + """Snapshot create auto-creates a crawl per run; same URL can appear multiple times.""" + env = cli_env(disable_extractors=True) + + # Add same URL twice + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=tmp_path, + env=env, + ) + run_archivebox_cmd( + ["snapshot", "create", "https://example.com"], + cwd=tmp_path, + env=env, + ) + + with use_archivebox_db(tmp_path): + count = Snapshot.objects.filter(url="https://example.com").count() + + assert count == 2, "Same URL should create separate snapshots across different crawls" diff --git a/archivebox/tests/test_cli_status.py b/archivebox/tests/test_cli_status.py index 5495cdc5..ec5da8ba 100644 --- a/archivebox/tests/test_cli_status.py +++ b/archivebox/tests/test_cli_status.py @@ -4,69 +4,49 @@ Comprehensive tests for archivebox status command. Verify status reports accurate collection state from DB and filesystem. """ -import os -import subprocess -from pathlib import Path - import pytest from archivebox.core.models import Snapshot -from archivebox.tests.conftest import run_queued_crawls +from archivebox.tests.conftest import find_snapshot_dir, run_archivebox_cmd, run_queued_crawls, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def _find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: - candidates = {snapshot_id} - if len(snapshot_id) == 32: - candidates.add(f"{snapshot_id[:8]}-{snapshot_id[8:12]}-{snapshot_id[12:16]}-{snapshot_id[16:20]}-{snapshot_id[20:]}") - elif len(snapshot_id) == 36 and "-" in snapshot_id: - candidates.add(snapshot_id.replace("-", "")) - - for needle in candidates: - for path in data_dir.rglob(needle): - if path.is_dir(): - return path - return None - - -def test_status_runs_successfully(tmp_path, process): +def test_status_runs_successfully(initialized_archive): """Test that status command runs without error.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) assert result.returncode == 0 assert len(result.stdout) > 100 -def test_status_shows_zero_snapshots_in_empty_archive(tmp_path, process): +def test_status_shows_zero_snapshots_in_empty_archive(initialized_archive): """Test status shows 0 snapshots in empty archive.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) output = result.stdout # Should indicate empty/zero state assert "0" in output -def test_status_shows_correct_snapshot_count(tmp_path, process, disable_extractors_dict): +def test_status_shows_correct_snapshot_count(initialized_archive): """Test that status shows accurate snapshot count from DB.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add 3 snapshots for url in ["https://example.com", "https://example.org", "https://example.net"]: - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", url], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", url], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) # Verify DB has 3 snapshots - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): db_count = Snapshot.objects.count() assert db_count == 3 @@ -74,159 +54,147 @@ def test_status_shows_correct_snapshot_count(tmp_path, process, disable_extracto assert "3" in result.stdout -def test_status_shows_archived_count(tmp_path, process, disable_extractors_dict): +def test_status_shows_archived_count(initialized_archive): """Test status distinguishes archived vs unarchived snapshots.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) # Should show archived/unarchived categories assert "archived" in result.stdout.lower() or "queued" in result.stdout.lower() -def test_status_shows_archive_directory_size(tmp_path, process): +def test_status_shows_archive_directory_size(initialized_archive): """Test status reports archive directory size.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) output = result.stdout # Should show size info assert "Size" in output or "size" in output -def test_status_counts_archive_directories(tmp_path, process, disable_extractors_dict): +def test_status_counts_archive_directories(initialized_archive): """Test status counts directories in archive/ folder.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) # Should show directory count assert "present" in result.stdout.lower() or "directories" in result.stdout -def test_status_detects_orphaned_directories(tmp_path, process, disable_extractors_dict): +def test_status_detects_orphaned_directories(initialized_archive): """Test status detects directories not in DB (orphaned).""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add a snapshot - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Create an orphaned directory - (tmp_path / "archive" / "fake_orphaned_dir").mkdir(parents=True, exist_ok=True) + (initialized_archive / "archive" / "fake_orphaned_dir").mkdir(parents=True, exist_ok=True) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) # Should mention orphaned dirs assert "orphan" in result.stdout.lower() or "1" in result.stdout -def test_status_counts_new_snapshot_output_dirs_as_archived(tmp_path, process, disable_extractors_dict): +def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive): """Test status reads archived/present counts from the current snapshot output layout.""" - os.chdir(tmp_path) - env = disable_extractors_dict.copy() + env = cli_env(disable_extractors=True) + env = env.copy() env["ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS"] = "true" - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], env=env, check=True, ) - run_queued_crawls(tmp_path, env) + run_queued_crawls(initialized_archive, env) - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url="https://example.com") - snapshot_dir = _find_snapshot_dir(tmp_path, str(snapshot_id)) + snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id)) assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}" title_dir = snapshot_dir / "title" title_dir.mkdir(parents=True, exist_ok=True) (title_dir / "title.txt").write_text("Example Domain") - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True, env=env) + result = run_archivebox_cmd(["status"], env=env) assert result.returncode == 0, result.stdout + result.stderr assert "archived: 1" in result.stdout assert "present: 1" in result.stdout -def test_status_shows_user_info(tmp_path, process): +def test_status_shows_user_info(initialized_archive): """Test status shows user/login information.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) output = result.stdout # Should show user section assert "user" in output.lower() or "login" in output.lower() -def test_status_reads_from_db_not_filesystem(tmp_path, process, disable_extractors_dict): +def test_status_reads_from_db_not_filesystem(initialized_archive): """Test that status uses DB as source of truth, not filesystem.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add snapshot to DB - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Verify DB has snapshot - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): db_count = Snapshot.objects.count() assert db_count == 1 # Status should reflect DB count - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) assert "1" in result.stdout -def test_status_shows_index_file_info(tmp_path, process): +def test_status_shows_index_file_info(initialized_archive): """Test status shows index file information.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) # Should mention index assert "index" in result.stdout.lower() or "Index" in result.stdout -def test_status_help_lists_available_options(tmp_path, process): +def test_status_help_lists_available_options(initialized_archive): """Test that status --help works and documents the command.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "status", "--help"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["status", "--help"], ) assert result.returncode == 0 assert "status" in result.stdout.lower() or "statistic" in result.stdout.lower() -def test_status_shows_data_directory_path(tmp_path, process): +def test_status_shows_data_directory_path(initialized_archive): """Test that status reports which collection directory it is inspecting.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) + result = run_archivebox_cmd(["status"]) - assert "archive" in result.stdout.lower() or str(tmp_path) in result.stdout + assert "archive" in result.stdout.lower() or str(initialized_archive) in result.stdout diff --git a/archivebox/tests/test_cli_tag.py b/archivebox/tests/test_cli_tag.py new file mode 100644 index 00000000..2af1f78d --- /dev/null +++ b/archivebox/tests/test_cli_tag.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +""" +Tests for archivebox tag command. + +TODO: expand beyond command discovery into create/list/update/delete behavior. +""" + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_tag_help_runs_successfully(tmp_path): + """The tag command should be registered and expose help.""" + + result = run_archivebox_cmd(["tag", "--help"]) + + assert result.returncode == 0 + assert "tag" in result.stdout.lower() + assert "list" in result.stdout diff --git a/archivebox/tests/test_cli_update.py b/archivebox/tests/test_cli_update.py index f6b7f2e7..41bc5c26 100644 --- a/archivebox/tests/test_cli_update.py +++ b/archivebox/tests/test_cli_update.py @@ -4,25 +4,20 @@ Comprehensive tests for archivebox update command. Verify update drains old dirs, reconciles DB, and queues snapshots. """ -import os -import subprocess - import pytest from archivebox.core.models import Snapshot -from archivebox.tests.conftest import run_queued_crawls +from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env + from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def test_update_runs_successfully_on_empty_archive(tmp_path, process): +def test_update_runs_successfully_on_empty_archive(initialized_archive): """Test that update runs without error on empty archive.""" - os.chdir(tmp_path) - result = subprocess.run( - ["archivebox", "update"], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["update"], timeout=30, ) @@ -30,53 +25,48 @@ def test_update_runs_successfully_on_empty_archive(tmp_path, process): assert result.returncode == 0 -def test_update_reconciles_existing_snapshots(tmp_path, process, disable_extractors_dict): +def test_update_reconciles_existing_snapshots(initialized_archive): """Test that update command reconciles existing snapshots.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add a snapshot (index-only for faster test) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Run update - should reconcile and queue - result = subprocess.run( - ["archivebox", "update"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["update"], + env=env, timeout=30, ) assert result.returncode == 0 -def test_update_specific_snapshot_by_filter(tmp_path, process, disable_extractors_dict): +def test_update_specific_snapshot_by_filter(initialized_archive): """Test updating specific snapshot using filter.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add multiple snapshots - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, timeout=90, ) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.org"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.org"], + env=env, timeout=90, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Update with filter pattern (uses filter_patterns argument) - result = subprocess.run( - ["archivebox", "update", "--filter-type=substring", "example.com"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["update", "--filter-type=substring", "example.com"], + env=env, timeout=30, ) @@ -84,65 +74,61 @@ def test_update_specific_snapshot_by_filter(tmp_path, process, disable_extractor assert result.returncode == 0 -def test_update_preserves_snapshot_count(tmp_path, process, disable_extractors_dict): +def test_update_preserves_snapshot_count(initialized_archive): """Test that update doesn't change snapshot count.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) # Add snapshots - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, timeout=90, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Count before update - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count_before = Snapshot.objects.count() assert count_before == 1 # Run update (should reconcile + queue, not create new snapshots) - subprocess.run( - ["archivebox", "update"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["update"], + env=env, timeout=30, ) # Count after update - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): count_after = Snapshot.objects.count() # Snapshot count should remain the same assert count_after == count_before -def test_update_seals_migrated_snapshots(tmp_path, process, disable_extractors_dict): +def test_update_seals_migrated_snapshots(initialized_archive): """Test that full update reconciles migrated snapshots without re-queuing them.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - subprocess.run( - ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"], - capture_output=True, - env=disable_extractors_dict, + run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com"], + env=env, timeout=90, ) - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(initialized_archive, env) # Run update - result = subprocess.run( - ["archivebox", "update"], - capture_output=True, - env=disable_extractors_dict, + result = run_archivebox_cmd( + ["update"], + env=env, timeout=30, ) assert result.returncode == 0 # Check that snapshot remains archived instead of being queued for a full re-crawl. - with use_archivebox_db(tmp_path): + with use_archivebox_db(initialized_archive): status = Snapshot.objects.values_list("status", flat=True).get() assert status == "sealed" diff --git a/archivebox/tests/test_cli_update_reindex_snapshots.py b/archivebox/tests/test_cli_update_reindex_snapshots.py index 550234cb..411e9be2 100644 --- a/archivebox/tests/test_cli_update_reindex_snapshots.py +++ b/archivebox/tests/test_cli_update_reindex_snapshots.py @@ -1,22 +1,20 @@ import json import os -import subprocess from datetime import datetime, timedelta +from archivebox.tests.conftest import run_archivebox_cmd, cli_env import pytest from django.utils import timezone from archivebox.core.models import Snapshot from archivebox.tests.test_orm_helpers import use_archivebox_db -from .fixtures import disable_extractors_dict, process pytestmark = pytest.mark.django_db(transaction=True) -FIXTURES = (disable_extractors_dict, process) - -def test_update_imports_orphaned_snapshots(tmp_path, process, disable_extractors_dict): +def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive): """Test that archivebox update imports real legacy archive directories.""" + env = cli_env(disable_extractors=True) legacy_timestamp = "1710000000" legacy_dir = tmp_path / "archive" / legacy_timestamp legacy_dir.mkdir(parents=True, exist_ok=True) @@ -34,11 +32,9 @@ def test_update_imports_orphaned_snapshots(tmp_path, process, disable_extractors ) # Run the migration phase only; default update also runs queued crawl work. - update_process = subprocess.run( - ["archivebox", "update", "--migrate-only"], - capture_output=True, - text=True, - env=disable_extractors_dict, + update_process = run_archivebox_cmd( + ["update", "--migrate-only"], + env=env, timeout=60, ) assert update_process.returncode == 0, update_process.stderr @@ -46,7 +42,7 @@ def test_update_imports_orphaned_snapshots(tmp_path, process, disable_extractors with use_archivebox_db(tmp_path): row = Snapshot.objects.values_list("url", "fs_version").get() - assert row == ("https://example.com", "0.9.0") + assert row == ("https://example.com", Snapshot._fs_current_version()) assert legacy_dir.is_symlink() migrated_dir = legacy_dir.resolve() diff --git a/archivebox/tests/test_cli_version.py b/archivebox/tests/test_cli_version.py index 0d524004..f94f79dc 100644 --- a/archivebox/tests/test_cli_version.py +++ b/archivebox/tests/test_cli_version.py @@ -6,44 +6,9 @@ Verify version output and system information reporting. import os import re -import sys import tempfile -import subprocess from pathlib import Path - -from .fixtures import process - -FIXTURES = (process,) - - -def _archivebox_cli() -> str: - cli = Path(sys.executable).with_name("archivebox") - return str(cli if cli.exists() else "archivebox") - - -def _run_real_cli( - args: list[str], - cwd: Path, - *, - home_dir: Path, - timeout: int = 180, - extra_env: dict[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - env = os.environ.copy() - env.pop("DATA_DIR", None) - env["HOME"] = str(home_dir) - env["USE_COLOR"] = "False" - env["SHOW_PROGRESS"] = "False" - if extra_env: - env.update(extra_env) - return subprocess.run( - [_archivebox_cli(), *args], - capture_output=True, - text=True, - cwd=cwd, - env=env, - timeout=timeout, - ) +from archivebox.tests.conftest import run_archivebox_cmd def _make_deep_collection_dir(tmp_path: Path) -> Path: @@ -66,8 +31,7 @@ def _extract_location_path(output: str, key: str) -> Path: def test_version_quiet_outputs_version_number(tmp_path): """Test that version --quiet outputs just the version number.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "version", "--quiet"], capture_output=True, text=True) + result = run_archivebox_cmd(["version", "--quiet"]) assert result.returncode == 0 version = result.stdout.strip() @@ -79,8 +43,7 @@ def test_version_quiet_outputs_version_number(tmp_path): def test_version_flag_outputs_version_number(tmp_path): """Test that top-level --version reports the package version.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "--version"], capture_output=True, text=True) + result = run_archivebox_cmd(["--version"]) assert result.returncode == 0 version = result.stdout.strip() @@ -88,10 +51,9 @@ def test_version_flag_outputs_version_number(tmp_path): assert len(version.split(".")) >= 2 -def test_version_shows_system_info_in_initialized_dir(tmp_path, process): +def test_version_shows_system_info_in_initialized_dir(tmp_path, initialized_archive): """Test that version shows system metadata in initialized directory.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "version"], capture_output=True, text=True) + result = run_archivebox_cmd(["version"]) output = result.stdout assert "ArchiveBox" in output @@ -99,20 +61,18 @@ def test_version_shows_system_info_in_initialized_dir(tmp_path, process): assert any(x in output for x in ["ARCH=", "OS=", "PYTHON="]) -def test_version_shows_binaries_after_init(tmp_path, process): +def test_version_shows_binaries_after_init(tmp_path, initialized_archive): """Test that version shows binary dependencies in initialized directory.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "version"], capture_output=True, text=True) + result = run_archivebox_cmd(["version"]) output = result.stdout # Should show binary section assert "Binary" in output or "Dependencies" in output -def test_version_shows_data_locations(tmp_path, process): +def test_version_shows_data_locations(tmp_path, initialized_archive): """Test that version shows data directory locations.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "version"], capture_output=True, text=True) + result = run_archivebox_cmd(["version"]) output = result.stdout # Should show paths @@ -123,9 +83,8 @@ def test_version_in_uninitialized_dir_still_works(tmp_path): """Test that version command works even without initialized data dir.""" empty_dir = tmp_path / "empty" empty_dir.mkdir() - os.chdir(empty_dir) - result = subprocess.run(["archivebox", "version", "--quiet"], capture_output=True, text=True) + result = run_archivebox_cmd(["version", "--quiet"], cwd=empty_dir) # Should still output version assert result.returncode == 0 @@ -140,11 +99,17 @@ def test_version_auto_selects_short_tmp_dir_for_deep_collection_path(tmp_path): with tempfile.TemporaryDirectory(prefix="abx-home-") as home_tmp: home_dir = Path(home_tmp) + env = { + "HOME": str(home_dir), + "USE_COLOR": "False", + "SHOW_PROGRESS": "False", + **extra_env, + } - init_result = _run_real_cli(["init", "--quick"], cwd=data_dir, home_dir=home_dir, extra_env=extra_env) + init_result = run_archivebox_cmd(["init", "--quick"], cwd=data_dir, env=env, timeout=180) assert init_result.returncode == 0, init_result.stdout + init_result.stderr - version_result = _run_real_cli(["version"], cwd=data_dir, home_dir=home_dir, extra_env=extra_env) + version_result = run_archivebox_cmd(["version"], cwd=data_dir, env=env, timeout=180) output = version_result.stdout + version_result.stderr assert version_result.returncode == 0, output @@ -163,8 +128,7 @@ def test_version_auto_selects_short_tmp_dir_for_deep_collection_path(tmp_path): def test_version_help_lists_quiet_flag(tmp_path): """Test that version --help documents the quiet output mode.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "version", "--help"], capture_output=True, text=True) + result = run_archivebox_cmd(["version", "--help"]) assert result.returncode == 0 assert "--quiet" in result.stdout or "-q" in result.stdout @@ -172,7 +136,6 @@ def test_version_help_lists_quiet_flag(tmp_path): def test_version_invalid_option_fails(tmp_path): """Test that invalid version options fail cleanly.""" - os.chdir(tmp_path) - result = subprocess.run(["archivebox", "version", "--invalid-option"], capture_output=True, text=True) + result = run_archivebox_cmd(["version", "--invalid-option"]) assert result.returncode != 0 diff --git a/archivebox/tests/test_config.py b/archivebox/tests/test_config.py deleted file mode 100644 index eec85da2..00000000 --- a/archivebox/tests/test_config.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python3 -"""Integration tests for archivebox config command.""" - -import os -import subprocess - -import pytest - - -def test_config_shows_all_config_values(tmp_path, process): - """Test that config without args shows all config values.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "config"], - capture_output=True, - text=True, - ) - - # Should show various config sections - assert "TIMEOUT" in result.stdout or "timeout" in result.stdout.lower() - # Config should show some output - assert len(result.stdout) > 100 - - -def test_config_get_specific_key(tmp_path, process): - """Test that --get retrieves a specific config value.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "config", "--get", "TIMEOUT"], - capture_output=True, - text=True, - ) - - # Should show the TIMEOUT value - assert "TIMEOUT" in result.stdout or result.returncode == 0 - - -def test_config_set_value_writes_to_config_file(tmp_path, process): - """Test that --set writes config value to ArchiveBox.conf file.""" - os.chdir(tmp_path) - - # Set a config value - result = subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=120"], - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - - # Read the config file directly to verify it was written - config_file = tmp_path / "ArchiveBox.conf" - if config_file.exists(): - config_content = config_file.read_text() - # Config should contain the set value - assert "TIMEOUT" in config_content or "timeout" in config_content.lower() - - -def test_config_set_and_get_roundtrip(tmp_path, process): - """Test that a value set with --set can be retrieved with --get.""" - os.chdir(tmp_path) - - # Set a value - set_result = subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT=999"], - capture_output=True, - text=True, - ) - - # Verify set was successful - assert set_result.returncode == 0 or "999" in set_result.stdout - - # Read the config file directly to verify - config_file = tmp_path / "ArchiveBox.conf" - if config_file.exists(): - config_content = config_file.read_text() - assert "999" in config_content or "TIMEOUT" in config_content - - -def test_config_search_finds_matching_keys(tmp_path, process): - """Test that --search finds config keys matching a pattern.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "config", "--search", "TIMEOUT"], - capture_output=True, - text=True, - ) - - # Should find TIMEOUT-related config - assert "TIMEOUT" in result.stdout or result.returncode == 0 - - -def test_config_invalid_key_fails(tmp_path, process): - """Test that setting an invalid config key fails.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "config", "--set", "INVALID_KEY_THAT_DOES_NOT_EXIST=value"], - capture_output=True, - text=True, - ) - - # Should fail - assert result.returncode != 0 or "failed" in result.stdout.lower() - - -def test_config_ignores_legacy_unknown_keys(tmp_path, process): - """Old ArchiveBox.conf keys should not prevent startup during upgrades.""" - os.chdir(tmp_path) - (tmp_path / "ArchiveBox.conf").write_text( - """ -[ARCHIVING_CONFIG] -MAX_MEDIA_SIZE = "750m" - -[SEARCH_BACKEND_CONFIG] -SEARCH_BACKEND_HOST_NAME = "sonic" -SEARCH_BACKEND_PASSWORD = "SecretPassword" -""", - ) - - result = subprocess.run( - ["archivebox", "version"], - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stderr - assert "Extra inputs are not permitted" not in result.stderr - - -def test_sonic_dir_is_allowed_inside_data_dir(): - from archivebox.config import CONSTANTS - - assert "sonic" in CONSTANTS.ALLOWED_IN_DATA_DIR - - -def test_config_set_requires_equals_sign(tmp_path, process): - """Test that --set requires KEY=VALUE format.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "config", "--set", "TIMEOUT"], - capture_output=True, - text=True, - ) - - # Should fail because there's no = sign - assert result.returncode != 0 - - -class TestConfigCLI: - """Test the CLI interface for config command.""" - - def test_cli_help(self, tmp_path, process): - """Test that --help works for config command.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "config", "--help"], - capture_output=True, - text=True, - ) - - assert result.returncode == 0 - assert "--get" in result.stdout - assert "--set" in result.stdout - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) diff --git a/archivebox/tests/test_retention.py b/archivebox/tests/test_config_DELETE_AFTER.py similarity index 87% rename from archivebox/tests/test_retention.py rename to archivebox/tests/test_config_DELETE_AFTER.py index a17972bc..f75af8e2 100644 --- a/archivebox/tests/test_retention.py +++ b/archivebox/tests/test_config_DELETE_AFTER.py @@ -1,12 +1,11 @@ import json -import os from pathlib import Path import pytest from django.contrib.auth import get_user_model from django.urls import reverse -from archivebox.tests.conftest import run_archivebox_cmd_cwd, run_queued_crawls +from archivebox.tests.conftest import run_archivebox_cmd, run_queued_crawls, cli_env pytestmark = pytest.mark.django_db(transaction=True) @@ -15,26 +14,28 @@ ADMIN_HOST = "admin.archivebox.localhost:8000" API_HOST = "api.archivebox.localhost:8000" -def test_delete_after_real_cli_and_orchestrator_paths_cover_all_retained_models(tmp_path, disable_extractors_dict): - os.chdir(tmp_path) - stdout, stderr, returncode = run_archivebox_cmd_cwd(["init", "--quick"], cwd=tmp_path, timeout=90) +def test_delete_after_real_cli_and_orchestrator_paths_cover_all_retained_models(tmp_path): + env = cli_env(disable_extractors=True) + _cmd_result = run_archivebox_cmd(["init", "--quick"], cwd=tmp_path, timeout=90) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr - cli_env = { - **disable_extractors_dict, + run_env = { + **env, "DELETE_AFTER": "1hr", "USE_COLOR": "False", "SHOW_PROGRESS": "False", } url = "https://example.com/delete-after-cli" - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["add", "--index-only", "--depth=0", url], cwd=tmp_path, timeout=120, - env=cli_env, + env=run_env, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox add failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" - run_queued_crawls(tmp_path, cli_env) + run_queued_crawls(tmp_path, run_env) lookup_script = f""" import json @@ -47,12 +48,13 @@ print(json.dumps({{ "snapshot_delete_at": bool(snapshot.delete_at), }})) """ - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["manage", "shell", "-c", lookup_script], cwd=tmp_path, timeout=90, - env=cli_env, + env=run_env, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"retention lookup failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" created = json.loads(stdout.strip().splitlines()[-1]) assert created["crawl_delete_at"] @@ -141,17 +143,19 @@ print(json.dumps({{ "archiveresult_dir": str(result.output_dir), }})) """ - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["manage", "shell", "-c", setup_script], cwd=tmp_path, timeout=90, - env=cli_env, + env=run_env, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"retention setup failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" retained = json.loads(stdout.strip().splitlines()[-1]) assert retained["crawl_id"] == created["crawl_id"] - stdout, stderr, returncode = run_archivebox_cmd_cwd(["run", "--crawl-id", retained["crawl_id"]], cwd=tmp_path, timeout=120, env=cli_env) + _cmd_result = run_archivebox_cmd(["run", "--crawl-id", retained["crawl_id"]], cwd=tmp_path, timeout=120, env=run_env) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox run failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" remaining_script = f""" @@ -166,12 +170,13 @@ print(json.dumps({{ "process": Process.objects.filter(id="{retained["process_id"]}").count(), }})) """ - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["manage", "shell", "-c", remaining_script], cwd=tmp_path, timeout=90, - env=cli_env, + env=run_env, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"retention remaining lookup failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" remaining = json.loads(stdout.strip().splitlines()[-1]) assert remaining == {"crawl": 0, "snapshot": 0, "archiveresult": 0, "process": 0} diff --git a/archivebox/tests/test_config_MAX_limits.py b/archivebox/tests/test_config_MAX_limits.py new file mode 100644 index 00000000..b3e44163 --- /dev/null +++ b/archivebox/tests/test_config_MAX_limits.py @@ -0,0 +1,295 @@ +"""Tests for MAX/SIZE crawl limit config behavior.""" + +import asyncio +import json +from pathlib import Path + +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.conftest import cli_env, run_archivebox_cmd +from archivebox.tests.test_orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + + +def test_create_snapshots_from_urls_respects_max_urls(admin_user): + crawl = Crawl.objects.create( + urls="\n".join( + [ + "https://example.com/root", + "https://example.com/about", + "https://example.com/contact", + ], + ), + config={"CRAWL_MAX_URLS": 2}, + created_by=admin_user, + ) + + created = crawl.create_snapshots_from_urls() + + assert [snapshot.url for snapshot in created] == [ + "https://example.com/root", + "https://example.com/about", + ] + assert crawl.snapshot_set.count() == 2 + assert crawl.remaining_snapshot_capacity() == 0 + assert crawl.limit_stop_reason() == "crawl_max_urls" + assert crawl.add_url({"url": "https://example.com/extra", "depth": 1}) is False + + +def test_crawl_stop_reason_keeps_specific_limit_reason_over_lifecycle_fallback(admin_user): + crawl = Crawl.objects.create( + urls="\n".join( + [ + "https://example.com/root", + "https://example.com/about", + ], + ), + config={"CRAWL_MAX_URLS": 1}, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + created_by=admin_user, + ) + Snapshot.objects.create( + url="https://example.com/root", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + timestamp="1700000000.011", + ) + + assert crawl.stop_reason() == "crawl_max_urls" + + +def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + + crawl = Crawl.objects.create( + urls="https://example.com", + max_depth=0, + config={"CRAWL_MAX_URLS": 5}, + created_by_id=get_or_create_system_user_pk(), + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + depth=0, + ) + parser_dir = Path(snapshot.output_dir) / "parse_html_urls" + parser_dir.mkdir(parents=True, exist_ok=True) + (parser_dir / "urls.jsonl").write_text( + "\n".join( + [ + json.dumps({"type": "Snapshot", "url": "https://example.com/child-a", "depth": 1}), + json.dumps({"type": "Snapshot", "url": "https://example.com/child-b", "depth": 1}), + "", + ], + ), + ) + + runner = CrawlRunner(crawl) + Crawl.objects.filter(id=crawl.id).update(max_depth=1) + payload = runner.load_snapshot_payload(str(snapshot.id)) + + asyncio.run(runner.enqueue_discovered_snapshots_from_outputs(payload)) + + child_snapshots = list(crawl.snapshot_set.filter(depth=1).order_by("url").values_list("url", "status")) + assert child_snapshots == [ + ("https://example.com/child-a", Snapshot.StatusChoices.QUEUED), + ("https://example.com/child-b", Snapshot.StatusChoices.QUEUED), + ] + + +def test_run_snapshot_seals_descendant_when_crawl_max_size_is_reached(tmp_path): + from abx_dl.events import CrawlStartEvent, SnapshotEvent + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + + crawl = Crawl.objects.create( + urls="https://example.com", + config={ + "LIB_DIR": str(tmp_path / "lib"), + "PLUGINS": "__archivebox_test_no_plugins__", + "CHROME_BINARY": "", + "CRAWL_MAX_SIZE": 16, + }, + created_by_id=get_or_create_system_user_pk(), + ) + root = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + depth=0, + status=Snapshot.StatusChoices.SEALED, + ) + child = Snapshot.objects.create( + url="https://example.com/child", + crawl=crawl, + depth=1, + parent_snapshot=root, + status=Snapshot.StatusChoices.QUEUED, + ) + state_dir = Path(crawl.output_dir) / ".abx-dl" + state_dir.mkdir(parents=True, exist_ok=True) + (state_dir / "limits.json").write_text( + json.dumps( + { + "admitted_snapshot_ids": [str(root.id), str(child.id)], + "counted_event_ids": ["proc-1"], + "total_size": 32, + "stop_reason": "crawl_max_size", + }, + ), + encoding="utf-8", + ) + + runner = CrawlRunner(crawl) + runner.load_run_state() + + async def run_child_from_crawl_start() -> list[SnapshotEvent]: + async def on_crawl_start(_event: CrawlStartEvent) -> None: + await runner.run_snapshot(str(child.id)) + + runner.bus.on(CrawlStartEvent, on_crawl_start) + crawl_start = runner.bus.emit( + CrawlStartEvent( + url=root.url, + snapshot_id=str(root.id), + output_dir=str(crawl.output_dir), + event_timeout=30, + event_handler_timeout=30, + ), + ) + await crawl_start.now() + await crawl_start.event_results_list() + await runner.bus.wait_until_idle() + return await runner.bus.filter(SnapshotEvent, child_of=crawl_start, past=True) + + snapshot_events = asyncio.run(run_child_from_crawl_start()) + + child.refresh_from_db() + assert child.status == Snapshot.StatusChoices.SEALED + assert child.retry_at is None + assert snapshot_events == [] + + +def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.snapshot_service import SnapshotService + from abx_dl.events import SnapshotCompletedEvent + from abx_dl.orchestrator import create_bus + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + config={"CRAWL_MAX_SIZE": 16}, + ) + root = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + ) + child = Snapshot.objects.create( + url="https://example.com/child", + crawl=crawl, + depth=1, + parent_snapshot_id=root.id, + status=Snapshot.StatusChoices.QUEUED, + ) + + state_dir = Path(crawl.output_dir) / ".abx-dl" + state_dir.mkdir(parents=True, exist_ok=True) + (state_dir / "limits.json").write_text( + json.dumps( + { + "admitted_snapshot_ids": [str(root.id), str(child.id)], + "counted_event_ids": ["proc-1"], + "total_size": 32, + "stop_reason": "crawl_max_size", + }, + ), + encoding="utf-8", + ) + + bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}") + service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: None) + try: + + async def emit_event() -> None: + await service.on_SnapshotCompletedEvent( + SnapshotCompletedEvent( + url=root.url, + snapshot_id=str(root.id), + output_dir=str(root.output_dir), + ), + ) + + asyncio.run(emit_event()) + finally: + asyncio.run(bus.wait_until_idle()) + asyncio.run(bus.destroy()) + + root.refresh_from_db() + child.refresh_from_db() + assert root.status == Snapshot.StatusChoices.SEALED + assert child.status == Snapshot.StatusChoices.SEALED + assert child.retry_at is None + + +def test_recursive_crawl_respects_max_urls(tmp_path, initialized_archive, recursive_test_site): + """Test that recursive discovery stops creating snapshots at max_urls.""" + env = cli_env(disable_extractors=True) + + env = env.copy() + env.update( + { + "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*", + "SAVE_WGET": "true", + "USE_CHROME": "false", + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + }, + ) + + result = run_archivebox_cmd( + [ + "add", + "--depth=2", + "--max-urls=4", + "--plugins=wget,parse_html_urls", + recursive_test_site["root_url"], + ], + env=env, + timeout=120, + ) + stdout, stderr = result.stdout, result.stderr + + if stderr: + print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n") + if stdout: + print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n") + + assert result.returncode == 0, result.stderr + + with use_archivebox_db(tmp_path): + crawl_obj = Crawl.objects.order_by("-created_at").first() + crawl = (crawl_obj.max_depth, crawl_obj.config["CRAWL_MAX_URLS"]) if crawl_obj else None + snapshot_rows = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "parent_snapshot_id")) + depth_counts = { + depth: Snapshot.objects.filter(depth=depth).count() for depth in set(Snapshot.objects.values_list("depth", flat=True)) + } + + assert crawl == (2, 4) + assert len(snapshot_rows) == 4 + assert depth_counts.get(0, 0) == 1 + assert depth_counts.get(1, 0) == 3 + assert depth_counts.get(2, 0) == 0 + assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1}) diff --git a/archivebox/tests/test_config_ONLY_NEW.py b/archivebox/tests/test_config_ONLY_NEW.py new file mode 100644 index 00000000..6df7b312 --- /dev/null +++ b/archivebox/tests/test_config_ONLY_NEW.py @@ -0,0 +1,87 @@ +"""Tests for ONLY_NEW crawl config behavior.""" + +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl + +pytestmark = pytest.mark.django_db + + +def test_create_snapshots_from_urls_respects_only_new_exact_url_matches(admin_user): + existing_crawl = Crawl.objects.create(urls="https://example.com/existing", created_by=admin_user) + Snapshot.objects.create( + url="https://example.com/existing", + crawl=existing_crawl, + timestamp="1700000000.001", + ) + crawl = Crawl.objects.create( + urls="\n".join( + [ + "https://example.com/existing", + "https://example.com/existing/", + "https://example.com/fresh", + ], + ), + config={"ONLY_NEW": True}, + created_by=admin_user, + ) + + created = crawl.create_snapshots_from_urls() + + assert [snapshot.url for snapshot in created] == [ + "https://example.com/existing/", + "https://example.com/fresh", + ] + assert Snapshot.objects.filter(url="https://example.com/existing").count() == 1 + + +def test_create_snapshots_from_urls_allows_existing_exact_url_when_only_new_false(admin_user): + existing_crawl = Crawl.objects.create(urls="https://example.com/existing", created_by=admin_user) + Snapshot.objects.create( + url="https://example.com/existing", + crawl=existing_crawl, + timestamp="1700000000.002", + ) + crawl = Crawl.objects.create( + urls="https://example.com/existing", + config={"ONLY_NEW": False}, + created_by=admin_user, + ) + + created = crawl.create_snapshots_from_urls() + + assert [snapshot.url for snapshot in created] == ["https://example.com/existing"] + assert Snapshot.objects.filter(url="https://example.com/existing").count() == 2 + + +def test_create_discovered_snapshots_respects_only_new_exact_url_matches(admin_user): + existing_crawl = Crawl.objects.create(urls="https://example.com/existing", created_by=admin_user) + Snapshot.objects.create( + url="https://example.com/existing", + crawl=existing_crawl, + timestamp="1700000000.003", + ) + crawl = Crawl.objects.create( + urls="https://example.com/root", + max_depth=1, + config={"ONLY_NEW": True}, + created_by=admin_user, + ) + parent = crawl.create_snapshots_from_urls()[0] + + created = crawl.create_discovered_snapshots( + parent, + [ + {"url": "https://example.com/existing"}, + {"url": "https://example.com/existing/"}, + {"url": "https://example.com/fresh"}, + ], + depth=1, + ) + + assert [snapshot.url for snapshot in created] == [ + "https://example.com/existing/", + "https://example.com/fresh", + ] + assert Snapshot.objects.filter(url="https://example.com/existing").count() == 1 diff --git a/archivebox/tests/test_title.py b/archivebox/tests/test_config_SAVE_TITLE.py similarity index 51% rename from archivebox/tests/test_title.py rename to archivebox/tests/test_config_SAVE_TITLE.py index 59257d77..34fcf93b 100644 --- a/archivebox/tests/test_title.py +++ b/archivebox/tests/test_config_SAVE_TITLE.py @@ -1,17 +1,13 @@ -import subprocess -import sys +from archivebox.tests.conftest import run_archivebox_cmd, cli_env import pytest from archivebox.core.models import Snapshot from archivebox.tests.test_orm_helpers import use_archivebox_db from .conftest import _find_system_browser -from .fixtures import disable_extractors_dict, process pytestmark = pytest.mark.django_db(transaction=True) -FIXTURES = (disable_extractors_dict, process) - def _install_chrome(tmp_path, env): env["CHROME_ISOLATION"] = "snapshot" @@ -20,26 +16,23 @@ def _install_chrome(tmp_path, env): env["CHROME_BINARY"] = str(system_browser) return - install_process = subprocess.run( - [sys.executable, "-m", "archivebox", "install", "chrome"], + install_process = run_archivebox_cmd( + ["install", "chrome"], cwd=tmp_path, - capture_output=True, - text=True, env=env, timeout=600, ) assert install_process.returncode == 0, install_process.stderr or install_process.stdout -def test_title_is_extracted(tmp_path, process, disable_extractors_dict): +def test_title_is_extracted(tmp_path, initialized_archive): """Test that title is extracted from the page.""" - disable_extractors_dict.update({"SAVE_TITLE": "true"}) - _install_chrome(tmp_path, disable_extractors_dict) - add_process = subprocess.run( - ["archivebox", "add", "--plugins=chrome,wget,title", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + env.update({"SAVE_TITLE": "true"}) + _install_chrome(tmp_path, env) + add_process = run_archivebox_cmd( + ["add", "--plugins=chrome,wget,title", "https://example.com"], + env=env, ) assert add_process.returncode == 0, add_process.stderr or add_process.stdout @@ -50,28 +43,25 @@ def test_title_is_extracted(tmp_path, process, disable_extractors_dict): assert "Example" in title -def test_title_is_htmlencoded_in_index_html(tmp_path, process, disable_extractors_dict): +def test_title_is_listed_by_search_alias(tmp_path, initialized_archive): """ https://github.com/ArchiveBox/ArchiveBox/issues/330 Unencoded content should not be rendered as it facilitates xss injections and breaks the layout. """ - disable_extractors_dict.update({"SAVE_TITLE": "true"}) - _install_chrome(tmp_path, disable_extractors_dict) - add_process = subprocess.run( - ["archivebox", "add", "--plugins=chrome,wget,title", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, + env = cli_env(disable_extractors=True) + env.update({"SAVE_TITLE": "true"}) + _install_chrome(tmp_path, env) + add_process = run_archivebox_cmd( + ["add", "--plugins=chrome,wget,title", "https://example.com"], + env=env, ) assert add_process.returncode == 0, add_process.stderr or add_process.stdout - list_process = subprocess.run( - ["archivebox", "search", "--html"], - capture_output=True, - text=True, + list_process = run_archivebox_cmd( + ["search"], + env=env, ) assert list_process.returncode == 0, list_process.stderr or list_process.stdout - # Should not contain unescaped HTML tags in output output = list_process.stdout assert "https://example.com" in output diff --git a/archivebox/tests/test_config_URL_filters.py b/archivebox/tests/test_config_URL_filters.py new file mode 100644 index 00000000..38d8964a --- /dev/null +++ b/archivebox/tests/test_config_URL_filters.py @@ -0,0 +1,60 @@ +"""Tests for URL_ALLOWLIST and URL_DENYLIST behavior.""" + +import pytest + +from archivebox.crawls.models import Crawl + +pytestmark = pytest.mark.django_db + + +def test_create_snapshots_from_urls_respects_url_allowlist_and_denylist(admin_user): + crawl = Crawl.objects.create( + urls="\n".join( + [ + "https://example.com/root", + "https://static.example.com/app.js", + "https://other.test/page", + ], + ), + created_by=admin_user, + config={ + "URL_ALLOWLIST": "example.com", + "URL_DENYLIST": "static.example.com", + }, + ) + + created = crawl.create_snapshots_from_urls() + + assert [snapshot.url for snapshot in created] == ["https://example.com/root"] + + +def test_url_filter_regex_lists_preserve_commas_and_split_on_newlines_only(admin_user): + crawl = Crawl.objects.create( + urls="\n".join( + [ + "https://example.com/root", + "https://example.com/path,with,commas", + "https://other.test/page", + ], + ), + created_by=admin_user, + config={ + "URL_ALLOWLIST": r"^https://example\.com/(root|path,with,commas)$" + "\n" + r"^https://other\.test/page$", + "URL_DENYLIST": r"^https://example\.com/path,with,commas$", + }, + ) + + assert crawl.get_url_allowlist(use_effective_config=False) == [ + r"^https://example\.com/(root|path,with,commas)$", + r"^https://other\.test/page$", + ] + assert crawl.get_url_denylist(use_effective_config=False) == [ + r"^https://example\.com/path,with,commas$", + ] + + created = crawl.create_snapshots_from_urls() + + assert [snapshot.url for snapshot in created] == [ + "https://example.com/root", + "https://other.test/page", + ] diff --git a/archivebox/tests/test_core_config.py b/archivebox/tests/test_core_config.py new file mode 100644 index 00000000..6b6c9438 --- /dev/null +++ b/archivebox/tests/test_core_config.py @@ -0,0 +1,5 @@ +from archivebox.config import CONSTANTS + + +def test_sonic_dir_is_allowed_inside_data_dir(): + assert "sonic" in CONSTANTS.ALLOWED_IN_DATA_DIR diff --git a/archivebox/tests/test_crawl.py b/archivebox/tests/test_crawl.py deleted file mode 100644 index 603b0425..00000000 --- a/archivebox/tests/test_crawl.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -"""Integration tests for archivebox crawl command.""" - -import os -import subprocess - -import pytest - -from archivebox.core.models import Snapshot -from archivebox.crawls.models import Crawl -from archivebox.tests.conftest import run_queued_crawls -from archivebox.tests.test_orm_helpers import use_archivebox_db - -pytestmark = pytest.mark.django_db(transaction=True) - - -def test_crawl_creates_crawl_object(tmp_path, process, disable_extractors_dict): - """Test that crawl command creates a Crawl object.""" - os.chdir(tmp_path) - - subprocess.run( - ["archivebox", "crawl", "--no-wait", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").first() - - assert crawl is not None, "Crawl object should be created" - - -def test_crawl_depth_sets_max_depth_in_crawl(tmp_path, process, disable_extractors_dict): - """Test that --depth option sets max_depth in the Crawl object.""" - os.chdir(tmp_path) - - subprocess.run( - ["archivebox", "crawl", "--depth=2", "--no-wait", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").first() - - assert crawl is not None - assert crawl.max_depth == 2, "Crawl max_depth should match --depth=2" - - -def test_crawl_creates_snapshot_for_url(tmp_path, process, disable_extractors_dict): - """Test that crawl creates a Snapshot for the input URL.""" - os.chdir(tmp_path) - - subprocess.run( - ["archivebox", "crawl", "--no-wait", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - run_queued_crawls(tmp_path, disable_extractors_dict) - - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.filter(url="https://example.com").first() - - assert snapshot is not None, "Snapshot should be created for input URL" - - -def test_crawl_links_snapshot_to_crawl(tmp_path, process, disable_extractors_dict): - """Test that Snapshot is linked to Crawl via crawl_id.""" - os.chdir(tmp_path) - - subprocess.run( - ["archivebox", "crawl", "--no-wait", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - run_queued_crawls(tmp_path, disable_extractors_dict) - - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").first() - assert crawl is not None - snapshot = Snapshot.objects.filter(url="https://example.com").first() - - assert snapshot is not None - assert snapshot.crawl_id == crawl.id, "Snapshot should be linked to Crawl" - - -def test_crawl_multiple_urls_creates_multiple_snapshots(tmp_path, process, disable_extractors_dict): - """Test that crawling multiple URLs creates multiple snapshots.""" - os.chdir(tmp_path) - - subprocess.run( - [ - "archivebox", - "crawl", - "--no-wait", - "https://example.com", - "https://iana.org", - ], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - run_queued_crawls(tmp_path, disable_extractors_dict) - - with use_archivebox_db(tmp_path): - urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) - - assert "https://example.com" in urls - assert "https://iana.org" in urls - - -def test_crawl_from_file_creates_snapshot(tmp_path, process, disable_extractors_dict): - """Test that crawl can create snapshots from a file of URLs.""" - os.chdir(tmp_path) - - # Write URLs to a file - urls_file = tmp_path / "urls.txt" - urls_file.write_text("https://example.com\n") - - subprocess.run( - ["archivebox", "crawl", "--no-wait", str(urls_file)], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - run_queued_crawls(tmp_path, disable_extractors_dict) - - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.first() - - # Should create at least one snapshot (the source file or the URL) - assert snapshot is not None, "Should create at least one snapshot" - - -def test_crawl_persists_input_urls_on_crawl(tmp_path, process, disable_extractors_dict): - """Test that crawl input URLs are stored on the Crawl record.""" - os.chdir(tmp_path) - - subprocess.run( - ["archivebox", "crawl", "--no-wait", "https://example.com"], - capture_output=True, - text=True, - env=disable_extractors_dict, - ) - - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").first() - - assert crawl is not None, "Crawl should be created for crawl input" - assert "https://example.com" in crawl.urls, "Crawl should persist input URLs" - - -class TestCrawlCLI: - """Test the CLI interface for crawl command.""" - - def test_cli_help(self, tmp_path, process): - """Test that --help works for crawl command.""" - os.chdir(tmp_path) - - result = subprocess.run( - ["archivebox", "crawl", "--help"], - capture_output=True, - text=True, - ) - - assert result.returncode == 0 - assert "create" in result.stdout - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) diff --git a/archivebox/tests/test_crawl_pause.py b/archivebox/tests/test_crawl_pause.py index 29382555..c73cd25a 100644 --- a/archivebox/tests/test_crawl_pause.py +++ b/archivebox/tests/test_crawl_pause.py @@ -1,251 +1,2 @@ -import os -import subprocess -import sys -import time - -import pytest -import requests - -from .conftest import ( - build_test_env, - create_admin_and_token, - get_crawl_runtime_state, - get_free_port, - init_archive, - start_server, - stop_server, - wait_for_http, - wait_for_snapshot_capture, -) - -pytestmark = pytest.mark.django_db(transaction=True) - - -def wait_for_crawl_snapshot_rows(cwd, crawl_id, timeout=45): - deadline = time.time() + timeout - latest_state = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - if latest_state["snapshots"]: - return latest_state - time.sleep(0.2) - raise AssertionError(f"timed out waiting for runner to create snapshots for crawl {crawl_id}: {latest_state}") - - -def wait_for_crawl_child_snapshots_paused_or_sealed(cwd, crawl_id, timeout=45): - deadline = time.time() + timeout - latest_state = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - snapshots = latest_state["snapshots"] - if snapshots and all(snapshot["status"] in {"paused", "sealed"} for snapshot in snapshots): - return latest_state - time.sleep(0.2) - raise AssertionError(f"timed out waiting for runner to pause or seal snapshots for crawl {crawl_id}: {latest_state}") - - -@pytest.mark.timeout(240) -def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(tmp_path, recursive_test_site): - os.chdir(tmp_path) - init_archive(tmp_path) - - port = get_free_port() - env = build_test_env(port, PLUGINS="wget", SAVE_WGET="True") - api_token = create_admin_and_token(tmp_path) - api_headers = { - "Host": f"api.archivebox.localhost:{port}", - "X-ArchiveBox-API-Key": api_token, - } - - try: - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - crawl_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/crawls/crawls", - headers=api_headers, - json={ - "urls": [recursive_test_site["root_url"]], - "max_depth": 0, - "tags": ["pause-resume-e2e"], - "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, - }, - timeout=10, - ) - assert crawl_response.status_code == 200, crawl_response.text - crawl_id = crawl_response.json()["id"] - wait_for_crawl_snapshot_rows(tmp_path, crawl_id) - - pause_response = requests.patch( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - json={"action": "pause"}, - timeout=10, - ) - assert pause_response.status_code == 200, pause_response.text - assert pause_response.json()["status"] == "paused" - - paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id) - assert paused_state["crawl_status"] == "paused" - assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"] - assert len(paused_state["snapshots"]) == 1 - snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed" - if snapshot_finished_before_pause: - assert any(result["status"] == "succeeded" for result in paused_state["results"]) - else: - assert paused_state["snapshots"][0]["status"] == "paused" - assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"] - - stop_server(tmp_path) - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - restarted_state = get_crawl_runtime_state(tmp_path, crawl_id) - assert restarted_state["crawl_status"] == "paused" - assert restarted_state["crawl_retry_at"] == restarted_state["retry_at_max"] - if snapshot_finished_before_pause: - assert restarted_state["snapshots"][0]["status"] == "sealed" - assert any(result["status"] == "succeeded" for result in restarted_state["results"]) - return - assert restarted_state["snapshots"][0]["status"] == "paused" - assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"] - assert not any(result["status"] == "succeeded" for result in restarted_state["results"]) - - resume_response = requests.patch( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - json={"action": "resume"}, - timeout=10, - ) - assert resume_response.status_code == 200, resume_response.text - assert resume_response.json()["status"] == "queued" - - captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) - assert "Root" in captured_text - assert "About" in captured_text - - final_state = get_crawl_runtime_state(tmp_path, crawl_id) - assert final_state["snapshots"][0]["status"] == "sealed" - wget_results = [result for result in final_state["results"] if result["plugin"] == "wget"] - assert wget_results - assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) - finally: - stop_server(tmp_path) - - -@pytest.mark.timeout(180) -def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(tmp_path, recursive_test_site): - os.chdir(tmp_path) - init_archive(tmp_path) - - port = get_free_port() - env = build_test_env(port, PLUGINS="wget", SAVE_WGET="True") - api_token = create_admin_and_token(tmp_path) - api_headers = { - "Host": f"api.archivebox.localhost:{port}", - "X-ArchiveBox-API-Key": api_token, - } - - try: - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - crawl_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/crawls/crawls", - headers=api_headers, - json={ - "urls": [recursive_test_site["root_url"]], - "max_depth": 0, - "tags": ["paused-index-e2e"], - "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, - }, - timeout=10, - ) - assert crawl_response.status_code == 200, crawl_response.text - crawl_id = crawl_response.json()["id"] - wait_for_crawl_snapshot_rows(tmp_path, crawl_id) - - pause_response = requests.patch( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - json={"action": "pause"}, - timeout=10, - ) - assert pause_response.status_code == 200, pause_response.text - assert pause_response.json()["status"] == "paused" - paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id) - snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed" - finally: - stop_server(tmp_path) - - if snapshot_finished_before_pause: - indexed_state = get_crawl_runtime_state(tmp_path, crawl_id) - assert indexed_state["crawl_status"] == "paused" - assert indexed_state["snapshots"][0]["status"] == "sealed" - return - - update_env = build_test_env( - port, - PLUGINS="search_backend_sqlite", - SEARCH_BACKEND_ENGINE="sqlite", - ) - update_process = subprocess.run( - [ - sys.executable, - "-m", - "archivebox", - "update", - "--index-only", - "--crawl-id", - crawl_id, - "--limit", - "1", - "--batch-size", - "1", - ], - cwd=tmp_path, - capture_output=True, - text=True, - env=update_env, - timeout=120, - ) - assert update_process.returncode == 0, update_process.stderr - - indexed_state = get_crawl_runtime_state(tmp_path, crawl_id) - assert indexed_state["crawl_status"] == "paused" - assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"] - assert indexed_state["snapshots"][0]["status"] == "paused" - assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"] - search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"] - assert search_results - assert all(result["status"] not in {"queued", "started", "paused"} for result in search_results) - - try: - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - still_paused_state = get_crawl_runtime_state(tmp_path, crawl_id) - assert still_paused_state["crawl_status"] == "paused" - assert still_paused_state["snapshots"][0]["status"] == "paused" - assert not any(result["plugin"] == "wget" and result["status"] == "succeeded" for result in still_paused_state["results"]) - - resume_response = requests.patch( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - json={"action": "resume"}, - timeout=10, - ) - assert resume_response.status_code == 200, resume_response.text - assert resume_response.json()["status"] == "queued" - - captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) - assert "Root" in captured_text - assert "About" in captured_text - - resumed_state = get_crawl_runtime_state(tmp_path, crawl_id) - assert resumed_state["snapshots"][0]["status"] == "sealed" - wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"] - assert wget_results - assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) - finally: - stop_server(tmp_path) +# test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume moved to test_api_v1_crawls_crawl_crawl_id.py. +# test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl moved to test_api_v1_crawls_crawl_crawl_id.py. diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index fdb27f5a..fea7d285 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -1,11 +1,9 @@ import asyncio -import json import sys from pathlib import Path import pytest from asgiref.sync import sync_to_async -from django.test import RequestFactory pytestmark = pytest.mark.django_db @@ -61,49 +59,6 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus(): @pytest.mark.django_db(transaction=True) -def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot - from archivebox.services.runner import CrawlRunner - - crawl = Crawl.objects.create( - urls="https://example.com", - max_depth=0, - config={"CRAWL_MAX_URLS": 5}, - created_by_id=get_or_create_system_user_pk(), - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.SEALED, - depth=0, - ) - parser_dir = Path(snapshot.output_dir) / "parse_html_urls" - parser_dir.mkdir(parents=True, exist_ok=True) - (parser_dir / "urls.jsonl").write_text( - "\n".join( - [ - json.dumps({"type": "Snapshot", "url": "https://example.com/child-a", "depth": 1}), - json.dumps({"type": "Snapshot", "url": "https://example.com/child-b", "depth": 1}), - "", - ], - ), - ) - - runner = CrawlRunner(crawl) - Crawl.objects.filter(id=crawl.id).update(max_depth=1) - payload = runner.load_snapshot_payload(str(snapshot.id)) - - asyncio.run(runner.enqueue_discovered_snapshots_from_outputs(payload)) - - child_snapshots = list(crawl.snapshot_set.filter(depth=1).order_by("url").values_list("url", "status")) - assert child_snapshots == [ - ("https://example.com/child-a", Snapshot.StatusChoices.QUEUED), - ("https://example.com/child-b", Snapshot.StatusChoices.QUEUED), - ] - - @pytest.mark.django_db(transaction=True) def test_snapshot_payload_uses_crawl_chrome_dirs_by_default(): from archivebox.base_models.models import get_or_create_system_user_pk @@ -723,80 +678,6 @@ def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot() @pytest.mark.django_db(transaction=True) -def test_run_snapshot_seals_descendant_when_crawl_max_size_is_reached(tmp_path): - from abx_dl.events import CrawlStartEvent, SnapshotEvent - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot - from archivebox.services.runner import CrawlRunner - - crawl = Crawl.objects.create( - urls="https://example.com", - config={ - "LIB_DIR": str(tmp_path / "lib"), - "PLUGINS": "__archivebox_test_no_plugins__", - "CHROME_BINARY": "", - "CRAWL_MAX_SIZE": 16, - }, - created_by_id=get_or_create_system_user_pk(), - ) - root = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - depth=0, - status=Snapshot.StatusChoices.SEALED, - ) - child = Snapshot.objects.create( - url="https://example.com/child", - crawl=crawl, - depth=1, - parent_snapshot=root, - status=Snapshot.StatusChoices.QUEUED, - ) - state_dir = Path(crawl.output_dir) / ".abx-dl" - state_dir.mkdir(parents=True, exist_ok=True) - (state_dir / "limits.json").write_text( - json.dumps( - { - "admitted_snapshot_ids": [str(root.id), str(child.id)], - "counted_event_ids": ["proc-1"], - "total_size": 32, - "stop_reason": "crawl_max_size", - }, - ), - encoding="utf-8", - ) - - runner = CrawlRunner(crawl) - runner.load_run_state() - - async def run_child_from_crawl_start() -> list[SnapshotEvent]: - async def on_crawl_start(_event: CrawlStartEvent) -> None: - await runner.run_snapshot(str(child.id)) - - runner.bus.on(CrawlStartEvent, on_crawl_start) - crawl_start = runner.bus.emit( - CrawlStartEvent( - url=root.url, - snapshot_id=str(root.id), - output_dir=str(crawl.output_dir), - event_timeout=30, - event_handler_timeout=30, - ), - ) - await crawl_start.now() - await crawl_start.event_results_list() - await runner.bus.wait_until_idle() - return await runner.bus.filter(SnapshotEvent, child_of=crawl_start, past=True) - - snapshot_events = asyncio.run(run_child_from_crawl_start()) - - child.refresh_from_db() - assert child.status == Snapshot.StatusChoices.SEALED - assert child.retry_at is None - assert snapshot_events == [] - - @pytest.mark.django_db(transaction=True) def test_run_pending_crawls_processes_queued_crawl_before_missing_binary_backlog(tmp_path): from archivebox.base_models.models import get_or_create_system_user_pk @@ -839,71 +720,6 @@ def test_run_pending_crawls_processes_queued_crawl_before_missing_binary_backlog @pytest.mark.django_db(transaction=True) -def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size(): - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot - from archivebox.services.snapshot_service import SnapshotService - from abx_dl.events import SnapshotCompletedEvent - from abx_dl.orchestrator import create_bus - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - config={"CRAWL_MAX_SIZE": 16}, - ) - root = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.STARTED, - ) - child = Snapshot.objects.create( - url="https://example.com/child", - crawl=crawl, - depth=1, - parent_snapshot_id=root.id, - status=Snapshot.StatusChoices.QUEUED, - ) - - state_dir = Path(crawl.output_dir) / ".abx-dl" - state_dir.mkdir(parents=True, exist_ok=True) - (state_dir / "limits.json").write_text( - json.dumps( - { - "admitted_snapshot_ids": [str(root.id), str(child.id)], - "counted_event_ids": ["proc-1"], - "total_size": 32, - "stop_reason": "crawl_max_size", - }, - ), - encoding="utf-8", - ) - - bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}") - service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: None) - try: - - async def emit_event() -> None: - await service.on_SnapshotCompletedEvent( - SnapshotCompletedEvent( - url=root.url, - snapshot_id=str(root.id), - output_dir=str(root.output_dir), - ), - ) - - asyncio.run(emit_event()) - finally: - asyncio.run(bus.wait_until_idle()) - asyncio.run(bus.destroy()) - - root.refresh_from_db() - child.refresh_from_db() - assert root.status == Snapshot.StatusChoices.SEALED - assert child.status == Snapshot.StatusChoices.SEALED - assert child.retry_at is None - - def test_sealed_crawl_does_not_create_discovered_snapshots(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -928,35 +744,7 @@ def test_sealed_crawl_does_not_create_discovered_snapshots(): assert crawl.snapshot_set.count() == 1 -def test_create_crawl_api_queues_crawl_without_spawning_runner(): - from django.contrib.auth import get_user_model - from archivebox.api.v1_crawls import CrawlCreateSchema, create_crawl - - user = get_user_model().objects.create_superuser( - username="runner-api-admin", - email="runner-api-admin@example.com", - password="testpassword", - ) - request = RequestFactory().post("/api/v1/crawls") - request.user = user - - crawl = create_crawl( - request, - CrawlCreateSchema( - urls=["https://example.com"], - max_depth=0, - tags=[], - tags_str="", - label="", - notes="", - config={}, - ), - ) - - assert str(crawl.id) - assert crawl.status == "queued" - assert crawl.retry_at is not None - assert crawl.snapshot_set.count() == 0 +# test_create_crawl_api_queues_crawl_without_spawning_runner moved to test_api_v1_crawls_crawls.py. def test_wait_for_snapshot_tasks_surfaces_already_failed_task(): diff --git a/archivebox/tests/test_crawl_service.py b/archivebox/tests/test_crawl_service.py index f44094d0..0783fa8e 100644 --- a/archivebox/tests/test_crawl_service.py +++ b/archivebox/tests/test_crawl_service.py @@ -1,28 +1,16 @@ -import os from pathlib import Path import pytest from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl -from archivebox.tests.conftest import run_archivebox_cmd_cwd +from archivebox.tests.conftest import run_archivebox_cmd from archivebox.tests.test_orm_helpers import use_archivebox_db -from .conftest import build_test_env, get_free_port, init_archive +from .conftest import cli_env, get_free_port, init_archive pytestmark = pytest.mark.django_db(transaction=True) -def _assert_command_ok(command: str, stdout: str, stderr: str, code: int) -> None: - assert code == 0, f"{command} failed with code {code}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" - - -def _latest_crawl_id(cwd: Path) -> str: - with use_archivebox_db(cwd): - crawl_id = Crawl.objects.order_by("-created_at").values_list("id", flat=True).first() - assert crawl_id is not None - return str(crawl_id) - - def _crawl_state(cwd: Path, crawl_id: str) -> dict[str, object]: with use_archivebox_db(cwd): crawl = Crawl.objects.select_related("created_by").get(id=crawl_id) @@ -49,11 +37,10 @@ def _crawl_state(cwd: Path, crawl_id: str) -> dict[str, object]: @pytest.mark.timeout(240) def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env( + env = cli_env( port, PLUGINS="wget,parse_html_urls", SAVE_WGET="True", @@ -64,7 +51,7 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p about_url = recursive_test_site["child_urls"][0] contact_url = recursive_test_site["child_urls"][2] - add_stdout, add_stderr, add_code = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( [ "add", "--bg", @@ -81,9 +68,13 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p env=env, timeout=120, ) - _assert_command_ok("archivebox add --bg", add_stdout, add_stderr, add_code) + add_stdout, add_stderr, add_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert add_code == 0, f"archivebox add --bg failed with code {add_code}\nSTDOUT:\n{add_stdout}\nSTDERR:\n{add_stderr}" - crawl_id = _latest_crawl_id(tmp_path) + with use_archivebox_db(tmp_path): + latest_crawl_id = Crawl.objects.order_by("-created_at").values_list("id", flat=True).first() + assert latest_crawl_id is not None + crawl_id = str(latest_crawl_id) queued_state = _crawl_state(tmp_path, crawl_id) assert queued_state["status"] == Crawl.StatusChoices.QUEUED assert queued_state["retry_at"] is not None @@ -91,13 +82,14 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p assert queued_state["config"]["URL_DENYLIST"] == "/contact$" assert queued_state["snapshots"] == [] - run_stdout, run_stderr, run_code = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["run", "--crawl-id", crawl_id], cwd=tmp_path, env=env, timeout=240, ) - _assert_command_ok("archivebox run --crawl-id", run_stdout, run_stderr, run_code) + run_stdout, run_stderr, run_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert run_code == 0, f"archivebox run --crawl-id failed with code {run_code}\nSTDOUT:\n{run_stdout}\nSTDERR:\n{run_stderr}" state = _crawl_state(tmp_path, crawl_id) snapshots = state["snapshots"] diff --git a/archivebox/tests/test_frozen_crawl_config.py b/archivebox/tests/test_frozen_crawl_config.py index 68e98b15..278967bf 100644 --- a/archivebox/tests/test_frozen_crawl_config.py +++ b/archivebox/tests/test_frozen_crawl_config.py @@ -2,7 +2,6 @@ import time from types import SimpleNamespace import pytest -from django.test import RequestFactory from django.utils import timezone pytestmark = pytest.mark.django_db(transaction=True) @@ -266,43 +265,7 @@ def test_crawl_config_projections_stay_under_hot_path_budget(): assert average_seconds < max_average_seconds, f"{name} averaged {average_seconds * 1000:.3f}ms" -def test_api_create_and_cli_add_store_full_frozen_config(archivebox_db): - from archivebox.api.v1_crawls import CrawlCreateSchema, CrawlSchema, create_crawl - from archivebox.cli.archivebox_add import add - from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED - - user = _user("frozen-config-api-admin") - request = RequestFactory().post("/api/v1/crawls") - request.user = user - - api_crawl = create_crawl( - request, - CrawlCreateSchema( - urls=["https://example.com/api"], - max_depth=0, - tags=[], - tags_str="", - label="API frozen config", - notes="", - config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 33, "SECRET_KEY": "must-not-freeze", "PUBLIC_ADD_VIEW": True}, - ), - ) - assert "CHECK_SSL_VALIDITY" in api_crawl.config - assert api_crawl.config["TIMEOUT"] == 33 - assert api_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET - assert "SECRET_KEY" not in api_crawl.config - assert "PUBLIC_ADD_VIEW" not in api_crawl.config - assert CrawlSchema.resolve_config(api_crawl)["TWOCAPTCHA_API_KEY"] == SENSITIVE_CONFIG_VALUE_REDACTED - - cli_crawl, _snapshots = add( - "https://example.com/cli", - bg=True, - created_by_id=user.pk, - config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 44}, - ) - assert "CHECK_SSL_VALIDITY" in cli_crawl.config - assert cli_crawl.config["TIMEOUT"] == 44 - assert cli_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET +# test_api_create_and_cli_add_store_full_frozen_config moved to test_api_v1_workflow_frozen_crawl_config_sources.py. def test_schedule_enqueue_refreezes_using_current_template_persona_defaults(archivebox_db): diff --git a/archivebox/tests/test_machine_service.py b/archivebox/tests/test_machine_service.py index d95abca3..cb5d8db9 100644 --- a/archivebox/tests/test_machine_service.py +++ b/archivebox/tests/test_machine_service.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest from archivebox.machine.models import Binary, Machine, Process -from archivebox.tests.conftest import run_archivebox_cmd_cwd +from archivebox.tests.conftest import run_archivebox_cmd from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) @@ -47,12 +47,13 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali _write_tool_shim(provider_bin_dir, "lit", "2.5.9") _link_real_tool(provider_bin_dir, "node") - stdout, stderr, returncode = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["install", "--binproviders=env", "liteparse"], cwd=initialized_archive, timeout=120, env=_runtime_env(initialized_archive, bootstrap_bin_dir), ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stdout + stderr assert "liteparse" in stdout @@ -114,12 +115,13 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali print("MACHINE_SERVICE_E2E_DONE") """, ) - shell_stdout, shell_stderr, shell_code = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["shell", "-c", machine_event_script], cwd=initialized_archive, timeout=60, env=_runtime_env(initialized_archive, bootstrap_bin_dir), ) + shell_stdout, shell_stderr, shell_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert shell_code == 0, shell_stdout + shell_stderr assert "MACHINE_SERVICE_E2E_DONE" in shell_stdout @@ -129,24 +131,26 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali assert machine.config["LITEPARSE_BINARY"] == str(installed_liteparse_path) assert machine.config["LITEPARSE_BINARY"] != "/tmp/user-config-must-not-persist" - version_stdout, version_stderr, version_code = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["version"], cwd=initialized_archive, timeout=60, env=_runtime_env(initialized_archive, bootstrap_bin_dir), ) + version_stdout, version_stderr, version_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert version_code == 0, version_stderr assert "lit" in version_stdout installed_liteparse_path.unlink() (initialized_archive / "lib" / "bin" / installed_liteparse_path.name).unlink(missing_ok=True) - cleanup_stdout, cleanup_stderr, cleanup_code = run_archivebox_cmd_cwd( + _cmd_result = run_archivebox_cmd( ["version"], cwd=initialized_archive, timeout=60, env=_runtime_env(initialized_archive, bootstrap_bin_dir), ) + cleanup_stdout, cleanup_stderr, cleanup_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert cleanup_code == 0, cleanup_stdout + cleanup_stderr with use_archivebox_db(initialized_archive): diff --git a/archivebox/tests/test_migrations_04_to_09.py b/archivebox/tests/test_migrations_04_to_09.py index 0a173a74..cada3774 100644 --- a/archivebox/tests/test_migrations_04_to_09.py +++ b/archivebox/tests/test_migrations_04_to_09.py @@ -14,7 +14,7 @@ import pytest from .migrations_helpers import ( SCHEMA_0_4, create_data_dir_structure, - run_archivebox, + run_archivebox_migration_cmd, seed_0_4_data, verify_snapshot_count, verify_snapshot_urls, @@ -46,7 +46,7 @@ def test_migration_preserves_snapshot_count(archive_04): work_dir, db_path, original_data = archive_04 expected_count = len(original_data["snapshots"]) - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_count(db_path, expected_count) @@ -58,7 +58,7 @@ def test_migration_preserves_snapshot_urls(archive_04): work_dir, db_path, original_data = archive_04 expected_urls = [s["url"] for s in original_data["snapshots"]] - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_urls(db_path, expected_urls) @@ -69,7 +69,7 @@ def test_migration_converts_string_tags_to_model(archive_04): """Migration should convert comma-separated tags to Tag model instances.""" work_dir, db_path, original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Collect unique tags from original data @@ -88,7 +88,7 @@ def test_migration_preserves_snapshot_titles(archive_04): """Migration should preserve all snapshot titles.""" work_dir, db_path, original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -105,10 +105,10 @@ def test_status_works_after_migration(archive_04): """Status command should work after migration.""" work_dir, _db_path, _original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["status"]) + result = run_archivebox_migration_cmd(work_dir, ["status"]) assert result.returncode == 0, f"Status failed after migration: {result.stderr}" @@ -116,10 +116,10 @@ def test_list_works_after_migration(archive_04): """List command should work and show ALL migrated snapshots.""" work_dir, _db_path, original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["list"]) + result = run_archivebox_migration_cmd(work_dir, ["list"]) assert result.returncode == 0, f"List failed after migration: {result.stderr}" # Verify ALL snapshots appear in output @@ -133,13 +133,13 @@ def test_add_works_after_migration(archive_04): """Adding new URLs should work after migration from 0.4.x.""" work_dir, db_path, _original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Try to add a new URL after migration - result = run_archivebox(work_dir, ["add", "--index-only", "https://example.com/new-page"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["add", "--index-only", "https://example.com/new-page"], timeout=45) assert result.returncode == 0, f"Add failed after migration: {result.stderr}" - result = run_archivebox(work_dir, ["run"], timeout=90) + result = run_archivebox_migration_cmd(work_dir, ["run"], timeout=90) assert result.returncode == 0, f"Run failed after migration: {result.stderr}" # Verify add queued the new crawl after migration. @@ -156,7 +156,7 @@ def test_new_schema_elements_created(archive_04): """Migration should create new 0.9.x schema elements.""" work_dir, db_path, _original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -175,7 +175,7 @@ def test_snapshots_have_new_fields(archive_04): """Migrated snapshots should have new 0.9.x fields.""" work_dir, db_path, _original_data = archive_04 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) diff --git a/archivebox/tests/test_migrations_07_to_09.py b/archivebox/tests/test_migrations_07_to_09.py index 07dd7b27..f3afa70f 100644 --- a/archivebox/tests/test_migrations_07_to_09.py +++ b/archivebox/tests/test_migrations_07_to_09.py @@ -15,7 +15,7 @@ import pytest from .migrations_helpers import ( SCHEMA_0_7, create_data_dir_structure, - run_archivebox, + run_archivebox_migration_cmd, seed_0_7_data, verify_all_snapshots_in_output, verify_archiveresult_count, @@ -51,7 +51,7 @@ def test_migration_preserves_snapshot_count(archive_07): work_dir, db_path, original_data = archive_07 expected_count = len(original_data["snapshots"]) - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_count(db_path, expected_count) @@ -63,7 +63,7 @@ def test_migration_preserves_snapshot_urls(archive_07): work_dir, db_path, original_data = archive_07 expected_urls = [s["url"] for s in original_data["snapshots"]] - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_urls(db_path, expected_urls) @@ -75,7 +75,7 @@ def test_migration_preserves_snapshot_titles(archive_07): work_dir, db_path, original_data = archive_07 expected_titles = {s["url"]: s["title"] for s in original_data["snapshots"]} - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_titles(db_path, expected_titles) @@ -87,7 +87,7 @@ def test_migration_preserves_tags(archive_07): work_dir, db_path, original_data = archive_07 expected_count = len(original_data["tags"]) - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_tag_count(db_path, expected_count) @@ -103,7 +103,7 @@ def test_migration_preserves_archiveresults(archive_07): key = (result["extractor"], result["status"]) expected_counts[key] = expected_counts.get(key, 0) + 1 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_archiveresult_count(db_path, expected_count) @@ -128,7 +128,7 @@ def test_migration_preserves_foreign_keys(archive_07): """Migration should maintain foreign key relationships.""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_foreign_keys(db_path) @@ -156,7 +156,7 @@ def test_migration_preserves_legacy_timestamp_meanings(archive_07): conn.commit() conn.close() - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -179,10 +179,10 @@ def test_update_saves_migrated_snapshots_without_foreign_key_errors(archive_07): """Migrated 0.7.x snapshots should be writable through the current ORM.""" work_dir, _db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["update"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60) output = result.stdout + result.stderr assert result.returncode == 0, f"Update failed after migration: {result.stderr}" assert "FOREIGN KEY constraint failed" not in output @@ -193,10 +193,10 @@ def test_status_works_after_migration(archive_07): """Status command should work after migration.""" work_dir, _db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["status"]) + result = run_archivebox_migration_cmd(work_dir, ["status"]) assert result.returncode == 0, f"Status failed after migration: {result.stderr}" @@ -204,10 +204,10 @@ def test_search_works_after_migration(archive_07): """Search command should find ALL migrated snapshots.""" work_dir, _db_path, original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["search"]) + result = run_archivebox_migration_cmd(work_dir, ["search"]) assert result.returncode == 0, f"Search failed after migration: {result.stderr}" # Verify ALL snapshots appear in output @@ -220,10 +220,10 @@ def test_list_works_after_migration(archive_07): """List command should work and show ALL migrated data.""" work_dir, _db_path, original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["snapshot", "list"]) + result = run_archivebox_migration_cmd(work_dir, ["snapshot", "list"]) assert result.returncode == 0, f"List failed after migration: {result.stderr}" # Verify ALL snapshots appear in output @@ -236,7 +236,7 @@ def test_new_schema_elements_created_after_migration(archive_07): """Migration should create new 0.9.x schema elements (crawls_crawl, etc.).""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -255,7 +255,7 @@ def test_snapshots_have_new_fields_after_migration(archive_07): """Migrated snapshots should have new 0.9.x fields (status, depth, etc.).""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -276,7 +276,7 @@ def test_add_works_after_migration(archive_07): """Adding new URLs should work after migration from 0.7.x.""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Verify that init created the crawls_crawl table before proceeding @@ -288,7 +288,7 @@ def test_add_works_after_migration(archive_07): assert table_exists, f"Init failed to create crawls_crawl table. Init stderr: {result.stderr[-500:]}" # Try to add a new URL after migration (use --index-only for speed) - result = run_archivebox(work_dir, ["add", "--index-only", "https://example.com/new-page"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["add", "--index-only", "https://example.com/new-page"], timeout=45) assert result.returncode == 0, f"Add failed after migration: {result.stderr}" # Verify a Crawl was created for the new URL @@ -305,7 +305,7 @@ def test_archiveresult_status_preserved_after_migration(archive_07): """Migration should preserve archive result status values.""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -326,10 +326,10 @@ def test_version_works_after_migration(archive_07): """Version command should work after migration.""" work_dir, _db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["version"]) + result = run_archivebox_migration_cmd(work_dir, ["version"]) assert result.returncode == 0, f"Version failed after migration: {result.stderr}" # Should show version info @@ -341,10 +341,10 @@ def test_help_works_after_migration(archive_07): """Help command should work after migration.""" work_dir, _db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["help"]) + result = run_archivebox_migration_cmd(work_dir, ["help"]) assert result.returncode == 0, f"Help failed after migration: {result.stderr}" # Should show available commands @@ -356,7 +356,7 @@ def test_no_duplicate_snapshots_after_migration(archive_07): """Migration should not create duplicate snapshots.""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Check for duplicate URLs @@ -376,7 +376,7 @@ def test_no_orphaned_archiveresults_after_migration(archive_07): """Migration should not leave orphaned ArchiveResults.""" work_dir, db_path, _original_data = archive_07 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_foreign_keys(db_path) @@ -388,7 +388,7 @@ def test_timestamps_preserved_after_migration(archive_07): work_dir, db_path, original_data = archive_07 original_timestamps = {s["url"]: s["timestamp"] for s in original_data["snapshots"]} - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -412,7 +412,7 @@ def test_tag_associations_preserved_after_migration(archive_07): original_count = cursor.fetchone()[0] conn.close() - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Count tag associations after migration diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py index cfb0f270..f045be55 100644 --- a/archivebox/tests/test_migrations_08_to_09.py +++ b/archivebox/tests/test_migrations_08_to_09.py @@ -21,7 +21,7 @@ from .migrations_helpers import ( SCHEMA_0_8, seed_0_8_data, seed_0_7_data, - run_archivebox, + run_archivebox_migration_cmd, create_data_dir_structure, verify_snapshot_count, verify_snapshot_urls, @@ -56,7 +56,7 @@ def test_migration_preserves_snapshot_count(migration_08_data): work_dir, db_path, original_data = migration_08_data expected_count = len(original_data["snapshots"]) - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_count(db_path, expected_count) @@ -68,7 +68,7 @@ def test_migration_preserves_snapshot_urls(migration_08_data): work_dir, db_path, original_data = migration_08_data expected_urls = [s["url"] for s in original_data["snapshots"]] - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_urls(db_path, expected_urls) @@ -78,7 +78,7 @@ def test_migration_preserves_snapshot_urls(migration_08_data): def test_migration_preserves_crawls(migration_08_data): """Migration should preserve all Crawl records and create default crawl if needed.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Count snapshots with NULL crawl_id in original data @@ -96,7 +96,7 @@ def test_migration_preserves_crawls(migration_08_data): def test_migration_preserves_snapshot_crawl_links(migration_08_data): """Migration should preserve snapshot-to-crawl relationships and assign default crawl to orphans.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -121,7 +121,7 @@ def test_migration_preserves_snapshot_crawl_links(migration_08_data): def test_migration_preserves_tags(migration_08_data): """Migration should preserve all tags.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_tag_count(db_path, len(original_data["tags"])) @@ -138,7 +138,7 @@ def test_migration_preserves_archiveresults(migration_08_data): key = (result["extractor"], status) expected_counts[key] = expected_counts.get(key, 0) + 1 - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_archiveresult_count(db_path, expected_count) @@ -162,7 +162,7 @@ def test_migration_preserves_archiveresults(migration_08_data): def test_migration_preserves_archiveresult_status(migration_08_data): """Migration should preserve archive result status values.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -182,20 +182,20 @@ def test_migration_preserves_archiveresult_status(migration_08_data): def test_status_works_after_migration(migration_08_data): """Status command should work after migration.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["status"]) + result = run_archivebox_migration_cmd(work_dir, ["status"]) assert result.returncode == 0, f"Status failed after migration: {result.stderr}" def test_list_works_after_migration(migration_08_data): """List command should work and show ALL migrated data.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["snapshot", "list"]) + result = run_archivebox_migration_cmd(work_dir, ["snapshot", "list"]) assert result.returncode == 0, f"List failed after migration: {result.stderr}" # Verify ALL snapshots appear in output @@ -207,10 +207,10 @@ def test_list_works_after_migration(migration_08_data): def test_search_works_after_migration(migration_08_data): """Search command should find ALL migrated snapshots.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["search"]) + result = run_archivebox_migration_cmd(work_dir, ["search"]) assert result.returncode == 0, f"Search failed after migration: {result.stderr}" # Verify ALL snapshots appear in output @@ -224,7 +224,7 @@ def test_migration_preserves_snapshot_titles(migration_08_data): work_dir, db_path, original_data = migration_08_data expected_titles = {s["url"]: s["title"] for s in original_data["snapshots"]} - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_snapshot_titles(db_path, expected_titles) @@ -234,7 +234,7 @@ def test_migration_preserves_snapshot_titles(migration_08_data): def test_migration_preserves_foreign_keys(migration_08_data): """Migration should maintain foreign key relationships.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_foreign_keys(db_path) @@ -264,7 +264,7 @@ def test_migration_preserves_08_timestamp_meanings(migration_08_data): conn.commit() conn.close() - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -296,7 +296,7 @@ def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(migration_08_ conn.commit() conn.close() - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -310,7 +310,7 @@ def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(migration_08_ assert hyphenated_crawls == 0 assert hyphenated_snapshot_refs == 0 - result = run_archivebox(work_dir, ["update"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60) output = result.stdout + result.stderr assert result.returncode == 0, f"Update failed after migration: {result.stderr}" assert "FOREIGN KEY constraint failed" not in output @@ -319,7 +319,7 @@ def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(migration_08_ def test_migration_removes_seed_id_column(migration_08_data): """Migration should remove seed_id column from archivebox.crawls.crawl.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -334,7 +334,7 @@ def test_migration_removes_seed_id_column(migration_08_data): def test_migration_removes_seed_table(migration_08_data): """Migration should remove crawls_seed table.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -349,7 +349,7 @@ def test_migration_removes_seed_table(migration_08_data): def test_add_works_after_migration(migration_08_data): """Adding new URLs should work after migration from 0.8.x.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) # Check that init actually ran and applied migrations assert "Applying" in result.stdout + result.stderr, ( f"Init did not apply migrations. stdout: {result.stdout[:500]}, stderr: {result.stderr[:500]}" @@ -364,7 +364,7 @@ def test_add_works_after_migration(migration_08_data): conn.close() # Try to add a new URL after migration (use --index-only for speed) - result = run_archivebox(work_dir, ["add", "--index-only", "https://example.com/new-page"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["add", "--index-only", "https://example.com/new-page"], timeout=45) assert result.returncode == 0, f"Add failed after migration: {result.stderr}" # Verify a new Crawl was created @@ -380,10 +380,10 @@ def test_add_works_after_migration(migration_08_data): def test_version_works_after_migration(migration_08_data): """Version command should work after migration.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["version"]) + result = run_archivebox_migration_cmd(work_dir, ["version"]) assert result.returncode == 0, f"Version failed after migration: {result.stderr}" # Should show version info @@ -394,7 +394,7 @@ def test_version_works_after_migration(migration_08_data): def test_migration_creates_process_records(migration_08_data): """Migration should create Process records for all ArchiveResults.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Verify Process records created @@ -406,7 +406,7 @@ def test_migration_creates_process_records(migration_08_data): def test_migration_creates_binary_records(migration_08_data): """Migration should create Binary records from cmd_version data.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -426,7 +426,7 @@ def test_migration_creates_binary_records(migration_08_data): def test_migration_preserves_cmd_data(migration_08_data): """Migration should preserve cmd data in Process.cmd field.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -446,7 +446,7 @@ def test_migration_preserves_cmd_data(migration_08_data): def test_no_duplicate_snapshots_after_migration(migration_08_data): """Migration should not create duplicate snapshots.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Check for duplicate URLs @@ -465,7 +465,7 @@ def test_no_duplicate_snapshots_after_migration(migration_08_data): def test_no_orphaned_archiveresults_after_migration(migration_08_data): """Migration should not leave orphaned ArchiveResults.""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" ok, msg = verify_foreign_keys(db_path) @@ -477,7 +477,7 @@ def test_timestamps_preserved_after_migration(migration_08_data): work_dir, db_path, original_data = migration_08_data original_timestamps = {s["url"]: s["timestamp"] for s in original_data["snapshots"]} - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -493,7 +493,7 @@ def test_timestamps_preserved_after_migration(migration_08_data): def test_crawl_data_preserved_after_migration(migration_08_data): """Migration should preserve crawl metadata (urls, label, status).""" work_dir, db_path, original_data = migration_08_data - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -520,7 +520,7 @@ def test_tag_associations_preserved_after_migration(migration_08_data): original_count = cursor.fetchone()[0] conn.close() - result = run_archivebox(work_dir, ["init"], timeout=45) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" # Count tag associations after migration @@ -548,9 +548,9 @@ def test_update_migrates_db_snapshot_when_legacy_index_missing(tmp_path): snapshot_dir.mkdir(parents=True, exist_ok=True) (snapshot_dir / "screenshot.png").write_text("existing-db-snapshot") - result = run_archivebox(work_dir, ["init"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=60) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["update"], timeout=120) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=120) assert result.returncode == 0, f"Update failed: {result.stderr}" migrated_files = list((work_dir / "archive" / "users").glob("*/snapshots/*/*/*/screenshot.png")) @@ -577,9 +577,9 @@ def test_update_recovers_orphan_with_corrupt_index_from_archive_org_url(tmp_path (snapshot_dir / "archive.org.txt").write_text(f"https://web.archive.org/web/20170531210128/{original_url}\n") (snapshot_dir / "output.pdf").write_text("orphan-output") - result = run_archivebox(work_dir, ["init"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=60) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["update"], timeout=120) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=120) assert result.returncode == 0, f"Update failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -620,9 +620,9 @@ def test_update_preserves_legacy_folder_timestamp_over_index_float_variant(tmp_p ) (snapshot_dir / "output.html").write_text("folder timestamp output") - result = run_archivebox(work_dir, ["init"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=60) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["update"], timeout=120) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=120) assert result.returncode == 0, f"Update failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -662,9 +662,9 @@ def test_update_preserves_distinct_legacy_dirs_with_integer_and_float_timestamps ) (snapshot_dir / f"{payload}.txt").write_text(payload) - result = run_archivebox(work_dir, ["init"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=60) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(work_dir, ["update"], timeout=120) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=120) assert result.returncode == 0, f"Update failed: {result.stderr}" conn = sqlite3.connect(str(db_path)) @@ -762,7 +762,7 @@ def test_archiveresult_files_preserved_after_migration(tmp_path): print(f"[*] Sample files found: {len(sample_paths_before)}") # Run init to trigger migration - result = run_archivebox(work_dir, ["init"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=60) assert result.returncode == 0, f"Init (migration) failed: {result.stderr}" # Count archive directories and files AFTER migration @@ -794,7 +794,7 @@ def test_archiveresult_files_preserved_after_migration(tmp_path): # Run update to trigger filesystem reorganization print("\n[*] Running archivebox update to reorganize filesystem...") - result = run_archivebox(work_dir, ["update"], timeout=120) + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=120) assert result.returncode == 0, f"Update failed: {result.stderr}" # Check new filesystem structure diff --git a/archivebox/tests/test_migrations_fresh.py b/archivebox/tests/test_migrations_fresh.py index e1cbd3a9..d3cacdda 100644 --- a/archivebox/tests/test_migrations_fresh.py +++ b/archivebox/tests/test_migrations_fresh.py @@ -11,16 +11,16 @@ from django.db.migrations.recorder import MigrationRecorder from archivebox.core.models import ArchiveResult, Snapshot, Tag from archivebox.crawls.models import Crawl from archivebox.tests.test_orm_helpers import use_archivebox_db -from archivebox.tests.conftest import run_queued_crawls +from archivebox.tests.conftest import run_queued_crawls, cli_env -from .migrations_helpers import run_archivebox +from .migrations_helpers import run_archivebox_migration_cmd pytestmark = pytest.mark.django_db(transaction=True) def test_init_creates_database(tmp_path): """Fresh init should create database and directories.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" # Verify database was created @@ -31,38 +31,40 @@ def test_init_creates_database(tmp_path): def test_status_after_init(tmp_path): """Status command should work after init.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(tmp_path, ["status"]) + result = run_archivebox_migration_cmd(tmp_path, ["status"]) assert result.returncode == 0, f"Status failed: {result.stderr}" -def test_add_url_after_init(tmp_path, disable_extractors_dict): +def test_add_url_after_init(tmp_path): """Should be able to add URLs after init with --index-only.""" - result = run_archivebox(tmp_path, ["init"]) + env = cli_env(disable_extractors=True) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" # Add a URL with --index-only for speed - result = run_archivebox(tmp_path, ["add", "--index-only", "https://example.com"]) + result = run_archivebox_migration_cmd(tmp_path, ["add", "--index-only", "https://example.com"]) assert result.returncode == 0, f"Add command failed: {result.stderr}" - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(tmp_path, env) with use_archivebox_db(tmp_path): assert Crawl.objects.count() >= 1, "No Crawl was created" assert Snapshot.objects.count() >= 1, "No Snapshot was created" -def test_list_after_add(tmp_path, disable_extractors_dict): +def test_list_after_add(tmp_path): """List command should show added snapshots.""" - result = run_archivebox(tmp_path, ["init"]) + env = cli_env(disable_extractors=True) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(tmp_path, ["add", "--index-only", "https://example.com"]) + result = run_archivebox_migration_cmd(tmp_path, ["add", "--index-only", "https://example.com"]) assert result.returncode == 0, f"Add failed: {result.stderr}" - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(tmp_path, env) - result = run_archivebox(tmp_path, ["list"]) + result = run_archivebox_migration_cmd(tmp_path, ["list"]) assert result.returncode == 0, f"List failed: {result.stderr}" # Verify the URL appears in output @@ -72,7 +74,7 @@ def test_list_after_add(tmp_path, disable_extractors_dict): def test_migrations_table_populated(tmp_path): """Django migrations table should be populated after init.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" with use_archivebox_db(tmp_path): @@ -84,7 +86,7 @@ def test_migrations_table_populated(tmp_path): def test_core_migrations_applied(tmp_path): """Core app migrations should be applied.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" with use_archivebox_db(tmp_path): @@ -97,7 +99,7 @@ def test_core_migrations_applied(tmp_path): def test_snapshot_table_has_required_columns(tmp_path): """Snapshot table should have all required columns.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" columns = {field.column for field in Snapshot._meta.local_fields} @@ -109,7 +111,7 @@ def test_snapshot_table_has_required_columns(tmp_path): def test_archiveresult_table_has_required_columns(tmp_path): """ArchiveResult table should have all required columns.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" columns = {field.column for field in ArchiveResult._meta.local_fields} @@ -121,7 +123,7 @@ def test_archiveresult_table_has_required_columns(tmp_path): def test_tag_table_has_required_columns(tmp_path): """Tag table should have all required columns.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" columns = {field.column for field in Tag._meta.local_fields} @@ -133,7 +135,7 @@ def test_tag_table_has_required_columns(tmp_path): def test_crawl_table_has_required_columns(tmp_path): """Crawl table should have all required columns.""" - result = run_archivebox(tmp_path, ["init"]) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" columns = {field.column for field in Crawl._meta.local_fields} @@ -146,18 +148,19 @@ def test_crawl_table_has_required_columns(tmp_path): assert "seed_id" not in columns, "seed_id column should not exist in 0.9.x" -def test_add_urls_separately(tmp_path, disable_extractors_dict): +def test_add_urls_separately(tmp_path): """Should be able to add multiple URLs one at a time.""" - result = run_archivebox(tmp_path, ["init"]) + env = cli_env(disable_extractors=True) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" # Add URLs one at a time - result = run_archivebox(tmp_path, ["add", "--index-only", "https://example.com"]) + result = run_archivebox_migration_cmd(tmp_path, ["add", "--index-only", "https://example.com"]) assert result.returncode == 0, f"Add 1 failed: {result.stderr}" - result = run_archivebox(tmp_path, ["add", "--index-only", "https://example.org"]) + result = run_archivebox_migration_cmd(tmp_path, ["add", "--index-only", "https://example.org"]) assert result.returncode == 0, f"Add 2 failed: {result.stderr}" - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(tmp_path, env) with use_archivebox_db(tmp_path): snapshot_count = Snapshot.objects.count() @@ -166,14 +169,15 @@ def test_add_urls_separately(tmp_path, disable_extractors_dict): assert crawl_count == 2, f"Expected 2 Crawls, got {crawl_count}" -def test_snapshots_linked_to_crawls(tmp_path, disable_extractors_dict): +def test_snapshots_linked_to_crawls(tmp_path): """Each snapshot should be linked to a crawl.""" - result = run_archivebox(tmp_path, ["init"]) + env = cli_env(disable_extractors=True) + result = run_archivebox_migration_cmd(tmp_path, ["init"]) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox(tmp_path, ["add", "--index-only", "https://example.com"]) + result = run_archivebox_migration_cmd(tmp_path, ["add", "--index-only", "https://example.com"]) assert result.returncode == 0, f"Add failed: {result.stderr}" - run_queued_crawls(tmp_path, disable_extractors_dict) + run_queued_crawls(tmp_path, env) with use_archivebox_db(tmp_path): row = Snapshot.objects.filter(url="https://example.com").values_list("crawl_id", flat=True).first() diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 657f4248..acd121f7 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -12,6 +12,7 @@ import pytest from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl from archivebox.machine.models import Process +from archivebox.tests.conftest import run_archivebox_cmd, cli_env from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) @@ -39,24 +40,26 @@ def stop_process(proc): def run_add_until(args, env, condition, timeout=120): - proc = subprocess.Popen( - args, + assert args[0] == "archivebox" + proc = run_archivebox_cmd( + args[1:], + cwd=Path.cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, env=env, + wait=False, ) assert wait_for_db_condition(timeout=timeout, condition=condition), f"Timed out waiting for condition while running: {' '.join(args)}" return stop_process(proc) -def test_background_hooks_dont_block_parser_extractors(tmp_path, process, recursive_test_site): +def test_background_hooks_dont_block_parser_extractors(tmp_path, initialized_archive, recursive_test_site): """Test that background hooks (.bg.) don't block other extractors from running.""" - os.chdir(tmp_path) - # Verify init succeeded - assert process.returncode == 0, f"archivebox init failed: {process.stderr}" + # Verify the initialized_archive fixture prepared the expected data dir. + assert initialized_archive == tmp_path + assert (initialized_archive / "index.sqlite3").exists() # Enable only parser extractors and background hooks for this test env = os.environ.copy() @@ -80,12 +83,13 @@ def test_background_hooks_dont_block_parser_extractors(tmp_path, process, recurs }, ) - proc = subprocess.Popen( - ["archivebox", "add", "--depth=1", "--plugins=favicon,parse_html_urls", recursive_test_site["root_url"]], + proc = run_archivebox_cmd( + ["add", "--depth=1", "--plugins=favicon,parse_html_urls", recursive_test_site["root_url"]], + cwd=tmp_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, env=env, + wait=False, ) assert wait_for_db_condition( @@ -133,9 +137,8 @@ def test_background_hooks_dont_block_parser_extractors(tmp_path, process, recurs ) -def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process, recursive_test_site): +def test_parser_extractors_emit_snapshot_jsonl(tmp_path, initialized_archive, recursive_test_site): """Test that parser extractors emit Snapshot JSONL to stdout.""" - os.chdir(tmp_path) env = os.environ.copy() env.update( @@ -158,10 +161,8 @@ def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process, recursive_test }, ) - result = subprocess.run( - ["archivebox", "add", "--depth=0", "--plugins=wget,parse_html_urls", recursive_test_site["root_url"]], - capture_output=True, - text=True, + result = run_archivebox_cmd( + ["add", "--depth=0", "--plugins=wget,parse_html_urls", recursive_test_site["root_url"]], env=env, timeout=60, ) @@ -196,9 +197,8 @@ def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process, recursive_test assert all(record.get("type") == "Snapshot" for record in records), f"Expected Snapshot JSONL records, got: {records}" -def test_recursive_crawl_creates_child_snapshots(tmp_path, process, recursive_test_site): +def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive, recursive_test_site): """Test that recursive crawling creates child snapshots with proper depth and parent_snapshot_id.""" - os.chdir(tmp_path) env = os.environ.copy() env.update( @@ -268,11 +268,11 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, process, recursive_te assert parent_id == root_id, f"Child snapshot {child_url} should have parent_snapshot_id={root_id}, got {parent_id}" -def test_recursive_crawl_respects_depth_limit(tmp_path, process, disable_extractors_dict, recursive_test_site): +def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, recursive_test_site): """Test that recursive crawling stops at max_depth.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - env = disable_extractors_dict.copy() + env = env.copy() env["URL_ALLOWLIST"] = r"127\.0\.0\.1[:/].*" stdout, stderr = run_add_until( @@ -304,63 +304,8 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, process, disable_extract assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}" -def test_recursive_crawl_respects_max_urls(tmp_path, process, disable_extractors_dict, recursive_test_site): - """Test that recursive discovery stops creating snapshots at max_urls.""" - os.chdir(tmp_path) - - env = disable_extractors_dict.copy() - env.update( - { - "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*", - "SAVE_WGET": "true", - "USE_CHROME": "false", - "USE_COLOR": "false", - "SHOW_PROGRESS": "false", - }, - ) - - result = subprocess.run( - [ - "archivebox", - "add", - "--depth=2", - "--max-urls=4", - "--plugins=wget,parse_html_urls", - recursive_test_site["root_url"], - ], - capture_output=True, - text=True, - env=env, - timeout=120, - ) - stdout, stderr = result.stdout, result.stderr - - if stderr: - print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n") - if stdout: - print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n") - - assert result.returncode == 0, result.stderr - - with use_archivebox_db(tmp_path): - crawl_obj = Crawl.objects.order_by("-created_at").first() - crawl = (crawl_obj.max_depth, crawl_obj.config["CRAWL_MAX_URLS"]) if crawl_obj else None - snapshot_rows = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "parent_snapshot_id")) - depth_counts = { - depth: Snapshot.objects.filter(depth=depth).count() for depth in set(Snapshot.objects.values_list("depth", flat=True)) - } - - assert crawl == (2, 4) - assert len(snapshot_rows) == 4 - assert depth_counts.get(0, 0) == 1 - assert depth_counts.get(1, 0) == 3 - assert depth_counts.get(2, 0) == 0 - assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1}) - - -def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, process, recursive_test_site): +def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, initialized_archive, recursive_test_site): """Run a real depth=2 crawl and verify DB, output files, and process side effects.""" - os.chdir(tmp_path) env = os.environ.copy() env.update( @@ -479,29 +424,27 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p assert any("wget" in (pwd or "") or "wget" in (cmd or "") for *_rest, pwd, cmd in process_rows) -def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, process, disable_extractors_dict): +def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive): """Test that Snapshot model has parent_snapshot field.""" - os.chdir(tmp_path) column_names = {field.column for field in Snapshot._meta.local_fields} assert "parent_snapshot_id" in column_names, f"Snapshot table should have parent_snapshot_id column. Columns: {column_names}" -def test_snapshot_depth_field_exists(tmp_path, process, disable_extractors_dict): +def test_snapshot_depth_field_exists(tmp_path, initialized_archive): """Test that Snapshot model has depth field.""" - os.chdir(tmp_path) column_names = {field.column for field in Snapshot._meta.local_fields} assert "depth" in column_names, f"Snapshot table should have depth column. Columns: {column_names}" -def test_root_snapshot_has_depth_zero(tmp_path, process, disable_extractors_dict, recursive_test_site): +def test_root_snapshot_has_depth_zero(tmp_path, initialized_archive, recursive_test_site): """Test that root snapshots are created with depth=0.""" - os.chdir(tmp_path) + env = cli_env(disable_extractors=True) - env = disable_extractors_dict.copy() + env = env.copy() env["URL_ALLOWLIST"] = r"127\.0\.0\.1[:/].*" stdout, stderr = run_add_until( @@ -518,9 +461,8 @@ def test_root_snapshot_has_depth_zero(tmp_path, process, disable_extractors_dict assert snapshot[1] == 0, f"Root snapshot should have depth=0, got {snapshot[1]}" -def test_archiveresult_worker_queue_filters_by_foreground_extractors(tmp_path, process, recursive_test_site): +def test_archiveresult_worker_queue_filters_by_foreground_extractors(tmp_path, initialized_archive, recursive_test_site): """Test that background hooks don't block foreground extractors from running.""" - os.chdir(tmp_path) env = os.environ.copy() env.update( diff --git a/archivebox/tests/test_search.py b/archivebox/tests/test_search.py index d51b62d3..51c8eaf7 100644 --- a/archivebox/tests/test_search.py +++ b/archivebox/tests/test_search.py @@ -2,10 +2,10 @@ import os import signal import socket import subprocess -import sys import time from types import SimpleNamespace from urllib.parse import urlencode +from archivebox.tests.conftest import run_archivebox_cmd import pytest from asgiref.sync import async_to_sync @@ -455,7 +455,7 @@ class TestSearchBackendsE2E: sock.bind(("127.0.0.1", 0)) return int(sock.getsockname()[1]) - def archivebox(*args: str, env: dict[str, str] | None = None, timeout: int = 120) -> subprocess.CompletedProcess[str]: + def archivebox(*args: str, env: dict[str, str] | None = None, timeout: int = 120): merged_env = os.environ.copy() merged_env.update( { @@ -468,12 +468,10 @@ class TestSearchBackendsE2E: ) if env: merged_env.update(env) - return subprocess.run( - [sys.executable, "-m", "archivebox", *args], + return run_archivebox_cmd( + list(args), cwd=data_dir, env=merged_env, - capture_output=True, - text=True, timeout=timeout, ) @@ -506,14 +504,14 @@ class TestSearchBackendsE2E: ) server_log = data_dir / "server.log" with server_log.open("w", encoding="utf-8") as log_file: - server = subprocess.Popen( - [sys.executable, "-m", "archivebox", "server", f"127.0.0.1:{http_port}"], + server = run_archivebox_cmd( + ["server", f"127.0.0.1:{http_port}"], cwd=data_dir, env=sonic_env, stdout=log_file, stderr=subprocess.STDOUT, - text=True, start_new_session=True, + wait=False, ) try: for _ in range(80): diff --git a/archivebox/tests/test_server_security_browser.py b/archivebox/tests/test_server_security_browser.py index 7882dabc..92622740 100644 --- a/archivebox/tests/test_server_security_browser.py +++ b/archivebox/tests/test_server_security_browser.py @@ -6,25 +6,23 @@ from __future__ import annotations import json import os import shutil -import signal -import socket import subprocess -import sys import textwrap import time from pathlib import Path from urllib.parse import urlencode import pytest -import requests from .conftest import _ensure_puppeteer, _find_cached_chrome, _find_system_browser, run_python_cwd from .conftest import ( - build_test_env, - run_archivebox_cmd_cwd, - start_server as start_daemon_server, + cli_env, + get_free_port, + run_archivebox_cmd, + start_archivebox_server as start_daemon_server, + stop_archivebox_process, stop_server as stop_daemon_server, - wait_for_http as wait_for_daemon_http, + wait_for_http, ) @@ -425,102 +423,6 @@ def _seed_archive(data_dir: Path) -> dict[str, object]: return json.loads(stdout.strip()) -def _get_free_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return sock.getsockname()[1] - - -def _wait_for_http( - port: int, - host: str, - timeout: float = 30.0, - process: subprocess.Popen[str] | None = None, -) -> None: - deadline = time.time() + timeout - last_error = "server did not answer" - while time.time() < deadline: - if process is not None and process.poll() is not None: - raise AssertionError(f"Server exited before becoming ready with code {process.returncode}") - try: - response = requests.get( - f"http://127.0.0.1:{port}/", - headers={"Host": host}, - timeout=2, - allow_redirects=False, - ) - if response.status_code < 500: - return - last_error = f"HTTP {response.status_code}" - except requests.RequestException as exc: - last_error = str(exc) - time.sleep(0.5) - raise AssertionError(f"Timed out waiting for {host}: {last_error}") - - -def _start_server(data_dir: Path, *, mode: str, port: int) -> subprocess.Popen[str]: - env = os.environ.copy() - env.pop("DATA_DIR", None) - env.update( - { - "PYTHONPATH": str(Path(__file__).resolve().parents[2]), - "BIND_ADDR": f"127.0.0.1:{port}", - "BASE_URL": f"http://archivebox.localhost:{port}", - "ALLOWED_HOSTS": "*", - "SERVER_SECURITY_MODE": mode, - "USE_COLOR": "False", - "SHOW_PROGRESS": "False", - "SAVE_ARCHIVEDOTORG": "False", - "SAVE_TITLE": "False", - "SAVE_FAVICON": "False", - "SAVE_WGET": "False", - "SAVE_WARC": "False", - "SAVE_PDF": "False", - "SAVE_SCREENSHOT": "False", - "SAVE_DOM": "False", - "SAVE_SINGLEFILE": "False", - "SAVE_READABILITY": "False", - "SAVE_MERCURY": "False", - "SAVE_GIT": "False", - "SAVE_YTDLP": "False", - "SAVE_HEADERS": "False", - "SAVE_HTMLTOTEXT": "False", - "USE_CHROME": "False", - }, - ) - process = subprocess.Popen( - [sys.executable, "-m", "archivebox", "server", "--debug", "--nothreading", f"127.0.0.1:{port}"], - cwd=data_dir, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - start_new_session=True, - ) - try: - _wait_for_http(port, f"archivebox.localhost:{port}", process=process) - except AssertionError as exc: - server_log = _stop_server(process) - raise AssertionError(f"{exc}\n\nSERVER LOG:\n{server_log}") from exc - return process - - -def _stop_server(process: subprocess.Popen[str]) -> str: - try: - if process.poll() is None: - os.killpg(process.pid, signal.SIGTERM) - try: - stdout, _ = process.communicate(timeout=3) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - stdout, _ = process.communicate(timeout=5) - else: - stdout, _ = process.communicate(timeout=5) - except ProcessLookupError: - stdout, _ = process.communicate(timeout=5) - return stdout - - def _build_probe_config(mode: str, port: int, fixture: dict[str, object], runtime: dict[str, Path]) -> dict[str, str]: snapshots = fixture["snapshots"] attacker = snapshots["attacker"] @@ -566,8 +468,51 @@ def _run_browser_probe( fixture: dict[str, object], tmp_path: Path, ) -> dict[str, object]: - port = _get_free_port() - process = _start_server(data_dir, mode=mode, port=port) + port = get_free_port() + server_env = os.environ.copy() + server_env.pop("DATA_DIR", None) + server_env.update( + { + "PYTHONPATH": str(Path(__file__).resolve().parents[2]), + "BIND_ADDR": f"127.0.0.1:{port}", + "BASE_URL": f"http://archivebox.localhost:{port}", + "ALLOWED_HOSTS": "*", + "SERVER_SECURITY_MODE": mode, + "USE_COLOR": "False", + "SHOW_PROGRESS": "False", + "SAVE_ARCHIVEDOTORG": "False", + "SAVE_TITLE": "False", + "SAVE_FAVICON": "False", + "SAVE_WGET": "False", + "SAVE_WARC": "False", + "SAVE_PDF": "False", + "SAVE_SCREENSHOT": "False", + "SAVE_DOM": "False", + "SAVE_SINGLEFILE": "False", + "SAVE_READABILITY": "False", + "SAVE_MERCURY": "False", + "SAVE_GIT": "False", + "SAVE_YTDLP": "False", + "SAVE_HEADERS": "False", + "SAVE_HTMLTOTEXT": "False", + "USE_CHROME": "False", + }, + ) + process = run_archivebox_cmd( + ["server", "--debug", "--nothreading", f"127.0.0.1:{port}"], + cwd=data_dir, + env=server_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + wait=False, + ) + try: + wait_for_http(port, f"archivebox.localhost:{port}", process=process) + except AssertionError as exc: + server_log = stop_archivebox_process(process) + raise AssertionError(f"{exc}\n\nSERVER LOG:\n{server_log}") from exc + probe_path = tmp_path / "server_security_probe.js" probe_path.write_text(PUPPETEER_PROBE_SCRIPT, encoding="utf-8") probe_config = _build_probe_config(mode, port, fixture, runtime) @@ -589,7 +534,7 @@ def _run_browser_probe( timeout=120, ) finally: - server_log = _stop_server(process) + server_log = stop_archivebox_process(process) assert result.returncode == 0, f"{result.stderr}\n\nSERVER LOG:\n{server_log}" return json.loads(result.stdout.strip()) @@ -797,8 +742,8 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi from archivebox.core.routes_util import get_snapshot_subdomain url = "https://example.com" - port = _get_free_port() - env = build_test_env( + port = get_free_port() + env = cli_env( port, PLUGINS="archivewebpage", URL_ALLOWLIST="", @@ -817,13 +762,14 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi try: start_daemon_server(initialized_archive, env=env, port=port) - wait_for_daemon_http(port, host=f"archivebox.localhost:{port}", path="/") - stdout, stderr, returncode = run_archivebox_cmd_cwd( + wait_for_http(port, host=f"archivebox.localhost:{port}", path="/") + _cmd_result = run_archivebox_cmd( ["add", "--bg", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url], cwd=initialized_archive, env=env, timeout=120, ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, f"archivebox add --bg failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" capture = _wait_for_archivewebpage_capture(initialized_archive, url, timeout=360) diff --git a/archivebox/tests/test_snapshot.py b/archivebox/tests/test_snapshot.py deleted file mode 100644 index 668c7ca9..00000000 --- a/archivebox/tests/test_snapshot.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 -"""Integration tests for archivebox snapshot command.""" - -import os -import subprocess -from archivebox.machine.models import Process - -import pytest - -from archivebox.core.models import Snapshot, Tag -from archivebox.tests.test_orm_helpers import use_archivebox_db - -pytestmark = pytest.mark.django_db(transaction=True) - - -def test_snapshot_creates_snapshot_with_correct_url(tmp_path, process, disable_extractors_dict): - """Test that snapshot stores the exact URL in the database.""" - os.chdir(tmp_path) - - subprocess.run( - ["archivebox", "snapshot", "create", "https://example.com"], - capture_output=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.select_related("crawl__created_by").get(url="https://example.com") - username = snapshot.crawl.created_by.username - - # Verify the crawl tree contains a relative symlink to the user-scoped snapshot output. - snapshots_root = tmp_path / "archive" / "users" / username / "snapshots" - crawl_root = tmp_path / "archive" / "users" / username / "crawls" - symlinks = [p for p in crawl_root.rglob("*") if p.is_symlink() and p.resolve().is_dir() and p.resolve().is_relative_to(snapshots_root)] - assert symlinks, "Snapshot symlink should exist under crawl dir" - link_path = symlinks[0] - - assert link_path.is_symlink(), "Snapshot symlink should exist under crawl dir" - link_target = os.readlink(link_path) - assert not os.path.isabs(link_target), "Symlink should be relative" - - -def test_snapshot_multiple_urls_creates_multiple_records(tmp_path, process, disable_extractors_dict): - """Test that multiple URLs each get their own snapshot record.""" - os.chdir(tmp_path) - - subprocess.run( - [ - "archivebox", - "snapshot", - "create", - "https://example.com", - "https://iana.org", - ], - capture_output=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - with use_archivebox_db(tmp_path): - urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) - - assert "https://example.com" in urls - assert "https://iana.org" in urls - assert len(urls) >= 2 - - -def test_snapshot_tag_creates_tag_and_links_to_snapshot(tmp_path, process, disable_extractors_dict): - """Test that --tag creates tag record and links it to the snapshot.""" - os.chdir(tmp_path) - - subprocess.run( - [ - "archivebox", - "snapshot", - "create", - "--tag=mytesttag", - "https://example.com", - ], - capture_output=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - with use_archivebox_db(tmp_path): - tag = Tag.objects.filter(name="mytesttag").first() - assert tag is not None, "Tag 'mytesttag' should exist in core_tag" - snapshot = Snapshot.objects.filter(url="https://example.com").first() - assert snapshot is not None - assert snapshot.tags.filter(pk=tag.pk).exists(), "Tag should be linked to snapshot via core_snapshot_tags" - - -def test_snapshot_jsonl_output_has_correct_structure(tmp_path, process, disable_extractors_dict): - """Test that JSONL output contains required fields with correct types.""" - os.chdir(tmp_path) - - # Pass URL as argument instead of stdin for more reliable behavior - result = subprocess.run( - ["archivebox", "snapshot", "create", "https://example.com"], - capture_output=True, - text=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - # Parse JSONL output lines - records = Process.parse_records_from_text(result.stdout) - snapshot_records = [r for r in records if r.get("type") == "Snapshot"] - - assert len(snapshot_records) >= 1, "Should output at least one Snapshot JSONL record" - - record = snapshot_records[0] - assert record.get("type") == "Snapshot" - assert "id" in record, "Snapshot record should have 'id' field" - assert "url" in record, "Snapshot record should have 'url' field" - assert record["url"] == "https://example.com" - - -def test_snapshot_with_tag_stores_tag_name(tmp_path, process, disable_extractors_dict): - """Test that title is stored when provided via tag option.""" - os.chdir(tmp_path) - - # Use command line args instead of stdin - subprocess.run( - ["archivebox", "snapshot", "create", "--tag=customtag", "https://example.com"], - capture_output=True, - text=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - with use_archivebox_db(tmp_path): - tag = Tag.objects.filter(name="customtag").first() - - assert tag is not None - assert tag.name == "customtag" - - -def test_snapshot_with_depth_sets_snapshot_depth(tmp_path, process, disable_extractors_dict): - """Test that --depth sets snapshot depth when creating snapshots.""" - os.chdir(tmp_path) - - subprocess.run( - [ - "archivebox", - "snapshot", - "create", - "--depth=1", - "https://example.com", - ], - capture_output=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.order_by("-created_at").first() - - assert snapshot is not None, "Snapshot should be created when depth is provided" - assert snapshot.depth == 1, "Snapshot depth should match --depth value" - - -def test_snapshot_allows_duplicate_urls_across_crawls(tmp_path, process, disable_extractors_dict): - """Snapshot create auto-creates a crawl per run; same URL can appear multiple times.""" - os.chdir(tmp_path) - - # Add same URL twice - subprocess.run( - ["archivebox", "snapshot", "create", "https://example.com"], - capture_output=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - subprocess.run( - ["archivebox", "snapshot", "create", "https://example.com"], - capture_output=True, - env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, - ) - - with use_archivebox_db(tmp_path): - count = Snapshot.objects.filter(url="https://example.com").count() - - assert count == 2, "Same URL should create separate snapshots across different crawls" - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) diff --git a/archivebox/tests/test_snapshot_pause.py b/archivebox/tests/test_snapshot_pause.py index d574dbfb..2a909373 100644 --- a/archivebox/tests/test_snapshot_pause.py +++ b/archivebox/tests/test_snapshot_pause.py @@ -1,219 +1 @@ -import json -import os -from pathlib import Path - -import pytest -from django.utils import timezone - -from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.crawls.models import Crawl -from archivebox.tests.test_orm_helpers import use_archivebox_db -from archivebox.workers.models import RETRY_AT_MAX - -from .conftest import create_admin_and_token, init_archive - -pytestmark = pytest.mark.django_db(transaction=True) - -API_HOST = "api.archivebox.localhost:8000" - - -def _api_headers(token: str) -> dict[str, str]: - return { - "HTTP_HOST": API_HOST, - "HTTP_X_ARCHIVEBOX_API_KEY": token, - } - - -def _json_response(response): - return json.loads(response.content.decode()) - - -def _post_json(client, path: str, token: str, payload: dict): - return client.post( - path, - data=json.dumps(payload), - content_type="application/json", - **_api_headers(token), - ) - - -def _patch_json(client, path: str, token: str, payload: dict): - return client.patch( - path, - data=json.dumps(payload), - content_type="application/json", - **_api_headers(token), - ) - - -def _seed_archiveresult( - snapshot: Snapshot, - *, - plugin: str, - hook_name: str, - status: str, - output_text: str = "", - output_path: str | None = None, -) -> ArchiveResult: - output_files = {} - output_size = 0 - output_mimetypes = "" - if output_path is not None: - output_bytes = output_text.encode() - absolute_path = Path(snapshot.output_dir) / output_path - absolute_path.parent.mkdir(parents=True, exist_ok=True) - absolute_path.write_bytes(output_bytes) - output_size = len(output_bytes) - output_mimetypes = "text/plain" - output_files[output_path] = { - "extension": Path(output_path).suffix.lstrip("."), - "mimetype": "text/plain", - "size": output_size, - } - - now = timezone.now() - return ArchiveResult.objects.create( - snapshot=snapshot, - plugin=plugin, - hook_name=hook_name, - status=status, - output_str=output_path or output_text, - output_files=output_files, - output_size=output_size, - output_mimetypes=output_mimetypes, - start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None, - end_ts=now if status in ArchiveResult.FINAL_STATES else None, - ) - - -def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows( - tmp_path, - client, - recursive_test_site, -): - os.chdir(tmp_path) - init_archive(tmp_path) - api_token = create_admin_and_token(tmp_path) - - with use_archivebox_db(tmp_path): - create_response = _post_json( - client, - "/api/v1/core/snapshots", - api_token, - { - "url": recursive_test_site["root_url"], - "depth": 0, - "title": "Snapshot pause target", - "tags": ["snapshot-pause-e2e"], - "status": "queued", - }, - ) - assert create_response.status_code == 200, create_response.content.decode() - snapshot_id = _json_response(create_response)["id"] - snapshot = Snapshot.objects.get(id=snapshot_id) - - queued_result = _seed_archiveresult( - snapshot, - plugin="manualqueue", - hook_name="on_Snapshot__manual_queue", - status=ArchiveResult.StatusChoices.QUEUED, - ) - started_result = _seed_archiveresult( - snapshot, - plugin="manualstart", - hook_name="on_Snapshot__manual_start", - status=ArchiveResult.StatusChoices.STARTED, - ) - succeeded_result = _seed_archiveresult( - snapshot, - plugin="manualdone", - hook_name="on_Snapshot__manual_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="finished result should stay finished", - output_path="manualdone/final.txt", - ) - failed_result = _seed_archiveresult( - snapshot, - plugin="manualfail", - hook_name="on_Snapshot__manual_fail", - status=ArchiveResult.StatusChoices.FAILED, - output_text="failed result should stay failed", - ) - - invalid_response = _patch_json( - client, - f"/api/v1/core/snapshot/{snapshot_id}", - api_token, - {"action": "hold"}, - ) - assert invalid_response.status_code == 400 - snapshot = Snapshot.objects.get(id=snapshot_id) - assert snapshot.status == Snapshot.StatusChoices.QUEUED - - pause_response = _patch_json( - client, - f"/api/v1/core/snapshot/{snapshot_id}", - api_token, - {"action": "pause"}, - ) - assert pause_response.status_code == 200, pause_response.content.decode() - assert _json_response(pause_response)["status"] == Snapshot.StatusChoices.PAUSED - - snapshot.refresh_from_db() - crawl = Crawl.objects.get(id=snapshot.crawl_id) - assert snapshot.status == Snapshot.StatusChoices.PAUSED - assert snapshot.retry_at == RETRY_AT_MAX - assert crawl.status == Crawl.StatusChoices.QUEUED - - active_rows = { - row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) - } - assert active_rows == { - "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - } - - finished_rows = { - row.plugin: (row.status, row.retry_at, row.output_size) - for row in ArchiveResult.objects.filter(id__in=[succeeded_result.id, failed_result.id]) - } - assert finished_rows["manualdone"][0] == ArchiveResult.StatusChoices.SUCCEEDED - assert finished_rows["manualdone"][1] is None - assert finished_rows["manualdone"][2] == len("finished result should stay finished") - assert finished_rows["manualfail"] == (ArchiveResult.StatusChoices.FAILED, None, 0) - - succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id) - output_path = Path(snapshot.output_dir) / next(iter(succeeded_row.output_files)) - assert output_path.read_text() == "finished result should stay finished" - - resume_response = _patch_json( - client, - f"/api/v1/core/snapshot/{snapshot_id}", - api_token, - {"action": "resume"}, - ) - assert resume_response.status_code == 200, resume_response.content.decode() - assert _json_response(resume_response)["status"] == Snapshot.StatusChoices.QUEUED - - snapshot.refresh_from_db() - crawl.refresh_from_db() - assert snapshot.status == Snapshot.StatusChoices.QUEUED - assert snapshot.retry_at is not None - assert snapshot.retry_at != RETRY_AT_MAX - assert crawl.status == Crawl.StatusChoices.QUEUED - assert crawl.retry_at is not None - assert crawl.retry_at != RETRY_AT_MAX - - resumed_rows = { - row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) - } - assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualqueue"][1] is not None - assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX - assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualstart"][1] is not None - assert resumed_rows["manualstart"][1] != RETRY_AT_MAX - - assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED - assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED - assert output_path.read_text() == "finished result should stay finished" +# test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows moved to test_api_v1_core_snapshot_snapshot_id.py. diff --git a/archivebox/tests/test_snapshot_service.py b/archivebox/tests/test_snapshot_service.py index eeb66ac1..055f77a8 100644 --- a/archivebox/tests/test_snapshot_service.py +++ b/archivebox/tests/test_snapshot_service.py @@ -1,34 +1,20 @@ import json -import os -import time from pathlib import Path import pytest -import requests from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.tests.conftest import run_archivebox_cmd_cwd +from archivebox.tests.conftest import run_archivebox_cmd from archivebox.tests.test_orm_helpers import use_archivebox_db -from archivebox.workers.models import RETRY_AT_MAX from .conftest import ( - build_test_env, - create_admin_and_token, - get_crawl_runtime_state, + cli_env, get_free_port, init_archive, - start_server, - stop_server, - wait_for_http, - wait_for_snapshot_capture, ) pytestmark = pytest.mark.django_db(transaction=True) -def _assert_command_ok(command: str, stdout: str, stderr: str, code: int) -> None: - assert code == 0, f"{command} failed with code {code}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" - - def _snapshot_state(cwd: Path, url: str) -> dict[str, object]: with use_archivebox_db(cwd): snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").get(url=url) @@ -54,57 +40,20 @@ def _snapshot_state(cwd: Path, url: str) -> dict[str, object]: } -def _paused_snapshot_state(cwd: Path, snapshot_id: str) -> dict[str, object]: - with use_archivebox_db(cwd): - snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) - succeeded_results = ArchiveResult.objects.filter(snapshot=snapshot, status=ArchiveResult.StatusChoices.SUCCEEDED).count() - return { - "status": snapshot.status, - "retry_at": snapshot.retry_at, - "crawl_status": snapshot.crawl.status, - "succeeded_results": succeeded_results, - "snapshot_dir": Path(snapshot.output_dir), - } - - -def _wait_for_paused_scheduler_marker(cwd: Path, snapshot_id: str, timeout: int = 60) -> dict[str, object]: - deadline = time.time() + timeout - last_state: dict[str, object] = {} - while time.time() < deadline: - last_state = _paused_snapshot_state(cwd, snapshot_id) - if last_state["status"] == Snapshot.StatusChoices.PAUSED and last_state["retry_at"] == RETRY_AT_MAX: - return last_state - if last_state["status"] == Snapshot.StatusChoices.SEALED: - return last_state - time.sleep(1) - raise AssertionError(f"paused snapshot did not settle back to retry_at=MAX: {last_state}") - - -def _wait_for_crawl_snapshot_rows(cwd: Path, crawl_id: str, timeout: int = 45) -> dict[str, object]: - deadline = time.time() + timeout - latest_state: dict[str, object] | None = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - if latest_state["snapshots"]: - return latest_state - time.sleep(0.2) - raise AssertionError(f"timed out waiting for snapshot rows for crawl {crawl_id}: {latest_state}") - - @pytest.mark.timeout(180) def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env(port, PLUGINS="wget", SAVE_WGET="True") - stdout, stderr, code = run_archivebox_cmd_cwd( + env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True") + _cmd_result = run_archivebox_cmd( ["add", "--depth=0", "--plugins=wget", recursive_test_site["root_url"]], cwd=tmp_path, env=env, timeout=180, ) - _assert_command_ok("archivebox add", stdout, stderr, code) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert code == 0, f"archivebox add failed with code {code}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" state = _snapshot_state(tmp_path, recursive_test_site["root_url"]) snapshot_dir = state["snapshot_dir"] @@ -133,85 +82,4 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re assert any(result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in state["results"]) -@pytest.mark.timeout(240) -def test_paused_snapshot_survives_server_restart_and_resumes_via_api(tmp_path, recursive_test_site): - os.chdir(tmp_path) - init_archive(tmp_path) - - port = get_free_port() - env = build_test_env(port, PLUGINS="wget", SAVE_WGET="True") - api_token = create_admin_and_token(tmp_path) - api_headers = { - "Host": f"api.archivebox.localhost:{port}", - "X-ArchiveBox-API-Key": api_token, - } - - try: - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - crawl_response = requests.post( - f"http://127.0.0.1:{port}/api/v1/crawls/crawls", - headers=api_headers, - json={ - "urls": [recursive_test_site["root_url"]], - "max_depth": 0, - "tags": ["snapshot-pause-restart-e2e"], - "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, - }, - timeout=10, - ) - assert crawl_response.status_code == 200, crawl_response.text - crawl_id = crawl_response.json()["id"] - crawl_state = _wait_for_crawl_snapshot_rows(tmp_path, crawl_id) - snapshot_id = crawl_state["snapshots"][0]["id"] - - pause_response = requests.patch( - f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", - headers=api_headers, - json={"action": "pause"}, - timeout=10, - ) - assert pause_response.status_code == 200, pause_response.text - - current_state = _paused_snapshot_state(tmp_path, snapshot_id) - if current_state["status"] == Snapshot.StatusChoices.SEALED: - assert current_state["succeeded_results"] > 0 - return - - paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id) - if paused_state["status"] == Snapshot.StatusChoices.SEALED: - assert paused_state["succeeded_results"] > 0 - return - assert paused_state["succeeded_results"] == 0 - assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html")) - - stop_server(tmp_path) - start_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") - - restarted_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id) - assert restarted_state["status"] == Snapshot.StatusChoices.PAUSED - assert restarted_state["succeeded_results"] == 0 - - resume_response = requests.patch( - f"http://127.0.0.1:{port}/api/v1/core/snapshot/{snapshot_id}", - headers=api_headers, - json={"action": "resume"}, - timeout=10, - ) - assert resume_response.status_code == 200, resume_response.text - assert resume_response.json()["status"] == Snapshot.StatusChoices.QUEUED - - captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) - assert "Root" in captured_text - assert "About" in captured_text - - final_state = _snapshot_state(tmp_path, recursive_test_site["root_url"]) - assert final_state["status"] == Snapshot.StatusChoices.SEALED - assert final_state["downloaded_at"] is not None - assert any( - result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in final_state["results"] - ) - finally: - stop_server(tmp_path) +# test_paused_snapshot_survives_server_restart_and_resumes_via_api moved to test_api_v1_core_snapshot_snapshot_id.py. diff --git a/archivebox/tests/test_tag_admin.py b/archivebox/tests/test_tag_admin.py deleted file mode 100644 index 89d362f4..00000000 --- a/archivebox/tests/test_tag_admin.py +++ /dev/null @@ -1,203 +0,0 @@ -import json -from datetime import datetime -from typing import cast - -import pytest -from django.contrib.auth import get_user_model -from django.contrib.auth.models import UserManager -from django.urls import reverse -from django.utils import timezone - - -pytestmark = pytest.mark.django_db - - -User = get_user_model() -ADMIN_HOST = "admin.archivebox.localhost:8000" - - -@pytest.fixture -def admin_user(db): - return cast(UserManager, User.objects).create_superuser( - username="tagadmin", - email="tagadmin@test.com", - password="testpassword", - ) - - -@pytest.fixture -def api_token(admin_user): - from archivebox.api.auth import get_or_create_api_token - - token = get_or_create_api_token(admin_user) - assert token is not None - return token.token - - -@pytest.fixture -def crawl(admin_user): - from archivebox.crawls.models import Crawl - - return Crawl.objects.create( - urls="https://example.com", - created_by=admin_user, - ) - - -@pytest.fixture -def tagged_data(crawl, admin_user): - from archivebox.core.models import Snapshot, Tag - - tag = Tag.objects.create(name="Alpha Research", created_by=admin_user) - first = Snapshot.objects.create( - url="https://example.com/one", - title="Example One", - crawl=crawl, - ) - second = Snapshot.objects.create( - url="https://example.com/two", - title="Example Two", - crawl=crawl, - ) - first.tags.add(tag) - second.tags.add(tag) - return tag, [first, second] - - -def test_tag_admin_changelist_renders_custom_ui(client, admin_user, tagged_data): - client.login(username="tagadmin", password="testpassword") - - response = client.get(reverse("admin:core_tag_changelist"), HTTP_HOST=ADMIN_HOST) - - assert response.status_code == 200 - assert b'id="tag-live-search"' in response.content - assert b'id="tag-sort-select"' in response.content - assert b'id="tag-created-by-select"' in response.content - assert b'id="tag-year-select"' in response.content - assert b"Alpha Research" in response.content - assert b'class="tag-card"' in response.content - - -def test_tag_admin_add_view_renders_similar_tag_reference(client, admin_user): - client.login(username="tagadmin", password="testpassword") - - response = client.get(reverse("admin:core_tag_add"), HTTP_HOST=ADMIN_HOST) - - assert response.status_code == 200 - assert b"Similar Tags" in response.content - assert b'data-tag-name-input="1"' in response.content - - -def test_tag_search_api_returns_card_payload(client, api_token, tagged_data): - tag, snapshots = tagged_data - - response = client.get( - reverse("api-1:search_tags"), - {"q": "Alpha", "api_key": api_token}, - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["sort"] == "created_desc" - assert payload["created_by"] == "" - assert payload["year"] == "" - assert payload["has_snapshots"] == "all" - assert payload["tags"][0]["id"] == tag.id - assert payload["tags"][0]["name"] == "Alpha Research" - assert payload["tags"][0]["num_snapshots"] == 2 - assert payload["tags"][0]["snapshots"] == [] - assert payload["tags"][0]["export_jsonl_url"].endswith(f"/api/v1/core/tag/{tag.id}/snapshots.jsonl") - assert payload["tags"][0]["filter_url"].endswith(f"/admin/core/snapshot/?tags__id__exact={tag.id}") - assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"} - - -def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data): - from archivebox.core.models import Snapshot, Tag - - other_user = cast(UserManager, User.objects).create_user( - username="tagother", - email="tagother@test.com", - password="unused", - ) - tag_with_snapshots = tagged_data[0] - empty_tag = Tag.objects.create(name="Zulu Empty", created_by=other_user) - alpha_tag = Tag.objects.create(name="Alpha Empty", created_by=other_user) - Snapshot.objects.create( - url="https://example.com/three", - title="Example Three", - crawl=crawl, - ).tags.add(alpha_tag) - - Tag.objects.filter(pk=empty_tag.pk).update(created_at=timezone.make_aware(datetime(2024, 1, 1, 12, 0, 0))) - Tag.objects.filter(pk=alpha_tag.pk).update(created_at=timezone.make_aware(datetime(2025, 1, 1, 12, 0, 0))) - Tag.objects.filter(pk=tag_with_snapshots.pk).update(created_at=timezone.make_aware(datetime(2026, 1, 1, 12, 0, 0))) - - response = client.get( - reverse("api-1:search_tags"), - { - "sort": "name_desc", - "created_by": str(other_user.pk), - "year": "2024", - "has_snapshots": "no", - "api_key": api_token, - }, - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["sort"] == "name_desc" - assert payload["created_by"] == str(other_user.pk) - assert payload["year"] == "2024" - assert payload["has_snapshots"] == "no" - assert [tag["name"] for tag in payload["tags"]] == ["Zulu Empty"] - - -def test_tag_rename_api_updates_name(client, api_token, tagged_data): - tag, _ = tagged_data - - response = client.post( - f"{reverse('api-1:rename_tag', args=[tag.id])}?api_key={api_token}", - data=json.dumps({"name": "Alpha Archive"}), - content_type="application/json", - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - - tag.refresh_from_db() - assert tag.name == "Alpha Archive" - - -def test_tag_snapshots_export_returns_jsonl(client, api_token, tagged_data): - tag, _ = tagged_data - - response = client.get( - reverse("api-1:tag_snapshots_export", args=[tag.id]), - {"api_key": api_token}, - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - assert response["Content-Type"].startswith("application/x-ndjson") - assert f"tag-{tag.slug}-snapshots.jsonl" in response["Content-Disposition"] - body = response.content.decode() - assert '"type": "Snapshot"' in body - assert '"tags": "Alpha Research"' in body - - -def test_tag_urls_export_returns_plain_text_urls(client, api_token, tagged_data): - tag, snapshots = tagged_data - - response = client.get( - reverse("api-1:tag_urls_export", args=[tag.id]), - {"api_key": api_token}, - HTTP_HOST=ADMIN_HOST, - ) - - assert response.status_code == 200 - assert response["Content-Type"].startswith("text/plain") - assert f"tag-{tag.slug}-urls.txt" in response["Content-Disposition"] - exported_urls = set(filter(None, response.content.decode().splitlines())) - assert exported_urls == {snapshot.url for snapshot in snapshots} diff --git a/archivebox/tests/test_takeover_util.py b/archivebox/tests/test_takeover_util.py index 7b847d26..34bf5c47 100644 --- a/archivebox/tests/test_takeover_util.py +++ b/archivebox/tests/test_takeover_util.py @@ -1,13 +1,766 @@ +#!/usr/bin/env python3 +"""Takeover utility tests and live command handoff flows.""" + import os import signal import subprocess import sys import time +from pathlib import Path import pytest +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.machine.models import Process +from archivebox.tests.conftest import ( + assert_no_processes_for_data_dir, + get_free_port, + kill_processes_for_data_dir, + cli_env, + pid_is_alive, + run_archivebox_cmd, + start_archivebox_server, + stop_archivebox_process, + supervisor_pid_from_log, + wait_for_http, + wait_for_log, + wait_for_log_count, + wait_for_log_pattern, + wait_for_pid_to_disappear, + wait_for_snapshot_capture, + wait_for_worker_pid_from_log, + worker_pid_from_log, +) +from archivebox.tests.test_orm_helpers import use_archivebox_db -pytestmark = pytest.mark.django_db +pytestmark = pytest.mark.django_db(transaction=True) + + +def _archive_pages_for_sqlite_reindexing(data_dir: Path, env: dict[str, str], root_url: str) -> None: + add_env = dict(env) + add_env["SEARCH_BACKEND_ENGINE"] = "ripgrep" + _cmd_result = run_archivebox_cmd( + [ + "add", + "--depth=2", + "--max-urls=20", + "--crawl-max-size=50mb", + "--plugins=wget,parse_html_urls", + root_url, + ], + cwd=data_dir, + env=add_env, + timeout=240, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + + with use_archivebox_db(data_dir): + assert Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).count() >= 1 + assert not ArchiveResult.objects.filter(plugin="search_backend_sqlite").exists() + + +@pytest.mark.timeout(360) +def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_path, initialized_archive, recursive_test_site): + + env = cli_env( + live=True, + SEARCH_BACKEND_ENGINE="sqlite", + SEARCH_BACKEND_SONIC_PORT=str(get_free_port()), + ) + root_url = recursive_test_site["root_url"] + _archive_pages_for_sqlite_reindexing(tmp_path, env, root_url) + + port = get_free_port() + server = None + try: + server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-update.log", env=env) + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + + update = run_archivebox_cmd( + ["update", "--index-only", "--batch-size=1"], + cwd=tmp_path, + env=env, + timeout=180, + ) + + assert update.returncode == 0, update.stderr or update.stdout + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + + search = run_archivebox_cmd( + ["list", "--search=contents", "--csv=url", "Root"], + cwd=tmp_path, + env=env, + timeout=60, + ) + + assert search.returncode == 0, search.stderr or search.stdout + assert root_url in search.stdout + finally: + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(420) +def test_behavior_update_yields_to_server_then_finishes_visible_indexing(tmp_path, initialized_archive, recursive_test_site): + + env = cli_env( + live=True, + SEARCH_BACKEND_ENGINE="sqlite", + SEARCH_BACKEND_SONIC_PORT=str(get_free_port()), + ) + root_url = recursive_test_site["root_url"] + _archive_pages_for_sqlite_reindexing(tmp_path, env, root_url) + + port = get_free_port() + update_proc = None + server = None + try: + update_log = tmp_path / "behavior-update-yields.log" + update_log_handle = update_log.open("w", encoding="utf-8") + update_proc = run_archivebox_cmd( + ["update", "--index-only", "--batch-size=1"], + cwd=tmp_path, + env=env, + stdout=update_log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + wait=False, + ) + update_log_handle.close() + wait_for_log(update_log, "[*] Reindexing", timeout=90) + + server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-takes-update.log", env=env) + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90) + + stop_archivebox_process(server, signal.SIGTERM) + server = None + update_proc.wait(timeout=180) + update_text = update_log.read_text(encoding="utf-8", errors="replace") + assert update_proc.returncode == 0, update_text + + search = run_archivebox_cmd( + ["list", "--search=contents", "--csv=url", "Root"], + cwd=tmp_path, + env=env, + timeout=60, + ) + + assert search.returncode == 0, search.stderr or search.stdout + assert root_url in search.stdout + finally: + if update_proc is not None and update_proc.poll() is None: + stop_archivebox_process(update_proc, signal.SIGTERM) + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(300) +def test_behavior_foreground_add_keeps_existing_server_http_visible(tmp_path, initialized_archive, recursive_test_site): + + port = get_free_port() + env = cli_env(live=True, server=True, port=port, SEARCH_BACKEND_ENGINE="ripgrep") + server = None + try: + server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-add.log", env=env) + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + + add = run_archivebox_cmd( + [ + "add", + "--depth=0", + "--plugins=wget", + recursive_test_site["root_url"], + ], + cwd=tmp_path, + env=env, + timeout=180, + ) + + assert add.returncode == 0, add.stderr or add.stdout + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=120) + assert "Root" in captured_text + finally: + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(300) +def test_behavior_background_add_returns_and_server_archives_visible_url(tmp_path, initialized_archive, recursive_test_site): + + port = get_free_port() + env = cli_env(live=True, server=True, port=port, SEARCH_BACKEND_ENGINE="ripgrep") + server = None + try: + server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-bg-add.log", env=env) + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + + add = run_archivebox_cmd( + [ + "add", + "--bg", + "--depth=0", + "--plugins=wget", + recursive_test_site["root_url"], + ], + cwd=tmp_path, + env=env, + timeout=60, + ) + + assert add.returncode == 0, add.stderr or add.stdout + assert "background runner will process" in add.stdout + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + finally: + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(300) +def test_behavior_daemonized_server_restarts_cleanly_after_forced_stop(tmp_path, initialized_archive): + + port = get_free_port() + env = cli_env(live=True, server=True, port=port) + try: + first = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True) + assert first.returncode == 0, first.stderr or first.stdout + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + second = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True) + assert second.returncode == 0, second.stderr or second.stdout + assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + finally: + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(240) +def test_live_second_server_takes_over_existing_server_process(tmp_path, initialized_archive): + + env = cli_env(live=True) + port = get_free_port() + first = None + second = None + try: + first = start_archivebox_server(tmp_path, port=port, log_name="server-first.log", env=env) + first_log = first.log_path + second = start_archivebox_server(tmp_path, port=port, log_name="server-second.log", env=env) + second_log = second.log_path + + assert first.poll() is None + first_text = first_log.read_text(encoding="utf-8", errors="replace") + second_text = second_log.read_text(encoding="utf-8", errors="replace") + assert "A newer archivebox process took over the orchestrator, server" in first_text + assert "Starting orchestrator, server" in second_text + + _cmd_result = run_archivebox_cmd( + ["status"], + cwd=tmp_path, + env=env, + timeout=60, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + + first_resumes = first_log.read_text(encoding="utf-8", errors="replace").count("Other newer archivebox process") + stop_archivebox_process(second, signal.SIGTERM) + second = None + wait_for_log_count(first_log, "Other newer archivebox process", first_resumes + 1, timeout=35) + assert first.poll() is None + finally: + if second is not None and second.poll() is None: + stop_archivebox_process(second, signal.SIGTERM) + if first is not None and first.poll() is None: + stop_archivebox_process(first, signal.SIGKILL) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(180) +def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, initialized_archive): + + env = cli_env(live=True) + port = get_free_port() + server = None + try: + server = start_archivebox_server(tmp_path, port=port, log_name="server-update-owner.log", env=env) + server_log = server.log_path + supervisor_pid_before = supervisor_pid_from_log(server_log) + daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne") + + _cmd_result = run_archivebox_cmd( + ["update", "--index-only", "--before=0"], + cwd=tmp_path, + env=env, + timeout=90, + ) + update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + + assert update_returncode == 0, update_stderr or update_stdout + assert server.poll() is None + assert pid_is_alive(supervisor_pid_before) + assert pid_is_alive(daphne_pid_before) + assert supervisor_pid_from_log(server_log) == supervisor_pid_before + assert worker_pid_from_log(server_log, "worker_daphne") == daphne_pid_before + assert "A newer archivebox process took over the orchestrator, server" not in server_log.read_text( + encoding="utf-8", + errors="replace", + ) + finally: + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(360) +def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tmp_path, initialized_archive, recursive_test_site): + + env = cli_env( + live=True, + SEARCH_BACKEND_ENGINE="sqlite", + SEARCH_BACKEND_SONIC_PORT=str(get_free_port()), + ) + _archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"]) + + port = get_free_port() + server = None + try: + server = start_archivebox_server(tmp_path, port=port, log_name="server-real-sqlite-update.log", env=env) + server_log = server.log_path + supervisor_pid_before = supervisor_pid_from_log(server_log) + daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne") + runner_pid_before = worker_pid_from_log(server_log, "worker_runner") + sonic_pid_before = worker_pid_from_log(server_log, "worker_sonic") + + _cmd_result = run_archivebox_cmd( + ["update", "--index-only", "--batch-size=1"], + cwd=tmp_path, + env=env, + timeout=180, + ) + update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + + assert update_returncode == 0, update_stderr or update_stdout + assert server.poll() is None + assert supervisor_pid_from_log(server_log) == supervisor_pid_before + assert pid_is_alive(daphne_pid_before) + assert pid_is_alive(sonic_pid_before) + assert worker_pid_from_log(server_log, "worker_daphne") == daphne_pid_before + assert worker_pid_from_log(server_log, "worker_sonic") == sonic_pid_before + assert "A newer archivebox process took over the orchestrator, server" not in server_log.read_text( + encoding="utf-8", + errors="replace", + ) + assert "Stopping older ArchiveBox runner process" in update_stdout + + deadline = time.time() + 90 + runner_pid_after = runner_pid_before + while time.time() < deadline: + with use_archivebox_db(tmp_path): + rows = list( + Process.objects.filter( + process_type=Process.TypeChoices.ORCHESTRATOR, + worker_type="worker_runner", + status=Process.StatusChoices.RUNNING, + ).values("pid"), + ) + for row in rows: + pid = int(row["pid"]) + if pid != runner_pid_before and pid_is_alive(pid): + runner_pid_after = pid + break + if runner_pid_after != runner_pid_before: + break + time.sleep(0.25) + assert runner_pid_after != runner_pid_before + + deadline = time.time() + 180 + indexed_results: list[str] = [] + while time.time() < deadline: + with use_archivebox_db(tmp_path): + indexed_results = list( + ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True), + ) + if indexed_results and all(status in ArchiveResult.FINAL_STATES for status in indexed_results): + break + time.sleep(0.25) + assert indexed_results + assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results) + + stop_archivebox_process(server, signal.SIGTERM) + server = None + wait_for_pid_to_disappear(daphne_pid_before, timeout=20) + wait_for_pid_to_disappear(sonic_pid_before, timeout=20) + wait_for_pid_to_disappear(runner_pid_after, timeout=20) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + finally: + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(420) +def test_live_update_yields_to_server_then_reclaims_real_sqlite_indexing(tmp_path, initialized_archive, recursive_test_site): + + env = cli_env( + live=True, + SEARCH_BACKEND_ENGINE="sqlite", + SEARCH_BACKEND_SONIC_PORT=str(get_free_port()), + ) + _archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"]) + + port = get_free_port() + update_proc = None + server = None + try: + update_log = tmp_path / "update-real-sqlite-owner.log" + update_log_handle = update_log.open("w", encoding="utf-8") + update_proc = run_archivebox_cmd( + ["update", "--index-only", "--batch-size=1"], + cwd=tmp_path, + env=env, + stdout=update_log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + wait=False, + ) + update_log_handle.close() + wait_for_log(update_log, "[*] Reindexing", timeout=90) + update_supervisor_match = wait_for_log_pattern(update_log, r"Supervisord connected \(pid=(\d+)\)", timeout=90) + update_supervisor_pid_before = int(update_supervisor_match.group(1)) + update_sonic_pid_before = wait_for_worker_pid_from_log(update_log, "worker_sonic", timeout=90) + update_runner_pid_before = wait_for_worker_pid_from_log(update_log, f"worker_runner_update_{update_proc.pid}", timeout=90) + assert pid_is_alive(update_supervisor_pid_before) + assert pid_is_alive(update_sonic_pid_before) + assert pid_is_alive(update_runner_pid_before) + assert "worker_daphne" not in update_log.read_text(encoding="utf-8", errors="replace") + + server = start_archivebox_server(tmp_path, port=port, log_name="server-takes-real-sqlite-update.log", env=env) + server_log = server.log_path + wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90) + assert update_proc.poll() is None + assert server.poll() is None + server_text = server_log.read_text(encoding="utf-8", errors="replace") + assert "Taking over orchestrator, sonic from older existing archivebox process" in server_text + assert "Starting orchestrator, server, sonic" in server_text + server_daphne_pid = worker_pid_from_log(server_log, "worker_daphne") + server_runner_pid = worker_pid_from_log(server_log, "worker_runner") + server_sonic_pid = worker_pid_from_log(server_log, "worker_sonic") + assert pid_is_alive(server_daphne_pid) + assert pid_is_alive(server_runner_pid) + assert pid_is_alive(server_sonic_pid) + wait_for_pid_to_disappear(update_supervisor_pid_before, timeout=30) + wait_for_pid_to_disappear(update_sonic_pid_before, timeout=30) + wait_for_pid_to_disappear(update_runner_pid_before, timeout=30) + + stop_archivebox_process(server, signal.SIGTERM) + server = None + update_proc.wait(timeout=180) + update_text = update_log.read_text(encoding="utf-8", errors="replace") + assert update_proc.returncode == 0, update_text + wait_for_pid_to_disappear(server_daphne_pid, timeout=20) + wait_for_pid_to_disappear(server_runner_pid, timeout=20) + wait_for_pid_to_disappear(server_sonic_pid, timeout=20) + + deadline = time.time() + 30 + indexed_results: list[str] = [] + while time.time() < deadline: + with use_archivebox_db(tmp_path): + indexed_results = list( + ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True), + ) + if indexed_results and all(status in ArchiveResult.FINAL_STATES for status in indexed_results): + break + time.sleep(0.25) + assert indexed_results + assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + finally: + if update_proc is not None and update_proc.poll() is None: + stop_archivebox_process(update_proc, signal.SIGTERM) + if server is not None and server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(420) +def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_archive): + + env = cli_env(live=True) + port = get_free_port() + servers: list[subprocess.Popen[str]] = [] + server_pids: list[int] = [] + daphne_pids: list[int] = [] + runner_pids: list[int] = [] + try: + for index in range(5): + server = start_archivebox_server(tmp_path, port=port, log_name=f"server-chaos-{index}.log", env=env) + log_path = server.log_path + servers.append(server) + server_pids.append(server.pid) + daphne_pids.append(worker_pid_from_log(log_path, "worker_daphne")) + runner_pids.append(worker_pid_from_log(log_path, "worker_runner")) + + if index > 0: + previous_server = servers[index - 1] + previous_log = (tmp_path / f"server-chaos-{index - 1}.log").read_text(encoding="utf-8", errors="replace") + current_log = log_path.read_text(encoding="utf-8", errors="replace") + assert previous_server.poll() is None + assert pid_is_alive(server_pids[index - 1]) + assert "A newer archivebox process took over the orchestrator, server" in previous_log + assert "Starting orchestrator, server" in current_log + wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15) + wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15) + + _cmd_result = run_archivebox_cmd( + ["status"], + cwd=tmp_path, + env=env, + timeout=60, + ) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert returncode == 0, stderr or stdout + time.sleep(5) + + assert servers[-1].poll() is None + assert all(server.poll() is None for server in servers) + listener = subprocess.run( + ["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"], + capture_output=True, + text=True, + timeout=10, + ) + assert listener.returncode == 0, listener.stderr or listener.stdout + assert listener.stdout.count(f":{port} (LISTEN)") == 1 + + previous_log_path = tmp_path / "server-chaos-3.log" + previous_takeovers = previous_log_path.read_text(encoding="utf-8", errors="replace").count( + "Other newer archivebox process", + ) + stop_archivebox_process(servers[-1], signal.SIGTERM) + wait_for_log_count(previous_log_path, "Other newer archivebox process", previous_takeovers + 1, timeout=35) + assert servers[3].poll() is None + finally: + for server in reversed(servers): + if server.poll() is None: + stop_archivebox_process(server, signal.SIGTERM) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(420) +def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initialized_archive): + plugins_root = tmp_path / "runtime_plugins" + marker_dir = tmp_path / "slow-plugin-markers" + plugin_dir = plugins_root / "slow_exit" + plugin_dir.mkdir(parents=True, exist_ok=True) + counter_hook = plugin_dir / "on_Snapshot__08_counter.sh" + counter_hook.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + f"marker_dir={str(marker_dir)!r}", + 'mkdir -p "$marker_dir/counter-seen"', + 'snapshot_key="${SNAPSHOT_ID:-$(basename "${SNAP_DIR:-unknown}")}"', + 'seen_file="$marker_dir/counter-seen/$snapshot_key"', + 'if [[ -e "$seen_file" ]]; then', + ' echo "$snapshot_key" >> "$marker_dir/counter-duplicates.txt"', + " exit 42", + "fi", + 'touch "$seen_file"', + 'echo "$snapshot_key" >> "$marker_dir/counter-runs.txt"', + "", + ], + ), + encoding="utf-8", + ) + counter_hook.chmod(0o755) + hook = plugin_dir / "on_Snapshot__09_slow_exit.sh" + hook.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + f"marker_dir={str(marker_dir)!r}", + 'mkdir -p "$marker_dir"', + 'echo $$ >> "$marker_dir/hook-pids.txt"', + 'touch "$marker_dir/hook-started"', + "trap 'touch \"$marker_dir/hook-stopped\"; exit 143' TERM INT HUP", + 'while [[ ! -f "$marker_dir/allow-finish" ]]; do sleep 0.1; done', + 'touch "$marker_dir/hook-finished"', + "", + ], + ), + encoding="utf-8", + ) + hook.chmod(0o755) + + env = cli_env(live=True, plugins_root=plugins_root) + port = get_free_port() + server = None + server2 = None + server3 = None + add_proc = None + add_proc2 = None + try: + server = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-1.log", env=env) + server_log = server.log_path + supervisor_pid_before = supervisor_pid_from_log(server_log) + + _cmd_result = run_archivebox_cmd( + ["update", "--index-only", "--batch-size=10"], + cwd=tmp_path, + env=env, + timeout=90, + ) + update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert update_returncode == 0, update_stderr or update_stdout + assert server.poll() is None + assert pid_is_alive(supervisor_pid_before) + assert supervisor_pid_from_log(server_log) == supervisor_pid_before + + add_log = tmp_path / "archivebox-add-1.log" + add_log_handle = add_log.open("w", encoding="utf-8") + add_proc = run_archivebox_cmd( + [ + "add", + "--depth=1", + "--max-urls=2", + "--crawl-max-size=50mb", + "--plugins=wget,parse_html_urls,slow_exit", + "https://example.com", + "https://blog.sweeting.me", + ], + cwd=tmp_path, + env=env, + stdout=add_log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + wait=False, + ) + add_log_handle.close() + + pid_file = marker_dir / "hook-pids.txt" + deadline = time.time() + 45 + hook_pids: list[int] = [] + while time.time() < deadline: + if pid_file.exists(): + hook_pids = [int(line.strip()) for line in pid_file.read_text().splitlines() if line.strip()] + if len(hook_pids) >= 1: + break + time.sleep(0.25) + assert len(hook_pids) >= 1 + + deadline = time.time() + 30 + snapshot_started = False + while time.time() < deadline: + with use_archivebox_db(tmp_path): + snapshot_started = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).exists() + if snapshot_started: + break + time.sleep(0.25) + assert snapshot_started + + os.kill(server.pid, signal.SIGTERM) + server.wait(timeout=20) + assert add_proc.poll() is None, "foreground add should keep owning its crawl after the server exits" + assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace") + + server2 = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env) + _server2_log = server2.log_path + (marker_dir / "allow-finish").touch() + stop_archivebox_process(add_proc, signal.SIGTERM, timeout=30) + add_output = add_log.read_text(encoding="utf-8", errors="replace") + assert "Runner error" not in add_output + + deadline = time.time() + 90 + crawls = [] + snapshots = [] + bad_results = [] + while time.time() < deadline: + with use_archivebox_db(tmp_path): + crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at")) + snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at")) + bad_results = list( + ArchiveResult.objects.filter( + status__in=[ + ArchiveResult.StatusChoices.FAILED, + ArchiveResult.StatusChoices.SKIPPED, + ], + ).values_list("plugin", "status", "output_str"), + ) + if ( + crawls + and snapshots + and all(status == Crawl.StatusChoices.SEALED for status, _retry_at in crawls) + and all(status == Snapshot.StatusChoices.SEALED for _url, status, _retry_at in snapshots) + and not bad_results + ): + break + time.sleep(0.25) + + os.kill(server2.pid, signal.SIGTERM) + server2.wait(timeout=20) + with use_archivebox_db(tmp_path): + crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at")) + snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at")) + bad_results = list( + ArchiveResult.objects.filter( + status__in=[ + ArchiveResult.StatusChoices.FAILED, + ArchiveResult.StatusChoices.SKIPPED, + ], + ).values_list("plugin", "status", "output_str"), + ) + assert crawls + assert snapshots + assert all(status == Crawl.StatusChoices.SEALED for status, _retry_at in crawls) + assert all(status == Snapshot.StatusChoices.SEALED for _url, status, _retry_at in snapshots) + counter_runs = (marker_dir / "counter-runs.txt").read_text(encoding="utf-8").splitlines() + assert counter_runs + assert len(counter_runs) == len(set(counter_runs)) + assert not (marker_dir / "counter-duplicates.txt").exists() + + # TODO: improve abx-dl's ability to explicitly resume from a given plugin / hook and skip ones before that + # current behavior: on retry, earlier sealed results are left untouched; the interrupted result is marked skipped + # and may have partial output saved to fs + # assertions that enforce the current behavior (uncommented): previous results are not run twice + interrupted + # result is marked skipped + assert bad_results == [("slow_exit", ArchiveResult.StatusChoices.SKIPPED, "")] + + # desired future behavior: earlier sealed results are left untouched, interrupted result is retried and cleanly + # overwrites on top of any previous partial output + # assert not bad_results + # assert (marker_dir / "hook-finished").exists() + finally: + for proc in (add_proc, add_proc2, server, server2, server3): + if proc is not None and proc.poll() is None: + stop_archivebox_process(proc, signal.SIGTERM, timeout=10) + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +# Utility-level takeover selection tests. def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path): @@ -53,6 +806,141 @@ def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path): proc.wait(timeout=5) +def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path): + from archivebox.machine.models import Machine, Process + from archivebox.core.takeover_util import runtime_stack_owner + + procs: list[subprocess.Popen[str]] = [] + try: + for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE): + proc = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + start_new_session=True, + ) + procs.append(proc) + Process.objects.create( + machine=Machine.current(), + process_type=process_type, + worker_type=process_type, + pwd=str(tmp_path), + cmd=[], + pid=proc.pid, + status=Process.StatusChoices.RUNNING, + ) + time.sleep(0.05) + + owner = runtime_stack_owner(data_dir=tmp_path) + + assert owner is not None + assert owner.process_type == Process.TypeChoices.SERVER + assert owner.pid == procs[0].pid + finally: + for proc in procs: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + for proc in procs: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait(timeout=5) + + +def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path): + from archivebox.machine.models import Machine, Process + from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner + + procs: list[subprocess.Popen[str]] = [] + try: + for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE): + proc = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + start_new_session=True, + ) + procs.append(proc) + Process.objects.create( + machine=Machine.current(), + process_type=process_type, + worker_type=process_type, + pwd=str(tmp_path), + cmd=[], + pid=proc.pid, + status=Process.StatusChoices.RUNNING, + ) + time.sleep(0.05) + + runtime_owner = runtime_stack_owner(data_dir=tmp_path) + runner_owner = foreground_runner_owner(data_dir=tmp_path) + + assert runtime_owner is not None + assert runtime_owner.process_type == Process.TypeChoices.SERVER + assert runner_owner is not None + assert runner_owner.process_type == Process.TypeChoices.UPDATE + assert runner_owner.pid == procs[-1].pid + finally: + for proc in procs: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + for proc in procs: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait(timeout=5) + + +def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path): + from archivebox.machine.models import Machine, Process + from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner + + procs: list[subprocess.Popen[str]] = [] + try: + for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER): + proc = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + start_new_session=True, + ) + procs.append(proc) + Process.objects.create( + machine=Machine.current(), + process_type=process_type, + worker_type=process_type, + pwd=str(tmp_path), + cmd=[], + pid=proc.pid, + status=Process.StatusChoices.RUNNING, + ) + time.sleep(0.05) + + runtime_owner = runtime_stack_owner(data_dir=tmp_path) + runner_owner = foreground_runner_owner(data_dir=tmp_path) + + assert runtime_owner is not None + assert runtime_owner.process_type == Process.TypeChoices.SERVER + assert runner_owner is not None + assert runner_owner.process_type == Process.TypeChoices.SERVER + assert runner_owner.pid == procs[-1].pid + finally: + for proc in procs: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + for proc in procs: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait(timeout=5) + + def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path): from archivebox.machine.models import Machine, Process from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner @@ -99,7 +987,7 @@ def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path) proc.wait(timeout=5) -def test_runtime_stack_owner_reaps_dead_newer_parent_and_promotes_next_live_command(tmp_path): +def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path): from archivebox.machine.models import Machine, Process from archivebox.core.takeover_util import runtime_stack_owner @@ -139,11 +1027,11 @@ def test_runtime_stack_owner_reaps_dead_newer_parent_and_promotes_next_live_comm owner = runtime_stack_owner(data_dir=tmp_path) - assert owner is not None - assert owner.id == older_row.id - assert owner.pid == procs[0].pid + assert owner is None newer_row.refresh_from_db() assert newer_row.status == Process.StatusChoices.EXITED + older_row.refresh_from_db() + assert older_row.status == Process.StatusChoices.RUNNING finally: for proc in procs: if proc.poll() is None: diff --git a/archivebox/tests/test_test_harness.py b/archivebox/tests/test_test_harness.py index d7bdae64..f4030c48 100644 --- a/archivebox/tests/test_test_harness.py +++ b/archivebox/tests/test_test_harness.py @@ -33,10 +33,7 @@ def test_in_process_archivebox_config_uses_temp_data_dir(): def test_cli_helpers_reject_repo_root_runtime_paths(): with pytest.raises(AssertionError, match="repo root"): - test_harness.run_archivebox_cmd(["version"], data_dir=test_harness.REPO_ROOT) - - with pytest.raises(AssertionError, match="repo root"): - test_harness.run_archivebox_cmd_cwd(["version"], cwd=test_harness.REPO_ROOT) + test_harness.run_archivebox_cmd(["version"], cwd=test_harness.REPO_ROOT) with pytest.raises(AssertionError, match="repo root"): test_harness.run_python_cwd("print('hello')", cwd=test_harness.REPO_ROOT) diff --git a/archivebox/tests/test_ui_add_view_runtime.py b/archivebox/tests/test_ui_add_view_runtime.py index 1bec66a8..b1cde087 100644 --- a/archivebox/tests/test_ui_add_view_runtime.py +++ b/archivebox/tests/test_ui_add_view_runtime.py @@ -1,4 +1,3 @@ -import os import re import time @@ -9,13 +8,13 @@ from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.tests.test_orm_helpers import use_archivebox_db from .conftest import ( - build_test_env, + cli_env, create_admin_and_token, get_depth_counts, get_free_port, init_archive, - run_archivebox_cmd_cwd, - start_server, + run_archivebox_cmd, + start_archivebox_server, stop_server, wait_for_http, ) @@ -25,11 +24,10 @@ pytestmark = pytest.mark.django_db(transaction=True) @pytest.mark.timeout(180) def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env( + env = cli_env( port, PLUGINS="wget", PUBLIC_ADD_VIEW="True", @@ -38,7 +36,7 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s create_admin_and_token(tmp_path) try: - start_server(tmp_path, env=env, port=port) + start_archivebox_server(tmp_path, env=env, port=port) _wait_for_worker_state(tmp_path, "worker_runner", "RUNNING") _stop_worker(tmp_path, "worker_runner") assert _worker_state(tmp_path, "worker_runner") != "RUNNING" @@ -119,7 +117,8 @@ supervisor = get_existing_supervisord_process() worker = get_worker(supervisor, {worker_name!r}) if supervisor else None print(json.dumps(worker)) """ - stdout, stderr, returncode = run_archivebox_cmd_cwd(["manage", "shell", "-c", script], cwd=cwd, timeout=60) + _cmd_result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr or stdout import json @@ -135,7 +134,8 @@ assert supervisor is not None stop_worker(supervisor, {worker_name!r}) print("stopped") """ - stdout, stderr, returncode = run_archivebox_cmd_cwd(["manage", "shell", "-c", script], cwd=cwd, timeout=60) + _cmd_result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60) + stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr or stdout @@ -152,15 +152,14 @@ def _wait_for_worker_state(cwd, worker_name: str, statename: str, timeout: int = @pytest.mark.timeout(180) def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env(port, PUBLIC_ADD_VIEW="True") + env = cli_env(port=port, server=True, PUBLIC_ADD_VIEW="True") create_admin_and_token(tmp_path) try: - start_server(tmp_path, env=env, port=port) + start_archivebox_server(tmp_path, env=env, port=port) session = requests.Session() wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") login_page = session.get( @@ -219,11 +218,10 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit @pytest.mark.timeout(240) def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursive_test_site): - os.chdir(tmp_path) init_archive(tmp_path) port = get_free_port() - env = build_test_env( + env = cli_env( port, PLUGINS="wget,parse_html_urls", PUBLIC_INDEX="True", @@ -232,7 +230,7 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv create_admin_and_token(tmp_path) try: - start_server(tmp_path, env=env, port=port) + start_archivebox_server(tmp_path, env=env, port=port) session = requests.Session() wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") login_page = session.get( diff --git a/archivebox/tests/test_ui_admin_apitoken.py b/archivebox/tests/test_ui_admin_apitoken.py new file mode 100644 index 00000000..8d38e5a6 --- /dev/null +++ b/archivebox/tests/test_ui_admin_apitoken.py @@ -0,0 +1,17 @@ +"""API token admin UI tests.""" + +import pytest +from django.urls import reverse + +from archivebox.tests.conftest import ADMIN_TEST_HOST + +pytestmark = pytest.mark.django_db + + +class TestAPITokenAdmin: + def test_api_token_admin_list_view_renders(self, client, admin_user): + client.force_login(admin_user) + response = client.get(reverse("admin:api_apitoken_changelist"), HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 200 + assert b"API Keys" in response.content diff --git a/archivebox/tests/test_ui_admin_archiveresult.py b/archivebox/tests/test_ui_admin_archiveresult.py new file mode 100644 index 00000000..f878f70e --- /dev/null +++ b/archivebox/tests/test_ui_admin_archiveresult.py @@ -0,0 +1,35 @@ +"""ArchiveResult admin UI tests.""" + +import pytest +from django.urls import reverse + +from archivebox.tests.conftest import ADMIN_TEST_HOST + +pytestmark = pytest.mark.django_db + + +class TestArchiveResultAdminListView: + def test_list_view_renders_readonly_tags_and_noresults_status(self, client, admin_user, snapshot): + from archivebox.core.models import ArchiveResult, Tag + + tag = Tag.objects.create(name="Alpha Research") + snapshot.tags.add(tag) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + status=ArchiveResult.StatusChoices.NORESULTS, + output_str="No title found", + ) + + client.force_login(admin_user) + response = client.get(reverse("admin:core_archiveresult_changelist"), HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 200 + assert b"Alpha Research" in response.content + assert b"tag-editor-inline readonly" in response.content + assert b"No Results" in response.content + + def test_archiveresult_model_has_retry_at_field(self): + from archivebox.core.models import ArchiveResult + + assert "retry_at" in {field.name for field in ArchiveResult._meta.fields} diff --git a/archivebox/tests/test_crawl_admin.py b/archivebox/tests/test_ui_admin_crawl.py similarity index 60% rename from archivebox/tests/test_crawl_admin.py rename to archivebox/tests/test_ui_admin_crawl.py index 9e0bb05f..0b357cf1 100644 --- a/archivebox/tests/test_crawl_admin.py +++ b/archivebox/tests/test_ui_admin_crawl.py @@ -1,47 +1,88 @@ +"""Crawl model and admin UI tests.""" + import re -from typing import cast import pytest -from django.contrib.auth import get_user_model -from django.contrib.auth.models import UserManager from django.urls import reverse from archivebox.crawls.admin import CrawlAdminForm from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot +from archivebox.tests.conftest import ADMIN_TEST_HOST pytestmark = pytest.mark.django_db -User = get_user_model() -ADMIN_HOST = "admin.archivebox.localhost:8000" +class TestCrawlScheduleAdmin: + def test_crawlschedule_change_view_renders_and_saves(self, client, admin_user, crawl): + from archivebox.crawls.models import CrawlSchedule + + schedule = CrawlSchedule.objects.create( + label="Nightly crawl", + notes="", + schedule="0 0 * * *", + template=crawl, + created_by=admin_user, + ) + client.force_login(admin_user) + + change_url = reverse("admin:crawls_crawlschedule_change", args=[schedule.pk]) + get_response = client.get(change_url, HTTP_HOST=ADMIN_TEST_HOST) + + assert get_response.status_code == 200 + assert b"Schedule Info" in get_response.content + assert b"No Crawls yet..." not in get_response.content + assert b"No Snapshots yet..." not in get_response.content + + post_response = client.post( + change_url, + { + "label": "Morning crawl", + "notes": "updated", + "schedule": "0 8 * * *", + "template": str(crawl.pk), + "created_by": str(admin_user.pk), + "_save": "Save", + }, + HTTP_HOST=ADMIN_TEST_HOST, + ) + + assert post_response.status_code == 302 + schedule.refresh_from_db() + assert schedule.label == "Morning crawl" + assert schedule.notes == "updated" + assert schedule.schedule == "0 8 * * *" + assert schedule.template_id == crawl.pk + assert schedule.created_by_id == admin_user.pk + + def test_crawlschedule_changelist_renders_snapshot_counts(self, client, admin_user, crawl, snapshot): + from archivebox.crawls.models import CrawlSchedule + + schedule = CrawlSchedule.objects.create( + label="Daily crawl", + notes="", + schedule="0 0 * * *", + template=crawl, + created_by=admin_user, + ) + crawl.schedule = schedule + crawl.save(update_fields=["schedule"]) + snapshot.crawl = crawl + snapshot.save(update_fields=["crawl"]) + + client.force_login(admin_user) + url = reverse("admin:crawls_crawlschedule_changelist") + response = client.get(url, HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 200 + assert b"Daily crawl" in response.content -@pytest.fixture -def admin_user(db): - return cast(UserManager, User.objects).create_superuser( - username="crawladmin", - email="crawladmin@test.com", - password="testpassword", - ) - - -@pytest.fixture -def crawl(admin_user): - return Crawl.objects.create( - urls="https://example.com\nhttps://example.org", - tags_str="alpha,beta", - created_by=admin_user, - ) - - -def test_crawl_admin_change_view_renders_tag_editor_widget(client, admin_user, crawl): - client.login(username="crawladmin", password="testpassword") - - response = client.get( +def test_crawl_admin_change_view_renders_tag_editor_widget(admin_client, crawl): + response = admin_client.get( reverse("admin:crawls_crawl_change", args=[crawl.pk]), - HTTP_HOST=ADMIN_HOST, + HTTP_HOST=ADMIN_TEST_HOST, ) assert response.status_code == 200 @@ -51,12 +92,28 @@ def test_crawl_admin_change_view_renders_tag_editor_widget(client, admin_user, c assert b"beta" in response.content -def test_crawl_admin_add_view_renders_url_filter_alias_fields(client, admin_user): - client.login(username="crawladmin", password="testpassword") +def test_crawl_admin_recrawl_object_action_is_post_only(admin_client, crawl): + action_url = reverse("admin:crawls_crawl_actions", kwargs={"pk": crawl.pk, "tool": "recrawl"}) - response = client.get( + change_response = admin_client.get(reverse("admin:crawls_crawl_change", args=[crawl.pk]), HTTP_HOST=ADMIN_TEST_HOST) + assert change_response.status_code == 200 + assert b'
    str: def worker_runtime_component(worker_name: str, *, config=None) -> str | None: - if worker_name in {RUNNER_WORKER["name"], RUNNER_WATCH_WORKER("")["name"]}: + if worker_name in {RUNNER_WORKER["name"], RUNNER_WATCH_WORKER("")["name"]} or worker_name.startswith("worker_runner_"): return "orchestrator" if worker_name in {SERVER_WORKER("", "")["name"], RUNSERVER_WORKER("", "", reload=False)["name"]}: return "server" diff --git a/bin/take_screenshot.js b/bin/take_screenshot.js index 91927243..fc32b709 100755 --- a/bin/take_screenshot.js +++ b/bin/take_screenshot.js @@ -15,6 +15,7 @@ Usage: Environment: SESSIONID Django session cookie value for admin.archivebox.localhost + SCREENSHOT_COOKIE_NAME Cookie name, defaults to sessionid SCREENSHOT_COOKIE_DOMAIN Cookie domain, defaults to admin.archivebox.localhost CHROME_BINARY Chromium/Chrome executable path PUPPETEER_EXECUTABLE_PATH Chromium/Chrome executable path @@ -90,7 +91,7 @@ async function main() { if (process.env.SESSIONID) { const cookie = { - name: 'sessionid', + name: process.env.SCREENSHOT_COOKIE_NAME || 'sessionid', value: process.env.SESSIONID, path: '/', }; diff --git a/pyproject.toml b/pyproject.toml index 2af70d71..c83f36d1 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,6 +142,7 @@ dev = [ "pytest>=8.3.3", "pytest-cov>=6.0.0", "pytest-httpserver>=1.1.0", + "pytest-timeout>=2.3.1", "coverage[toml]>=7.6.0", "bottle>=0.13.1", ### LINTING