From aaea35f9fd41fbfb2372637a2a11af2b7374a06c Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 1 Sep 2026 12:39:58 -0700 Subject: [PATCH] Preserve lifecycle compatibility after runtime unification --- archivebox/cli/archivebox_add.py | 2 +- archivebox/machine/models.py | 22 +- archivebox/plugins/hooks.py | 12 +- archivebox/services/process_service.py | 10 +- archivebox/services/runner.py | 11 + archivebox/tests/test_crawl_runner.py | 39 + archivebox/tests/test_hooks.py | 34 + docs/ArchiveBox-Architecture-Diagrams.md | 33 +- .../archivebox/archivebox.core.models.md | 383 ++++----- .../archivebox/archivebox.crawls.models.md | 222 ++--- .../archivebox/archivebox.machine.models.md | 371 +++------ .../archivebox/archivebox.workers.models.md | 775 ++++++------------ 12 files changed, 694 insertions(+), 1220 deletions(-) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 596cd497..1f5152ae 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -213,7 +213,7 @@ def add( print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]") print(f" [dim]First URL: {first_url}[/dim]") - # 3. The CrawlMachine will create Snapshots from all URLs when started + # 3. The runner will create Snapshots from all URLs after claiming the Crawl # Parser extractors run on snapshots and discover more URLs # Discovered URLs become child Snapshots (depth+1) diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 0b774a08..07ba0030 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -524,8 +524,10 @@ class Binary(ModelWithHealthStats, ModelWithQueue): Installation is synchronous during queued→installed transition. If installation fails, Binary stays in queued with retry_at set for later retry. - State machine calls run(), which emits an abxpkg BinaryRequestEvent through - the ArchiveBox runner and installs the binary using the specified providers. + BinaryService claims queued rows with a conditional update, then run() + emits an abxpkg BinaryRequestEvent and persists the resolved installation. + The database row is the only lifecycle state; there is intentionally no + second in-memory state machine to reconcile after a worker interruption. """ class StatusChoices(models.TextChoices): @@ -564,7 +566,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue): version = models.CharField(max_length=32, default="", null=False, blank=True) sha256 = models.CharField(max_length=64, default="", null=False, blank=True) - # State machine fields + # Durable queue lifecycle fields status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16) retry_at = ModelWithQueue.RetryAtField( default=timezone.now, @@ -990,7 +992,9 @@ class Process(ModelWithDeleteAfter, models.Model): - running: Process actively executing - exited: Process completed (check exit_code for success/failure) - State machine calls launch() to spawn the process and monitors its lifecycle. + Direct Process methods and the abx-dl event projector own these transitions. + Keeping the DB row as the only lifecycle state makes interrupted subprocess + recovery observable without reconciling a second in-memory state machine. """ class StatusChoices(models.TextChoices): @@ -1149,7 +1153,7 @@ class Process(ModelWithDeleteAfter, models.Model): # Reverse relation to ArchiveResult (OneToOne from AR side) # archiveresult: OneToOneField defined on ArchiveResult model - # State machine fields + # Durable process lifecycle fields status = models.CharField( max_length=16, choices=StatusChoices.choices, @@ -2264,8 +2268,8 @@ class Process(ModelWithDeleteAfter, models.Model): """ Gracefully terminate process: SIGTERM → wait → SIGKILL. - This consolidates the scattered SIGTERM/SIGKILL logic from: - - crawls/models.py Crawl.cleanup() + This consolidates SIGTERM/SIGKILL logic used by: + - workers/management/commands/runner_watch.py - workers/pid_utils.py stop_worker() - supervisord_util.py stop_existing_supervisord_process() @@ -2328,8 +2332,8 @@ class Process(ModelWithDeleteAfter, models.Model): Uses parallel polling approach - sends SIGTERM to all processes at once, then polls all simultaneously with individual deadline tracking. - This consolidates the scattered child-killing logic from: - - crawls/models.py Crawl.cleanup() os.killpg() + This consolidates child-killing logic used by: + - core/takeover_util.py - supervisord_util.py stop_existing_supervisord_process() Args: diff --git a/archivebox/plugins/hooks.py b/archivebox/plugins/hooks.py index 5a6d3cca..2f7364d1 100644 --- a/archivebox/plugins/hooks.py +++ b/archivebox/plugins/hooks.py @@ -165,8 +165,14 @@ def run_hook( from archivebox.services.process_service import parse_event_datetime from abx_dl.orchestrator import create_bus - if parent is not None: - kwargs.setdefault("_parent_process_id", str(parent.id)) + # Preserve the old direct-call contract: hooks are children of an explicit + # parent, or of the current ArchiveBox process when one can be identified. + # This belongs on the DB projection adapter, not in hook CLI arguments. + if parent is None: + try: + parent = Process.current() + except Exception: + parent = None config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")} env, resolved = _hook_environment(config, **config_scope) hook = _catalog_hook(script) @@ -175,7 +181,7 @@ def run_hook( timeout = min(int(timeout or 300), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS)) bus = create_bus(name=f"ArchiveBoxHook_{hook.plugin_name}", total_timeout=float(timeout) + 30.0) - PersistedProcessService(bus) + PersistedProcessService(bus, parent_process_id=str(parent.id) if parent is not None else None) async def execute_and_close(): try: diff --git a/archivebox/services/process_service.py b/archivebox/services/process_service.py index 31f31bd2..ab919149 100644 --- a/archivebox/services/process_service.py +++ b/archivebox/services/process_service.py @@ -61,8 +61,12 @@ class ProcessService(BaseService): ] EMITS: ClassVar[list[type[BaseEvent]]] = [] - def __init__(self, bus): + def __init__(self, bus, *, parent_process_id: str | None = None): self._iface = None + # A direct run_hook() call owns a private bus, so its caller-supplied + # parent applies to every process projected from that bus. Crawl buses + # leave this unset and derive their hierarchy from lifecycle events. + self.parent_process_id = parent_process_id self._completed_queue: asyncio.Queue[ProcessCompletedEvent | None] = asyncio.Queue() self._completed_worker: asyncio.Task | None = None super().__init__(bus) @@ -100,6 +104,7 @@ class ProcessService(BaseService): process = await Process.objects.acreate( machine=iface.machine, iface=iface, + parent_id=self.parent_process_id, process_type=process_type, worker_type=worker_type, pwd=event.output_dir, @@ -133,6 +138,7 @@ class ProcessService(BaseService): hook_path=event.hook_path, ) await Process.objects.filter(id=process.id).aupdate( + parent_id=self.parent_process_id or process.parent_id, pwd=process.pwd, cmd=process.cmd, env=process.env, @@ -219,6 +225,7 @@ class ProcessService(BaseService): await Process.objects.acreate( machine=iface.machine, iface=iface, + parent_id=self.parent_process_id, process_type=process_type, worker_type=worker_type, pwd=event.output_dir, @@ -239,6 +246,7 @@ class ProcessService(BaseService): updates = { "machine_id": iface.machine_id, "iface_id": iface.id, + "parent_id": self.parent_process_id or process.parent_id, "pwd": event.output_dir, "env": process_env, "pid": event.pid or process.pid, diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index a43bc912..471af73c 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -602,6 +602,17 @@ class CrawlRunner: self.selected_plugins = sorted(runtime_plugins) or None if self.crawl.is_paused: return [] + system_task = self.crawl.get_system_task() + if system_task == "archivebox://update": + # Scheduled maintenance crawls are control-plane work, not URL + # input. The pre-unified crawl lifecycle delegated this sentinel + # directly to the database maintenance scan. Preserve that contract so + # the unified runner does not turn it into an archivebox://internal + # Snapshot and feed the literal sentinel through parser plugins. + from archivebox.cli.archivebox_update import process_all_db_snapshots + + process_all_db_snapshots() + return [] if self.initial_snapshot_ids: # Explicit ids select normal runnable work, except for the one # sealed-search backfill admitted by allow_maintenance_on_inactive_crawl. diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index 8d541c3c..fa2a5ee8 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -271,6 +271,45 @@ def test_snapshot_started_state_keeps_retry_at_lease(): assert snapshot.retry_at > before +@pytest.mark.django_db(transaction=True) +def test_system_update_crawl_runs_database_maintenance_without_snapshot_work(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + from django.utils import timezone + + owner_id = get_or_create_system_user_pk() + archived_crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=owner_id, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + archived_snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=archived_crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + Snapshot.objects.filter(pk=archived_snapshot.pk).update(fs_version=0, retry_at=None) + maintenance_crawl = Crawl.objects.create( + urls="archivebox://update", + created_by_id=owner_id, + status=Crawl.StatusChoices.QUEUED, + retry_at=timezone.now(), + ) + + snapshot_ids = CrawlRunner(maintenance_crawl, show_progress=False).load_run_state() + + maintenance_crawl.refresh_from_db() + archived_snapshot.refresh_from_db() + assert snapshot_ids == [] + assert maintenance_crawl.snapshot_set.count() == 0 + assert archived_snapshot.status == Snapshot.StatusChoices.SEALED + assert archived_snapshot.retry_at is not None + + @pytest.mark.django_db(transaction=True) def test_crawl_start_event_keeps_retry_at_lease(): from abx_dl.events import CrawlStartEvent diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index 063f706d..621a3931 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -73,6 +73,11 @@ class TestBackgroundHookDetection: assert any(hook.name == "on_Snapshot__35_wget.finite.bg.py" for hook in background_hooks) assert any(hook.name == "on_Snapshot__93_hashes.py" for hook in foreground_hooks) + def test_legacy_background_marker_remains_supported_for_user_plugins(self): + from archivebox.plugins.hooks import is_background_hook + + assert is_background_hook("on_Snapshot__50_custom__background.py") is True + @pytest.mark.django_db(transaction=True) class TestJSONLParsing: @@ -182,6 +187,35 @@ class TestJSONLParsing: assert len(records) == 1 assert records[0]["type"] == "ArchiveResult" + def test_direct_hook_preserves_explicit_parent_process(self, tmp_path): + """The compatibility adapter must retain the caller's process hierarchy.""" + from archivebox.machine.models import Machine, Process + from archivebox.plugins.hooks import run_hook + + parent = Process.objects.create( + machine=Machine.current(), + process_type=Process.TypeChoices.CLI, + status=Process.StatusChoices.RUNNING, + ) + snap_dir = tmp_path / "parented-snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("parented hook input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + + process = run_hook( + hook_path, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + parent=parent, + url="https://example.com/parented-hook", + ) + + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + assert process.parent_id == parent.id + class TestRequiredBinaryConfigHandling: """Test that required_binaries keep configured XYZ_BINARY values intact.""" diff --git a/docs/ArchiveBox-Architecture-Diagrams.md b/docs/ArchiveBox-Architecture-Diagrams.md index 35fb0100..daad4aa7 100644 --- a/docs/ArchiveBox-Architecture-Diagrams.md +++ b/docs/ArchiveBox-Architecture-Diagrams.md @@ -4,8 +4,8 @@ This page is a map of the current execution and persistence paths. The implement - `archivebox/cli/` for CLI entry points - `archivebox/services/runner.py` for crawl and snapshot execution -- `archivebox/crawls/models.py` for the `Crawl` model and state machine -- `archivebox/core/models.py` for `Snapshot`, `ArchiveResult`, and the `Snapshot` state machine +- `archivebox/crawls/models.py` for the `Crawl` model and its atomic queue transitions +- `archivebox/core/models.py` for `Snapshot`, `ArchiveResult`, and Snapshot queue transitions - `archivebox/services/` for bus event projectors - `abxpkg` and `abx-plugins` for binary resolution and plugin hooks @@ -57,44 +57,49 @@ flowchart LR The database is the source of truth for model state. Snapshot directories contain captured artifacts and rendered metadata. Older collections may also contain legacy timestamp-named snapshot directories. -## `Crawl` State Machine +## `Crawl` Queue Lifecycle -Implemented by `Crawl` and `CrawlMachine` in `archivebox/crawls/models.py`. +Implemented directly by `Crawl` in `archivebox/crawls/models.py`. The database +row is the durable state; the runner claims `retry_at` with a conditional update +before it performs side effects, then calls the model's explicit lifecycle +methods. There is deliberately no second in-memory state machine that can drift +from the row owned by another process. ```mermaid stateDiagram-v2 [*] --> QUEUED - QUEUED --> STARTED: tick and valid URLs - QUEUED --> QUEUED: tick and not ready + QUEUED --> STARTED: runner claim and valid URLs + QUEUED --> QUEUED: claimed but not ready QUEUED --> SEALED: all existing snapshots finished STARTED --> SEALED: all snapshots finished QUEUED --> PAUSED: pause requested STARTED --> PAUSED: pause requested PAUSED --> QUEUED: resume requested - PAUSED --> PAUSED: tick + PAUSED --> PAUSED: not runnable QUEUED --> SEALED: explicit seal STARTED --> SEALED: explicit seal PAUSED --> SEALED: explicit seal SEALED --> [*] ``` -A crawl owns a set of snapshots. Entering `STARTED` creates or discovers those snapshots; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. +A crawl owns a set of snapshots. The runner creates or discovers those snapshots and projects crawl events while the row is `STARTED`; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. The `archivebox://update` sentinel remains control-plane work: the runner invokes database maintenance directly and never turns it into a Snapshot. -## `Snapshot` State Machine +## `Snapshot` Queue Lifecycle -Implemented by `Snapshot` and `SnapshotMachine` in `archivebox/core/models.py`. +Implemented directly by `Snapshot` in `archivebox/core/models.py`, using the +same conditional `retry_at` claim protocol as `Crawl`. ```mermaid stateDiagram-v2 [*] --> QUEUED - QUEUED --> STARTED: tick and URL is ready - QUEUED --> QUEUED: tick and not ready + QUEUED --> STARTED: runner claim and URL is ready + QUEUED --> QUEUED: claimed but not ready QUEUED --> SEALED: all existing results finished STARTED --> SEALED: all hook results finished QUEUED --> PAUSED: pause requested STARTED --> PAUSED: pause requested PAUSED --> QUEUED: resume requested - PAUSED --> PAUSED: tick + PAUSED --> PAUSED: not runnable QUEUED --> SEALED: explicit seal STARTED --> SEALED: explicit seal PAUSED --> SEALED: explicit seal @@ -105,7 +110,7 @@ The runner creates one queued `ArchiveResult` per selected hook, executes those ## `ArchiveResult` Projection -`ArchiveResult` is not driven by a separate Python state machine. The runner creates queued rows, and `ArchiveResultService` projects `ArchiveResultEvent` and `ProcessCompletedEvent` data into them. +`ArchiveResult` is not driven by a separate in-memory state machine. The runner creates queued rows, and `ArchiveResultService` projects `ArchiveResultEvent` and `ProcessCompletedEvent` data into them. ```mermaid flowchart LR diff --git a/docs/apidocs/archivebox/archivebox.core.models.md b/docs/apidocs/archivebox/archivebox.core.models.md index c95ae91c..588197de 100644 --- a/docs/apidocs/archivebox/archivebox.core.models.md +++ b/docs/apidocs/archivebox/archivebox.core.models.md @@ -33,16 +33,28 @@ ``` * - {py:obj}`Snapshot ` - -* - {py:obj}`SnapshotMachine ` - - ```{autodoc2-docstring} archivebox.core.models.SnapshotMachine - :summary: - ``` * - {py:obj}`ArchiveResult ` - ```` ### API +````{py:exception} SnapshotMigrationError() +:canonical: archivebox.core.models.SnapshotMigrationError + +Bases: {py:obj}`RuntimeError` + +```{autodoc2-docstring} archivebox.core.models.SnapshotMigrationError +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.models.SnapshotMigrationError.__init__ +``` + +```` + `````{py:class} UngroupedSubquery(queryset, output_field=None, **extra) :canonical: archivebox.core.models.UngroupedSubquery @@ -280,6 +292,7 @@ Bases: {py:obj}`django.db.models.Model` ````{py:attribute} unique_together :canonical: archivebox.core.models.SnapshotTag.Meta.unique_together +:type: typing.ClassVar[list[tuple[str, str]]] :value: > [('snapshot', 'tag')] @@ -321,6 +334,7 @@ Bases: {py:obj}`django.db.models.QuerySet` ````{py:attribute} FILTER_TYPES :canonical: archivebox.core.models.SnapshotQuerySet.FILTER_TYPES +:type: typing.ClassVar[dict[str, typing.Any]] :value: > None @@ -438,7 +452,17 @@ Bases: {py:obj}`models.Manager.from_queryset`\({py:obj}`SnapshotQuerySet`\) ``````{py:class} Snapshot(*args, **kwargs) :canonical: archivebox.core.models.Snapshot -Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithQueue` + +````{py:attribute} BROWSER_EXTENSION_UPLOAD_HOOK_NAME +:canonical: archivebox.core.models.Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME +:value: > + 'on_Snapshot__archivebox_browser_extension_upload' + +```{autodoc2-docstring} archivebox.core.models.Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME +``` + +```` ````{py:attribute} INTERNAL_INPUT_URL :canonical: archivebox.core.models.Snapshot.INTERNAL_INPUT_URL @@ -651,16 +675,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar ```` -````{py:attribute} state_machine_name -:canonical: archivebox.core.models.Snapshot.state_machine_name -:value: > - 'archivebox.core.models.SnapshotMachine' - -```{autodoc2-docstring} archivebox.core.models.Snapshot.state_machine_name -``` - -```` - ````{py:attribute} state_field_name :canonical: archivebox.core.models.Snapshot.state_field_name :value: > @@ -691,6 +705,46 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar ```` +````{py:attribute} INITIAL_STATE +:canonical: archivebox.core.models.Snapshot.INITIAL_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot.INITIAL_STATE +``` + +```` + +````{py:attribute} ACTIVE_STATE +:canonical: archivebox.core.models.Snapshot.ACTIVE_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot.ACTIVE_STATE +``` + +```` + +````{py:attribute} FINAL_STATES +:canonical: archivebox.core.models.Snapshot.FINAL_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.core.models.Snapshot.FINAL_STATES +``` + +```` + +````{py:attribute} FINAL_OR_ACTIVE_STATES +:canonical: archivebox.core.models.Snapshot.FINAL_OR_ACTIVE_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.core.models.Snapshot.FINAL_OR_ACTIVE_STATES +``` + +```` + ````{py:attribute} active_state :canonical: archivebox.core.models.Snapshot.active_state :value: > @@ -796,7 +850,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar `````{py:class} Meta :canonical: archivebox.core.models.Snapshot.Meta -Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithQueue.Meta` ````{py:attribute} app_label :canonical: archivebox.core.models.Snapshot.Meta.app_label @@ -830,6 +884,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ````{py:attribute} indexes :canonical: archivebox.core.models.Snapshot.Meta.indexes +:type: typing.ClassVar[list[django.db.models.Index]] :value: > None @@ -840,6 +895,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ````{py:attribute} constraints :canonical: archivebox.core.models.Snapshot.Meta.constraints +:type: typing.ClassVar[list[django.db.models.BaseConstraint]] :value: > None @@ -946,6 +1002,30 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` +````{py:method} start_processing() -> bool +:canonical: archivebox.core.models.Snapshot.start_processing + +```{autodoc2-docstring} archivebox.core.models.Snapshot.start_processing +``` + +```` + +````{py:method} seal() -> bool +:canonical: archivebox.core.models.Snapshot.seal + +```{autodoc2-docstring} archivebox.core.models.Snapshot.seal +``` + +```` + +````{py:method} advance_lifecycle() -> bool +:canonical: archivebox.core.models.Snapshot.advance_lifecycle + +```{autodoc2-docstring} archivebox.core.models.Snapshot.advance_lifecycle +``` + +```` + ````{py:method} cancel() -> None :canonical: archivebox.core.models.Snapshot.cancel @@ -1042,6 +1122,17 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` +````{py:attribute} _FS_VERSION_MIGRATION_PATHS +:canonical: archivebox.core.models.Snapshot._FS_VERSION_MIGRATION_PATHS +:type: typing.ClassVar[dict[str, str]] +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot._FS_VERSION_MIGRATION_PATHS +``` + +```` + ````{py:property} fs_migration_needed :canonical: archivebox.core.models.Snapshot.fs_migration_needed :type: bool @@ -1100,6 +1191,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` +````{py:method} hydrate_archiveresult_output_metadata(snapshot_dir: pathlib.Path | None = None) -> int +:canonical: archivebox.core.models.Snapshot.hydrate_archiveresult_output_metadata + +```{autodoc2-docstring} archivebox.core.models.Snapshot.hydrate_archiveresult_output_metadata +``` + +```` + ````{py:method} _fs_migrate_legacy_to_0_9_0(source_dir: pathlib.Path | None = None, target_dir: pathlib.Path | None = None, config: ArchiveBoxBaseConfig | None = None) :canonical: archivebox.core.models.Snapshot._fs_migrate_legacy_to_0_9_0 @@ -1311,7 +1410,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} tags_str(nocache=True) -> str | None +````{py:method} tags_str() -> str | None :canonical: archivebox.core.models.Snapshot.tags_str ```{autodoc2-docstring} archivebox.core.models.Snapshot.tags_str @@ -1319,7 +1418,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} icons(path: str | None = None) -> str +````{py:method} icons(path: str | None = None, prefix: str = '/', quote_paths: bool = False) -> str :canonical: archivebox.core.models.Snapshot.icons ```{autodoc2-docstring} archivebox.core.models.Snapshot.icons @@ -1404,6 +1503,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` +````{py:method} remove_legacy_archive_symlink() -> None +:canonical: archivebox.core.models.Snapshot.remove_legacy_archive_symlink + +```{autodoc2-docstring} archivebox.core.models.Snapshot.remove_legacy_archive_symlink +``` + +```` + ````{py:method} legacy_archive_path() -> str :canonical: archivebox.core.models.Snapshot.legacy_archive_path @@ -1468,10 +1575,10 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} cleanup() -:canonical: archivebox.core.models.Snapshot.cleanup +````{py:method} finalize_output_metadata() -> None +:canonical: archivebox.core.models.Snapshot.finalize_output_metadata -```{autodoc2-docstring} archivebox.core.models.Snapshot.cleanup +```{autodoc2-docstring} archivebox.core.models.Snapshot.finalize_output_metadata ``` ```` @@ -1653,7 +1760,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} discover_outputs(include_filesystem_fallback: bool = True) -> list[dict] +````{py:method} discover_outputs(include_filesystem_fallback: bool = True, archive_results: list[archivebox.core.models.Snapshot.discover_outputs.ArchiveResult] | None = None) -> list[dict] :canonical: archivebox.core.models.Snapshot.discover_outputs ```{autodoc2-docstring} archivebox.core.models.Snapshot.discover_outputs @@ -1661,7 +1768,16 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} to_dict(extended: bool = False) -> dict[str, typing.Any] +````{py:property} static_archive_path +:canonical: archivebox.core.models.Snapshot.static_archive_path +:type: str + +```{autodoc2-docstring} archivebox.core.models.Snapshot.static_archive_path +``` + +```` + +````{py:method} to_dict(extended: bool = False, static_export: bool = False) -> dict[str, typing.Any] :canonical: archivebox.core.models.Snapshot.to_dict ```{autodoc2-docstring} archivebox.core.models.Snapshot.to_dict @@ -1693,6 +1809,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` +````{py:method} get_html_details_context(request=None, *, static_export_dir: pathlib.Path | None = None) -> dict[str, typing.Any] +:canonical: archivebox.core.models.Snapshot.get_html_details_context + +```{autodoc2-docstring} archivebox.core.models.Snapshot.get_html_details_context +``` + +```` + ````{py:method} write_html_details(out_dir: pathlib.Path | str | None = None) -> None :canonical: archivebox.core.models.Snapshot.write_html_details @@ -1701,7 +1825,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} get_detail_page_auxiliary_items(outputs: list[dict] | None = None, hidden_card_plugins: set[str] | None = None) -> tuple[list[dict[str, object]], list[dict[str, object]]] +````{py:method} get_detail_page_auxiliary_items(outputs: list[dict] | None = None, hidden_card_plugins: set[str] | None = None, archive_results: list[archivebox.core.models.Snapshot.get_detail_page_auxiliary_items.ArchiveResult] | None = None) -> tuple[list[dict[str, object]], list[dict[str, object]]] :canonical: archivebox.core.models.Snapshot.get_detail_page_auxiliary_items ```{autodoc2-docstring} archivebox.core.models.Snapshot.get_detail_page_auxiliary_items @@ -1720,185 +1844,12 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob `````` -`````{py:class} SnapshotMachine(obj, *args, **kwargs) -:canonical: archivebox.core.models.SnapshotMachine - -Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.__init__ -``` - -````{py:attribute} model_attr_name -:canonical: archivebox.core.models.SnapshotMachine.model_attr_name -:value: > - 'snapshot' - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.model_attr_name -``` - -```` - -````{py:attribute} queued -:canonical: archivebox.core.models.SnapshotMachine.queued -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.queued -``` - -```` - -````{py:attribute} started -:canonical: archivebox.core.models.SnapshotMachine.started -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.started -``` - -```` - -````{py:attribute} paused -:canonical: archivebox.core.models.SnapshotMachine.paused -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.paused -``` - -```` - -````{py:attribute} sealed -:canonical: archivebox.core.models.SnapshotMachine.sealed -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.sealed -``` - -```` - -````{py:attribute} tick -:canonical: archivebox.core.models.SnapshotMachine.tick -:value: > - None - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.tick -``` - -```` - -````{py:attribute} seal -:canonical: archivebox.core.models.SnapshotMachine.seal -:value: > - None - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.seal -``` - -```` - -````{py:attribute} pause_requested -:canonical: archivebox.core.models.SnapshotMachine.pause_requested -:value: > - None - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.pause_requested -``` - -```` - -````{py:attribute} resume_requested -:canonical: archivebox.core.models.SnapshotMachine.resume_requested -:value: > - 'to(...)' - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.resume_requested -``` - -```` - -````{py:attribute} snapshot -:canonical: archivebox.core.models.SnapshotMachine.snapshot -:type: archivebox.core.models.Snapshot -:value: > - None - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.snapshot -``` - -```` - -````{py:method} can_start() -> bool -:canonical: archivebox.core.models.SnapshotMachine.can_start - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.can_start -``` - -```` - -````{py:method} is_finished() -> bool -:canonical: archivebox.core.models.SnapshotMachine.is_finished - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.is_finished -``` - -```` - -````{py:method} has_finished_archive_results() -> bool -:canonical: archivebox.core.models.SnapshotMachine.has_finished_archive_results - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.has_finished_archive_results -``` - -```` - -````{py:method} enter_queued() -:canonical: archivebox.core.models.SnapshotMachine.enter_queued - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_queued -``` - -```` - -````{py:method} enter_paused() -:canonical: archivebox.core.models.SnapshotMachine.enter_paused - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_paused -``` - -```` - -````{py:method} enter_started() -:canonical: archivebox.core.models.SnapshotMachine.enter_started - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_started -``` - -```` - -````{py:method} enter_sealed() -:canonical: archivebox.core.models.SnapshotMachine.enter_sealed - -```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_sealed -``` - -```` - -````` - ``````{py:class} ArchiveResult(*args, **kwargs) :canonical: archivebox.core.models.ArchiveResult Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithNotes` -`````{py:class} StatusChoices(*args, **kwds) +`````{py:class} StatusChoices() :canonical: archivebox.core.models.ArchiveResult.StatusChoices Bases: {py:obj}`django.db.models.TextChoices` @@ -2107,24 +2058,6 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} cached_snapshot_ids_with_majority_status(status: str | collections.abc.Iterable[str], *, timeout: int = 60) -> tuple[str, ...] -:canonical: archivebox.core.models.ArchiveResult.cached_snapshot_ids_with_majority_status -:classmethod: - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.cached_snapshot_ids_with_majority_status -``` - -```` - -````{py:method} clear_majority_status_cache() -> None -:canonical: archivebox.core.models.ArchiveResult.clear_majority_status_cache -:classmethod: - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.clear_majority_status_cache -``` - -```` - ````{py:attribute} id :canonical: archivebox.core.models.ArchiveResult.id :value: > @@ -2346,7 +2279,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ````{py:attribute} verbose_name_plural :canonical: archivebox.core.models.ArchiveResult.Meta.verbose_name_plural :value: > - 'Archive Results Log' + 'Archive Results' ```{autodoc2-docstring} archivebox.core.models.ArchiveResult.Meta.verbose_name_plural ``` @@ -2355,6 +2288,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ````{py:attribute} indexes :canonical: archivebox.core.models.ArchiveResult.Meta.indexes +:type: typing.ClassVar[list[django.db.models.Index]] :value: > None @@ -2365,6 +2299,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ````{py:attribute} constraints :canonical: archivebox.core.models.ArchiveResult.Meta.constraints +:type: typing.ClassVar[list[django.db.models.BaseConstraint]] :value: > None @@ -2444,10 +2379,10 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} delete(*args, **kwargs) -:canonical: archivebox.core.models.ArchiveResult.delete +````{py:method} schedule_delete_cleanup(*, using: str | None = None) -> None +:canonical: archivebox.core.models.ArchiveResult.schedule_delete_cleanup -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.delete +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.schedule_delete_cleanup ``` ```` @@ -2765,14 +2700,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} update_from_output() -:canonical: archivebox.core.models.ArchiveResult.update_from_output - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.update_from_output -``` - -```` - ````{py:method} _set_binary_from_cmd(cmd: list) -> None :canonical: archivebox.core.models.ArchiveResult._set_binary_from_cmd diff --git a/docs/apidocs/archivebox/archivebox.crawls.models.md b/docs/apidocs/archivebox/archivebox.crawls.models.md index 4554d830..dfb1b5e5 100644 --- a/docs/apidocs/archivebox/archivebox.crawls.models.md +++ b/docs/apidocs/archivebox/archivebox.crawls.models.md @@ -19,8 +19,6 @@ - * - {py:obj}`Crawl ` - -* - {py:obj}`CrawlMachine ` - - ```` ### API @@ -235,7 +233,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID.Meta`, {py:obj}`arch ``````{py:class} Crawl(*args, **kwargs) :canonical: archivebox.crawls.models.Crawl -Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithQueue` ````{py:attribute} id :canonical: archivebox.crawls.models.Crawl.id @@ -387,16 +385,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar ```` -````{py:attribute} state_machine_name -:canonical: archivebox.crawls.models.Crawl.state_machine_name -:value: > - 'archivebox.crawls.models.CrawlMachine' - -```{autodoc2-docstring} archivebox.crawls.models.Crawl.state_machine_name -``` - -```` - ````{py:attribute} retry_at_field_name :canonical: archivebox.crawls.models.Crawl.retry_at_field_name :value: > @@ -427,6 +415,46 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar ```` +````{py:attribute} INITIAL_STATE +:canonical: archivebox.crawls.models.Crawl.INITIAL_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.INITIAL_STATE +``` + +```` + +````{py:attribute} ACTIVE_STATE +:canonical: archivebox.crawls.models.Crawl.ACTIVE_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.ACTIVE_STATE +``` + +```` + +````{py:attribute} FINAL_STATES +:canonical: archivebox.crawls.models.Crawl.FINAL_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.FINAL_STATES +``` + +```` + +````{py:attribute} FINAL_OR_ACTIVE_STATES +:canonical: archivebox.crawls.models.Crawl.FINAL_OR_ACTIVE_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.FINAL_OR_ACTIVE_STATES +``` + +```` + ````{py:attribute} active_state :canonical: archivebox.crawls.models.Crawl.active_state :value: > @@ -492,7 +520,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar `````{py:class} Meta :canonical: archivebox.crawls.models.Crawl.Meta -Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithQueue.Meta` ````{py:attribute} app_label :canonical: archivebox.crawls.models.Crawl.Meta.app_label @@ -940,14 +968,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} run() -> Snapshot | None -:canonical: archivebox.crawls.models.Crawl.run - -```{autodoc2-docstring} archivebox.crawls.models.Crawl.run -``` - -```` - ````{py:method} is_finished() -> bool :canonical: archivebox.crawls.models.Crawl.is_finished @@ -956,176 +976,52 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob ```` -````{py:method} cleanup() -:canonical: archivebox.crawls.models.Crawl.cleanup - -```{autodoc2-docstring} archivebox.crawls.models.Crawl.cleanup -``` - -```` - -`````` - -`````{py:class} CrawlMachine(obj, *args, **kwargs) -:canonical: archivebox.crawls.models.CrawlMachine - -Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` - -````{py:attribute} crawl -:canonical: archivebox.crawls.models.CrawlMachine.crawl -:type: archivebox.crawls.models.Crawl -:value: > - None - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.crawl -``` - -```` - -````{py:attribute} model_attr_name -:canonical: archivebox.crawls.models.CrawlMachine.model_attr_name -:value: > - 'crawl' - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.model_attr_name -``` - -```` - -````{py:attribute} queued -:canonical: archivebox.crawls.models.CrawlMachine.queued -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.queued -``` - -```` - -````{py:attribute} started -:canonical: archivebox.crawls.models.CrawlMachine.started -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.started -``` - -```` - -````{py:attribute} paused -:canonical: archivebox.crawls.models.CrawlMachine.paused -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.paused -``` - -```` - -````{py:attribute} sealed -:canonical: archivebox.crawls.models.CrawlMachine.sealed -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.sealed -``` - -```` - -````{py:attribute} tick -:canonical: archivebox.crawls.models.CrawlMachine.tick -:value: > - None - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.tick -``` - -```` - -````{py:attribute} seal -:canonical: archivebox.crawls.models.CrawlMachine.seal -:value: > - None - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.seal -``` - -```` - -````{py:attribute} pause_requested -:canonical: archivebox.crawls.models.CrawlMachine.pause_requested -:value: > - None - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.pause_requested -``` - -```` - -````{py:attribute} resume_requested -:canonical: archivebox.crawls.models.CrawlMachine.resume_requested -:value: > - 'to(...)' - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.resume_requested -``` - -```` - ````{py:method} can_start() -> bool -:canonical: archivebox.crawls.models.CrawlMachine.can_start +:canonical: archivebox.crawls.models.Crawl.can_start -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.can_start -``` - -```` - -````{py:method} is_finished() -> bool -:canonical: archivebox.crawls.models.CrawlMachine.is_finished - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.is_finished +```{autodoc2-docstring} archivebox.crawls.models.Crawl.can_start ``` ```` ````{py:method} has_finished_snapshots() -> bool -:canonical: archivebox.crawls.models.CrawlMachine.has_finished_snapshots +:canonical: archivebox.crawls.models.Crawl.has_finished_snapshots -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.has_finished_snapshots +```{autodoc2-docstring} archivebox.crawls.models.Crawl.has_finished_snapshots ``` ```` -````{py:method} enter_queued() -:canonical: archivebox.crawls.models.CrawlMachine.enter_queued +````{py:method} mark_started() -> bool +:canonical: archivebox.crawls.models.Crawl.mark_started -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_queued +```{autodoc2-docstring} archivebox.crawls.models.Crawl.mark_started ``` ```` -````{py:method} enter_started() -:canonical: archivebox.crawls.models.CrawlMachine.enter_started +````{py:method} seal() -> bool +:canonical: archivebox.crawls.models.Crawl.seal -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_started +```{autodoc2-docstring} archivebox.crawls.models.Crawl.seal ``` ```` -````{py:method} enter_paused() -:canonical: archivebox.crawls.models.CrawlMachine.enter_paused +````{py:method} advance_lifecycle() -> bool +:canonical: archivebox.crawls.models.Crawl.advance_lifecycle -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_paused +```{autodoc2-docstring} archivebox.crawls.models.Crawl.advance_lifecycle ``` ```` -````{py:method} enter_sealed() -:canonical: archivebox.crawls.models.CrawlMachine.enter_sealed +````{py:method} cleanup_runtime() -> None +:canonical: archivebox.crawls.models.Crawl.cleanup_runtime -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_sealed +```{autodoc2-docstring} archivebox.crawls.models.Crawl.cleanup_runtime ``` ```` -````` +`````` diff --git a/docs/apidocs/archivebox/archivebox.machine.models.md b/docs/apidocs/archivebox/archivebox.machine.models.md index e7787988..c6a7404d 100644 --- a/docs/apidocs/archivebox/archivebox.machine.models.md +++ b/docs/apidocs/archivebox/archivebox.machine.models.md @@ -43,14 +43,6 @@ - ```{autodoc2-docstring} archivebox.machine.models.Process :summary: ``` -* - {py:obj}`BinaryMachine ` - - ```{autodoc2-docstring} archivebox.machine.models.BinaryMachine - :summary: - ``` -* - {py:obj}`ProcessMachine ` - - ```{autodoc2-docstring} archivebox.machine.models.ProcessMachine - :summary: - ``` ```` ### Functions @@ -59,6 +51,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`get_current_pid_namespace ` + - ```{autodoc2-docstring} archivebox.machine.models.get_current_pid_namespace + :summary: + ``` * - {py:obj}`_default_exit_code_for_unowned_process ` - ```{autodoc2-docstring} archivebox.machine.models._default_exit_code_for_unowned_process :summary: @@ -135,6 +131,10 @@ - ```{autodoc2-docstring} archivebox.machine.models.START_TIME_TOLERANCE :summary: ``` +* - {py:obj}`PROCESS_PID_NAMESPACE_KEY ` + - ```{autodoc2-docstring} archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY + :summary: + ``` ```` ### API @@ -264,6 +264,23 @@ ```` +````{py:data} PROCESS_PID_NAMESPACE_KEY +:canonical: archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY +:value: > + '_ARCHIVEBOX_PID_NAMESPACE' + +```{autodoc2-docstring} archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY +``` + +```` + +````{py:function} get_current_pid_namespace() -> str +:canonical: archivebox.machine.models.get_current_pid_namespace + +```{autodoc2-docstring} archivebox.machine.models.get_current_pid_namespace +``` +```` + ````{py:function} _default_exit_code_for_unowned_process(process_type: str) -> int :canonical: archivebox.machine.models._default_exit_code_for_unowned_process @@ -853,7 +870,7 @@ Bases: {py:obj}`django.db.models.Manager` ``````{py:class} Binary(*args, **kwargs) :canonical: archivebox.machine.models.Binary -Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` +Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithQueue` ```{autodoc2-docstring} archivebox.machine.models.Binary ``` @@ -864,7 +881,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`ar ```{autodoc2-docstring} archivebox.machine.models.Binary.__init__ ``` -`````{py:class} StatusChoices(*args, **kwds) +`````{py:class} StatusChoices() :canonical: archivebox.machine.models.Binary.StatusChoices Bases: {py:obj}`django.db.models.TextChoices` @@ -1052,13 +1069,42 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:attribute} state_machine_name -:canonical: archivebox.machine.models.Binary.state_machine_name -:type: str | None +````{py:attribute} INITIAL_STATE +:canonical: archivebox.machine.models.Binary.INITIAL_STATE :value: > - 'archivebox.machine.models.BinaryMachine' + None -```{autodoc2-docstring} archivebox.machine.models.Binary.state_machine_name +```{autodoc2-docstring} archivebox.machine.models.Binary.INITIAL_STATE +``` + +```` + +````{py:attribute} ACTIVE_STATE +:canonical: archivebox.machine.models.Binary.ACTIVE_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Binary.ACTIVE_STATE +``` + +```` + +````{py:attribute} FINAL_STATES +:canonical: archivebox.machine.models.Binary.FINAL_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.machine.models.Binary.FINAL_STATES +``` + +```` + +````{py:attribute} FINAL_OR_ACTIVE_STATES +:canonical: archivebox.machine.models.Binary.FINAL_OR_ACTIVE_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.machine.models.Binary.FINAL_OR_ACTIVE_STATES ``` ```` @@ -1097,7 +1143,7 @@ Bases: {py:obj}`django.db.models.TextChoices` `````{py:class} Meta :canonical: archivebox.machine.models.Binary.Meta -Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithQueue.Meta` ````{py:attribute} app_label :canonical: archivebox.machine.models.Binary.Meta.app_label @@ -1155,6 +1201,15 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:ob ```` +````{py:property} can_install +:canonical: archivebox.machine.models.Binary.can_install +:type: bool + +```{autodoc2-docstring} archivebox.machine.models.Binary.can_install +``` + +```` + ````{py:method} binary_info() -> dict :canonical: archivebox.machine.models.Binary.binary_info @@ -1205,6 +1260,30 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:ob ```` +````{py:method} install() -> bool +:canonical: archivebox.machine.models.Binary.install + +```{autodoc2-docstring} archivebox.machine.models.Binary.install +``` + +```` + +````{py:method} advance_lifecycle() -> bool +:canonical: archivebox.machine.models.Binary.advance_lifecycle + +```{autodoc2-docstring} archivebox.machine.models.Binary.advance_lifecycle +``` + +```` + +````{py:method} install_claimed(*, lock_seconds: int = 600) -> bool +:canonical: archivebox.machine.models.Binary.install_claimed + +```{autodoc2-docstring} archivebox.machine.models.Binary.install_claimed +``` + +```` + ````{py:method} cleanup() :canonical: archivebox.machine.models.Binary.cleanup @@ -1279,7 +1358,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`dj ```{autodoc2-docstring} archivebox.machine.models.Process.__init__ ``` -`````{py:class} StatusChoices(*args, **kwds) +`````{py:class} StatusChoices() :canonical: archivebox.machine.models.Process.StatusChoices Bases: {py:obj}`django.db.models.TextChoices` @@ -1316,7 +1395,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ````` -`````{py:class} TypeChoices(*args, **kwds) +`````{py:class} TypeChoices() :canonical: archivebox.machine.models.Process.TypeChoices Bases: {py:obj}`django.db.models.TextChoices` @@ -1698,17 +1777,6 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:attribute} state_machine_name -:canonical: archivebox.machine.models.Process.state_machine_name -:type: str -:value: > - 'archivebox.machine.models.ProcessMachine' - -```{autodoc2-docstring} archivebox.machine.models.Process.state_machine_name -``` - -```` - ````{py:attribute} delete_after_final_statuses :canonical: archivebox.machine.models.Process.delete_after_final_statuses :value: > @@ -1997,6 +2065,15 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta` ```` +````{py:property} shares_pid_namespace +:canonical: archivebox.machine.models.Process.shares_pid_namespace +:type: bool + +```{autodoc2-docstring} archivebox.machine.models.Process.shares_pid_namespace +``` + +```` + ````{py:property} proc :canonical: archivebox.machine.models.Process.proc :type: psutil.Process | None @@ -2242,239 +2319,3 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta` ```` `````` - -`````{py:class} BinaryMachine(obj, *args, **kwargs) -:canonical: archivebox.machine.models.BinaryMachine - -Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.__init__ -``` - -````{py:attribute} model_attr_name -:canonical: archivebox.machine.models.BinaryMachine.model_attr_name -:value: > - 'binary' - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.model_attr_name -``` - -```` - -````{py:attribute} binary -:canonical: archivebox.machine.models.BinaryMachine.binary -:type: archivebox.machine.models.Binary -:value: > - None - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.binary -``` - -```` - -````{py:attribute} queued -:canonical: archivebox.machine.models.BinaryMachine.queued -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.queued -``` - -```` - -````{py:attribute} installed -:canonical: archivebox.machine.models.BinaryMachine.installed -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.installed -``` - -```` - -````{py:attribute} tick -:canonical: archivebox.machine.models.BinaryMachine.tick -:value: > - None - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.tick -``` - -```` - -````{py:method} can_install() -> bool -:canonical: archivebox.machine.models.BinaryMachine.can_install - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.can_install -``` - -```` - -````{py:method} enter_queued() -:canonical: archivebox.machine.models.BinaryMachine.enter_queued - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.enter_queued -``` - -```` - -````{py:method} on_install() -:canonical: archivebox.machine.models.BinaryMachine.on_install - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.on_install -``` - -```` - -````{py:method} enter_installed() -:canonical: archivebox.machine.models.BinaryMachine.enter_installed - -```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.enter_installed -``` - -```` - -````` - -`````{py:class} ProcessMachine(obj, *args, **kwargs) -:canonical: archivebox.machine.models.ProcessMachine - -Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.__init__ -``` - -````{py:attribute} model_attr_name -:canonical: archivebox.machine.models.ProcessMachine.model_attr_name -:value: > - 'process' - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.model_attr_name -``` - -```` - -````{py:attribute} process -:canonical: archivebox.machine.models.ProcessMachine.process -:type: archivebox.machine.models.Process -:value: > - None - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.process -``` - -```` - -````{py:attribute} queued -:canonical: archivebox.machine.models.ProcessMachine.queued -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.queued -``` - -```` - -````{py:attribute} running -:canonical: archivebox.machine.models.ProcessMachine.running -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.running -``` - -```` - -````{py:attribute} exited -:canonical: archivebox.machine.models.ProcessMachine.exited -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.exited -``` - -```` - -````{py:attribute} tick -:canonical: archivebox.machine.models.ProcessMachine.tick -:value: > - None - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.tick -``` - -```` - -````{py:attribute} launch -:canonical: archivebox.machine.models.ProcessMachine.launch -:value: > - 'to(...)' - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.launch -``` - -```` - -````{py:attribute} kill -:canonical: archivebox.machine.models.ProcessMachine.kill -:value: > - 'to(...)' - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.kill -``` - -```` - -````{py:method} can_start() -> bool -:canonical: archivebox.machine.models.ProcessMachine.can_start - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.can_start -``` - -```` - -````{py:method} is_exited() -> bool -:canonical: archivebox.machine.models.ProcessMachine.is_exited - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.is_exited -``` - -```` - -````{py:method} enter_queued() -:canonical: archivebox.machine.models.ProcessMachine.enter_queued - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.enter_queued -``` - -```` - -````{py:method} enter_running() -:canonical: archivebox.machine.models.ProcessMachine.enter_running - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.enter_running -``` - -```` - -````{py:method} enter_exited() -:canonical: archivebox.machine.models.ProcessMachine.enter_exited - -```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.enter_exited -``` - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.workers.models.md b/docs/apidocs/archivebox/archivebox.workers.models.md index 15bd67e3..8f52c06f 100644 --- a/docs/apidocs/archivebox/archivebox.workers.models.md +++ b/docs/apidocs/archivebox/archivebox.workers.models.md @@ -17,14 +17,8 @@ * - {py:obj}`DefaultStatusChoices ` - -* - {py:obj}`ModelStateMachine ` - - -* - {py:obj}`BaseModelWithStateMachine ` - - -* - {py:obj}`ModelWithStateMachine ` - - -* - {py:obj}`BaseStateMachine ` - - ```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine +* - {py:obj}`ModelWithQueue ` + - ```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue :summary: ``` ```` @@ -67,19 +61,11 @@ - ```{autodoc2-docstring} archivebox.workers.models.PACKAGE_ROOT :summary: ``` -* - {py:obj}`ObjectState ` - - ```{autodoc2-docstring} archivebox.workers.models.ObjectState - :summary: - ``` -* - {py:obj}`ObjectStateList ` - - ```{autodoc2-docstring} archivebox.workers.models.ObjectStateList - :summary: - ``` ```` ### API -`````{py:class} DefaultStatusChoices(*args, **kwds) +`````{py:class} DefaultStatusChoices() :canonical: archivebox.workers.models.DefaultStatusChoices Bases: {py:obj}`django.db.models.TextChoices` @@ -208,550 +194,267 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:data} ObjectState -:canonical: archivebox.workers.models.ObjectState -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.ObjectState -``` - -```` - -````{py:data} ObjectStateList -:canonical: archivebox.workers.models.ObjectStateList -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.ObjectStateList -``` - -```` - -`````{py:class} ModelStateMachine -:canonical: archivebox.workers.models.ModelStateMachine - -Bases: {py:obj}`typing.Protocol` - -````{py:method} tick() -> typing.Any -:canonical: archivebox.workers.models.ModelStateMachine.tick - -```{autodoc2-docstring} archivebox.workers.models.ModelStateMachine.tick -``` - -```` - -````{py:method} pause_requested() -> typing.Any -:canonical: archivebox.workers.models.ModelStateMachine.pause_requested - -```{autodoc2-docstring} archivebox.workers.models.ModelStateMachine.pause_requested -``` - -```` - -````{py:method} resume_requested() -> typing.Any -:canonical: archivebox.workers.models.ModelStateMachine.resume_requested - -```{autodoc2-docstring} archivebox.workers.models.ModelStateMachine.resume_requested -``` - -```` - -````` - -``````{py:class} BaseModelWithStateMachine(*args, **kwargs) -:canonical: archivebox.workers.models.BaseModelWithStateMachine +``````{py:class} ModelWithQueue(*args, **kwargs) +:canonical: archivebox.workers.models.ModelWithQueue Bases: {py:obj}`django.db.models.Model` -````{py:attribute} StatusChoices -:canonical: archivebox.workers.models.BaseModelWithStateMachine.StatusChoices -:type: typing.ClassVar[type[archivebox.workers.models.DefaultStatusChoices]] -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StatusChoices -``` - -```` - -````{py:attribute} state_machine_name -:canonical: archivebox.workers.models.BaseModelWithStateMachine.state_machine_name -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.state_machine_name -``` - -```` - -````{py:attribute} state_field_name -:canonical: archivebox.workers.models.BaseModelWithStateMachine.state_field_name -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.state_field_name -``` - -```` - -````{py:attribute} state_machine_attr -:canonical: archivebox.workers.models.BaseModelWithStateMachine.state_machine_attr -:type: str -:value: > - 'sm' - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.state_machine_attr -``` - -```` - -````{py:attribute} bind_events_as_methods -:canonical: archivebox.workers.models.BaseModelWithStateMachine.bind_events_as_methods -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.bind_events_as_methods -``` - -```` - -````{py:attribute} warn_on_save_outside_runner -:canonical: archivebox.workers.models.BaseModelWithStateMachine.warn_on_save_outside_runner -:type: typing.ClassVar[bool] -:value: > - True - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.warn_on_save_outside_runner -``` - -```` - -````{py:attribute} active_state -:canonical: archivebox.workers.models.BaseModelWithStateMachine.active_state -:type: archivebox.workers.models.ObjectState -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.active_state -``` - -```` - -````{py:attribute} retry_at_field_name -:canonical: archivebox.workers.models.BaseModelWithStateMachine.retry_at_field_name -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.retry_at_field_name -``` - -```` - -`````{py:class} Meta -:canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta - -Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` - -````{py:attribute} app_label -:canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta.app_label -:value: > - 'workers' - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.Meta.app_label -``` - -```` - -````{py:attribute} abstract -:canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta.abstract -:value: > - True - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.Meta.abstract -``` - -```` - -````` - -````{py:property} sm -:canonical: archivebox.workers.models.BaseModelWithStateMachine.sm -:type: statemachine.StateMachine - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.sm -``` - -```` - -````{py:method} status_counts(queryset: django.db.models.QuerySet | None = None, statuses: collections.abc.Iterable[str] | None = None) -> dict[str, int] -:canonical: archivebox.workers.models.BaseModelWithStateMachine.status_counts -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.status_counts -``` - -```` - -````{py:method} check(sender=None, **kwargs) -:canonical: archivebox.workers.models.BaseModelWithStateMachine.check -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.check -``` - -```` - -````{py:method} _state_to_str(state: archivebox.workers.models.ObjectState) -> str -:canonical: archivebox.workers.models.BaseModelWithStateMachine._state_to_str -:staticmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine._state_to_str -``` - -```` - -````{py:property} RETRY_AT -:canonical: archivebox.workers.models.BaseModelWithStateMachine.RETRY_AT -:type: datetime.datetime - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.RETRY_AT -``` - -```` - -````{py:property} STATE -:canonical: archivebox.workers.models.BaseModelWithStateMachine.STATE -:type: str - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.STATE -``` - -```` - -````{py:method} bump_retry_at(seconds: int = 10) -:canonical: archivebox.workers.models.BaseModelWithStateMachine.bump_retry_at - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.bump_retry_at -``` - -```` - -````{py:property} is_paused -:canonical: archivebox.workers.models.BaseModelWithStateMachine.is_paused -:type: bool - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.is_paused -``` - -```` - -````{py:method} safe_update(update_fields: dict[str, typing.Any], *, refresh: bool = True, extra_filter: dict[str, typing.Any] | None = None) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.safe_update - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.safe_update -``` - -```` - -````{py:method} save(*args, **kwargs) -:canonical: archivebox.workers.models.BaseModelWithStateMachine.save - -```` - -````{py:method} pause(*, save: bool = True) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.pause - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.pause -``` - -```` - -````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.resume - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.resume -``` - -```` - -````{py:method} update_and_requeue(*, refresh: bool = True, **kwargs) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.update_and_requeue - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.update_and_requeue -``` - -```` - -````{py:method} get_queue() -:canonical: archivebox.workers.models.BaseModelWithStateMachine.get_queue -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.get_queue -``` - -```` - -````{py:method} claim_for_worker(obj: archivebox.workers.models.BaseModelWithStateMachine, lock_seconds: int = 60) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.claim_for_worker -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.claim_for_worker -``` - -```` - -````{py:method} claim_processing_lock(lock_seconds: int = 60) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.claim_processing_lock - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.claim_processing_lock -``` - -```` - -````{py:method} tick_claimed(lock_seconds: int = 60) -> bool -:canonical: archivebox.workers.models.BaseModelWithStateMachine.tick_claimed - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.tick_claimed -``` - -```` - -````{py:method} ACTIVE_STATE() -> str -:canonical: archivebox.workers.models.BaseModelWithStateMachine.ACTIVE_STATE - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.ACTIVE_STATE -``` - -```` - -````{py:method} INITIAL_STATE() -> str -:canonical: archivebox.workers.models.BaseModelWithStateMachine.INITIAL_STATE - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.INITIAL_STATE -``` - -```` - -````{py:method} FINAL_STATES() -> list[str] -:canonical: archivebox.workers.models.BaseModelWithStateMachine.FINAL_STATES - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.FINAL_STATES -``` - -```` - -````{py:method} FINAL_OR_ACTIVE_STATES() -> list[str] -:canonical: archivebox.workers.models.BaseModelWithStateMachine.FINAL_OR_ACTIVE_STATES - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.FINAL_OR_ACTIVE_STATES -``` - -```` - -````{py:method} extend_choices(base_choices: type[django.db.models.TextChoices]) -:canonical: archivebox.workers.models.BaseModelWithStateMachine.extend_choices -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.extend_choices -``` - -```` - -````{py:method} StatusField(**kwargs) -> django.db.models.CharField -:canonical: archivebox.workers.models.BaseModelWithStateMachine.StatusField -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StatusField -``` - -```` - -````{py:method} RetryAtField(**kwargs) -> django.db.models.DateTimeField -:canonical: archivebox.workers.models.BaseModelWithStateMachine.RetryAtField -:classmethod: - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.RetryAtField -``` - -```` - -````{py:method} StateMachineClass() -> type[statemachine.StateMachine] -:canonical: archivebox.workers.models.BaseModelWithStateMachine.StateMachineClass - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StateMachineClass -``` - -```` - -`````` - -``````{py:class} ModelWithStateMachine(*args, **kwargs) -:canonical: archivebox.workers.models.ModelWithStateMachine - -Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` - -````{py:attribute} StatusChoices -:canonical: archivebox.workers.models.ModelWithStateMachine.StatusChoices -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.StatusChoices -``` - -```` - -````{py:attribute} status -:canonical: archivebox.workers.models.ModelWithStateMachine.status -:type: django.db.models.CharField -:value: > - 'StatusField(...)' - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.status -``` - -```` - -````{py:attribute} retry_at -:canonical: archivebox.workers.models.ModelWithStateMachine.retry_at -:type: django.db.models.DateTimeField -:value: > - 'RetryAtField(...)' - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.retry_at -``` - -```` - -````{py:attribute} state_machine_name -:canonical: archivebox.workers.models.ModelWithStateMachine.state_machine_name -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.state_machine_name -``` - -```` - -````{py:attribute} state_field_name -:canonical: archivebox.workers.models.ModelWithStateMachine.state_field_name -:type: str -:value: > - 'status' - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.state_field_name -``` - -```` - -````{py:attribute} state_machine_attr -:canonical: archivebox.workers.models.ModelWithStateMachine.state_machine_attr -:type: str -:value: > - 'sm' - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.state_machine_attr -``` - -```` - -````{py:attribute} bind_events_as_methods -:canonical: archivebox.workers.models.ModelWithStateMachine.bind_events_as_methods -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.bind_events_as_methods -``` - -```` - -````{py:attribute} active_state -:canonical: archivebox.workers.models.ModelWithStateMachine.active_state -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.active_state -``` - -```` - -````{py:attribute} retry_at_field_name -:canonical: archivebox.workers.models.ModelWithStateMachine.retry_at_field_name -:type: str -:value: > - 'retry_at' - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.retry_at_field_name -``` - -```` - -`````{py:class} Meta -:canonical: archivebox.workers.models.ModelWithStateMachine.Meta - -Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` - -````{py:attribute} abstract -:canonical: archivebox.workers.models.ModelWithStateMachine.Meta.abstract -:value: > - True - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.Meta.abstract -``` - -```` - -````` - -`````` - -`````{py:class} BaseStateMachine(obj, *args, **kwargs) -:canonical: archivebox.workers.models.BaseStateMachine - -Bases: {py:obj}`statemachine.StateMachine` - -```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue ``` ```{rubric} Initialization ``` -```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine.__init__ +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.__init__ ``` -````{py:attribute} model_attr_name -:canonical: archivebox.workers.models.BaseStateMachine.model_attr_name -:type: str +````{py:attribute} StatusChoices +:canonical: archivebox.workers.models.ModelWithQueue.StatusChoices +:type: typing.ClassVar[type[django.db.models.TextChoices]] :value: > - 'obj' + None -```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine.model_attr_name +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.StatusChoices ``` ```` -````{py:method} _register_callbacks(listeners: list[object]) -:canonical: archivebox.workers.models.BaseStateMachine._register_callbacks +````{py:attribute} INITIAL_STATE +:canonical: archivebox.workers.models.ModelWithQueue.INITIAL_STATE +:type: typing.ClassVar[str] +:value: > + None -```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine._register_callbacks +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.INITIAL_STATE ``` ```` -````{py:method} __repr__() -> str -:canonical: archivebox.workers.models.BaseStateMachine.__repr__ +````{py:attribute} ACTIVE_STATE +:canonical: archivebox.workers.models.ModelWithQueue.ACTIVE_STATE +:type: typing.ClassVar[str] +:value: > + None + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.ACTIVE_STATE +``` ```` -````{py:method} __str__() -> str -:canonical: archivebox.workers.models.BaseStateMachine.__str__ +````{py:attribute} FINAL_STATES +:canonical: archivebox.workers.models.ModelWithQueue.FINAL_STATES +:type: typing.ClassVar[tuple[str, ...]] +:value: > + () + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.FINAL_STATES +``` + +```` + +````{py:attribute} warn_on_save_outside_runner +:canonical: archivebox.workers.models.ModelWithQueue.warn_on_save_outside_runner +:type: typing.ClassVar[bool] +:value: > + True + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.warn_on_save_outside_runner +``` + +```` + +````{py:attribute} status +:canonical: archivebox.workers.models.ModelWithQueue.status +:type: django.db.models.CharField +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.status +``` + +```` + +````{py:attribute} retry_at +:canonical: archivebox.workers.models.ModelWithQueue.retry_at +:type: django.db.models.DateTimeField +:value: > + 'DateTimeField(...)' + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.retry_at +``` + +```` + +`````{py:class} Meta +:canonical: archivebox.workers.models.ModelWithQueue.Meta + +Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` + +````{py:attribute} app_label +:canonical: archivebox.workers.models.ModelWithQueue.Meta.app_label +:value: > + 'workers' + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.Meta.app_label +``` + +```` + +````{py:attribute} abstract +:canonical: archivebox.workers.models.ModelWithQueue.Meta.abstract +:value: > + True + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.Meta.abstract +``` ```` ````` + +````{py:attribute} FINAL_OR_ACTIVE_STATES +:canonical: archivebox.workers.models.ModelWithQueue.FINAL_OR_ACTIVE_STATES +:type: typing.ClassVar[tuple[str, ...]] +:value: > + () + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.FINAL_OR_ACTIVE_STATES +``` + +```` + +````{py:method} status_counts(queryset: django.db.models.QuerySet | None = None, statuses: collections.abc.Iterable[str] | None = None) -> dict[str, int] +:canonical: archivebox.workers.models.ModelWithQueue.status_counts +:classmethod: + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.status_counts +``` + +```` + +````{py:property} RETRY_AT +:canonical: archivebox.workers.models.ModelWithQueue.RETRY_AT +:type: datetime.datetime | None + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.RETRY_AT +``` + +```` + +````{py:property} STATE +:canonical: archivebox.workers.models.ModelWithQueue.STATE +:type: str + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.STATE +``` + +```` + +````{py:method} bump_retry_at(seconds: int = 10) -> None +:canonical: archivebox.workers.models.ModelWithQueue.bump_retry_at + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.bump_retry_at +``` + +```` + +````{py:property} is_paused +:canonical: archivebox.workers.models.ModelWithQueue.is_paused +:type: bool + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.is_paused +``` + +```` + +````{py:method} safe_update(update_fields: dict[str, typing.Any], *, refresh: bool = True, extra_filter: dict[str, typing.Any] | None = None) -> bool +:canonical: archivebox.workers.models.ModelWithQueue.safe_update + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.safe_update +``` + +```` + +````{py:method} save(*args: typing.Any, **kwargs: typing.Any) -> None +:canonical: archivebox.workers.models.ModelWithQueue.save + +```` + +````{py:method} pause(*, save: bool = True) -> bool +:canonical: archivebox.workers.models.ModelWithQueue.pause + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.pause +``` + +```` + +````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool +:canonical: archivebox.workers.models.ModelWithQueue.resume + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.resume +``` + +```` + +````{py:method} update_and_requeue(*, refresh: bool = True, **kwargs: typing.Any) -> bool +:canonical: archivebox.workers.models.ModelWithQueue.update_and_requeue + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.update_and_requeue +``` + +```` + +````{py:method} get_queue() +:canonical: archivebox.workers.models.ModelWithQueue.get_queue +:classmethod: + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.get_queue +``` + +```` + +````{py:method} claim_for_worker(obj: archivebox.workers.models.ModelWithQueue, lock_seconds: int = 60) -> bool +:canonical: archivebox.workers.models.ModelWithQueue.claim_for_worker +:classmethod: + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.claim_for_worker +``` + +```` + +````{py:method} claim_processing_lock(lock_seconds: int = 60) -> bool +:canonical: archivebox.workers.models.ModelWithQueue.claim_processing_lock + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.claim_processing_lock +``` + +```` + +````{py:method} extend_choices(base_choices: type[django.db.models.TextChoices]) +:canonical: archivebox.workers.models.ModelWithQueue.extend_choices +:classmethod: + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.extend_choices +``` + +```` + +````{py:method} StatusField(**kwargs: typing.Any) -> django.db.models.CharField +:canonical: archivebox.workers.models.ModelWithQueue.StatusField +:classmethod: + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.StatusField +``` + +```` + +````{py:method} RetryAtField(**kwargs: typing.Any) -> django.db.models.DateTimeField +:canonical: archivebox.workers.models.ModelWithQueue.RetryAtField +:classmethod: + +```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.RetryAtField +``` + +```` + +``````