From f4932430db9be6db9f0196ca4fdea649757c3c95 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sun, 31 May 2026 03:40:25 -0700 Subject: [PATCH] docs: regenerate sphinx apidocs to reflect current source tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean rebuild of docs/apidocs/archivebox/ via autodoc2: - Removes 19 stale module pages whose source files no longer exist (cli_utils, host_utils, schedule_utils — renamed to *_util; actors, ideas, debugging, folders, legacy, progress_layout, tests_piping, config_tags, personas.runtime/views, orchestrator*, worker, tasks). - Adds 29 new module pages for code that was added since the previous generation but not yet documented. - Updates 100 existing pages to reflect API surface changes (e.g. ENABLED_PLUGINS Field removed, SNAPSHOTS_PER_PAGE clamp removed, default bumped to 50, etc.). 156 source modules <-> 156 apidoc files (zero drift). Build clean under sphinx-build -W --keep-going. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../archivebox/archivebox.api.admin.md | 44 +- .../apidocs/archivebox/archivebox.api.auth.md | 119 +- docs/apidocs/archivebox/archivebox.api.md | 2 + .../apidocs/archivebox/archivebox.api.urls.md | 19 + .../archivebox/archivebox.api.v1_auth.md | 8 +- .../archivebox/archivebox.api.v1_cli.md | 212 ++-- .../archivebox/archivebox.api.v1_core.md | 1009 +++++++++++++++-- .../archivebox/archivebox.api.v1_crawls.md | 93 +- .../archivebox/archivebox.api.v1_machine.md | 64 +- .../archivebox/archivebox.api.v1_personas.md | 405 +++++++ .../archivebox/archivebox.api.webhooks.md | 64 ++ .../archivebox.base_models.admin.md | 204 +++- .../archivebox.base_models.models.md | 145 ++- .../archivebox.cli.archivebox_add.md | 13 +- ...archivebox.cli.archivebox_archiveresult.md | 23 +- .../archivebox.cli.archivebox_binary.md | 8 +- .../archivebox.cli.archivebox_config.md | 11 + .../archivebox.cli.archivebox_crawl.md | 10 +- .../archivebox.cli.archivebox_extract.md | 2 +- .../archivebox.cli.archivebox_help.md | 11 + .../archivebox.cli.archivebox_init.md | 11 + .../archivebox.cli.archivebox_list.md | 2 +- .../archivebox.cli.archivebox_machine.md | 4 +- .../archivebox.cli.archivebox_persona.md | 78 +- .../archivebox.cli.archivebox_pluginmap.md | 51 +- .../archivebox.cli.archivebox_process.md | 4 +- .../archivebox.cli.archivebox_remove.md | 2 +- .../archivebox.cli.archivebox_run.md | 32 +- .../archivebox.cli.archivebox_schedule.md | 2 +- .../archivebox.cli.archivebox_search.md | 47 +- .../archivebox.cli.archivebox_server.md | 118 +- .../archivebox.cli.archivebox_shell.md | 4 +- .../archivebox.cli.archivebox_snapshot.md | 21 +- .../archivebox.cli.archivebox_status.md | 22 + .../archivebox.cli.archivebox_tag.md | 10 +- .../archivebox.cli.archivebox_update.md | 63 +- .../archivebox.cli.archivebox_version.md | 24 +- .../archivebox/archivebox.cli.cli_util.md | 31 + .../archivebox/archivebox.cli.cli_utils.md | 31 - docs/apidocs/archivebox/archivebox.cli.md | 17 +- .../archivebox/archivebox.cli.tests_piping.md | 768 ------------- .../archivebox.config.collection.md | 208 ++-- .../archivebox/archivebox.config.common.md | 918 +++++++++++---- .../archivebox/archivebox.config.configset.md | 211 ++-- .../archivebox/archivebox.config.constants.md | 103 +- .../archivebox/archivebox.config.django.md | 25 +- .../archivebox/archivebox.config.ldap.md | 30 +- docs/apidocs/archivebox/archivebox.config.md | 28 +- .../archivebox/archivebox.config.paths.md | 36 +- .../archivebox.config.permissions.md | 2 +- .../archivebox/archivebox.config.version.md | 32 +- .../archivebox/archivebox.config.views.md | 292 ++++- .../archivebox/archivebox.core.actors.md | 8 - .../archivebox.core.admin_archiveresults.md | 197 +++- .../archivebox/archivebox.core.admin_site.md | 56 + .../archivebox.core.admin_snapshots.md | 659 ++++++++++- .../archivebox/archivebox.core.admin_tags.md | 165 ++- .../archivebox/archivebox.core.admin_users.md | 44 + .../archivebox/archivebox.core.apps.md | 22 - .../archivebox/archivebox.core.asgi.md | 19 + .../archivebox.core.context_processors.md | 31 + .../archivebox/archivebox.core.forms.md | 528 ++++++++- .../archivebox/archivebox.core.host_util.md | 510 +++++++++ .../archivebox/archivebox.core.host_utils.md | 372 ------ ...box.core.management.commands.archivebox.md | 49 + .../archivebox.core.management.commands.md | 17 + .../archivebox/archivebox.core.management.md | 17 + docs/apidocs/archivebox/archivebox.core.md | 22 +- .../archivebox/archivebox.core.middleware.md | 46 +- .../archivebox/archivebox.core.models.md | 1005 ++++++++++------ .../archivebox/archivebox.core.permissions.md | 211 ++++ .../archivebox.core.recovery_util.md | 31 + .../archivebox/archivebox.core.settings.md | 63 +- .../archivebox.core.settings_logging.md | 50 + .../archivebox.core.shutdown_util.md | 151 +++ .../archivebox/archivebox.core.tag_util.md | 290 +++++ .../archivebox.core.takeover_util.md | 235 ++++ ...rchivebox.core.templatetags.config_tags.md | 53 - .../archivebox.core.templatetags.core_tags.md | 351 +++++- .../archivebox.core.templatetags.md | 1 - .../archivebox/archivebox.core.urls.md | 47 + .../archivebox/archivebox.core.views.md | 291 ++++- .../archivebox/archivebox.core.widgets.md | 44 +- .../archivebox/archivebox.crawls.admin.md | 474 +++++++- docs/apidocs/archivebox/archivebox.crawls.md | 2 +- .../archivebox/archivebox.crawls.models.md | 419 ++++++- .../archivebox.crawls.schedule_util.md | 76 ++ .../archivebox.crawls.schedule_utils.md | 76 -- docs/apidocs/archivebox/archivebox.hooks.md | 279 +++-- docs/apidocs/archivebox/archivebox.ideas.md | 17 - .../archivebox.ideas.process_plugin.md | 633 ----------- .../archivebox/archivebox.ldap.auth.md | 2 +- .../archivebox/archivebox.machine.admin.md | 153 ++- .../archivebox/archivebox.machine.detect.md | 17 +- .../archivebox/archivebox.machine.env_util.md | 111 ++ docs/apidocs/archivebox/archivebox.machine.md | 1 + .../archivebox/archivebox.machine.models.md | 424 +++++-- .../archivebox/archivebox.mcp.server.md | 8 +- docs/apidocs/archivebox/archivebox.md | 116 +- .../archivebox/archivebox.misc.checks.md | 10 +- docs/apidocs/archivebox/archivebox.misc.db.md | 119 +- .../archivebox/archivebox.misc.debugging.md | 31 - .../archivebox/archivebox.misc.folders.md | 31 - .../archivebox/archivebox.misc.hashing.md | 6 +- .../archivebox/archivebox.misc.jsonl.md | 35 +- .../archivebox/archivebox.misc.legacy.md | 125 -- .../archivebox/archivebox.misc.logging.md | 46 +- .../archivebox.misc.logging_util.md | 342 +----- docs/apidocs/archivebox/archivebox.misc.md | 4 - .../archivebox.misc.monkey_patches.md | 29 +- .../archivebox/archivebox.misc.paginators.md | 84 +- .../archivebox.misc.progress_layout.md | 455 -------- .../archivebox.misc.serve_static.md | 264 ++++- .../archivebox.misc.shell_welcome_message.md | 26 - .../archivebox/archivebox.misc.system.md | 81 +- .../archivebox/archivebox.misc.toml_util.md | 2 +- .../archivebox/archivebox.misc.util.md | 293 +++-- .../archivebox/archivebox.personas.admin.md | 203 ++++ .../archivebox/archivebox.personas.forms.md | 177 +++ .../archivebox.personas.importers.md | 627 ++++++++++ .../apidocs/archivebox/archivebox.personas.md | 4 +- .../archivebox/archivebox.personas.models.md | 142 ++- .../archivebox/archivebox.personas.runtime.md | 111 -- .../archivebox/archivebox.personas.views.md | 8 - .../archivebox/archivebox.search.admin.md | 116 ++ docs/apidocs/archivebox/archivebox.search.md | 161 ++- .../archivebox.search.sonic_daemon.md | 31 + ...hivebox.services.archive_result_service.md | 207 ++++ .../archivebox.services.binary_service.md | 67 ++ .../archivebox.services.crawl_service.md | 85 ++ .../archivebox.services.machine_service.md | 110 ++ .../apidocs/archivebox/archivebox.services.md | 50 + .../archivebox.services.process_service.md | 152 +++ .../archivebox/archivebox.services.runner.md | 551 +++++++++ .../archivebox.services.snapshot_service.md | 97 ++ .../archivebox.services.tag_service.md | 58 + .../archivebox/archivebox.uuid_compat.md | 2 +- .../archivebox.workers.management.commands.md | 4 +- ...orkers.management.commands.orchestrator.md | 49 - ....management.commands.orchestrator_watch.md | 49 - ...orkers.management.commands.runner_watch.md | 49 + ...anagement.commands.supervisord_watchdog.md | 49 + docs/apidocs/archivebox/archivebox.workers.md | 3 - .../archivebox/archivebox.workers.models.md | 207 +++- .../archivebox.workers.orchestrator.md | 311 ----- .../archivebox.workers.supervisord_util.md | 452 +++++++- .../archivebox/archivebox.workers.tasks.md | 53 - .../archivebox/archivebox.workers.worker.md | 549 --------- 148 files changed, 14428 insertions(+), 6375 deletions(-) create mode 100644 docs/apidocs/archivebox/archivebox.api.v1_personas.md create mode 100644 docs/apidocs/archivebox/archivebox.api.webhooks.md create mode 100644 docs/apidocs/archivebox/archivebox.cli.cli_util.md delete mode 100644 docs/apidocs/archivebox/archivebox.cli.cli_utils.md delete mode 100644 docs/apidocs/archivebox/archivebox.cli.tests_piping.md delete mode 100644 docs/apidocs/archivebox/archivebox.core.actors.md create mode 100644 docs/apidocs/archivebox/archivebox.core.context_processors.md create mode 100644 docs/apidocs/archivebox/archivebox.core.host_util.md delete mode 100644 docs/apidocs/archivebox/archivebox.core.host_utils.md create mode 100644 docs/apidocs/archivebox/archivebox.core.management.commands.archivebox.md create mode 100644 docs/apidocs/archivebox/archivebox.core.management.commands.md create mode 100644 docs/apidocs/archivebox/archivebox.core.management.md create mode 100644 docs/apidocs/archivebox/archivebox.core.permissions.md create mode 100644 docs/apidocs/archivebox/archivebox.core.recovery_util.md create mode 100644 docs/apidocs/archivebox/archivebox.core.shutdown_util.md create mode 100644 docs/apidocs/archivebox/archivebox.core.tag_util.md create mode 100644 docs/apidocs/archivebox/archivebox.core.takeover_util.md delete mode 100644 docs/apidocs/archivebox/archivebox.core.templatetags.config_tags.md create mode 100644 docs/apidocs/archivebox/archivebox.crawls.schedule_util.md delete mode 100644 docs/apidocs/archivebox/archivebox.crawls.schedule_utils.md delete mode 100644 docs/apidocs/archivebox/archivebox.ideas.md delete mode 100644 docs/apidocs/archivebox/archivebox.ideas.process_plugin.md create mode 100644 docs/apidocs/archivebox/archivebox.machine.env_util.md delete mode 100644 docs/apidocs/archivebox/archivebox.misc.debugging.md delete mode 100644 docs/apidocs/archivebox/archivebox.misc.folders.md delete mode 100644 docs/apidocs/archivebox/archivebox.misc.legacy.md delete mode 100644 docs/apidocs/archivebox/archivebox.misc.progress_layout.md create mode 100644 docs/apidocs/archivebox/archivebox.personas.forms.md create mode 100644 docs/apidocs/archivebox/archivebox.personas.importers.md delete mode 100644 docs/apidocs/archivebox/archivebox.personas.runtime.md delete mode 100644 docs/apidocs/archivebox/archivebox.personas.views.md create mode 100644 docs/apidocs/archivebox/archivebox.search.sonic_daemon.md create mode 100644 docs/apidocs/archivebox/archivebox.services.archive_result_service.md create mode 100644 docs/apidocs/archivebox/archivebox.services.binary_service.md create mode 100644 docs/apidocs/archivebox/archivebox.services.crawl_service.md create mode 100644 docs/apidocs/archivebox/archivebox.services.machine_service.md create mode 100644 docs/apidocs/archivebox/archivebox.services.md create mode 100644 docs/apidocs/archivebox/archivebox.services.process_service.md create mode 100644 docs/apidocs/archivebox/archivebox.services.runner.md create mode 100644 docs/apidocs/archivebox/archivebox.services.snapshot_service.md create mode 100644 docs/apidocs/archivebox/archivebox.services.tag_service.md delete mode 100644 docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator.md delete mode 100644 docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator_watch.md create mode 100644 docs/apidocs/archivebox/archivebox.workers.management.commands.runner_watch.md create mode 100644 docs/apidocs/archivebox/archivebox.workers.management.commands.supervisord_watchdog.md delete mode 100644 docs/apidocs/archivebox/archivebox.workers.orchestrator.md delete mode 100644 docs/apidocs/archivebox/archivebox.workers.tasks.md delete mode 100644 docs/apidocs/archivebox/archivebox.workers.worker.md diff --git a/docs/apidocs/archivebox/archivebox.api.admin.md b/docs/apidocs/archivebox/archivebox.api.admin.md index c8ae3c1e..2b62166d 100644 --- a/docs/apidocs/archivebox/archivebox.api.admin.md +++ b/docs/apidocs/archivebox/archivebox.api.admin.md @@ -17,6 +17,8 @@ * - {py:obj}`APITokenAdmin ` - +* - {py:obj}`OutboundWebhookAdminForm ` + - * - {py:obj}`CustomWebhookAdmin ` - ```` @@ -27,6 +29,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_webhook_fields ` + - ```{autodoc2-docstring} archivebox.api.admin._webhook_fields + :summary: + ``` * - {py:obj}`register_admin ` - ```{autodoc2-docstring} archivebox.api.admin.register_admin :summary: @@ -35,6 +41,13 @@ ### API +````{py:function} _webhook_fields(*names: str) -> tuple[str, ...] +:canonical: archivebox.api.admin._webhook_fields + +```{autodoc2-docstring} archivebox.api.admin._webhook_fields +``` +```` + `````{py:class} APITokenAdmin(model, admin_site) :canonical: archivebox.api.admin.APITokenAdmin @@ -122,11 +135,28 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````` +```{py:class} OutboundWebhookAdminForm(*args, **kwargs) +:canonical: archivebox.api.admin.OutboundWebhookAdminForm + +Bases: {py:obj}`signal_webhooks.admin.WebhookModelForm` + +``` + `````{py:class} CustomWebhookAdmin(model, admin_site) :canonical: archivebox.api.admin.CustomWebhookAdmin Bases: {py:obj}`signal_webhooks.admin.WebhookAdmin`, {py:obj}`archivebox.base_models.admin.BaseModelAdmin` +````{py:attribute} form +:canonical: archivebox.api.admin.CustomWebhookAdmin.form +:value: > + None + +```{autodoc2-docstring} archivebox.api.admin.CustomWebhookAdmin.form +``` + +```` + ````{py:attribute} list_display :canonical: archivebox.api.admin.CustomWebhookAdmin.list_display :value: > @@ -140,7 +170,7 @@ Bases: {py:obj}`signal_webhooks.admin.WebhookAdmin`, {py:obj}`archivebox.base_mo ````{py:attribute} sort_fields :canonical: archivebox.api.admin.CustomWebhookAdmin.sort_fields :value: > - ('created_at', 'created_by', 'id', 'referenced_model', 'endpoint', 'last_success', 'last_error') + '_webhook_fields(...)' ```{autodoc2-docstring} archivebox.api.admin.CustomWebhookAdmin.sort_fields ``` @@ -150,7 +180,7 @@ Bases: {py:obj}`signal_webhooks.admin.WebhookAdmin`, {py:obj}`archivebox.base_mo ````{py:attribute} readonly_fields :canonical: archivebox.api.admin.CustomWebhookAdmin.readonly_fields :value: > - ('created_at', 'modified_at') + '_webhook_fields(...)' ```{autodoc2-docstring} archivebox.api.admin.CustomWebhookAdmin.readonly_fields ``` @@ -167,9 +197,17 @@ Bases: {py:obj}`signal_webhooks.admin.WebhookAdmin`, {py:obj}`archivebox.base_mo ```` +````{py:method} lookup_allowed(lookup: str, value: str, request: django.http.HttpRequest | None = None) -> bool +:canonical: archivebox.api.admin.CustomWebhookAdmin.lookup_allowed + +```{autodoc2-docstring} archivebox.api.admin.CustomWebhookAdmin.lookup_allowed +``` + +```` + ````` -````{py:function} register_admin(admin_site) +````{py:function} register_admin(admin_site: django.contrib.admin.AdminSite) -> None :canonical: archivebox.api.admin.register_admin ```{autodoc2-docstring} archivebox.api.admin.register_admin diff --git a/docs/apidocs/archivebox/archivebox.api.auth.md b/docs/apidocs/archivebox/archivebox.api.auth.md index 9c26a7cc..1bf22651 100644 --- a/docs/apidocs/archivebox/archivebox.api.auth.md +++ b/docs/apidocs/archivebox/archivebox.api.auth.md @@ -15,14 +15,6 @@ :class: autosummary longtable :align: left -* - {py:obj}`APITokenAuthCheck ` - - ```{autodoc2-docstring} archivebox.api.auth.APITokenAuthCheck - :summary: - ``` -* - {py:obj}`UserPassAuthCheck ` - - ```{autodoc2-docstring} archivebox.api.auth.UserPassAuthCheck - :summary: - ``` * - {py:obj}`HeaderTokenAuth ` - ```{autodoc2-docstring} archivebox.api.auth.HeaderTokenAuth :summary: @@ -35,14 +27,6 @@ - ```{autodoc2-docstring} archivebox.api.auth.QueryParamTokenAuth :summary: ``` -* - {py:obj}`UsernameAndPasswordAuth ` - - ```{autodoc2-docstring} archivebox.api.auth.UsernameAndPasswordAuth - :summary: - ``` -* - {py:obj}`DjangoSessionAuth ` - - ```{autodoc2-docstring} archivebox.api.auth.DjangoSessionAuth - :summary: - ``` ```` ### Functions @@ -63,6 +47,10 @@ - ```{autodoc2-docstring} archivebox.api.auth.auth_using_password :summary: ``` +* - {py:obj}`_require_superuser ` + - ```{autodoc2-docstring} archivebox.api.auth._require_superuser + :summary: + ``` ```` ### Data @@ -79,63 +67,38 @@ ### API -````{py:function} get_or_create_api_token(user) +````{py:function} get_or_create_api_token(user: django.contrib.auth.models.User | None) :canonical: archivebox.api.auth.get_or_create_api_token ```{autodoc2-docstring} archivebox.api.auth.get_or_create_api_token ``` ```` -````{py:function} auth_using_token(token, request: typing.Optional[django.http.HttpRequest] = None) -> typing.Optional[django.contrib.auth.models.AbstractBaseUser] +````{py:function} auth_using_token(token: str | None, request: django.http.HttpRequest | None = None) -> django.contrib.auth.models.User | None :canonical: archivebox.api.auth.auth_using_token ```{autodoc2-docstring} archivebox.api.auth.auth_using_token ``` ```` -````{py:function} auth_using_password(username, password, request: typing.Optional[django.http.HttpRequest] = None) -> typing.Optional[django.contrib.auth.models.AbstractBaseUser] +````{py:function} auth_using_password(username: str | None, password: str | None, request: django.http.HttpRequest | None = None) -> django.contrib.auth.models.User | None :canonical: archivebox.api.auth.auth_using_password ```{autodoc2-docstring} archivebox.api.auth.auth_using_password ``` ```` -`````{py:class} APITokenAuthCheck -:canonical: archivebox.api.auth.APITokenAuthCheck +````{py:function} _require_superuser(user: django.contrib.auth.models.User | None, request: django.http.HttpRequest, auth_method: str) -> django.contrib.auth.models.User | None +:canonical: archivebox.api.auth._require_superuser -```{autodoc2-docstring} archivebox.api.auth.APITokenAuthCheck +```{autodoc2-docstring} archivebox.api.auth._require_superuser ``` - -````{py:method} authenticate(request: django.http.HttpRequest, key: typing.Optional[str] = None) -> typing.Optional[django.contrib.auth.models.AbstractBaseUser] -:canonical: archivebox.api.auth.APITokenAuthCheck.authenticate - -```{autodoc2-docstring} archivebox.api.auth.APITokenAuthCheck.authenticate -``` - ```` -````` - -`````{py:class} UserPassAuthCheck -:canonical: archivebox.api.auth.UserPassAuthCheck - -```{autodoc2-docstring} archivebox.api.auth.UserPassAuthCheck -``` - -````{py:method} authenticate(request: django.http.HttpRequest, username: typing.Optional[str] = None, password: typing.Optional[str] = None) -> typing.Optional[django.contrib.auth.models.AbstractBaseUser] -:canonical: archivebox.api.auth.UserPassAuthCheck.authenticate - -```{autodoc2-docstring} archivebox.api.auth.UserPassAuthCheck.authenticate -``` - -```` - -````` - `````{py:class} HeaderTokenAuth() :canonical: archivebox.api.auth.HeaderTokenAuth -Bases: {py:obj}`archivebox.api.auth.APITokenAuthCheck`, {py:obj}`ninja.security.APIKeyHeader` +Bases: {py:obj}`ninja.security.APIKeyHeader` ```{autodoc2-docstring} archivebox.api.auth.HeaderTokenAuth ``` @@ -156,12 +119,20 @@ Bases: {py:obj}`archivebox.api.auth.APITokenAuthCheck`, {py:obj}`ninja.security. ```` +````{py:method} authenticate(request: django.http.HttpRequest, key: str | None) -> django.contrib.auth.models.User | None +:canonical: archivebox.api.auth.HeaderTokenAuth.authenticate + +```{autodoc2-docstring} archivebox.api.auth.HeaderTokenAuth.authenticate +``` + +```` + ````` -````{py:class} BearerTokenAuth() +`````{py:class} BearerTokenAuth() :canonical: archivebox.api.auth.BearerTokenAuth -Bases: {py:obj}`archivebox.api.auth.APITokenAuthCheck`, {py:obj}`ninja.security.HttpBearer` +Bases: {py:obj}`ninja.security.HttpBearer` ```{autodoc2-docstring} archivebox.api.auth.BearerTokenAuth ``` @@ -172,12 +143,20 @@ Bases: {py:obj}`archivebox.api.auth.APITokenAuthCheck`, {py:obj}`ninja.security. ```{autodoc2-docstring} archivebox.api.auth.BearerTokenAuth.__init__ ``` +````{py:method} authenticate(request: django.http.HttpRequest, token: str) -> django.contrib.auth.models.User | None +:canonical: archivebox.api.auth.BearerTokenAuth.authenticate + +```{autodoc2-docstring} archivebox.api.auth.BearerTokenAuth.authenticate +``` + ```` +````` + `````{py:class} QueryParamTokenAuth() :canonical: archivebox.api.auth.QueryParamTokenAuth -Bases: {py:obj}`archivebox.api.auth.APITokenAuthCheck`, {py:obj}`ninja.security.APIKeyQuery` +Bases: {py:obj}`ninja.security.APIKeyQuery` ```{autodoc2-docstring} archivebox.api.auth.QueryParamTokenAuth ``` @@ -198,42 +177,10 @@ Bases: {py:obj}`archivebox.api.auth.APITokenAuthCheck`, {py:obj}`ninja.security. ```` -````` +````{py:method} authenticate(request: django.http.HttpRequest, key: str | None) -> django.contrib.auth.models.User | None +:canonical: archivebox.api.auth.QueryParamTokenAuth.authenticate -````{py:class} UsernameAndPasswordAuth() -:canonical: archivebox.api.auth.UsernameAndPasswordAuth - -Bases: {py:obj}`archivebox.api.auth.UserPassAuthCheck`, {py:obj}`ninja.security.HttpBasicAuth` - -```{autodoc2-docstring} archivebox.api.auth.UsernameAndPasswordAuth -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.api.auth.UsernameAndPasswordAuth.__init__ -``` - -```` - -`````{py:class} DjangoSessionAuth -:canonical: archivebox.api.auth.DjangoSessionAuth - -```{autodoc2-docstring} archivebox.api.auth.DjangoSessionAuth -``` - -````{py:method} __call__(request: django.http.HttpRequest) -> typing.Optional[django.contrib.auth.models.AbstractBaseUser] -:canonical: archivebox.api.auth.DjangoSessionAuth.__call__ - -```{autodoc2-docstring} archivebox.api.auth.DjangoSessionAuth.__call__ -``` - -```` - -````{py:method} authenticate(request: django.http.HttpRequest, **kwargs) -> typing.Optional[django.contrib.auth.models.AbstractBaseUser] -:canonical: archivebox.api.auth.DjangoSessionAuth.authenticate - -```{autodoc2-docstring} archivebox.api.auth.DjangoSessionAuth.authenticate +```{autodoc2-docstring} archivebox.api.auth.QueryParamTokenAuth.authenticate ``` ```` diff --git a/docs/apidocs/archivebox/archivebox.api.md b/docs/apidocs/archivebox/archivebox.api.md index 71a28a5e..bbe586b2 100644 --- a/docs/apidocs/archivebox/archivebox.api.md +++ b/docs/apidocs/archivebox/archivebox.api.md @@ -21,6 +21,8 @@ archivebox.api.v1_cli archivebox.api.v1_core archivebox.api.apps archivebox.api.admin +archivebox.api.v1_personas +archivebox.api.webhooks archivebox.api.urls archivebox.api.v1_machine archivebox.api.middleware diff --git a/docs/apidocs/archivebox/archivebox.api.urls.md b/docs/apidocs/archivebox/archivebox.api.urls.md index c73d37a8..c7503fd0 100644 --- a/docs/apidocs/archivebox/archivebox.api.urls.md +++ b/docs/apidocs/archivebox/archivebox.api.urls.md @@ -9,6 +9,18 @@ ## Module Contents +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`archive_redirect_view ` + - ```{autodoc2-docstring} archivebox.api.urls.archive_redirect_view + :summary: + ``` +```` + ### Data ````{list-table} @@ -23,6 +35,13 @@ ### API +````{py:function} archive_redirect_view(request: django.http.HttpRequest, url: str) -> django.http.HttpResponseRedirect +:canonical: archivebox.api.urls.archive_redirect_view + +```{autodoc2-docstring} archivebox.api.urls.archive_redirect_view +``` +```` + ````{py:data} urlpatterns :canonical: archivebox.api.urls.urlpatterns :value: > diff --git a/docs/apidocs/archivebox/archivebox.api.v1_auth.md b/docs/apidocs/archivebox/archivebox.api.v1_auth.md index 8649c62c..1285fa26 100644 --- a/docs/apidocs/archivebox/archivebox.api.v1_auth.md +++ b/docs/apidocs/archivebox/archivebox.api.v1_auth.md @@ -81,7 +81,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} username :canonical: archivebox.api.v1_auth.PasswordAuthSchema.username -:type: typing.Optional[str] +:type: str | None :value: > None @@ -92,7 +92,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} password :canonical: archivebox.api.v1_auth.PasswordAuthSchema.password -:type: typing.Optional[str] +:type: str | None :value: > None @@ -103,7 +103,7 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} get_api_token(request, auth_data: archivebox.api.v1_auth.PasswordAuthSchema) +````{py:function} get_api_token(request: django.http.HttpRequest, auth_data: archivebox.api.v1_auth.PasswordAuthSchema) :canonical: archivebox.api.v1_auth.get_api_token ```{autodoc2-docstring} archivebox.api.v1_auth.get_api_token @@ -137,7 +137,7 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} check_api_token(request, token_data: archivebox.api.v1_auth.TokenAuthSchema) +````{py:function} check_api_token(request: django.http.HttpRequest, token_data: archivebox.api.v1_auth.TokenAuthSchema) :canonical: archivebox.api.v1_auth.check_api_token ```{autodoc2-docstring} archivebox.api.v1_auth.check_api_token diff --git a/docs/apidocs/archivebox/archivebox.api.v1_cli.md b/docs/apidocs/archivebox/archivebox.api.v1_cli.md index d8e11fa6..ebec6094 100644 --- a/docs/apidocs/archivebox/archivebox.api.v1_cli.md +++ b/docs/apidocs/archivebox/archivebox.api.v1_cli.md @@ -117,7 +117,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} errors :canonical: archivebox.api.v1_cli.CLICommandResponseSchema.errors -:type: typing.List[str] +:type: list[str] :value: > None @@ -353,7 +353,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} urls :canonical: archivebox.api.v1_cli.AddCommandSchema.urls -:type: typing.List[str] +:type: list[str] :value: > None @@ -362,6 +362,17 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:attribute} snapshot_ids +:canonical: archivebox.api.v1_cli.AddCommandSchema.snapshot_ids +:type: list[str] | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.snapshot_ids +``` + +```` + ````{py:attribute} tag :canonical: archivebox.api.v1_cli.AddCommandSchema.tag :type: str @@ -383,6 +394,50 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:attribute} max_urls +:canonical: archivebox.api.v1_cli.AddCommandSchema.max_urls +:type: int +:value: > + 0 + +```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.max_urls +``` + +```` + +````{py:attribute} crawl_max_size +:canonical: archivebox.api.v1_cli.AddCommandSchema.crawl_max_size +:type: int +:value: > + 0 + +```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.crawl_max_size +``` + +```` + +````{py:attribute} crawl_timeout +:canonical: archivebox.api.v1_cli.AddCommandSchema.crawl_timeout +:type: int +:value: > + 0 + +```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.crawl_timeout +``` + +```` + +````{py:attribute} snapshot_max_size +:canonical: archivebox.api.v1_cli.AddCommandSchema.snapshot_max_size +:type: int +:value: > + 0 + +```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.snapshot_max_size +``` + +```` + ````{py:attribute} parser :canonical: archivebox.api.v1_cli.AddCommandSchema.parser :type: str @@ -404,24 +459,13 @@ Bases: {py:obj}`ninja.Schema` ```` -````{py:attribute} update -:canonical: archivebox.api.v1_cli.AddCommandSchema.update -:type: bool +````{py:attribute} only_new +:canonical: archivebox.api.v1_cli.AddCommandSchema.only_new +:type: bool | None :value: > None -```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.update -``` - -```` - -````{py:attribute} overwrite -:canonical: archivebox.api.v1_cli.AddCommandSchema.overwrite -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.overwrite +```{autodoc2-docstring} archivebox.api.v1_cli.AddCommandSchema.only_new ``` ```` @@ -446,51 +490,18 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} resume :canonical: archivebox.api.v1_cli.UpdateCommandSchema.resume -:type: typing.Optional[float] +:type: str | None :value: > - 0 + None ```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.resume ``` ```` -````{py:attribute} only_new -:canonical: archivebox.api.v1_cli.UpdateCommandSchema.only_new -:type: bool -:value: > - None - -```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.only_new -``` - -```` - -````{py:attribute} index_only -:canonical: archivebox.api.v1_cli.UpdateCommandSchema.index_only -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.index_only -``` - -```` - -````{py:attribute} overwrite -:canonical: archivebox.api.v1_cli.UpdateCommandSchema.overwrite -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.overwrite -``` - -```` - ````{py:attribute} after :canonical: archivebox.api.v1_cli.UpdateCommandSchema.after -:type: typing.Optional[float] +:type: float | None :value: > 0 @@ -501,7 +512,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} before :canonical: archivebox.api.v1_cli.UpdateCommandSchema.before -:type: typing.Optional[float] +:type: float | None :value: > 999999999999999 @@ -510,20 +521,9 @@ Bases: {py:obj}`ninja.Schema` ```` -````{py:attribute} status -:canonical: archivebox.api.v1_cli.UpdateCommandSchema.status -:type: typing.Optional[archivebox.api.v1_cli.StatusChoices] -:value: > - None - -```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.status -``` - -```` - ````{py:attribute} filter_type :canonical: archivebox.api.v1_cli.UpdateCommandSchema.filter_type -:type: typing.Optional[str] +:type: str | None :value: > None @@ -534,7 +534,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} filter_patterns :canonical: archivebox.api.v1_cli.UpdateCommandSchema.filter_patterns -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > ['https://example.com'] @@ -543,12 +543,24 @@ Bases: {py:obj}`ninja.Schema` ```` -````{py:attribute} plugins -:canonical: archivebox.api.v1_cli.UpdateCommandSchema.plugins -:type: typing.Optional[str] -:value: +````{py:attribute} batch_size +:canonical: archivebox.api.v1_cli.UpdateCommandSchema.batch_size +:type: int +:value: > + 100 -```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.plugins +```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.batch_size +``` + +```` + +````{py:attribute} continuous +:canonical: archivebox.api.v1_cli.UpdateCommandSchema.continuous +:type: bool +:value: > + False + +```{autodoc2-docstring} archivebox.api.v1_cli.UpdateCommandSchema.continuous ``` ```` @@ -562,7 +574,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} import_path :canonical: archivebox.api.v1_cli.ScheduleCommandSchema.import_path -:type: typing.Optional[str] +:type: str | None :value: > None @@ -628,7 +640,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} every :canonical: archivebox.api.v1_cli.ScheduleCommandSchema.every -:type: typing.Optional[str] +:type: str | None :value: > None @@ -658,24 +670,13 @@ Bases: {py:obj}`ninja.Schema` ```` -````{py:attribute} overwrite -:canonical: archivebox.api.v1_cli.ScheduleCommandSchema.overwrite -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.api.v1_cli.ScheduleCommandSchema.overwrite -``` - -```` - -````{py:attribute} update -:canonical: archivebox.api.v1_cli.ScheduleCommandSchema.update -:type: bool +````{py:attribute} only_new +:canonical: archivebox.api.v1_cli.ScheduleCommandSchema.only_new +:type: bool | None :value: > None -```{autodoc2-docstring} archivebox.api.v1_cli.ScheduleCommandSchema.update +```{autodoc2-docstring} archivebox.api.v1_cli.ScheduleCommandSchema.only_new ``` ```` @@ -700,7 +701,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} filter_patterns :canonical: archivebox.api.v1_cli.ListCommandSchema.filter_patterns -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > ['https://example.com'] @@ -733,7 +734,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} after :canonical: archivebox.api.v1_cli.ListCommandSchema.after -:type: typing.Optional[float] +:type: float | None :value: > 0 @@ -744,7 +745,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} before :canonical: archivebox.api.v1_cli.ListCommandSchema.before -:type: typing.Optional[float] +:type: float | None :value: > 999999999999999 @@ -815,20 +816,9 @@ Bases: {py:obj}`ninja.Schema` Bases: {py:obj}`ninja.Schema` -````{py:attribute} delete -:canonical: archivebox.api.v1_cli.RemoveCommandSchema.delete -:type: bool -:value: > - True - -```{autodoc2-docstring} archivebox.api.v1_cli.RemoveCommandSchema.delete -``` - -```` - ````{py:attribute} after :canonical: archivebox.api.v1_cli.RemoveCommandSchema.after -:type: typing.Optional[float] +:type: float | None :value: > 0 @@ -839,7 +829,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} before :canonical: archivebox.api.v1_cli.RemoveCommandSchema.before -:type: typing.Optional[float] +:type: float | None :value: > 999999999999999 @@ -861,7 +851,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} filter_patterns :canonical: archivebox.api.v1_cli.RemoveCommandSchema.filter_patterns -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > ['https://example.com'] @@ -872,35 +862,35 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} cli_add(request, args: archivebox.api.v1_cli.AddCommandSchema) +````{py:function} cli_add(request: django.http.HttpRequest, args: archivebox.api.v1_cli.AddCommandSchema) :canonical: archivebox.api.v1_cli.cli_add ```{autodoc2-docstring} archivebox.api.v1_cli.cli_add ``` ```` -````{py:function} cli_update(request, args: archivebox.api.v1_cli.UpdateCommandSchema) +````{py:function} cli_update(request: django.http.HttpRequest, args: archivebox.api.v1_cli.UpdateCommandSchema) :canonical: archivebox.api.v1_cli.cli_update ```{autodoc2-docstring} archivebox.api.v1_cli.cli_update ``` ```` -````{py:function} cli_schedule(request, args: archivebox.api.v1_cli.ScheduleCommandSchema) +````{py:function} cli_schedule(request: django.http.HttpRequest, args: archivebox.api.v1_cli.ScheduleCommandSchema) :canonical: archivebox.api.v1_cli.cli_schedule ```{autodoc2-docstring} archivebox.api.v1_cli.cli_schedule ``` ```` -````{py:function} cli_search(request, args: archivebox.api.v1_cli.ListCommandSchema) +````{py:function} cli_search(request: django.http.HttpRequest, args: archivebox.api.v1_cli.ListCommandSchema) :canonical: archivebox.api.v1_cli.cli_search ```{autodoc2-docstring} archivebox.api.v1_cli.cli_search ``` ```` -````{py:function} cli_remove(request, args: archivebox.api.v1_cli.RemoveCommandSchema) +````{py:function} cli_remove(request: django.http.HttpRequest, args: archivebox.api.v1_cli.RemoveCommandSchema) :canonical: archivebox.api.v1_cli.cli_remove ```{autodoc2-docstring} archivebox.api.v1_cli.cli_remove diff --git a/docs/apidocs/archivebox/archivebox.api.v1_core.md b/docs/apidocs/archivebox/archivebox.api.v1_core.md index 5268cbc6..1a2f5312 100644 --- a/docs/apidocs/archivebox/archivebox.api.v1_core.md +++ b/docs/apidocs/archivebox/archivebox.api.v1_core.md @@ -41,6 +41,18 @@ - * - {py:obj}`TagCreateResponseSchema ` - +* - {py:obj}`TagSearchSnapshotSchema ` + - +* - {py:obj}`TagSearchCardSchema ` + - +* - {py:obj}`TagSearchResponseSchema ` + - +* - {py:obj}`TagUpdateSchema ` + - +* - {py:obj}`TagUpdateResponseSchema ` + - +* - {py:obj}`TagDeleteResponseSchema ` + - * - {py:obj}`TagSnapshotRequestSchema ` - * - {py:obj}`TagSnapshotResponseSchema ` @@ -57,18 +69,98 @@ - ```{autodoc2-docstring} archivebox.api.v1_core.get_archiveresults :summary: ``` +* - {py:obj}`_uuid_ref_query ` + - ```{autodoc2-docstring} archivebox.api.v1_core._uuid_ref_query + :summary: + ``` * - {py:obj}`get_archiveresult ` - ```{autodoc2-docstring} archivebox.api.v1_core.get_archiveresult :summary: ``` +* - {py:obj}`_normalize_uploaded_archiveresult_plugin ` + - ```{autodoc2-docstring} archivebox.api.v1_core._normalize_uploaded_archiveresult_plugin + :summary: + ``` +* - {py:obj}`_normalize_uploaded_archiveresult_output_path ` + - ```{autodoc2-docstring} archivebox.api.v1_core._normalize_uploaded_archiveresult_output_path + :summary: + ``` +* - {py:obj}`_parse_archiveresult_output_json ` + - ```{autodoc2-docstring} archivebox.api.v1_core._parse_archiveresult_output_json + :summary: + ``` +* - {py:obj}`_get_archiveresult_upload_data ` + - ```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_data + :summary: + ``` +* - {py:obj}`_get_archiveresult_upload_files ` + - ```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_files + :summary: + ``` +* - {py:obj}`_get_archiveresult_upload_form_values ` + - ```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_form_values + :summary: + ``` +* - {py:obj}`_get_archiveresult_upload_form_value ` + - ```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_form_value + :summary: + ``` +* - {py:obj}`_parse_archiveresult_upload_int ` + - ```{autodoc2-docstring} archivebox.api.v1_core._parse_archiveresult_upload_int + :summary: + ``` +* - {py:obj}`_summarize_archiveresult_output_files ` + - ```{autodoc2-docstring} archivebox.api.v1_core._summarize_archiveresult_output_files + :summary: + ``` +* - {py:obj}`_get_snapshot_by_ref ` + - ```{autodoc2-docstring} archivebox.api.v1_core._get_snapshot_by_ref + :summary: + ``` +* - {py:obj}`_queue_archiveresult_snapshot_maintenance ` + - ```{autodoc2-docstring} archivebox.api.v1_core._queue_archiveresult_snapshot_maintenance + :summary: + ``` +* - {py:obj}`_merge_archiveresult_output_file_maps ` + - ```{autodoc2-docstring} archivebox.api.v1_core._merge_archiveresult_output_file_maps + :summary: + ``` +* - {py:obj}`_write_archiveresult_files ` + - ```{autodoc2-docstring} archivebox.api.v1_core._write_archiveresult_files + :summary: + ``` +* - {py:obj}`create_archiveresult ` + - ```{autodoc2-docstring} archivebox.api.v1_core.create_archiveresult + :summary: + ``` +* - {py:obj}`patch_archiveresult ` + - ```{autodoc2-docstring} archivebox.api.v1_core.patch_archiveresult + :summary: + ``` * - {py:obj}`normalize_tag_list ` - ```{autodoc2-docstring} archivebox.api.v1_core.normalize_tag_list :summary: ``` +* - {py:obj}`_parse_rss_before ` + - ```{autodoc2-docstring} archivebox.api.v1_core._parse_rss_before + :summary: + ``` +* - {py:obj}`_filter_snapshots_for_rss ` + - ```{autodoc2-docstring} archivebox.api.v1_core._filter_snapshots_for_rss + :summary: + ``` +* - {py:obj}`_snapshots_rss_response ` + - ```{autodoc2-docstring} archivebox.api.v1_core._snapshots_rss_response + :summary: + ``` * - {py:obj}`get_snapshots ` - ```{autodoc2-docstring} archivebox.api.v1_core.get_snapshots :summary: ``` +* - {py:obj}`get_snapshots_rss ` + - ```{autodoc2-docstring} archivebox.api.v1_core.get_snapshots_rss + :summary: + ``` * - {py:obj}`get_snapshot ` - ```{autodoc2-docstring} archivebox.api.v1_core.get_snapshot :summary: @@ -97,6 +189,22 @@ - ```{autodoc2-docstring} archivebox.api.v1_core.get_any :summary: ``` +* - {py:obj}`_get_snapshot_for_tag_edit ` + - ```{autodoc2-docstring} archivebox.api.v1_core._get_snapshot_for_tag_edit + :summary: + ``` +* - {py:obj}`search_tags ` + - ```{autodoc2-docstring} archivebox.api.v1_core.search_tags + :summary: + ``` +* - {py:obj}`_public_tag_listing_enabled ` + - ```{autodoc2-docstring} archivebox.api.v1_core._public_tag_listing_enabled + :summary: + ``` +* - {py:obj}`_request_has_tag_autocomplete_access ` + - ```{autodoc2-docstring} archivebox.api.v1_core._request_has_tag_autocomplete_access + :summary: + ``` * - {py:obj}`tags_autocomplete ` - ```{autodoc2-docstring} archivebox.api.v1_core.tags_autocomplete :summary: @@ -105,6 +213,22 @@ - ```{autodoc2-docstring} archivebox.api.v1_core.tags_create :summary: ``` +* - {py:obj}`rename_tag ` + - ```{autodoc2-docstring} archivebox.api.v1_core.rename_tag + :summary: + ``` +* - {py:obj}`delete_tag ` + - ```{autodoc2-docstring} archivebox.api.v1_core.delete_tag + :summary: + ``` +* - {py:obj}`tag_urls_export ` + - ```{autodoc2-docstring} archivebox.api.v1_core.tag_urls_export + :summary: + ``` +* - {py:obj}`tag_snapshots_export ` + - ```{autodoc2-docstring} archivebox.api.v1_core.tag_snapshots_export + :summary: + ``` * - {py:obj}`tags_add_to_snapshot ` - ```{autodoc2-docstring} archivebox.api.v1_core.tags_add_to_snapshot :summary: @@ -125,6 +249,14 @@ - ```{autodoc2-docstring} archivebox.api.v1_core.router :summary: ``` +* - {py:obj}`ARCHIVERESULT_UPLOAD_HOOK_NAME ` + - ```{autodoc2-docstring} archivebox.api.v1_core.ARCHIVERESULT_UPLOAD_HOOK_NAME + :summary: + ``` +* - {py:obj}`ARCHIVERESULT_UPLOAD_PLUGIN_RE ` + - ```{autodoc2-docstring} archivebox.api.v1_core.ARCHIVERESULT_UPLOAD_PLUGIN_RE + :summary: + ``` ```` ### API @@ -139,6 +271,26 @@ ```` +````{py:data} ARCHIVERESULT_UPLOAD_HOOK_NAME +:canonical: archivebox.api.v1_core.ARCHIVERESULT_UPLOAD_HOOK_NAME +:value: > + 'on_Snapshot__archivebox_browser_extension_upload' + +```{autodoc2-docstring} archivebox.api.v1_core.ARCHIVERESULT_UPLOAD_HOOK_NAME +``` + +```` + +````{py:data} ARCHIVERESULT_UPLOAD_PLUGIN_RE +:canonical: archivebox.api.v1_core.ARCHIVERESULT_UPLOAD_PLUGIN_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.api.v1_core.ARCHIVERESULT_UPLOAD_PLUGIN_RE +``` + +```` + ``````{py:class} CustomPagination(*, pass_parameter: typing.Optional[str] = None, **kwargs: typing.Any) :canonical: archivebox.api.v1_core.CustomPagination @@ -147,7 +299,7 @@ Bases: {py:obj}`ninja.pagination.PaginationBase` `````{py:class} Input(/, **data: typing.Any) :canonical: archivebox.api.v1_core.CustomPagination.Input -Bases: {py:obj}`ninja.Schema` +Bases: {py:obj}`ninja.pagination.PaginationBase.Input` ````{py:attribute} limit :canonical: archivebox.api.v1_core.CustomPagination.Input.limit @@ -187,7 +339,18 @@ Bases: {py:obj}`ninja.Schema` `````{py:class} Output(/, **data: typing.Any) :canonical: archivebox.api.v1_core.CustomPagination.Output -Bases: {py:obj}`ninja.Schema` +Bases: {py:obj}`ninja.pagination.PaginationBase.Output` + +````{py:attribute} count +:canonical: archivebox.api.v1_core.CustomPagination.Output.count +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.CustomPagination.Output.count +``` + +```` ````{py:attribute} total_items :canonical: archivebox.api.v1_core.CustomPagination.Output.total_items @@ -257,7 +420,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} items :canonical: archivebox.api.v1_core.CustomPagination.Output.items -:type: typing.List[typing.Any] +:type: list[typing.Any] :value: > None @@ -268,7 +431,7 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:method} paginate_queryset(queryset, pagination: Input, **params) +````{py:method} paginate_queryset(queryset, pagination: Input, request: django.http.HttpRequest, **params) :canonical: archivebox.api.v1_core.CustomPagination.paginate_queryset ```{autodoc2-docstring} archivebox.api.v1_core.CustomPagination.paginate_queryset @@ -450,7 +613,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} output_json :canonical: archivebox.api.v1_core.MinimalArchiveResultSchema.output_json -:type: dict | None +:type: dict[str, typing.Any] | None :value: > None @@ -461,7 +624,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} output_files :canonical: archivebox.api.v1_core.MinimalArchiveResultSchema.output_files -:type: dict | None +:type: dict[str, dict[str, typing.Any]] | None :value: > None @@ -532,6 +695,24 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:method} resolve_output_files(obj) +:canonical: archivebox.api.v1_core.MinimalArchiveResultSchema.resolve_output_files +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_core.MinimalArchiveResultSchema.resolve_output_files +``` + +```` + +````{py:method} resolve_output_mimetypes(obj) -> str +:canonical: archivebox.api.v1_core.MinimalArchiveResultSchema.resolve_output_mimetypes +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_core.MinimalArchiveResultSchema.resolve_output_mimetypes +``` + +```` + ````` `````{py:class} ArchiveResultSchema(/, **data: typing.Any) @@ -585,7 +766,7 @@ Bases: {py:obj}`archivebox.api.v1_core.MinimalArchiveResultSchema` ````{py:attribute} snapshot_tags :canonical: archivebox.api.v1_core.ArchiveResultSchema.snapshot_tags -:type: typing.List[str] +:type: list[str] :value: > None @@ -639,9 +820,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} id :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup(['id__startswith', 'snapshot__id__startswith', 'snapshot__timestamp__startswith'])] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.id ``` @@ -650,9 +831,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} search :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.search -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup(['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'plugin', 'output_str__icontains', 'id__startswith', 'snapshot__id__startswith', 'snapshot__timestamp__startswith'])] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.search ``` @@ -661,9 +842,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} snapshot_id :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.snapshot_id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup(['snapshot__id__startswith', 'snapshot__timestamp__startswith'])] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.snapshot_id ``` @@ -672,9 +853,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} snapshot_url :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.snapshot_url -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('snapshot__url__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.snapshot_url ``` @@ -683,9 +864,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} snapshot_tag :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.snapshot_tag -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('snapshot__tags__name__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.snapshot_tag ``` @@ -694,9 +875,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} status :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.status -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('status')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.status ``` @@ -705,9 +886,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} output_str :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.output_str -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('output_str__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.output_str ``` @@ -716,9 +897,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} plugin :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.plugin -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('plugin__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.plugin ``` @@ -727,9 +908,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} hook_name :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.hook_name -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('hook_name__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.hook_name ``` @@ -738,9 +919,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} process_id :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.process_id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('process__id__startswith')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.process_id ``` @@ -749,9 +930,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} cmd :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.cmd -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('cmd__0__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.cmd ``` @@ -760,9 +941,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} pwd :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.pwd -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('pwd__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.pwd ``` @@ -771,9 +952,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} cmd_version :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.cmd_version -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('cmd_version')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.cmd_version ``` @@ -782,9 +963,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_at :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.created_at -:type: typing.Optional[datetime.datetime] +:type: typing.Annotated[datetime.datetime | None, FilterLookup('created_at')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.created_at ``` @@ -793,9 +974,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_at__gte :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.created_at__gte -:type: typing.Optional[datetime.datetime] +:type: typing.Annotated[datetime.datetime | None, FilterLookup('created_at__gte')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.created_at__gte ``` @@ -804,9 +985,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_at__lt :canonical: archivebox.api.v1_core.ArchiveResultFilterSchema.created_at__lt -:type: typing.Optional[datetime.datetime] +:type: typing.Annotated[datetime.datetime | None, FilterLookup('created_at__lt')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.ArchiveResultFilterSchema.created_at__lt ``` @@ -815,20 +996,132 @@ Bases: {py:obj}`ninja.FilterSchema` ````` -````{py:function} get_archiveresults(request, filters: archivebox.api.v1_core.ArchiveResultFilterSchema = Query(...)) +````{py:function} get_archiveresults(request: django.http.HttpRequest, filters: ninja.Query[archivebox.api.v1_core.ArchiveResultFilterSchema]) :canonical: archivebox.api.v1_core.get_archiveresults ```{autodoc2-docstring} archivebox.api.v1_core.get_archiveresults ``` ```` -````{py:function} get_archiveresult(request, archiveresult_id: str) +````{py:function} _uuid_ref_query(field_name: str, ref: str) -> django.db.models.Q +:canonical: archivebox.api.v1_core._uuid_ref_query + +```{autodoc2-docstring} archivebox.api.v1_core._uuid_ref_query +``` +```` + +````{py:function} get_archiveresult(request: django.http.HttpRequest, archiveresult_id: str) :canonical: archivebox.api.v1_core.get_archiveresult ```{autodoc2-docstring} archivebox.api.v1_core.get_archiveresult ``` ```` +````{py:function} _normalize_uploaded_archiveresult_plugin(plugin: str) -> str +:canonical: archivebox.api.v1_core._normalize_uploaded_archiveresult_plugin + +```{autodoc2-docstring} archivebox.api.v1_core._normalize_uploaded_archiveresult_plugin +``` +```` + +````{py:function} _normalize_uploaded_archiveresult_output_path(output_path: str, *, filename: str) -> str +:canonical: archivebox.api.v1_core._normalize_uploaded_archiveresult_output_path + +```{autodoc2-docstring} archivebox.api.v1_core._normalize_uploaded_archiveresult_output_path +``` +```` + +````{py:function} _parse_archiveresult_output_json(output_json: str | None) -> dict[str, typing.Any] | None +:canonical: archivebox.api.v1_core._parse_archiveresult_output_json + +```{autodoc2-docstring} archivebox.api.v1_core._parse_archiveresult_output_json +``` +```` + +````{py:function} _get_archiveresult_upload_data(request: django.http.HttpRequest) +:canonical: archivebox.api.v1_core._get_archiveresult_upload_data + +```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_data +``` +```` + +````{py:function} _get_archiveresult_upload_files(request: django.http.HttpRequest, *, allow_empty: bool = False) -> list[ninja.UploadedFile] +:canonical: archivebox.api.v1_core._get_archiveresult_upload_files + +```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_files +``` +```` + +````{py:function} _get_archiveresult_upload_form_values(request: django.http.HttpRequest, *field_names: str) -> list[str] +:canonical: archivebox.api.v1_core._get_archiveresult_upload_form_values + +```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_form_values +``` +```` + +````{py:function} _get_archiveresult_upload_form_value(request: django.http.HttpRequest, *field_names: str) -> str +:canonical: archivebox.api.v1_core._get_archiveresult_upload_form_value + +```{autodoc2-docstring} archivebox.api.v1_core._get_archiveresult_upload_form_value +``` +```` + +````{py:function} _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int | None = None) -> int +:canonical: archivebox.api.v1_core._parse_archiveresult_upload_int + +```{autodoc2-docstring} archivebox.api.v1_core._parse_archiveresult_upload_int +``` +```` + +````{py:function} _summarize_archiveresult_output_files(output_files: dict[str, dict[str, typing.Any]]) -> tuple[int, str] +:canonical: archivebox.api.v1_core._summarize_archiveresult_output_files + +```{autodoc2-docstring} archivebox.api.v1_core._summarize_archiveresult_output_files +``` +```` + +````{py:function} _get_snapshot_by_ref(snapshot_id: str) +:canonical: archivebox.api.v1_core._get_snapshot_by_ref + +```{autodoc2-docstring} archivebox.api.v1_core._get_snapshot_by_ref +``` +```` + +````{py:function} _queue_archiveresult_snapshot_maintenance(snapshot: archivebox.core.models.Snapshot) -> None +:canonical: archivebox.api.v1_core._queue_archiveresult_snapshot_maintenance + +```{autodoc2-docstring} archivebox.api.v1_core._queue_archiveresult_snapshot_maintenance +``` +```` + +````{py:function} _merge_archiveresult_output_file_maps(results: list[archivebox.core.models.ArchiveResult]) -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.api.v1_core._merge_archiveresult_output_file_maps + +```{autodoc2-docstring} archivebox.api.v1_core._merge_archiveresult_output_file_maps +``` +```` + +````{py:function} _write_archiveresult_files(request: django.http.HttpRequest, snapshot: archivebox.core.models.Snapshot, plugin_name: str, *, existing_output_files: dict[str, dict[str, typing.Any]] | None = None, allow_empty: bool = False) -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.api.v1_core._write_archiveresult_files + +```{autodoc2-docstring} archivebox.api.v1_core._write_archiveresult_files +``` +```` + +````{py:function} create_archiveresult(request: django.http.HttpRequest, snapshot_id: str = Form(...), plugin: str = Form(...), output_str: str = Form(''), hook_name: str = Form(ARCHIVERESULT_UPLOAD_HOOK_NAME), status: str = Form(str(ArchiveResult.StatusChoices.SUCCEEDED)), output_json: str = Form('')) +:canonical: archivebox.api.v1_core.create_archiveresult + +```{autodoc2-docstring} archivebox.api.v1_core.create_archiveresult +``` +```` + +````{py:function} patch_archiveresult(request: django.http.HttpRequest, archiveresult_id: str) +:canonical: archivebox.api.v1_core.patch_archiveresult + +```{autodoc2-docstring} archivebox.api.v1_core.patch_archiveresult +``` +```` + `````{py:class} SnapshotSchema(/, **data: typing.Any) :canonical: archivebox.api.v1_core.SnapshotSchema @@ -935,7 +1228,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} downloaded_at :canonical: archivebox.api.v1_core.SnapshotSchema.downloaded_at -:type: typing.Optional[datetime.datetime] +:type: datetime.datetime | None :value: > None @@ -957,7 +1250,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tags :canonical: archivebox.api.v1_core.SnapshotSchema.tags -:type: typing.List[str] +:type: list[str] :value: > None @@ -968,7 +1261,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} title :canonical: archivebox.api.v1_core.SnapshotSchema.title -:type: typing.Optional[str] +:type: str | None :value: > None @@ -999,6 +1292,28 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:attribute} archive_size +:canonical: archivebox.api.v1_core.SnapshotSchema.archive_size +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.SnapshotSchema.archive_size +``` + +```` + +````{py:attribute} output_size +:canonical: archivebox.api.v1_core.SnapshotSchema.output_size +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.SnapshotSchema.output_size +``` + +```` + ````{py:attribute} num_archiveresults :canonical: archivebox.api.v1_core.SnapshotSchema.num_archiveresults :type: int @@ -1012,7 +1327,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} archiveresults :canonical: archivebox.api.v1_core.SnapshotSchema.archiveresults -:type: typing.List[archivebox.api.v1_core.MinimalArchiveResultSchema] +:type: list[archivebox.api.v1_core.MinimalArchiveResultSchema] :value: > None @@ -1048,6 +1363,24 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:method} resolve_archive_size(obj) +:canonical: archivebox.api.v1_core.SnapshotSchema.resolve_archive_size +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_core.SnapshotSchema.resolve_archive_size +``` + +```` + +````{py:method} resolve_output_size(obj) +:canonical: archivebox.api.v1_core.SnapshotSchema.resolve_output_size +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_core.SnapshotSchema.resolve_output_size +``` + +```` + ````{py:method} resolve_num_archiveresults(obj, context) :canonical: archivebox.api.v1_core.SnapshotSchema.resolve_num_archiveresults :staticmethod: @@ -1073,6 +1406,17 @@ Bases: {py:obj}`ninja.Schema` Bases: {py:obj}`ninja.Schema` +````{py:attribute} action +:canonical: archivebox.api.v1_core.SnapshotUpdateSchema.action +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.SnapshotUpdateSchema.action +``` + +```` + ````{py:attribute} status :canonical: archivebox.api.v1_core.SnapshotUpdateSchema.status :type: str | None @@ -1097,7 +1441,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tags :canonical: archivebox.api.v1_core.SnapshotUpdateSchema.tags -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > None @@ -1126,7 +1470,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} crawl_id :canonical: archivebox.api.v1_core.SnapshotCreateSchema.crawl_id -:type: typing.Optional[str] +:type: str | None :value: > None @@ -1148,7 +1492,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} title :canonical: archivebox.api.v1_core.SnapshotCreateSchema.title -:type: typing.Optional[str] +:type: str | None :value: > None @@ -1159,7 +1503,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tags :canonical: archivebox.api.v1_core.SnapshotCreateSchema.tags -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > None @@ -1170,7 +1514,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} status :canonical: archivebox.api.v1_core.SnapshotCreateSchema.status -:type: typing.Optional[str] +:type: str | None :value: > None @@ -1232,13 +1576,34 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} normalize_tag_list(tags: typing.Optional[typing.List[str]] = None) -> typing.List[str] +````{py:function} normalize_tag_list(tags: list[str] | None = None) -> list[str] :canonical: archivebox.api.v1_core.normalize_tag_list ```{autodoc2-docstring} archivebox.api.v1_core.normalize_tag_list ``` ```` +````{py:function} _parse_rss_before(before: str | None) -> datetime.datetime +:canonical: archivebox.api.v1_core._parse_rss_before + +```{autodoc2-docstring} archivebox.api.v1_core._parse_rss_before +``` +```` + +````{py:function} _filter_snapshots_for_rss(*, crawl_id: str = '', created_by: str = '', before: str | None = None, limit: int = 50) +:canonical: archivebox.api.v1_core._filter_snapshots_for_rss + +```{autodoc2-docstring} archivebox.api.v1_core._filter_snapshots_for_rss +``` +```` + +````{py:function} _snapshots_rss_response(request: django.http.HttpRequest, *, snapshots, title: str = 'ArchiveBox Snapshots') -> django.http.HttpResponse +:canonical: archivebox.api.v1_core._snapshots_rss_response + +```{autodoc2-docstring} archivebox.api.v1_core._snapshots_rss_response +``` +```` + `````{py:class} SnapshotFilterSchema(/, **data: typing.Any) :canonical: archivebox.api.v1_core.SnapshotFilterSchema @@ -1246,9 +1611,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} id :canonical: archivebox.api.v1_core.SnapshotFilterSchema.id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup(['id__istartswith', 'id__iendswith', 'timestamp__startswith'])] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.id ``` @@ -1257,9 +1622,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_by_id :canonical: archivebox.api.v1_core.SnapshotFilterSchema.created_by_id -:type: str +:type: typing.Annotated[str | None, FilterLookup('crawl__created_by_id')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.created_by_id ``` @@ -1268,9 +1633,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_by_username :canonical: archivebox.api.v1_core.SnapshotFilterSchema.created_by_username -:type: str +:type: typing.Annotated[str | None, FilterLookup('crawl__created_by__username__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.created_by_username ``` @@ -1279,9 +1644,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_at__gte :canonical: archivebox.api.v1_core.SnapshotFilterSchema.created_at__gte -:type: datetime.datetime +:type: typing.Annotated[datetime.datetime | None, FilterLookup('created_at__gte')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.created_at__gte ``` @@ -1290,9 +1655,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_at__lt :canonical: archivebox.api.v1_core.SnapshotFilterSchema.created_at__lt -:type: datetime.datetime +:type: typing.Annotated[datetime.datetime | None, FilterLookup('created_at__lt')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.created_at__lt ``` @@ -1301,9 +1666,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} created_at :canonical: archivebox.api.v1_core.SnapshotFilterSchema.created_at -:type: datetime.datetime +:type: typing.Annotated[datetime.datetime | None, FilterLookup('created_at')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.created_at ``` @@ -1312,9 +1677,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} modified_at :canonical: archivebox.api.v1_core.SnapshotFilterSchema.modified_at -:type: datetime.datetime +:type: typing.Annotated[datetime.datetime | None, FilterLookup('modified_at')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.modified_at ``` @@ -1323,9 +1688,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} modified_at__gte :canonical: archivebox.api.v1_core.SnapshotFilterSchema.modified_at__gte -:type: datetime.datetime +:type: typing.Annotated[datetime.datetime | None, FilterLookup('modified_at__gte')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.modified_at__gte ``` @@ -1334,9 +1699,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} modified_at__lt :canonical: archivebox.api.v1_core.SnapshotFilterSchema.modified_at__lt -:type: datetime.datetime +:type: typing.Annotated[datetime.datetime | None, FilterLookup('modified_at__lt')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.modified_at__lt ``` @@ -1345,9 +1710,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} search :canonical: archivebox.api.v1_core.SnapshotFilterSchema.search -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup(['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__istartswith', 'id__iendswith', 'timestamp__startswith'])] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.search ``` @@ -1356,9 +1721,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} url :canonical: archivebox.api.v1_core.SnapshotFilterSchema.url -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('url')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.url ``` @@ -1367,9 +1732,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} tag :canonical: archivebox.api.v1_core.SnapshotFilterSchema.tag -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('tags__name')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.tag ``` @@ -1378,9 +1743,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} title :canonical: archivebox.api.v1_core.SnapshotFilterSchema.title -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('title__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.title ``` @@ -1389,9 +1754,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} timestamp :canonical: archivebox.api.v1_core.SnapshotFilterSchema.timestamp -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('timestamp__startswith')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.timestamp ``` @@ -1400,9 +1765,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} bookmarked_at__gte :canonical: archivebox.api.v1_core.SnapshotFilterSchema.bookmarked_at__gte -:type: typing.Optional[datetime.datetime] +:type: typing.Annotated[datetime.datetime | None, FilterLookup('bookmarked_at__gte')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.bookmarked_at__gte ``` @@ -1411,9 +1776,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} bookmarked_at__lt :canonical: archivebox.api.v1_core.SnapshotFilterSchema.bookmarked_at__lt -:type: typing.Optional[datetime.datetime] +:type: typing.Annotated[datetime.datetime | None, FilterLookup('bookmarked_at__lt')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_core.SnapshotFilterSchema.bookmarked_at__lt ``` @@ -1422,35 +1787,42 @@ Bases: {py:obj}`ninja.FilterSchema` ````` -````{py:function} get_snapshots(request, filters: archivebox.api.v1_core.SnapshotFilterSchema = Query(...), with_archiveresults: bool = False) +````{py:function} get_snapshots(request: django.http.HttpRequest, filters: ninja.Query[archivebox.api.v1_core.SnapshotFilterSchema], with_archiveresults: bool = False) :canonical: archivebox.api.v1_core.get_snapshots ```{autodoc2-docstring} archivebox.api.v1_core.get_snapshots ``` ```` -````{py:function} get_snapshot(request, snapshot_id: str, with_archiveresults: bool = True) +````{py:function} get_snapshots_rss(request: django.http.HttpRequest, crawl_id: str = '', created_by: str = '', limit: int = 50, before: str | None = None) +:canonical: archivebox.api.v1_core.get_snapshots_rss + +```{autodoc2-docstring} archivebox.api.v1_core.get_snapshots_rss +``` +```` + +````{py:function} get_snapshot(request: django.http.HttpRequest, snapshot_id: str, with_archiveresults: bool = True) :canonical: archivebox.api.v1_core.get_snapshot ```{autodoc2-docstring} archivebox.api.v1_core.get_snapshot ``` ```` -````{py:function} create_snapshot(request, data: archivebox.api.v1_core.SnapshotCreateSchema) +````{py:function} create_snapshot(request: django.http.HttpRequest, data: archivebox.api.v1_core.SnapshotCreateSchema) :canonical: archivebox.api.v1_core.create_snapshot ```{autodoc2-docstring} archivebox.api.v1_core.create_snapshot ``` ```` -````{py:function} patch_snapshot(request, snapshot_id: str, data: archivebox.api.v1_core.SnapshotUpdateSchema) +````{py:function} patch_snapshot(request: django.http.HttpRequest, snapshot_id: str, data: archivebox.api.v1_core.SnapshotUpdateSchema) :canonical: archivebox.api.v1_core.patch_snapshot ```{autodoc2-docstring} archivebox.api.v1_core.patch_snapshot ``` ```` -````{py:function} delete_snapshot(request, snapshot_id: str) +````{py:function} delete_snapshot(request: django.http.HttpRequest, snapshot_id: str) :canonical: archivebox.api.v1_core.delete_snapshot ```{autodoc2-docstring} archivebox.api.v1_core.delete_snapshot @@ -1539,17 +1911,6 @@ Bases: {py:obj}`ninja.Schema` ```` -````{py:attribute} slug -:canonical: archivebox.api.v1_core.TagSchema.slug -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.api.v1_core.TagSchema.slug -``` - -```` - ````{py:attribute} num_snapshots :canonical: archivebox.api.v1_core.TagSchema.num_snapshots :type: int @@ -1563,7 +1924,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} snapshots :canonical: archivebox.api.v1_core.TagSchema.snapshots -:type: typing.List[archivebox.api.v1_core.SnapshotSchema] +:type: list[archivebox.api.v1_core.SnapshotSchema] :value: > None @@ -1610,21 +1971,21 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} get_tags(request) +````{py:function} get_tags(request: django.http.HttpRequest) :canonical: archivebox.api.v1_core.get_tags ```{autodoc2-docstring} archivebox.api.v1_core.get_tags ``` ```` -````{py:function} get_tag(request, tag_id: str, with_snapshots: bool = True) +````{py:function} get_tag(request: django.http.HttpRequest, tag_id: str, with_snapshots: bool = True) :canonical: archivebox.api.v1_core.get_tag ```{autodoc2-docstring} archivebox.api.v1_core.get_tag ``` ```` -````{py:function} get_any(request, id: str) +````{py:function} get_any(request: django.http.HttpRequest, id: str) :canonical: archivebox.api.v1_core.get_any ```{autodoc2-docstring} archivebox.api.v1_core.get_any @@ -1638,7 +1999,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tags :canonical: archivebox.api.v1_core.TagAutocompleteSchema.tags -:type: typing.List[dict] +:type: list[dict] :value: > None @@ -1718,6 +2079,378 @@ Bases: {py:obj}`ninja.Schema` ````` +`````{py:class} TagSearchSnapshotSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} id +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.id +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.id +``` + +```` + +````{py:attribute} title +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.title +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.title +``` + +```` + +````{py:attribute} url +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.url +``` + +```` + +````{py:attribute} favicon_url +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.favicon_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.favicon_url +``` + +```` + +````{py:attribute} admin_url +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.admin_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.admin_url +``` + +```` + +````{py:attribute} archive_url +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.archive_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.archive_url +``` + +```` + +````{py:attribute} downloaded_at +:canonical: archivebox.api.v1_core.TagSearchSnapshotSchema.downloaded_at +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchSnapshotSchema.downloaded_at +``` + +```` + +````` + +`````{py:class} TagSearchCardSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_core.TagSearchCardSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} id +:canonical: archivebox.api.v1_core.TagSearchCardSchema.id +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.id +``` + +```` + +````{py:attribute} name +:canonical: archivebox.api.v1_core.TagSearchCardSchema.name +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.name +``` + +```` + +````{py:attribute} slug +:canonical: archivebox.api.v1_core.TagSearchCardSchema.slug +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.slug +``` + +```` + +````{py:attribute} num_snapshots +:canonical: archivebox.api.v1_core.TagSearchCardSchema.num_snapshots +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.num_snapshots +``` + +```` + +````{py:attribute} filter_url +:canonical: archivebox.api.v1_core.TagSearchCardSchema.filter_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.filter_url +``` + +```` + +````{py:attribute} edit_url +:canonical: archivebox.api.v1_core.TagSearchCardSchema.edit_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.edit_url +``` + +```` + +````{py:attribute} export_urls_url +:canonical: archivebox.api.v1_core.TagSearchCardSchema.export_urls_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.export_urls_url +``` + +```` + +````{py:attribute} export_jsonl_url +:canonical: archivebox.api.v1_core.TagSearchCardSchema.export_jsonl_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.export_jsonl_url +``` + +```` + +````{py:attribute} rename_url +:canonical: archivebox.api.v1_core.TagSearchCardSchema.rename_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.rename_url +``` + +```` + +````{py:attribute} delete_url +:canonical: archivebox.api.v1_core.TagSearchCardSchema.delete_url +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.delete_url +``` + +```` + +````{py:attribute} snapshots +:canonical: archivebox.api.v1_core.TagSearchCardSchema.snapshots +:type: list[archivebox.api.v1_core.TagSearchSnapshotSchema] +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchCardSchema.snapshots +``` + +```` + +````` + +`````{py:class} TagSearchResponseSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_core.TagSearchResponseSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} tags +:canonical: archivebox.api.v1_core.TagSearchResponseSchema.tags +:type: list[archivebox.api.v1_core.TagSearchCardSchema] +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchResponseSchema.tags +``` + +```` + +````{py:attribute} sort +:canonical: archivebox.api.v1_core.TagSearchResponseSchema.sort +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchResponseSchema.sort +``` + +```` + +````{py:attribute} created_by +:canonical: archivebox.api.v1_core.TagSearchResponseSchema.created_by +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchResponseSchema.created_by +``` + +```` + +````{py:attribute} year +:canonical: archivebox.api.v1_core.TagSearchResponseSchema.year +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchResponseSchema.year +``` + +```` + +````{py:attribute} has_snapshots +:canonical: archivebox.api.v1_core.TagSearchResponseSchema.has_snapshots +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagSearchResponseSchema.has_snapshots +``` + +```` + +````` + +`````{py:class} TagUpdateSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_core.TagUpdateSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} name +:canonical: archivebox.api.v1_core.TagUpdateSchema.name +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagUpdateSchema.name +``` + +```` + +````` + +`````{py:class} TagUpdateResponseSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_core.TagUpdateResponseSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} success +:canonical: archivebox.api.v1_core.TagUpdateResponseSchema.success +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagUpdateResponseSchema.success +``` + +```` + +````{py:attribute} tag_id +:canonical: archivebox.api.v1_core.TagUpdateResponseSchema.tag_id +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagUpdateResponseSchema.tag_id +``` + +```` + +````{py:attribute} tag_name +:canonical: archivebox.api.v1_core.TagUpdateResponseSchema.tag_name +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagUpdateResponseSchema.tag_name +``` + +```` + +````` + +`````{py:class} TagDeleteResponseSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_core.TagDeleteResponseSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} success +:canonical: archivebox.api.v1_core.TagDeleteResponseSchema.success +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagDeleteResponseSchema.success +``` + +```` + +````{py:attribute} tag_id +:canonical: archivebox.api.v1_core.TagDeleteResponseSchema.tag_id +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagDeleteResponseSchema.tag_id +``` + +```` + +````{py:attribute} deleted_count +:canonical: archivebox.api.v1_core.TagDeleteResponseSchema.deleted_count +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_core.TagDeleteResponseSchema.deleted_count +``` + +```` + +````` + `````{py:class} TagSnapshotRequestSchema(/, **data: typing.Any) :canonical: archivebox.api.v1_core.TagSnapshotRequestSchema @@ -1736,7 +2469,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tag_name :canonical: archivebox.api.v1_core.TagSnapshotRequestSchema.tag_name -:type: typing.Optional[str] +:type: str | None :value: > None @@ -1747,7 +2480,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tag_id :canonical: archivebox.api.v1_core.TagSnapshotRequestSchema.tag_id -:type: typing.Optional[int] +:type: int | None :value: > None @@ -1798,28 +2531,84 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} tags_autocomplete(request, q: str = '') +````{py:function} _get_snapshot_for_tag_edit(snapshot_ref: str) -> archivebox.core.models.Snapshot +:canonical: archivebox.api.v1_core._get_snapshot_for_tag_edit + +```{autodoc2-docstring} archivebox.api.v1_core._get_snapshot_for_tag_edit +``` +```` + +````{py:function} search_tags(request: django.http.HttpRequest, q: str = '', sort: str = 'created_desc', created_by: str = '', year: str = '', has_snapshots: str = 'all') +:canonical: archivebox.api.v1_core.search_tags + +```{autodoc2-docstring} archivebox.api.v1_core.search_tags +``` +```` + +````{py:function} _public_tag_listing_enabled() -> bool +:canonical: archivebox.api.v1_core._public_tag_listing_enabled + +```{autodoc2-docstring} archivebox.api.v1_core._public_tag_listing_enabled +``` +```` + +````{py:function} _request_has_tag_autocomplete_access(request: django.http.HttpRequest) -> bool +:canonical: archivebox.api.v1_core._request_has_tag_autocomplete_access + +```{autodoc2-docstring} archivebox.api.v1_core._request_has_tag_autocomplete_access +``` +```` + +````{py:function} tags_autocomplete(request: django.http.HttpRequest, q: str = '') :canonical: archivebox.api.v1_core.tags_autocomplete ```{autodoc2-docstring} archivebox.api.v1_core.tags_autocomplete ``` ```` -````{py:function} tags_create(request, data: archivebox.api.v1_core.TagCreateSchema) +````{py:function} tags_create(request: django.http.HttpRequest, data: archivebox.api.v1_core.TagCreateSchema) :canonical: archivebox.api.v1_core.tags_create ```{autodoc2-docstring} archivebox.api.v1_core.tags_create ``` ```` -````{py:function} tags_add_to_snapshot(request, data: archivebox.api.v1_core.TagSnapshotRequestSchema) +````{py:function} rename_tag(request: django.http.HttpRequest, tag_id: int, data: archivebox.api.v1_core.TagUpdateSchema) +:canonical: archivebox.api.v1_core.rename_tag + +```{autodoc2-docstring} archivebox.api.v1_core.rename_tag +``` +```` + +````{py:function} delete_tag(request: django.http.HttpRequest, tag_id: int) +:canonical: archivebox.api.v1_core.delete_tag + +```{autodoc2-docstring} archivebox.api.v1_core.delete_tag +``` +```` + +````{py:function} tag_urls_export(request: django.http.HttpRequest, tag_id: int) +:canonical: archivebox.api.v1_core.tag_urls_export + +```{autodoc2-docstring} archivebox.api.v1_core.tag_urls_export +``` +```` + +````{py:function} tag_snapshots_export(request: django.http.HttpRequest, tag_id: int) +:canonical: archivebox.api.v1_core.tag_snapshots_export + +```{autodoc2-docstring} archivebox.api.v1_core.tag_snapshots_export +``` +```` + +````{py:function} tags_add_to_snapshot(request: django.http.HttpRequest, data: archivebox.api.v1_core.TagSnapshotRequestSchema) :canonical: archivebox.api.v1_core.tags_add_to_snapshot ```{autodoc2-docstring} archivebox.api.v1_core.tags_add_to_snapshot ``` ```` -````{py:function} tags_remove_from_snapshot(request, data: archivebox.api.v1_core.TagSnapshotRequestSchema) +````{py:function} tags_remove_from_snapshot(request: django.http.HttpRequest, data: archivebox.api.v1_core.TagSnapshotRequestSchema) :canonical: archivebox.api.v1_core.tags_remove_from_snapshot ```{autodoc2-docstring} archivebox.api.v1_core.tags_remove_from_snapshot diff --git a/docs/apidocs/archivebox/archivebox.api.v1_crawls.md b/docs/apidocs/archivebox/archivebox.api.v1_crawls.md index 7800fc2b..0f72db86 100644 --- a/docs/apidocs/archivebox/archivebox.api.v1_crawls.md +++ b/docs/apidocs/archivebox/archivebox.api.v1_crawls.md @@ -47,6 +47,22 @@ - ```{autodoc2-docstring} archivebox.api.v1_crawls.get_crawl :summary: ``` +* - {py:obj}`crawl_file ` + - ```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file + :summary: + ``` +* - {py:obj}`crawl_file_root ` + - ```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file_root + :summary: + ``` +* - {py:obj}`crawl_file_nested_1 ` + - ```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file_nested_1 + :summary: + ``` +* - {py:obj}`crawl_file_nested_2 ` + - ```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file_nested_2 + :summary: + ``` * - {py:obj}`patch_crawl ` - ```{autodoc2-docstring} archivebox.api.v1_crawls.patch_crawl :summary: @@ -174,6 +190,17 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:attribute} is_paused +:canonical: archivebox.api.v1_crawls.CrawlSchema.is_paused +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_crawls.CrawlSchema.is_paused +``` + +```` + ````{py:attribute} urls :canonical: archivebox.api.v1_crawls.CrawlSchema.urls :type: str @@ -236,6 +263,15 @@ Bases: {py:obj}`ninja.Schema` ```` +````{py:method} resolve_config(obj) +:canonical: archivebox.api.v1_crawls.CrawlSchema.resolve_config +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_crawls.CrawlSchema.resolve_config +``` + +```` + ````{py:method} resolve_snapshots(obj, context) :canonical: archivebox.api.v1_crawls.CrawlSchema.resolve_snapshots :staticmethod: @@ -252,6 +288,17 @@ Bases: {py:obj}`ninja.Schema` Bases: {py:obj}`ninja.Schema` +````{py:attribute} action +:canonical: archivebox.api.v1_crawls.CrawlUpdateSchema.action +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_crawls.CrawlUpdateSchema.action +``` + +```` + ````{py:attribute} status :canonical: archivebox.api.v1_crawls.CrawlUpdateSchema.status :type: str | None @@ -276,7 +323,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tags :canonical: archivebox.api.v1_crawls.CrawlUpdateSchema.tags -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > None @@ -305,7 +352,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} urls :canonical: archivebox.api.v1_crawls.CrawlCreateSchema.urls -:type: typing.List[str] +:type: list[str] :value: > None @@ -327,7 +374,7 @@ Bases: {py:obj}`ninja.Schema` ````{py:attribute} tags :canonical: archivebox.api.v1_crawls.CrawlCreateSchema.tags -:type: typing.Optional[typing.List[str]] +:type: list[str] | None :value: > None @@ -430,42 +477,70 @@ Bases: {py:obj}`ninja.Schema` ````` -````{py:function} normalize_tag_list(tags: typing.Optional[typing.List[str]] = None, tags_str: str = '') -> typing.List[str] +````{py:function} normalize_tag_list(tags: list[str] | None = None, tags_str: str = '') -> list[str] :canonical: archivebox.api.v1_crawls.normalize_tag_list ```{autodoc2-docstring} archivebox.api.v1_crawls.normalize_tag_list ``` ```` -````{py:function} get_crawls(request) +````{py:function} get_crawls(request: django.http.HttpRequest) :canonical: archivebox.api.v1_crawls.get_crawls ```{autodoc2-docstring} archivebox.api.v1_crawls.get_crawls ``` ```` -````{py:function} create_crawl(request, data: archivebox.api.v1_crawls.CrawlCreateSchema) +````{py:function} create_crawl(request: django.http.HttpRequest, data: archivebox.api.v1_crawls.CrawlCreateSchema) :canonical: archivebox.api.v1_crawls.create_crawl ```{autodoc2-docstring} archivebox.api.v1_crawls.create_crawl ``` ```` -````{py:function} get_crawl(request, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False) +````{py:function} get_crawl(request: django.http.HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False) :canonical: archivebox.api.v1_crawls.get_crawl ```{autodoc2-docstring} archivebox.api.v1_crawls.get_crawl ``` ```` -````{py:function} patch_crawl(request, crawl_id: str, data: archivebox.api.v1_crawls.CrawlUpdateSchema) +````{py:function} crawl_file(request: django.http.HttpRequest, crawl_id: str, path: str) +:canonical: archivebox.api.v1_crawls.crawl_file + +```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file +``` +```` + +````{py:function} crawl_file_root(request: django.http.HttpRequest, crawl_id: str, filename: str) +:canonical: archivebox.api.v1_crawls.crawl_file_root + +```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file_root +``` +```` + +````{py:function} crawl_file_nested_1(request: django.http.HttpRequest, crawl_id: str, folder: str, filename: str) +:canonical: archivebox.api.v1_crawls.crawl_file_nested_1 + +```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file_nested_1 +``` +```` + +````{py:function} crawl_file_nested_2(request: django.http.HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str) +:canonical: archivebox.api.v1_crawls.crawl_file_nested_2 + +```{autodoc2-docstring} archivebox.api.v1_crawls.crawl_file_nested_2 +``` +```` + +````{py:function} patch_crawl(request: django.http.HttpRequest, crawl_id: str, data: archivebox.api.v1_crawls.CrawlUpdateSchema) :canonical: archivebox.api.v1_crawls.patch_crawl ```{autodoc2-docstring} archivebox.api.v1_crawls.patch_crawl ``` ```` -````{py:function} delete_crawl(request, crawl_id: str) +````{py:function} delete_crawl(request: django.http.HttpRequest, crawl_id: str) :canonical: archivebox.api.v1_crawls.delete_crawl ```{autodoc2-docstring} archivebox.api.v1_crawls.delete_crawl diff --git a/docs/apidocs/archivebox/archivebox.api.v1_machine.md b/docs/apidocs/archivebox/archivebox.api.v1_machine.md index c5b64260..6a5df1c7 100644 --- a/docs/apidocs/archivebox/archivebox.api.v1_machine.md +++ b/docs/apidocs/archivebox/archivebox.api.v1_machine.md @@ -317,9 +317,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} id :canonical: archivebox.api.v1_machine.MachineFilterSchema.id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('id__startswith')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.id ``` @@ -328,9 +328,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} hostname :canonical: archivebox.api.v1_machine.MachineFilterSchema.hostname -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('hostname__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.hostname ``` @@ -339,9 +339,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} os_platform :canonical: archivebox.api.v1_machine.MachineFilterSchema.os_platform -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('os_platform__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.os_platform ``` @@ -350,9 +350,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} os_arch :canonical: archivebox.api.v1_machine.MachineFilterSchema.os_arch -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('os_arch')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.os_arch ``` @@ -361,9 +361,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} hw_in_docker :canonical: archivebox.api.v1_machine.MachineFilterSchema.hw_in_docker -:type: typing.Optional[bool] +:type: typing.Annotated[bool | None, FilterLookup('hw_in_docker')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.hw_in_docker ``` @@ -372,9 +372,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} hw_in_vm :canonical: archivebox.api.v1_machine.MachineFilterSchema.hw_in_vm -:type: typing.Optional[bool] +:type: typing.Annotated[bool | None, FilterLookup('hw_in_vm')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.hw_in_vm ``` @@ -383,9 +383,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} bin_providers :canonical: archivebox.api.v1_machine.MachineFilterSchema.bin_providers -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('bin_providers__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.MachineFilterSchema.bin_providers ``` @@ -611,9 +611,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} id :canonical: archivebox.api.v1_machine.BinaryFilterSchema.id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('id__startswith')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.BinaryFilterSchema.id ``` @@ -622,9 +622,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} name :canonical: archivebox.api.v1_machine.BinaryFilterSchema.name -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('name__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.BinaryFilterSchema.name ``` @@ -633,9 +633,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} binprovider :canonical: archivebox.api.v1_machine.BinaryFilterSchema.binprovider -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('binprovider')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.BinaryFilterSchema.binprovider ``` @@ -644,9 +644,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} status :canonical: archivebox.api.v1_machine.BinaryFilterSchema.status -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('status')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.BinaryFilterSchema.status ``` @@ -655,9 +655,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} machine_id :canonical: archivebox.api.v1_machine.BinaryFilterSchema.machine_id -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('machine_id__startswith')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.BinaryFilterSchema.machine_id ``` @@ -666,9 +666,9 @@ Bases: {py:obj}`ninja.FilterSchema` ````{py:attribute} version :canonical: archivebox.api.v1_machine.BinaryFilterSchema.version -:type: typing.Optional[str] +:type: typing.Annotated[str | None, FilterLookup('version__icontains')] :value: > - 'Field(...)' + None ```{autodoc2-docstring} archivebox.api.v1_machine.BinaryFilterSchema.version ``` @@ -677,42 +677,42 @@ Bases: {py:obj}`ninja.FilterSchema` ````` -````{py:function} get_machines(request, filters: archivebox.api.v1_machine.MachineFilterSchema = Query(...)) +````{py:function} get_machines(request: django.http.HttpRequest, filters: ninja.Query[archivebox.api.v1_machine.MachineFilterSchema]) :canonical: archivebox.api.v1_machine.get_machines ```{autodoc2-docstring} archivebox.api.v1_machine.get_machines ``` ```` -````{py:function} get_current_machine(request) +````{py:function} get_current_machine(request: django.http.HttpRequest) :canonical: archivebox.api.v1_machine.get_current_machine ```{autodoc2-docstring} archivebox.api.v1_machine.get_current_machine ``` ```` -````{py:function} get_machine(request, machine_id: str) +````{py:function} get_machine(request: django.http.HttpRequest, machine_id: str) :canonical: archivebox.api.v1_machine.get_machine ```{autodoc2-docstring} archivebox.api.v1_machine.get_machine ``` ```` -````{py:function} get_binaries(request, filters: archivebox.api.v1_machine.BinaryFilterSchema = Query(...)) +````{py:function} get_binaries(request: django.http.HttpRequest, filters: ninja.Query[archivebox.api.v1_machine.BinaryFilterSchema]) :canonical: archivebox.api.v1_machine.get_binaries ```{autodoc2-docstring} archivebox.api.v1_machine.get_binaries ``` ```` -````{py:function} get_binary(request, binary_id: str) +````{py:function} get_binary(request: django.http.HttpRequest, binary_id: str) :canonical: archivebox.api.v1_machine.get_binary ```{autodoc2-docstring} archivebox.api.v1_machine.get_binary ``` ```` -````{py:function} get_binaries_by_name(request, name: str) +````{py:function} get_binaries_by_name(request: django.http.HttpRequest, name: str) :canonical: archivebox.api.v1_machine.get_binaries_by_name ```{autodoc2-docstring} archivebox.api.v1_machine.get_binaries_by_name diff --git a/docs/apidocs/archivebox/archivebox.api.v1_personas.md b/docs/apidocs/archivebox/archivebox.api.v1_personas.md new file mode 100644 index 00000000..d35a4883 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.api.v1_personas.md @@ -0,0 +1,405 @@ +# {py:mod}`archivebox.api.v1_personas` + +```{py:module} archivebox.api.v1_personas +``` + +```{autodoc2-docstring} archivebox.api.v1_personas +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`PersonaBrowserSettingsSchema ` + - +* - {py:obj}`PersonaSyncSchema ` + - +* - {py:obj}`PersonaSchema ` + - +* - {py:obj}`PersonaSyncResponseSchema ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`browser_settings_to_config ` + - ```{autodoc2-docstring} archivebox.api.v1_personas.browser_settings_to_config + :summary: + ``` +* - {py:obj}`find_persona ` + - ```{autodoc2-docstring} archivebox.api.v1_personas.find_persona + :summary: + ``` +* - {py:obj}`get_personas ` + - ```{autodoc2-docstring} archivebox.api.v1_personas.get_personas + :summary: + ``` +* - {py:obj}`sync_persona ` + - ```{autodoc2-docstring} archivebox.api.v1_personas.sync_persona + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`router ` + - ```{autodoc2-docstring} archivebox.api.v1_personas.router + :summary: + ``` +```` + +### API + +````{py:data} router +:canonical: archivebox.api.v1_personas.router +:value: > + 'Router(...)' + +```{autodoc2-docstring} archivebox.api.v1_personas.router +``` + +```` + +`````{py:class} PersonaBrowserSettingsSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} user_agent +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema.user_agent +:type: str +:value: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaBrowserSettingsSchema.user_agent +``` + +```` + +````{py:attribute} viewport_size +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema.viewport_size +:type: str +:value: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaBrowserSettingsSchema.viewport_size +``` + +```` + +````{py:attribute} viewport_device_scale_factor +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema.viewport_device_scale_factor +:type: float | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaBrowserSettingsSchema.viewport_device_scale_factor +``` + +```` + +````{py:attribute} language +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema.language +:type: str +:value: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaBrowserSettingsSchema.language +``` + +```` + +````{py:attribute} timezone +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema.timezone +:type: str +:value: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaBrowserSettingsSchema.timezone +``` + +```` + +````{py:attribute} geolocation +:canonical: archivebox.api.v1_personas.PersonaBrowserSettingsSchema.geolocation +:type: dict[str, typing.Any] | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaBrowserSettingsSchema.geolocation +``` + +```` + +````` + +`````{py:class} PersonaSyncSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_personas.PersonaSyncSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} extension_persona_id +:canonical: archivebox.api.v1_personas.PersonaSyncSchema.extension_persona_id +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncSchema.extension_persona_id +``` + +```` + +````{py:attribute} name +:canonical: archivebox.api.v1_personas.PersonaSyncSchema.name +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncSchema.name +``` + +```` + +````{py:attribute} settings +:canonical: archivebox.api.v1_personas.PersonaSyncSchema.settings +:type: archivebox.api.v1_personas.PersonaBrowserSettingsSchema +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncSchema.settings +``` + +```` + +````{py:attribute} cookies_txt +:canonical: archivebox.api.v1_personas.PersonaSyncSchema.cookies_txt +:type: str +:value: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncSchema.cookies_txt +``` + +```` + +````{py:attribute} auth_json +:canonical: archivebox.api.v1_personas.PersonaSyncSchema.auth_json +:type: dict[str, typing.Any] +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncSchema.auth_json +``` + +```` + +````` + +`````{py:class} PersonaSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_personas.PersonaSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} TYPE +:canonical: archivebox.api.v1_personas.PersonaSchema.TYPE +:type: str +:value: > + 'personas.models.Persona' + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.TYPE +``` + +```` + +````{py:attribute} id +:canonical: archivebox.api.v1_personas.PersonaSchema.id +:type: uuid.UUID +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.id +``` + +```` + +````{py:attribute} name +:canonical: archivebox.api.v1_personas.PersonaSchema.name +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.name +``` + +```` + +````{py:attribute} created_at +:canonical: archivebox.api.v1_personas.PersonaSchema.created_at +:type: datetime.datetime +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.created_at +``` + +```` + +````{py:attribute} created_by_id +:canonical: archivebox.api.v1_personas.PersonaSchema.created_by_id +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.created_by_id +``` + +```` + +````{py:attribute} created_by_username +:canonical: archivebox.api.v1_personas.PersonaSchema.created_by_username +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.created_by_username +``` + +```` + +````{py:attribute} config +:canonical: archivebox.api.v1_personas.PersonaSchema.config +:type: dict[str, typing.Any] | None +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.config +``` + +```` + +````{py:method} resolve_created_by_id(obj) +:canonical: archivebox.api.v1_personas.PersonaSchema.resolve_created_by_id +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.resolve_created_by_id +``` + +```` + +````{py:method} resolve_created_by_username(obj) -> str +:canonical: archivebox.api.v1_personas.PersonaSchema.resolve_created_by_username +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.resolve_created_by_username +``` + +```` + +````{py:method} resolve_config(obj) +:canonical: archivebox.api.v1_personas.PersonaSchema.resolve_config +:staticmethod: + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSchema.resolve_config +``` + +```` + +````` + +`````{py:class} PersonaSyncResponseSchema(/, **data: typing.Any) +:canonical: archivebox.api.v1_personas.PersonaSyncResponseSchema + +Bases: {py:obj}`ninja.Schema` + +````{py:attribute} success +:canonical: archivebox.api.v1_personas.PersonaSyncResponseSchema.success +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncResponseSchema.success +``` + +```` + +````{py:attribute} created +:canonical: archivebox.api.v1_personas.PersonaSyncResponseSchema.created +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncResponseSchema.created +``` + +```` + +````{py:attribute} persona +:canonical: archivebox.api.v1_personas.PersonaSyncResponseSchema.persona +:type: archivebox.api.v1_personas.PersonaSchema +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncResponseSchema.persona +``` + +```` + +````{py:attribute} cookies_file_written +:canonical: archivebox.api.v1_personas.PersonaSyncResponseSchema.cookies_file_written +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncResponseSchema.cookies_file_written +``` + +```` + +````{py:attribute} auth_file_written +:canonical: archivebox.api.v1_personas.PersonaSyncResponseSchema.auth_file_written +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.api.v1_personas.PersonaSyncResponseSchema.auth_file_written +``` + +```` + +````` + +````{py:function} browser_settings_to_config(extension_persona_id: str, settings: archivebox.api.v1_personas.PersonaBrowserSettingsSchema) -> dict[str, typing.Any] +:canonical: archivebox.api.v1_personas.browser_settings_to_config + +```{autodoc2-docstring} archivebox.api.v1_personas.browser_settings_to_config +``` +```` + +````{py:function} find_persona(extension_persona_id: str, name: str) -> archivebox.personas.models.Persona | None +:canonical: archivebox.api.v1_personas.find_persona + +```{autodoc2-docstring} archivebox.api.v1_personas.find_persona +``` +```` + +````{py:function} get_personas(request: django.http.HttpRequest) +:canonical: archivebox.api.v1_personas.get_personas + +```{autodoc2-docstring} archivebox.api.v1_personas.get_personas +``` +```` + +````{py:function} sync_persona(request: django.http.HttpRequest, payload: archivebox.api.v1_personas.PersonaSyncSchema) +:canonical: archivebox.api.v1_personas.sync_persona + +```{autodoc2-docstring} archivebox.api.v1_personas.sync_persona +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.api.webhooks.md b/docs/apidocs/archivebox/archivebox.api.webhooks.md new file mode 100644 index 00000000..96802163 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.api.webhooks.md @@ -0,0 +1,64 @@ +# {py:mod}`archivebox.api.webhooks` + +```{py:module} archivebox.api.webhooks +``` + +```{autodoc2-docstring} archivebox.api.webhooks +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`warning_error_handler ` + - ```{autodoc2-docstring} archivebox.api.webhooks.warning_error_handler + :summary: + ``` +* - {py:obj}`transaction_on_commit_task_handler ` + - ```{autodoc2-docstring} archivebox.api.webhooks.transaction_on_commit_task_handler + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`logger ` + - ```{autodoc2-docstring} archivebox.api.webhooks.logger + :summary: + ``` +```` + +### API + +````{py:data} logger +:canonical: archivebox.api.webhooks.logger +:value: > + 'getLogger(...)' + +```{autodoc2-docstring} archivebox.api.webhooks.logger +``` + +```` + +````{py:function} warning_error_handler(hook: typing.Any, error: Exception | None) -> None +:canonical: archivebox.api.webhooks.warning_error_handler + +```{autodoc2-docstring} archivebox.api.webhooks.warning_error_handler +``` +```` + +````{py:function} transaction_on_commit_task_handler(hook: collections.abc.Callable[..., None], **kwargs: typing.Any) -> None +:canonical: archivebox.api.webhooks.transaction_on_commit_task_handler + +```{autodoc2-docstring} archivebox.api.webhooks.transaction_on_commit_task_handler +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.base_models.admin.md b/docs/apidocs/archivebox/archivebox.base_models.admin.md index a71a0a75..86a3ab40 100644 --- a/docs/apidocs/archivebox/archivebox.base_models.admin.md +++ b/docs/apidocs/archivebox/archivebox.base_models.admin.md @@ -15,6 +15,12 @@ :class: autosummary longtable :align: left +* - {py:obj}`HexUUIDConverter ` + - ```{autodoc2-docstring} archivebox.base_models.admin.HexUUIDConverter + :summary: + ``` +* - {py:obj}`ConfigOption ` + - * - {py:obj}`KeyValueWidget ` - ```{autodoc2-docstring} archivebox.base_models.admin.KeyValueWidget :summary: @@ -29,6 +35,135 @@ ### API +`````{py:class} HexUUIDConverter +:canonical: archivebox.base_models.admin.HexUUIDConverter + +```{autodoc2-docstring} archivebox.base_models.admin.HexUUIDConverter +``` + +````{py:attribute} regex +:canonical: archivebox.base_models.admin.HexUUIDConverter.regex +:value: > + '[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}' + +```{autodoc2-docstring} archivebox.base_models.admin.HexUUIDConverter.regex +``` + +```` + +````{py:method} to_python(value: str) -> str +:canonical: archivebox.base_models.admin.HexUUIDConverter.to_python + +```{autodoc2-docstring} archivebox.base_models.admin.HexUUIDConverter.to_python +``` + +```` + +````{py:method} to_url(value) -> str +:canonical: archivebox.base_models.admin.HexUUIDConverter.to_url + +```{autodoc2-docstring} archivebox.base_models.admin.HexUUIDConverter.to_url +``` + +```` + +````` + +`````{py:class} ConfigOption() +:canonical: archivebox.base_models.admin.ConfigOption + +Bases: {py:obj}`typing.TypedDict` + +````{py:attribute} plugin +:canonical: archivebox.base_models.admin.ConfigOption.plugin +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.plugin +``` + +```` + +````{py:attribute} type +:canonical: archivebox.base_models.admin.ConfigOption.type +:type: str | list[str] +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.type +``` + +```` + +````{py:attribute} default +:canonical: archivebox.base_models.admin.ConfigOption.default +:type: object +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.default +``` + +```` + +````{py:attribute} description +:canonical: archivebox.base_models.admin.ConfigOption.description +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.description +``` + +```` + +````{py:attribute} enum +:canonical: archivebox.base_models.admin.ConfigOption.enum +:type: typing.NotRequired[list[object]] +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.enum +``` + +```` + +````{py:attribute} pattern +:canonical: archivebox.base_models.admin.ConfigOption.pattern +:type: typing.NotRequired[str] +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.pattern +``` + +```` + +````{py:attribute} minimum +:canonical: archivebox.base_models.admin.ConfigOption.minimum +:type: typing.NotRequired[int | float] +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.minimum +``` + +```` + +````{py:attribute} maximum +:canonical: archivebox.base_models.admin.ConfigOption.maximum +:type: typing.NotRequired[int | float] +:value: > + None + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigOption.maximum +``` + +```` + +````` + ``````{py:class} KeyValueWidget(attrs=None) :canonical: archivebox.base_models.admin.KeyValueWidget @@ -45,8 +180,7 @@ Bases: {py:obj}`django.forms.Widget` ````{py:attribute} template_name :canonical: archivebox.base_models.admin.KeyValueWidget.template_name -:value: > - None +:value: ```{autodoc2-docstring} archivebox.base_models.admin.KeyValueWidget.template_name ``` @@ -81,7 +215,7 @@ Bases: {py:obj}`django.forms.Widget` ````` -````{py:method} _get_config_options() +````{py:method} _get_config_options() -> dict[str, archivebox.base_models.admin.ConfigOption] :canonical: archivebox.base_models.admin.KeyValueWidget._get_config_options ```{autodoc2-docstring} archivebox.base_models.admin.KeyValueWidget._get_config_options @@ -89,12 +223,20 @@ Bases: {py:obj}`django.forms.Widget` ```` -````{py:method} render(name, value, attrs=None, renderer=None) +````{py:method} _parse_value(value: object) -> dict[str, object] +:canonical: archivebox.base_models.admin.KeyValueWidget._parse_value + +```{autodoc2-docstring} archivebox.base_models.admin.KeyValueWidget._parse_value +``` + +```` + +````{py:method} render(name: str, value: object, attrs: collections.abc.Mapping[str, str] | None = None, renderer: django.forms.renderers.BaseRenderer | None = None) -> django.utils.safestring.SafeString :canonical: archivebox.base_models.admin.KeyValueWidget.render ```` -````{py:method} _render_row(widget_id, idx, key, value) +````{py:method} _render_row(widget_id: str, key: str, value: str) -> str :canonical: archivebox.base_models.admin.KeyValueWidget._render_row ```{autodoc2-docstring} archivebox.base_models.admin.KeyValueWidget._render_row @@ -102,7 +244,7 @@ Bases: {py:obj}`django.forms.Widget` ```` -````{py:method} _escape(s) +````{py:method} _escape(s: object) -> str :canonical: archivebox.base_models.admin.KeyValueWidget._escape ```{autodoc2-docstring} archivebox.base_models.admin.KeyValueWidget._escape @@ -110,20 +252,28 @@ Bases: {py:obj}`django.forms.Widget` ```` -````{py:method} value_from_datadict(data, files, name) +````{py:method} value_from_datadict(data: django.http.QueryDict | collections.abc.Mapping[str, object], files: object, name: str) -> str :canonical: archivebox.base_models.admin.KeyValueWidget.value_from_datadict ```` `````` -`````{py:class} ConfigEditorMixin +`````{py:class} ConfigEditorMixin(model, admin_site) :canonical: archivebox.base_models.admin.ConfigEditorMixin +Bases: {py:obj}`django.contrib.admin.ModelAdmin` + ```{autodoc2-docstring} archivebox.base_models.admin.ConfigEditorMixin ``` -````{py:method} formfield_for_dbfield(db_field, request, **kwargs) +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigEditorMixin.__init__ +``` + +````{py:method} formfield_for_dbfield(db_field: django.db.models.Field, request: django.http.HttpRequest, **kwargs: object) -> django.forms.Field | None :canonical: archivebox.base_models.admin.ConfigEditorMixin.formfield_for_dbfield ```{autodoc2-docstring} archivebox.base_models.admin.ConfigEditorMixin.formfield_for_dbfield @@ -131,6 +281,14 @@ Bases: {py:obj}`django.forms.Widget` ```` +````{py:method} save_model(request: django.http.HttpRequest, obj, form, change) +:canonical: archivebox.base_models.admin.ConfigEditorMixin.save_model + +```{autodoc2-docstring} archivebox.base_models.admin.ConfigEditorMixin.save_model +``` + +```` + ````` `````{py:class} BaseModelAdmin(model, admin_site) @@ -158,9 +316,35 @@ Bases: {py:obj}`django_object_actions.DjangoObjectActions`, {py:obj}`django.cont ```` -````{py:method} get_form(request, obj=None, **kwargs) +````{py:attribute} show_search_mode_selector +:canonical: archivebox.base_models.admin.BaseModelAdmin.show_search_mode_selector +:value: > + False + +```{autodoc2-docstring} archivebox.base_models.admin.BaseModelAdmin.show_search_mode_selector +``` + +```` + +````{py:method} get_default_search_mode() -> str +:canonical: archivebox.base_models.admin.BaseModelAdmin.get_default_search_mode + +```{autodoc2-docstring} archivebox.base_models.admin.BaseModelAdmin.get_default_search_mode +``` + +```` + +````{py:method} get_form(request: django.http.HttpRequest, obj: django.db.models.Model | None = None, change: bool = False, **kwargs: object) :canonical: archivebox.base_models.admin.BaseModelAdmin.get_form ```` +````{py:method} get_urls() +:canonical: archivebox.base_models.admin.BaseModelAdmin.get_urls + +```{autodoc2-docstring} archivebox.base_models.admin.BaseModelAdmin.get_urls +``` + +```` + ````` diff --git a/docs/apidocs/archivebox/archivebox.base_models.models.md b/docs/apidocs/archivebox/archivebox.base_models.models.md index 3d5e90a5..3d7cedc3 100644 --- a/docs/apidocs/archivebox/archivebox.base_models.models.md +++ b/docs/apidocs/archivebox/archivebox.base_models.models.md @@ -33,6 +33,8 @@ - ```{autodoc2-docstring} archivebox.base_models.models.ModelWithConfig :summary: ``` +* - {py:obj}`ModelWithDeleteAfter ` + - * - {py:obj}`ModelWithOutputDir ` - ```` @@ -141,7 +143,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ````` -````{py:method} __str__() +````{py:method} __str__() -> str :canonical: archivebox.base_models.models.ModelWithUUID.__str__ ```` @@ -202,8 +204,7 @@ Bases: {py:obj}`django.db.models.Model` `````{py:class} Meta :canonical: archivebox.base_models.models.ModelWithNotes.Meta -```{autodoc2-docstring} archivebox.base_models.models.ModelWithNotes.Meta -``` +Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ````{py:attribute} abstract :canonical: archivebox.base_models.models.ModelWithNotes.Meta.abstract @@ -256,8 +257,7 @@ Bases: {py:obj}`django.db.models.Model` `````{py:class} Meta :canonical: archivebox.base_models.models.ModelWithHealthStats.Meta -```{autodoc2-docstring} archivebox.base_models.models.ModelWithHealthStats.Meta -``` +Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ````{py:attribute} abstract :canonical: archivebox.base_models.models.ModelWithHealthStats.Meta.abstract @@ -317,8 +317,7 @@ Bases: {py:obj}`django.db.models.Model` `````{py:class} Meta :canonical: archivebox.base_models.models.ModelWithConfig.Meta -```{autodoc2-docstring} archivebox.base_models.models.ModelWithConfig.Meta -``` +Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ````{py:attribute} abstract :canonical: archivebox.base_models.models.ModelWithConfig.Meta.abstract @@ -334,6 +333,90 @@ Bases: {py:obj}`django.db.models.Model` `````` +``````{py:class} ModelWithDeleteAfter(*args, **kwargs) +:canonical: archivebox.base_models.models.ModelWithDeleteAfter + +Bases: {py:obj}`django.db.models.Model` + +````{py:attribute} delete_after_final_statuses +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.delete_after_final_statuses +:type: tuple[str, ...] +:value: > + () + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.delete_after_final_statuses +``` + +```` + +````{py:attribute} delete_at +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.delete_at +:value: > + 'DateTimeField(...)' + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.delete_at +``` + +```` + +`````{py:class} Meta +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.Meta + +Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` + +````{py:attribute} abstract +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.Meta.abstract +:value: > + True + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.Meta.abstract +``` + +```` + +````` + +````{py:method} save(*args, **kwargs) +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.save + +```` + +````{py:method} get_delete_after_config_value() +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.get_delete_after_config_value + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.get_delete_after_config_value +``` + +```` + +````{py:method} set_delete_at_from_config(config_value=None) -> bool +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.set_delete_at_from_config + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.set_delete_at_from_config +``` + +```` + +````{py:method} missing_delete_at_candidates() +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.missing_delete_at_candidates +:classmethod: + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.missing_delete_at_candidates +``` + +```` + +````{py:method} delete_expired(*, batch_size: int = 100, backfill_missing: bool = True) -> int +:canonical: archivebox.base_models.models.ModelWithDeleteAfter.delete_expired +:classmethod: + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithDeleteAfter.delete_expired +``` + +```` + +`````` + ``````{py:class} ModelWithOutputDir(*args, **kwargs) :canonical: archivebox.base_models.models.ModelWithOutputDir @@ -342,8 +425,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID` `````{py:class} Meta :canonical: archivebox.base_models.models.ModelWithOutputDir.Meta -```{autodoc2-docstring} archivebox.base_models.models.ModelWithOutputDir.Meta -``` +Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID` ````{py:attribute} abstract :canonical: archivebox.base_models.models.ModelWithOutputDir.Meta.abstract @@ -357,6 +439,16 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID` ````` +````{py:attribute} _delete_signal_registered +:canonical: archivebox.base_models.models.ModelWithOutputDir._delete_signal_registered +:value: > + False + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithOutputDir._delete_signal_registered +``` + +```` + ````{py:method} save(*args, **kwargs) :canonical: archivebox.base_models.models.ModelWithOutputDir.save @@ -399,4 +491,39 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID` ```` +````{py:method} output_paths_for_delete() -> tuple[pathlib.Path, ...] +:canonical: archivebox.base_models.models.ModelWithOutputDir.output_paths_for_delete + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithOutputDir.output_paths_for_delete +``` + +```` + +````{py:method} validate_output_paths_for_delete(paths) -> tuple[pathlib.Path, ...] +:canonical: archivebox.base_models.models.ModelWithOutputDir.validate_output_paths_for_delete +:classmethod: + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithOutputDir.validate_output_paths_for_delete +``` + +```` + +````{py:method} delete_output_paths(paths) -> None +:canonical: archivebox.base_models.models.ModelWithOutputDir.delete_output_paths +:classmethod: + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithOutputDir.delete_output_paths +``` + +```` + +````{py:method} register_delete_signal() -> None +:canonical: archivebox.base_models.models.ModelWithOutputDir.register_delete_signal +:classmethod: + +```{autodoc2-docstring} archivebox.base_models.models.ModelWithOutputDir.register_delete_signal +``` + +```` + `````` diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_add.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_add.md index 91391810..988bea56 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_add.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_add.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_collect_input_urls ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_add._collect_input_urls + :summary: + ``` * - {py:obj}`add ` - ```{autodoc2-docstring} archivebox.cli.archivebox_add.add :summary: @@ -49,7 +53,14 @@ ```` -````{py:function} add(urls: str | list[str], depth: int | str = 0, tag: str = '', parser: str = 'auto', plugins: str = '', persona: str = 'Default', overwrite: bool = False, update: bool = not ARCHIVING_CONFIG.ONLY_NEW, index_only: bool = False, bg: bool = False, created_by_id: int | None = None) -> django.db.models.QuerySet[archivebox.core.models.Snapshot] +````{py:function} _collect_input_urls(args: tuple[str, ...]) -> list[str] +:canonical: archivebox.cli.archivebox_add._collect_input_urls + +```{autodoc2-docstring} archivebox.cli.archivebox_add._collect_input_urls +``` +```` + +````{py:function} add(urls: str | list[str], snapshot_ids: list[str] | None = None, depth: int | str = 0, max_urls: int = 0, crawl_max_size: int | str = 0, crawl_timeout: int = 0, snapshot_max_size: int | str = 0, crawl_max_concurrent_snapshots: int | None = None, tag: str = '', url_allowlist: str = '', url_denylist: str = '', parser: str = 'auto', plugins: str = '', persona: str = 'Default', index_only: bool = False, bg: bool = False, created_by_id: int | None = None, config: dict[str, typing.Any] | None = None) -> tuple[archivebox.crawls.models.Crawl, django.db.models.QuerySet[archivebox.core.models.Snapshot]] :canonical: archivebox.cli.archivebox_add.add ```{autodoc2-docstring} archivebox.cli.archivebox_add.add diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_archiveresult.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_archiveresult.md index c1ca2bd4..8b01f3da 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_archiveresult.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_archiveresult.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`build_archiveresult_request ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.build_archiveresult_request + :summary: + ``` * - {py:obj}`create_archiveresults ` - ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.create_archiveresults :summary: @@ -77,21 +81,28 @@ ```` -````{py:function} create_archiveresults(snapshot_id: typing.Optional[str] = None, plugin: typing.Optional[str] = None, status: str = 'queued') -> int +````{py:function} build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = '', status: str = 'queued') -> dict +:canonical: archivebox.cli.archivebox_archiveresult.build_archiveresult_request + +```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.build_archiveresult_request +``` +```` + +````{py:function} create_archiveresults(snapshot_id: str | None = None, plugin: str | None = None, status: str = 'queued') -> int :canonical: archivebox.cli.archivebox_archiveresult.create_archiveresults ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.create_archiveresults ``` ```` -````{py:function} list_archiveresults(status: typing.Optional[str] = None, plugin: typing.Optional[str] = None, snapshot_id: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_archiveresults(status: str | None = None, plugin: str | None = None, snapshot_id: str | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_archiveresult.list_archiveresults ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.list_archiveresults ``` ```` -````{py:function} update_archiveresults(status: typing.Optional[str] = None) -> int +````{py:function} update_archiveresults(status: str | None = None) -> int :canonical: archivebox.cli.archivebox_archiveresult.update_archiveresults ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.update_archiveresults @@ -112,21 +123,21 @@ ``` ```` -````{py:function} create_cmd(snapshot_id: typing.Optional[str], plugin: typing.Optional[str], status: str) +````{py:function} create_cmd(snapshot_id: str | None, plugin: str | None, status: str) :canonical: archivebox.cli.archivebox_archiveresult.create_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.create_cmd ``` ```` -````{py:function} list_cmd(status: typing.Optional[str], plugin: typing.Optional[str], snapshot_id: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(status: str | None, plugin: str | None, snapshot_id: str | None, limit: int | None) :canonical: archivebox.cli.archivebox_archiveresult.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.list_cmd ``` ```` -````{py:function} update_cmd(status: typing.Optional[str]) +````{py:function} update_cmd(status: str | None) :canonical: archivebox.cli.archivebox_archiveresult.update_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_archiveresult.update_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_binary.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_binary.md index f4cca17d..005a1929 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_binary.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_binary.md @@ -84,14 +84,14 @@ ``` ```` -````{py:function} list_binaries(name: typing.Optional[str] = None, abspath__icontains: typing.Optional[str] = None, version__icontains: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_binaries(name: str | None = None, abspath__icontains: str | None = None, version__icontains: str | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_binary.list_binaries ```{autodoc2-docstring} archivebox.cli.archivebox_binary.list_binaries ``` ```` -````{py:function} update_binaries(version: typing.Optional[str] = None, abspath: typing.Optional[str] = None) -> int +````{py:function} update_binaries(version: str | None = None, abspath: str | None = None) -> int :canonical: archivebox.cli.archivebox_binary.update_binaries ```{autodoc2-docstring} archivebox.cli.archivebox_binary.update_binaries @@ -119,14 +119,14 @@ ``` ```` -````{py:function} list_cmd(name: typing.Optional[str], abspath__icontains: typing.Optional[str], version__icontains: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(name: str | None, abspath__icontains: str | None, version__icontains: str | None, limit: int | None) :canonical: archivebox.cli.archivebox_binary.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_binary.list_cmd ``` ```` -````{py:function} update_cmd(version: typing.Optional[str], abspath: typing.Optional[str]) +````{py:function} update_cmd(version: str | None, abspath: str | None) :canonical: archivebox.cli.archivebox_binary.update_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_binary.update_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_config.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_config.md index 0282e059..dfd22fc9 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_config.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_config.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_format_toml ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_config._format_toml + :summary: + ``` * - {py:obj}`config ` - ```{autodoc2-docstring} archivebox.cli.archivebox_config.config :summary: @@ -27,6 +31,13 @@ ### API +````{py:function} _format_toml(config: dict) -> str +:canonical: archivebox.cli.archivebox_config._format_toml + +```{autodoc2-docstring} archivebox.cli.archivebox_config._format_toml +``` +```` + ````{py:function} config(*keys, get: bool = False, set: bool = False, search: bool = False, reset: bool = False, **kwargs) -> None :canonical: archivebox.cli.archivebox_config.config diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_crawl.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_crawl.md index 798b95d3..9fc676b9 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_crawl.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_crawl.md @@ -77,21 +77,21 @@ ```` -````{py:function} create_crawl(urls: typing.Iterable[str], depth: int = 0, tag: str = '', status: str = 'queued', created_by_id: typing.Optional[int] = None) -> int +````{py:function} create_crawl(urls: collections.abc.Iterable[str], depth: int = 0, tag: str = '', status: str = 'queued', created_by_id: int | None = None) -> int :canonical: archivebox.cli.archivebox_crawl.create_crawl ```{autodoc2-docstring} archivebox.cli.archivebox_crawl.create_crawl ``` ```` -````{py:function} list_crawls(status: typing.Optional[str] = None, urls__icontains: typing.Optional[str] = None, max_depth: typing.Optional[int] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_crawls(status: str | None = None, urls__icontains: str | None = None, max_depth: int | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_crawl.list_crawls ```{autodoc2-docstring} archivebox.cli.archivebox_crawl.list_crawls ``` ```` -````{py:function} update_crawls(status: typing.Optional[str] = None, max_depth: typing.Optional[int] = None) -> int +````{py:function} update_crawls(status: str | None = None, max_depth: int | None = None) -> int :canonical: archivebox.cli.archivebox_crawl.update_crawls ```{autodoc2-docstring} archivebox.cli.archivebox_crawl.update_crawls @@ -119,14 +119,14 @@ ``` ```` -````{py:function} list_cmd(status: typing.Optional[str], urls__icontains: typing.Optional[str], max_depth: typing.Optional[int], limit: typing.Optional[int]) +````{py:function} list_cmd(status: str | None, urls__icontains: str | None, max_depth: int | None, limit: int | None) :canonical: archivebox.cli.archivebox_crawl.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_crawl.list_cmd ``` ```` -````{py:function} update_cmd(status: typing.Optional[str], max_depth: typing.Optional[int]) +````{py:function} update_cmd(status: str | None, max_depth: int | None) :canonical: archivebox.cli.archivebox_crawl.update_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_crawl.update_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_extract.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_extract.md index 819fccd6..21f65192 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_extract.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_extract.md @@ -64,7 +64,7 @@ ``` ```` -````{py:function} run_plugins(args: tuple, plugins: str = '', wait: bool = True) -> int +````{py:function} run_plugins(args: tuple, records: list[dict] | None = None, plugins: str = '', wait: bool = True, emit_results: bool = True, show_progress: bool = True) -> int :canonical: archivebox.cli.archivebox_extract.run_plugins ```{autodoc2-docstring} archivebox.cli.archivebox_extract.run_plugins diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_help.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_help.md index 76edf9cd..f591c6e0 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_help.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_help.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_command_doc ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_help._command_doc + :summary: + ``` * - {py:obj}`help ` - ```{autodoc2-docstring} archivebox.cli.archivebox_help.help :summary: @@ -49,6 +53,13 @@ ```` +````{py:function} _command_doc(cmd: str, import_path: str) -> str +:canonical: archivebox.cli.archivebox_help._command_doc + +```{autodoc2-docstring} archivebox.cli.archivebox_help._command_doc +``` +```` + ````{py:function} help() -> None :canonical: archivebox.cli.archivebox_help.help diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_init.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_init.md index 418511b5..0861d312 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_init.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_init.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_display_data_path ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_init._display_data_path + :summary: + ``` * - {py:obj}`init ` - ```{autodoc2-docstring} archivebox.cli.archivebox_init.init :summary: @@ -27,6 +31,13 @@ ### API +````{py:function} _display_data_path(path: pathlib.Path, data_dir: pathlib.Path) -> str +:canonical: archivebox.cli.archivebox_init._display_data_path + +```{autodoc2-docstring} archivebox.cli.archivebox_init._display_data_path +``` +```` + ````{py:function} init(force: bool = False, quick: bool = False, install: bool = False) -> None :canonical: archivebox.cli.archivebox_init.init diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_list.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_list.md index 0cd16b14..8d4a0354 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_list.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_list.md @@ -45,7 +45,7 @@ ```` -````{py:function} main(status: typing.Optional[str], url__icontains: typing.Optional[str], url__istartswith: typing.Optional[str], tag: typing.Optional[str], crawl_id: typing.Optional[str], limit: typing.Optional[int]) -> None +````{py:function} main(status: str | None, url__icontains: str | None, url__istartswith: str | None, tag: str | None, crawl_id: str | None, limit: int | None, sort: str | None, csv: str | None, with_headers: bool, search: str | None, query: tuple[str, ...]) -> None :canonical: archivebox.cli.archivebox_list.main ```{autodoc2-docstring} archivebox.cli.archivebox_list.main diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_machine.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_machine.md index 0d761d98..3aec1fa7 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_machine.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_machine.md @@ -53,7 +53,7 @@ ```` -````{py:function} list_machines(hostname__icontains: typing.Optional[str] = None, os_platform: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_machines(hostname__icontains: str | None = None, os_platform: str | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_machine.list_machines ```{autodoc2-docstring} archivebox.cli.archivebox_machine.list_machines @@ -67,7 +67,7 @@ ``` ```` -````{py:function} list_cmd(hostname__icontains: typing.Optional[str], os_platform: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None) :canonical: archivebox.cli.archivebox_machine.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_machine.list_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_persona.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_persona.md index 4ea72183..6c4689ec 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_persona.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_persona.md @@ -31,22 +31,6 @@ - ```{autodoc2-docstring} archivebox.cli.archivebox_persona.get_browser_binary :summary: ``` -* - {py:obj}`_parse_netscape_cookies ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_persona._parse_netscape_cookies - :summary: - ``` -* - {py:obj}`_write_netscape_cookies ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_persona._write_netscape_cookies - :summary: - ``` -* - {py:obj}`_merge_netscape_cookies ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_persona._merge_netscape_cookies - :summary: - ``` -* - {py:obj}`extract_cookies_via_cdp ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_persona.extract_cookies_via_cdp - :summary: - ``` * - {py:obj}`validate_persona_name ` - ```{autodoc2-docstring} archivebox.cli.archivebox_persona.validate_persona_name :summary: @@ -111,10 +95,6 @@ - ```{autodoc2-docstring} archivebox.cli.archivebox_persona.CHROMIUM_BROWSERS :summary: ``` -* - {py:obj}`NETSCAPE_COOKIE_HEADER ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_persona.NETSCAPE_COOKIE_HEADER - :summary: - ``` ```` ### API @@ -129,28 +109,28 @@ ```` -````{py:function} get_chrome_user_data_dir() -> typing.Optional[pathlib.Path] +````{py:function} get_chrome_user_data_dir() -> pathlib.Path | None :canonical: archivebox.cli.archivebox_persona.get_chrome_user_data_dir ```{autodoc2-docstring} archivebox.cli.archivebox_persona.get_chrome_user_data_dir ``` ```` -````{py:function} get_brave_user_data_dir() -> typing.Optional[pathlib.Path] +````{py:function} get_brave_user_data_dir() -> pathlib.Path | None :canonical: archivebox.cli.archivebox_persona.get_brave_user_data_dir ```{autodoc2-docstring} archivebox.cli.archivebox_persona.get_brave_user_data_dir ``` ```` -````{py:function} get_edge_user_data_dir() -> typing.Optional[pathlib.Path] +````{py:function} get_edge_user_data_dir() -> pathlib.Path | None :canonical: archivebox.cli.archivebox_persona.get_edge_user_data_dir ```{autodoc2-docstring} archivebox.cli.archivebox_persona.get_edge_user_data_dir ``` ```` -````{py:function} get_browser_binary(browser: str) -> typing.Optional[str] +````{py:function} get_browser_binary(browser: str) -> str | None :canonical: archivebox.cli.archivebox_persona.get_browser_binary ```{autodoc2-docstring} archivebox.cli.archivebox_persona.get_browser_binary @@ -177,44 +157,6 @@ ```` -````{py:data} NETSCAPE_COOKIE_HEADER -:canonical: archivebox.cli.archivebox_persona.NETSCAPE_COOKIE_HEADER -:value: > - ['# Netscape HTTP Cookie File', '# https://curl.se/docs/http-cookies.html', '# This file was generat... - -```{autodoc2-docstring} archivebox.cli.archivebox_persona.NETSCAPE_COOKIE_HEADER -``` - -```` - -````{py:function} _parse_netscape_cookies(path: pathlib.Path) -> OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]] -:canonical: archivebox.cli.archivebox_persona._parse_netscape_cookies - -```{autodoc2-docstring} archivebox.cli.archivebox_persona._parse_netscape_cookies -``` -```` - -````{py:function} _write_netscape_cookies(path: pathlib.Path, cookies: OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]) -> None -:canonical: archivebox.cli.archivebox_persona._write_netscape_cookies - -```{autodoc2-docstring} archivebox.cli.archivebox_persona._write_netscape_cookies -``` -```` - -````{py:function} _merge_netscape_cookies(existing_file: pathlib.Path, new_file: pathlib.Path) -> None -:canonical: archivebox.cli.archivebox_persona._merge_netscape_cookies - -```{autodoc2-docstring} archivebox.cli.archivebox_persona._merge_netscape_cookies -``` -```` - -````{py:function} extract_cookies_via_cdp(user_data_dir: pathlib.Path, output_file: pathlib.Path, profile_dir: str | None = None, chrome_binary: str | None = None) -> bool -:canonical: archivebox.cli.archivebox_persona.extract_cookies_via_cdp - -```{autodoc2-docstring} archivebox.cli.archivebox_persona.extract_cookies_via_cdp -``` -```` - ````{py:function} validate_persona_name(name: str) -> tuple[bool, str] :canonical: archivebox.cli.archivebox_persona.validate_persona_name @@ -229,21 +171,21 @@ ``` ```` -````{py:function} create_personas(names: typing.Iterable[str], import_from: typing.Optional[str] = None, profile: typing.Optional[str] = None) -> int +````{py:function} create_personas(names: collections.abc.Iterable[str], import_from: str | None = None, profile: str | None = None) -> int :canonical: archivebox.cli.archivebox_persona.create_personas ```{autodoc2-docstring} archivebox.cli.archivebox_persona.create_personas ``` ```` -````{py:function} list_personas(name: typing.Optional[str] = None, name__icontains: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_personas(name: str | None = None, name__icontains: str | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_persona.list_personas ```{autodoc2-docstring} archivebox.cli.archivebox_persona.list_personas ``` ```` -````{py:function} update_personas(name: typing.Optional[str] = None) -> int +````{py:function} update_personas(name: str | None = None) -> int :canonical: archivebox.cli.archivebox_persona.update_personas ```{autodoc2-docstring} archivebox.cli.archivebox_persona.update_personas @@ -264,21 +206,21 @@ ``` ```` -````{py:function} create_cmd(names: tuple, import_from: typing.Optional[str], profile: typing.Optional[str]) +````{py:function} create_cmd(names: tuple, import_from: str | None, profile: str | None) :canonical: archivebox.cli.archivebox_persona.create_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_persona.create_cmd ``` ```` -````{py:function} list_cmd(name: typing.Optional[str], name__icontains: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(name: str | None, name__icontains: str | None, limit: int | None) :canonical: archivebox.cli.archivebox_persona.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_persona.list_cmd ``` ```` -````{py:function} update_cmd(name: typing.Optional[str]) +````{py:function} update_cmd(name: str | None) :canonical: archivebox.cli.archivebox_persona.update_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_persona.update_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_pluginmap.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_pluginmap.md index bb112f26..d2ef4d7e 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_pluginmap.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_pluginmap.md @@ -31,63 +31,24 @@ :class: autosummary longtable :align: left -* - {py:obj}`CRAWL_MACHINE_DIAGRAM ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.CRAWL_MACHINE_DIAGRAM - :summary: - ``` -* - {py:obj}`SNAPSHOT_MACHINE_DIAGRAM ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.SNAPSHOT_MACHINE_DIAGRAM - :summary: - ``` -* - {py:obj}`ARCHIVERESULT_MACHINE_DIAGRAM ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.ARCHIVERESULT_MACHINE_DIAGRAM - :summary: - ``` -* - {py:obj}`BINARY_MACHINE_DIAGRAM ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.BINARY_MACHINE_DIAGRAM +* - {py:obj}`EVENT_FLOW_DIAGRAM ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.EVENT_FLOW_DIAGRAM :summary: ``` ```` ### API -````{py:data} CRAWL_MACHINE_DIAGRAM -:canonical: archivebox.cli.archivebox_pluginmap.CRAWL_MACHINE_DIAGRAM +````{py:data} EVENT_FLOW_DIAGRAM +:canonical: archivebox.cli.archivebox_pluginmap.EVENT_FLOW_DIAGRAM :value: -```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.CRAWL_MACHINE_DIAGRAM +```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.EVENT_FLOW_DIAGRAM ``` ```` -````{py:data} SNAPSHOT_MACHINE_DIAGRAM -:canonical: archivebox.cli.archivebox_pluginmap.SNAPSHOT_MACHINE_DIAGRAM -:value: - -```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.SNAPSHOT_MACHINE_DIAGRAM -``` - -```` - -````{py:data} ARCHIVERESULT_MACHINE_DIAGRAM -:canonical: archivebox.cli.archivebox_pluginmap.ARCHIVERESULT_MACHINE_DIAGRAM -:value: - -```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.ARCHIVERESULT_MACHINE_DIAGRAM -``` - -```` - -````{py:data} BINARY_MACHINE_DIAGRAM -:canonical: archivebox.cli.archivebox_pluginmap.BINARY_MACHINE_DIAGRAM -:value: - -```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.BINARY_MACHINE_DIAGRAM -``` - -```` - -````{py:function} pluginmap(show_disabled: bool = False, model: typing.Optional[str] = None, quiet: bool = False) -> dict +````{py:function} pluginmap(show_disabled: bool = False, event: str | None = None, quiet: bool = False) -> dict :canonical: archivebox.cli.archivebox_pluginmap.pluginmap ```{autodoc2-docstring} archivebox.cli.archivebox_pluginmap.pluginmap diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_process.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_process.md index e4d08db4..84e9138d 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_process.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_process.md @@ -53,7 +53,7 @@ ```` -````{py:function} list_processes(binary_name: typing.Optional[str] = None, machine_id: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_processes(binary_name: str | None = None, machine_id: str | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_process.list_processes ```{autodoc2-docstring} archivebox.cli.archivebox_process.list_processes @@ -67,7 +67,7 @@ ``` ```` -````{py:function} list_cmd(binary_name: typing.Optional[str], machine_id: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None) :canonical: archivebox.cli.archivebox_process.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_process.list_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_remove.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_remove.md index c4157f15..54cb744f 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_remove.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_remove.md @@ -49,7 +49,7 @@ ```` -````{py:function} remove(filter_patterns: typing.Iterable[str] = (), filter_type: str = 'exact', snapshots: django.db.models.QuerySet | None = None, after: float | None = None, before: float | None = None, yes: bool = False, delete: bool = False, out_dir: pathlib.Path = DATA_DIR) -> django.db.models.QuerySet +````{py:function} remove(filter_patterns: collections.abc.Iterable[str] = (), filter_type: str = 'exact', snapshots: django.db.models.QuerySet | None = None, after: float | None = None, before: float | None = None, yes: bool = False, out_dir: pathlib.Path = DATA_DIR) -> django.db.models.QuerySet :canonical: archivebox.cli.archivebox_remove.remove ```{autodoc2-docstring} archivebox.cli.archivebox_remove.remove diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_run.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_run.md index e86c3cae..024d53fb 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_run.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_run.md @@ -19,18 +19,18 @@ - ```{autodoc2-docstring} archivebox.cli.archivebox_run.process_stdin_records :summary: ``` -* - {py:obj}`run_orchestrator ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_run.run_orchestrator - :summary: - ``` -* - {py:obj}`run_snapshot_worker ` - - ```{autodoc2-docstring} archivebox.cli.archivebox_run.run_snapshot_worker +* - {py:obj}`run_runner ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_run.run_runner :summary: ``` * - {py:obj}`main ` - ```{autodoc2-docstring} archivebox.cli.archivebox_run.main :summary: ``` +* - {py:obj}`run_snapshot_worker ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_run.run_snapshot_worker + :summary: + ``` ```` ### Data @@ -64,10 +64,17 @@ ``` ```` -````{py:function} run_orchestrator(daemon: bool = False) -> int -:canonical: archivebox.cli.archivebox_run.run_orchestrator +````{py:function} run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int +:canonical: archivebox.cli.archivebox_run.run_runner -```{autodoc2-docstring} archivebox.cli.archivebox_run.run_orchestrator +```{autodoc2-docstring} archivebox.cli.archivebox_run.run_runner +``` +```` + +````{py:function} main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool) +:canonical: archivebox.cli.archivebox_run.main + +```{autodoc2-docstring} archivebox.cli.archivebox_run.main ``` ```` @@ -77,10 +84,3 @@ ```{autodoc2-docstring} archivebox.cli.archivebox_run.run_snapshot_worker ``` ```` - -````{py:function} main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, worker_type: str) -:canonical: archivebox.cli.archivebox_run.main - -```{autodoc2-docstring} archivebox.cli.archivebox_run.main -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_schedule.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_schedule.md index 4376bfe6..8cd31cbc 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_schedule.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_schedule.md @@ -27,7 +27,7 @@ ### API -````{py:function} schedule(add: bool = False, show: bool = False, clear: bool = False, foreground: bool = False, run_all: bool = False, quiet: bool = False, every: str | None = None, tag: str = '', depth: int | str = 0, overwrite: bool = False, update: bool = not ARCHIVING_CONFIG.ONLY_NEW, import_path: str | None = None) +````{py:function} schedule(add: bool = False, show: bool = False, clear: bool = False, foreground: bool = False, run_all: bool = False, quiet: bool = False, every: str | None = None, tag: str = '', depth: int | str = 0, import_path: str | None = None, config: dict[str, object] | None = None) :canonical: archivebox.cli.archivebox_schedule.schedule ```{autodoc2-docstring} archivebox.cli.archivebox_schedule.schedule diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_search.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_search.md index 191093b4..951ec10e 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_search.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_search.md @@ -15,6 +15,22 @@ :class: autosummary longtable :align: left +* - {py:obj}`_apply_pattern_filters ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_search._apply_pattern_filters + :summary: + ``` +* - {py:obj}`_snapshots_to_json ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_search._snapshots_to_json + :summary: + ``` +* - {py:obj}`_snapshots_to_csv ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_search._snapshots_to_csv + :summary: + ``` +* - {py:obj}`_snapshots_to_html ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_search._snapshots_to_html + :summary: + ``` * - {py:obj}`get_snapshots ` - ```{autodoc2-docstring} archivebox.cli.archivebox_search.get_snapshots :summary: @@ -63,6 +79,7 @@ ````{py:data} LINK_FILTERS :canonical: archivebox.cli.archivebox_search.LINK_FILTERS +:type: dict[str, collections.abc.Callable[[str], django.db.models.Q]] :value: > None @@ -81,7 +98,35 @@ ```` -````{py:function} get_snapshots(snapshots: typing.Optional[django.db.models.QuerySet] = None, filter_patterns: typing.Optional[typing.List[str]] = None, filter_type: str = 'substring', after: typing.Optional[float] = None, before: typing.Optional[float] = None, out_dir: pathlib.Path = DATA_DIR) -> django.db.models.QuerySet +````{py:function} _apply_pattern_filters(snapshots: django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot], filter_patterns: list[str], filter_type: str) -> django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot] +:canonical: archivebox.cli.archivebox_search._apply_pattern_filters + +```{autodoc2-docstring} archivebox.cli.archivebox_search._apply_pattern_filters +``` +```` + +````{py:function} _snapshots_to_json(snapshots: django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot], *, with_headers: bool) -> str +:canonical: archivebox.cli.archivebox_search._snapshots_to_json + +```{autodoc2-docstring} archivebox.cli.archivebox_search._snapshots_to_json +``` +```` + +````{py:function} _snapshots_to_csv(snapshots: django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot], *, cols: list[str], with_headers: bool) -> str +:canonical: archivebox.cli.archivebox_search._snapshots_to_csv + +```{autodoc2-docstring} archivebox.cli.archivebox_search._snapshots_to_csv +``` +```` + +````{py:function} _snapshots_to_html(snapshots: django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot], *, with_headers: bool) -> str +:canonical: archivebox.cli.archivebox_search._snapshots_to_html + +```{autodoc2-docstring} archivebox.cli.archivebox_search._snapshots_to_html +``` +```` + +````{py:function} get_snapshots(snapshots: django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot] | None = None, filter_patterns: list[str] | None = None, filter_type: str = 'substring', after: float | None = None, before: float | None = None, out_dir: pathlib.Path = DATA_DIR) -> django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot] :canonical: archivebox.cli.archivebox_search.get_snapshots ```{autodoc2-docstring} archivebox.cli.archivebox_search.get_snapshots diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_server.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_server.md index 4ad8cdf5..5a806298 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_server.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_server.md @@ -15,6 +15,30 @@ :class: autosummary longtable :align: left +* - {py:obj}`_is_ipv4_literal ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._is_ipv4_literal + :summary: + ``` +* - {py:obj}`_is_ipv6_literal ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._is_ipv6_literal + :summary: + ``` +* - {py:obj}`_bind_host_looks_like_ip ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._bind_host_looks_like_ip + :summary: + ``` +* - {py:obj}`_split_bind_spec ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._split_bind_spec + :summary: + ``` +* - {py:obj}`_parse_and_validate_bind_spec ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._parse_and_validate_bind_spec + :summary: + ``` +* - {py:obj}`_print_server_startup_warnings ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._print_server_startup_warnings + :summary: + ``` * - {py:obj}`server ` - ```{autodoc2-docstring} archivebox.cli.archivebox_server.server :summary: @@ -25,9 +49,101 @@ ``` ```` +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_IPV4_RE ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._IPV4_RE + :summary: + ``` +* - {py:obj}`_IPV6_CHARS_RE ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._IPV6_CHARS_RE + :summary: + ``` +* - {py:obj}`_LOCAL_BIND_HOSTS ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_server._LOCAL_BIND_HOSTS + :summary: + ``` +```` + ### API -````{py:function} server(runserver_args: typing.Iterable[str] = (SERVER_CONFIG.BIND_ADDR, ), reload: bool = False, init: bool = False, debug: bool = False, daemonize: bool = False, nothreading: bool = False) -> None +````{py:data} _IPV4_RE +:canonical: archivebox.cli.archivebox_server._IPV4_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.cli.archivebox_server._IPV4_RE +``` + +```` + +````{py:data} _IPV6_CHARS_RE +:canonical: archivebox.cli.archivebox_server._IPV6_CHARS_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.cli.archivebox_server._IPV6_CHARS_RE +``` + +```` + +````{py:data} _LOCAL_BIND_HOSTS +:canonical: archivebox.cli.archivebox_server._LOCAL_BIND_HOSTS +:value: > + 'frozenset(...)' + +```{autodoc2-docstring} archivebox.cli.archivebox_server._LOCAL_BIND_HOSTS +``` + +```` + +````{py:function} _is_ipv4_literal(host: str) -> bool +:canonical: archivebox.cli.archivebox_server._is_ipv4_literal + +```{autodoc2-docstring} archivebox.cli.archivebox_server._is_ipv4_literal +``` +```` + +````{py:function} _is_ipv6_literal(host: str) -> bool +:canonical: archivebox.cli.archivebox_server._is_ipv6_literal + +```{autodoc2-docstring} archivebox.cli.archivebox_server._is_ipv6_literal +``` +```` + +````{py:function} _bind_host_looks_like_ip(host: str) -> bool +:canonical: archivebox.cli.archivebox_server._bind_host_looks_like_ip + +```{autodoc2-docstring} archivebox.cli.archivebox_server._bind_host_looks_like_ip +``` +```` + +````{py:function} _split_bind_spec(spec: str) -> tuple[str, str] +:canonical: archivebox.cli.archivebox_server._split_bind_spec + +```{autodoc2-docstring} archivebox.cli.archivebox_server._split_bind_spec +``` +```` + +````{py:function} _parse_and_validate_bind_spec(spec: str) -> tuple[str, str] +:canonical: archivebox.cli.archivebox_server._parse_and_validate_bind_spec + +```{autodoc2-docstring} archivebox.cli.archivebox_server._parse_and_validate_bind_spec +``` +```` + +````{py:function} _print_server_startup_warnings(config, host: str, port: str) -> None +:canonical: archivebox.cli.archivebox_server._print_server_startup_warnings + +```{autodoc2-docstring} archivebox.cli.archivebox_server._print_server_startup_warnings +``` +```` + +````{py:function} server(runserver_args: collections.abc.Iterable[str] | None = None, reload: bool = False, debug: bool = False, daemonize: bool = False, nothreading: bool = False) -> None :canonical: archivebox.cli.archivebox_server.server ```{autodoc2-docstring} archivebox.cli.archivebox_server.server diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_shell.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_shell.md index 02b76fc8..9b952d3c 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_shell.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_shell.md @@ -27,14 +27,14 @@ ### API -````{py:function} shell(args: typing.Iterable[str] = ()) -> None +````{py:function} shell(args: collections.abc.Iterable[str] = ()) -> None :canonical: archivebox.cli.archivebox_shell.shell ```{autodoc2-docstring} archivebox.cli.archivebox_shell.shell ``` ```` -````{py:function} main(args: typing.Iterable[str] = ()) -> None +````{py:function} main(args: collections.abc.Iterable[str] = ()) -> None :canonical: archivebox.cli.archivebox_shell.main ```{autodoc2-docstring} archivebox.cli.archivebox_shell.main diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_snapshot.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_snapshot.md index ce882327..a3159215 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_snapshot.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_snapshot.md @@ -19,6 +19,10 @@ - ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.create_snapshots :summary: ``` +* - {py:obj}`build_snapshot_queryset ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.build_snapshot_queryset + :summary: + ``` * - {py:obj}`list_snapshots ` - ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.list_snapshots :summary: @@ -77,21 +81,28 @@ ```` -````{py:function} create_snapshots(urls: typing.Iterable[str], tag: str = '', status: str = 'queued', depth: int = 0, created_by_id: typing.Optional[int] = None) -> int +````{py:function} create_snapshots(urls: collections.abc.Iterable[str], tag: str = '', status: str = 'queued', depth: int = 0, created_by_id: int | None = None) -> int :canonical: archivebox.cli.archivebox_snapshot.create_snapshots ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.create_snapshots ``` ```` -````{py:function} list_snapshots(status: typing.Optional[str] = None, url__icontains: typing.Optional[str] = None, url__istartswith: typing.Optional[str] = None, tag: typing.Optional[str] = None, crawl_id: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} build_snapshot_queryset(*, status: str | None = None, url__icontains: str | None = None, url__istartswith: str | None = None, tag: str | None = None, crawl_id: str | None = None, sort: str | None = None, search: str | None = None, query: str | None = None, limit: int | None = None) -> django.db.models.QuerySet +:canonical: archivebox.cli.archivebox_snapshot.build_snapshot_queryset + +```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.build_snapshot_queryset +``` +```` + +````{py:function} list_snapshots(status: str | None = None, url__icontains: str | None = None, url__istartswith: str | None = None, tag: str | None = None, crawl_id: str | None = None, limit: int | None = None, sort: str | None = None, csv: str | None = None, with_headers: bool = False, search: str | None = None, query: str | None = None) -> int :canonical: archivebox.cli.archivebox_snapshot.list_snapshots ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.list_snapshots ``` ```` -````{py:function} update_snapshots(status: typing.Optional[str] = None, tag: typing.Optional[str] = None) -> int +````{py:function} update_snapshots(status: str | None = None, tag: str | None = None) -> int :canonical: archivebox.cli.archivebox_snapshot.update_snapshots ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.update_snapshots @@ -119,14 +130,14 @@ ``` ```` -````{py:function} list_cmd(status: typing.Optional[str], url__icontains: typing.Optional[str], url__istartswith: typing.Optional[str], tag: typing.Optional[str], crawl_id: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(status: str | None, url__icontains: str | None, url__istartswith: str | None, tag: str | None, crawl_id: str | None, limit: int | None, sort: str | None, csv: str | None, with_headers: bool, search: str | None, query: tuple[str, ...]) :canonical: archivebox.cli.archivebox_snapshot.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.list_cmd ``` ```` -````{py:function} update_cmd(status: typing.Optional[str], tag: typing.Optional[str]) +````{py:function} update_cmd(status: str | None, tag: str | None) :canonical: archivebox.cli.archivebox_snapshot.update_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_snapshot.update_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_status.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_status.md index 875d3d47..cc037f6e 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_status.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_status.md @@ -25,8 +25,30 @@ ``` ```` +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`MAX_STATUS_FS_DIR_SCAN ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_status.MAX_STATUS_FS_DIR_SCAN + :summary: + ``` +```` + ### API +````{py:data} MAX_STATUS_FS_DIR_SCAN +:canonical: archivebox.cli.archivebox_status.MAX_STATUS_FS_DIR_SCAN +:value: > + 5000 + +```{autodoc2-docstring} archivebox.cli.archivebox_status.MAX_STATUS_FS_DIR_SCAN +``` + +```` + ````{py:function} status(out_dir: pathlib.Path = DATA_DIR) -> None :canonical: archivebox.cli.archivebox_status.status diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_tag.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_tag.md index ba1a9dc9..f36ffb19 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_tag.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_tag.md @@ -77,21 +77,21 @@ ```` -````{py:function} create_tags(names: typing.Iterable[str]) -> int +````{py:function} create_tags(names: collections.abc.Iterable[str]) -> int :canonical: archivebox.cli.archivebox_tag.create_tags ```{autodoc2-docstring} archivebox.cli.archivebox_tag.create_tags ``` ```` -````{py:function} list_tags(name: typing.Optional[str] = None, name__icontains: typing.Optional[str] = None, limit: typing.Optional[int] = None) -> int +````{py:function} list_tags(name: str | None = None, name__icontains: str | None = None, limit: int | None = None) -> int :canonical: archivebox.cli.archivebox_tag.list_tags ```{autodoc2-docstring} archivebox.cli.archivebox_tag.list_tags ``` ```` -````{py:function} update_tags(name: typing.Optional[str] = None) -> int +````{py:function} update_tags(name: str | None = None) -> int :canonical: archivebox.cli.archivebox_tag.update_tags ```{autodoc2-docstring} archivebox.cli.archivebox_tag.update_tags @@ -119,14 +119,14 @@ ``` ```` -````{py:function} list_cmd(name: typing.Optional[str], name__icontains: typing.Optional[str], limit: typing.Optional[int]) +````{py:function} list_cmd(name: str | None, name__icontains: str | None, limit: int | None) :canonical: archivebox.cli.archivebox_tag.list_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_tag.list_cmd ``` ```` -````{py:function} update_cmd(name: typing.Optional[str]) +````{py:function} update_cmd(name: str | None) :canonical: archivebox.cli.archivebox_tag.update_cmd ```{autodoc2-docstring} archivebox.cli.archivebox_tag.update_cmd diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_update.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_update.md index e9588294..3df75787 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_update.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_update.md @@ -15,6 +15,22 @@ :class: autosummary longtable :align: left +* - {py:obj}`_get_snapshot_crawl ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_update._get_snapshot_crawl + :summary: + ``` +* - {py:obj}`_get_search_indexing_plugins ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_update._get_search_indexing_plugins + :summary: + ``` +* - {py:obj}`_build_filtered_snapshots_queryset ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_update._build_filtered_snapshots_queryset + :summary: + ``` +* - {py:obj}`reindex_snapshots ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_update.reindex_snapshots + :summary: + ``` * - {py:obj}`update ` - ```{autodoc2-docstring} archivebox.cli.archivebox_update.update :summary: @@ -39,6 +55,10 @@ - ```{autodoc2-docstring} archivebox.cli.archivebox_update.print_combined_stats :summary: ``` +* - {py:obj}`print_index_stats ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_update.print_index_stats + :summary: + ``` * - {py:obj}`main ` - ```{autodoc2-docstring} archivebox.cli.archivebox_update.main :summary: @@ -47,28 +67,56 @@ ### API -````{py:function} update(filter_patterns: typing.Iterable[str] = (), filter_type: str = 'exact', before: float | None = None, after: float | None = None, resume: str | None = None, batch_size: int = 100, continuous: bool = False) -> None +````{py:function} _get_snapshot_crawl(snapshot: archivebox.core.models.Snapshot) -> archivebox.crawls.models.Crawl | None +:canonical: archivebox.cli.archivebox_update._get_snapshot_crawl + +```{autodoc2-docstring} archivebox.cli.archivebox_update._get_snapshot_crawl +``` +```` + +````{py:function} _get_search_indexing_plugins() -> list[str] +:canonical: archivebox.cli.archivebox_update._get_search_indexing_plugins + +```{autodoc2-docstring} archivebox.cli.archivebox_update._get_search_indexing_plugins +``` +```` + +````{py:function} _build_filtered_snapshots_queryset(*, filter_patterns: collections.abc.Iterable[str], filter_type: str, status: str | None = None, url__icontains: str | None = None, url__istartswith: str | None = None, tag: str | None = None, crawl_id: str | None = None, limit: int | None = None, sort: str | None = None, search: str | None = None, before: float | None = None, after: float | None = None, resume: str | None = None) +:canonical: archivebox.cli.archivebox_update._build_filtered_snapshots_queryset + +```{autodoc2-docstring} archivebox.cli.archivebox_update._build_filtered_snapshots_queryset +``` +```` + +````{py:function} reindex_snapshots(snapshots: django.db.models.QuerySet[archivebox.core.models.Snapshot, archivebox.core.models.Snapshot], *, search_plugins: list[str], batch_size: int, collect_ids: bool = False, wait_for_turn=None) -> dict[str, typing.Any] +:canonical: archivebox.cli.archivebox_update.reindex_snapshots + +```{autodoc2-docstring} archivebox.cli.archivebox_update.reindex_snapshots +``` +```` + +````{py:function} update(filter_patterns: collections.abc.Iterable[str] = (), filter_type: str = 'exact', status: str | None = None, url__icontains: str | None = None, url__istartswith: str | None = None, tag: str | None = None, crawl_id: str | None = None, limit: int | None = None, sort: str | None = None, search: str | None = None, before: float | None = None, after: float | None = None, resume: str | None = None, batch_size: int = 500, continuous: bool = False, index_only: bool = False, migrate_only: bool = False, stop_daemon_stack: bool = True) -> None :canonical: archivebox.cli.archivebox_update.update ```{autodoc2-docstring} archivebox.cli.archivebox_update.update ``` ```` -````{py:function} drain_old_archive_dirs(resume_from: str = None, batch_size: int = 100) -> dict +````{py:function} drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500) -> dict[str, int] :canonical: archivebox.cli.archivebox_update.drain_old_archive_dirs ```{autodoc2-docstring} archivebox.cli.archivebox_update.drain_old_archive_dirs ``` ```` -````{py:function} process_all_db_snapshots(batch_size: int = 100) -> dict +````{py:function} process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int] :canonical: archivebox.cli.archivebox_update.process_all_db_snapshots ```{autodoc2-docstring} archivebox.cli.archivebox_update.process_all_db_snapshots ``` ```` -````{py:function} process_filtered_snapshots(filter_patterns: typing.Iterable[str], filter_type: str, before: float | None, after: float | None, batch_size: int) -> dict +````{py:function} process_filtered_snapshots(filter_patterns: collections.abc.Iterable[str], filter_type: str, status: str | None, url__icontains: str | None, url__istartswith: str | None, tag: str | None, crawl_id: str | None, limit: int | None, sort: str | None, search: str | None, before: float | None, after: float | None, resume: str | None, batch_size: int, queue_for_archiving: bool = True, wait_for_turn=None) -> dict[str, typing.Any] :canonical: archivebox.cli.archivebox_update.process_filtered_snapshots ```{autodoc2-docstring} archivebox.cli.archivebox_update.process_filtered_snapshots @@ -89,6 +137,13 @@ ``` ```` +````{py:function} print_index_stats(stats: dict[str, typing.Any]) -> None +:canonical: archivebox.cli.archivebox_update.print_index_stats + +```{autodoc2-docstring} archivebox.cli.archivebox_update.print_index_stats +``` +```` + ````{py:function} main(**kwargs) :canonical: archivebox.cli.archivebox_update.main diff --git a/docs/apidocs/archivebox/archivebox.cli.archivebox_version.md b/docs/apidocs/archivebox/archivebox.cli.archivebox_version.md index 5362813f..f7809fc2 100644 --- a/docs/apidocs/archivebox/archivebox.cli.archivebox_version.md +++ b/docs/apidocs/archivebox/archivebox.cli.archivebox_version.md @@ -15,6 +15,14 @@ :class: autosummary longtable :align: left +* - {py:obj}`_format_binary_abspath ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_version._format_binary_abspath + :summary: + ``` +* - {py:obj}`_render_binary_abspath ` + - ```{autodoc2-docstring} archivebox.cli.archivebox_version._render_binary_abspath + :summary: + ``` * - {py:obj}`version ` - ```{autodoc2-docstring} archivebox.cli.archivebox_version.version :summary: @@ -27,7 +35,21 @@ ### API -````{py:function} version(quiet: bool = False, binaries: typing.Iterable[str] = ()) -> list[str] +````{py:function} _format_binary_abspath(abspath: str, *, pwd: pathlib.Path, lib_dir: pathlib.Path, personas_dir: pathlib.Path, home: pathlib.Path) -> str +:canonical: archivebox.cli.archivebox_version._format_binary_abspath + +```{autodoc2-docstring} archivebox.cli.archivebox_version._format_binary_abspath +``` +```` + +````{py:function} _render_binary_abspath(abspath: str) +:canonical: archivebox.cli.archivebox_version._render_binary_abspath + +```{autodoc2-docstring} archivebox.cli.archivebox_version._render_binary_abspath +``` +```` + +````{py:function} version(quiet: bool = False, binaries: collections.abc.Iterable[str] = ()) -> list[str] :canonical: archivebox.cli.archivebox_version.version ```{autodoc2-docstring} archivebox.cli.archivebox_version.version diff --git a/docs/apidocs/archivebox/archivebox.cli.cli_util.md b/docs/apidocs/archivebox/archivebox.cli.cli_util.md new file mode 100644 index 00000000..c70f084f --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.cli.cli_util.md @@ -0,0 +1,31 @@ +# {py:mod}`archivebox.cli.cli_util` + +```{py:module} archivebox.cli.cli_util +``` + +```{autodoc2-docstring} archivebox.cli.cli_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`apply_filters ` + - ```{autodoc2-docstring} archivebox.cli.cli_util.apply_filters + :summary: + ``` +```` + +### API + +````{py:function} apply_filters(queryset, filter_kwargs: dict, limit: int | None = None) +:canonical: archivebox.cli.cli_util.apply_filters + +```{autodoc2-docstring} archivebox.cli.cli_util.apply_filters +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.cli.cli_utils.md b/docs/apidocs/archivebox/archivebox.cli.cli_utils.md deleted file mode 100644 index c8590834..00000000 --- a/docs/apidocs/archivebox/archivebox.cli.cli_utils.md +++ /dev/null @@ -1,31 +0,0 @@ -# {py:mod}`archivebox.cli.cli_utils` - -```{py:module} archivebox.cli.cli_utils -``` - -```{autodoc2-docstring} archivebox.cli.cli_utils -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`apply_filters ` - - ```{autodoc2-docstring} archivebox.cli.cli_utils.apply_filters - :summary: - ``` -```` - -### API - -````{py:function} apply_filters(queryset, filter_kwargs: dict, limit: typing.Optional[int] = None) -:canonical: archivebox.cli.cli_utils.apply_filters - -```{autodoc2-docstring} archivebox.cli.cli_utils.apply_filters -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.cli.md b/docs/apidocs/archivebox/archivebox.cli.md index 937ac959..79187d74 100644 --- a/docs/apidocs/archivebox/archivebox.cli.md +++ b/docs/apidocs/archivebox/archivebox.cli.md @@ -20,9 +20,7 @@ archivebox.cli.archivebox_archiveresult archivebox.cli.archivebox_process archivebox.cli.archivebox_tag archivebox.cli.archivebox_config -archivebox.cli.cli_utils archivebox.cli.archivebox_server -archivebox.cli.tests_piping archivebox.cli.archivebox_binary archivebox.cli.archivebox_snapshot archivebox.cli.archivebox_pluginmap @@ -40,6 +38,7 @@ archivebox.cli.archivebox_persona archivebox.cli.archivebox_snapshot_compat archivebox.cli.archivebox_add archivebox.cli.archivebox_status +archivebox.cli.cli_util archivebox.cli.archivebox_run archivebox.cli.archivebox_init archivebox.cli.archivebox_help @@ -86,6 +85,10 @@ archivebox.cli.archivebox_manage - ```{autodoc2-docstring} archivebox.cli.__command__ :summary: ``` +* - {py:obj}`STDERR ` + - ```{autodoc2-docstring} archivebox.cli.STDERR + :summary: + ``` ```` ### API @@ -100,6 +103,16 @@ archivebox.cli.archivebox_manage ```` +````{py:data} STDERR +:canonical: archivebox.cli.STDERR +:value: > + 'Console(...)' + +```{autodoc2-docstring} archivebox.cli.STDERR +``` + +```` + `````{py:class} ArchiveBoxGroup(name: str | None = None, commands: collections.abc.MutableMapping[str, click.core.Command] | collections.abc.Sequence[click.core.Command] | None = None, invoke_without_command: bool = False, no_args_is_help: bool | None = None, subcommand_metavar: str | None = None, chain: bool = False, result_callback: typing.Callable[..., typing.Any] | None = None, **kwargs: typing.Any) :canonical: archivebox.cli.ArchiveBoxGroup diff --git a/docs/apidocs/archivebox/archivebox.cli.tests_piping.md b/docs/apidocs/archivebox/archivebox.cli.tests_piping.md deleted file mode 100644 index 7106e308..00000000 --- a/docs/apidocs/archivebox/archivebox.cli.tests_piping.md +++ /dev/null @@ -1,768 +0,0 @@ -# {py:mod}`archivebox.cli.tests_piping` - -```{py:module} archivebox.cli.tests_piping -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`TestJSONLParsing ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing - :summary: - ``` -* - {py:obj}`TestReadArgsOrStdin ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin - :summary: - ``` -* - {py:obj}`TestCrawlCommand ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand - :summary: - ``` -* - {py:obj}`TestSnapshotCommand ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand - :summary: - ``` -* - {py:obj}`TestArchiveResultCommand ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand - :summary: - ``` -* - {py:obj}`TestURLCollection ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection - :summary: - ``` -* - {py:obj}`TestPipingWorkflowIntegration ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration - :summary: - ``` -* - {py:obj}`TestDepthWorkflows ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows - :summary: - ``` -* - {py:obj}`TestParserPluginWorkflows ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows - :summary: - ``` -* - {py:obj}`TestEdgeCases ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases - :summary: - ``` -* - {py:obj}`TestPassThroughBehavior ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior - :summary: - ``` -* - {py:obj}`TestPipelineAccumulation ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipelineAccumulation - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`TEST_CONFIG ` - - ```{autodoc2-docstring} archivebox.cli.tests_piping.TEST_CONFIG - :summary: - ``` -```` - -### API - -````{py:data} TEST_CONFIG -:canonical: archivebox.cli.tests_piping.TEST_CONFIG -:value: > - None - -```{autodoc2-docstring} archivebox.cli.tests_piping.TEST_CONFIG -``` - -```` - -`````{py:class} TestJSONLParsing(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestJSONLParsing - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.__init__ -``` - -````{py:method} test_parse_plain_url() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_plain_url - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_plain_url -``` - -```` - -````{py:method} test_parse_jsonl_snapshot() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_jsonl_snapshot - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_jsonl_snapshot -``` - -```` - -````{py:method} test_parse_jsonl_crawl() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_jsonl_crawl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_jsonl_crawl -``` - -```` - -````{py:method} test_parse_jsonl_with_id() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_jsonl_with_id - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_jsonl_with_id -``` - -```` - -````{py:method} test_parse_uuid_as_snapshot_id() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_uuid_as_snapshot_id - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_uuid_as_snapshot_id -``` - -```` - -````{py:method} test_parse_empty_line() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_empty_line - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_empty_line -``` - -```` - -````{py:method} test_parse_comment_line() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_comment_line - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_comment_line -``` - -```` - -````{py:method} test_parse_invalid_url() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_invalid_url - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_invalid_url -``` - -```` - -````{py:method} test_parse_file_url() -:canonical: archivebox.cli.tests_piping.TestJSONLParsing.test_parse_file_url - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestJSONLParsing.test_parse_file_url -``` - -```` - -````` - -`````{py:class} TestReadArgsOrStdin(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestReadArgsOrStdin - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin.__init__ -``` - -````{py:method} test_read_from_args() -:canonical: archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_from_args - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_from_args -``` - -```` - -````{py:method} test_read_from_stdin() -:canonical: archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_from_stdin - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_from_stdin -``` - -```` - -````{py:method} test_read_jsonl_from_stdin() -:canonical: archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_jsonl_from_stdin - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_jsonl_from_stdin -``` - -```` - -````{py:method} test_read_crawl_jsonl_from_stdin() -:canonical: archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_crawl_jsonl_from_stdin - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin.test_read_crawl_jsonl_from_stdin -``` - -```` - -````{py:method} test_skip_tty_stdin() -:canonical: archivebox.cli.tests_piping.TestReadArgsOrStdin.test_skip_tty_stdin - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestReadArgsOrStdin.test_skip_tty_stdin -``` - -```` - -````` - -`````{py:class} TestCrawlCommand(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestCrawlCommand - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand.__init__ -``` - -````{py:method} setUp() -:canonical: archivebox.cli.tests_piping.TestCrawlCommand.setUp - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand.setUp -``` - -```` - -````{py:method} tearDown() -:canonical: archivebox.cli.tests_piping.TestCrawlCommand.tearDown - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand.tearDown -``` - -```` - -````{py:method} test_crawl_accepts_url() -:canonical: archivebox.cli.tests_piping.TestCrawlCommand.test_crawl_accepts_url - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand.test_crawl_accepts_url -``` - -```` - -````{py:method} test_crawl_output_format() -:canonical: archivebox.cli.tests_piping.TestCrawlCommand.test_crawl_output_format - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestCrawlCommand.test_crawl_output_format -``` - -```` - -````` - -`````{py:class} TestSnapshotCommand(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestSnapshotCommand - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand.__init__ -``` - -````{py:method} setUp() -:canonical: archivebox.cli.tests_piping.TestSnapshotCommand.setUp - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand.setUp -``` - -```` - -````{py:method} tearDown() -:canonical: archivebox.cli.tests_piping.TestSnapshotCommand.tearDown - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand.tearDown -``` - -```` - -````{py:method} test_snapshot_accepts_url() -:canonical: archivebox.cli.tests_piping.TestSnapshotCommand.test_snapshot_accepts_url - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand.test_snapshot_accepts_url -``` - -```` - -````{py:method} test_snapshot_accepts_crawl_jsonl() -:canonical: archivebox.cli.tests_piping.TestSnapshotCommand.test_snapshot_accepts_crawl_jsonl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand.test_snapshot_accepts_crawl_jsonl -``` - -```` - -````{py:method} test_snapshot_accepts_jsonl_with_metadata() -:canonical: archivebox.cli.tests_piping.TestSnapshotCommand.test_snapshot_accepts_jsonl_with_metadata - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestSnapshotCommand.test_snapshot_accepts_jsonl_with_metadata -``` - -```` - -````` - -`````{py:class} TestArchiveResultCommand(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestArchiveResultCommand - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand.__init__ -``` - -````{py:method} setUp() -:canonical: archivebox.cli.tests_piping.TestArchiveResultCommand.setUp - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand.setUp -``` - -```` - -````{py:method} tearDown() -:canonical: archivebox.cli.tests_piping.TestArchiveResultCommand.tearDown - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand.tearDown -``` - -```` - -````{py:method} test_archiveresult_accepts_snapshot_id() -:canonical: archivebox.cli.tests_piping.TestArchiveResultCommand.test_archiveresult_accepts_snapshot_id - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand.test_archiveresult_accepts_snapshot_id -``` - -```` - -````{py:method} test_archiveresult_accepts_jsonl_snapshot() -:canonical: archivebox.cli.tests_piping.TestArchiveResultCommand.test_archiveresult_accepts_jsonl_snapshot - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand.test_archiveresult_accepts_jsonl_snapshot -``` - -```` - -````{py:method} test_archiveresult_gathers_snapshot_ids() -:canonical: archivebox.cli.tests_piping.TestArchiveResultCommand.test_archiveresult_gathers_snapshot_ids - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestArchiveResultCommand.test_archiveresult_gathers_snapshot_ids -``` - -```` - -````` - -`````{py:class} TestURLCollection(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestURLCollection - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection.__init__ -``` - -````{py:method} setUp() -:canonical: archivebox.cli.tests_piping.TestURLCollection.setUp - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection.setUp -``` - -```` - -````{py:method} tearDown() -:canonical: archivebox.cli.tests_piping.TestURLCollection.tearDown - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection.tearDown -``` - -```` - -````{py:method} test_collect_urls_from_plugins() -:canonical: archivebox.cli.tests_piping.TestURLCollection.test_collect_urls_from_plugins - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection.test_collect_urls_from_plugins -``` - -```` - -````{py:method} test_collect_urls_preserves_metadata() -:canonical: archivebox.cli.tests_piping.TestURLCollection.test_collect_urls_preserves_metadata - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection.test_collect_urls_preserves_metadata -``` - -```` - -````{py:method} test_collect_urls_empty_dir() -:canonical: archivebox.cli.tests_piping.TestURLCollection.test_collect_urls_empty_dir - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestURLCollection.test_collect_urls_empty_dir -``` - -```` - -````` - -`````{py:class} TestPipingWorkflowIntegration(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.__init__ -``` - -````{py:method} setUpClass() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.setUpClass -:classmethod: - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.setUpClass -``` - -```` - -````{py:method} tearDownClass() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.tearDownClass -:classmethod: - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.tearDownClass -``` - -```` - -````{py:method} test_crawl_creates_and_outputs_jsonl() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_crawl_creates_and_outputs_jsonl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_crawl_creates_and_outputs_jsonl -``` - -```` - -````{py:method} test_snapshot_accepts_crawl_jsonl() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_snapshot_accepts_crawl_jsonl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_snapshot_accepts_crawl_jsonl -``` - -```` - -````{py:method} test_snapshot_creates_and_outputs_jsonl() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_snapshot_creates_and_outputs_jsonl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_snapshot_creates_and_outputs_jsonl -``` - -```` - -````{py:method} test_extract_accepts_snapshot_from_previous_command() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_extract_accepts_snapshot_from_previous_command - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_extract_accepts_snapshot_from_previous_command -``` - -```` - -````{py:method} test_full_pipeline_crawl_snapshot_extract() -:canonical: archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_full_pipeline_crawl_snapshot_extract - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipingWorkflowIntegration.test_full_pipeline_crawl_snapshot_extract -``` - -```` - -````` - -`````{py:class} TestDepthWorkflows(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestDepthWorkflows - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows.__init__ -``` - -````{py:method} setUpClass() -:canonical: archivebox.cli.tests_piping.TestDepthWorkflows.setUpClass -:classmethod: - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows.setUpClass -``` - -```` - -````{py:method} tearDownClass() -:canonical: archivebox.cli.tests_piping.TestDepthWorkflows.tearDownClass -:classmethod: - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows.tearDownClass -``` - -```` - -````{py:method} test_depth_0_workflow() -:canonical: archivebox.cli.tests_piping.TestDepthWorkflows.test_depth_0_workflow - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows.test_depth_0_workflow -``` - -```` - -````{py:method} test_depth_metadata_in_crawl() -:canonical: archivebox.cli.tests_piping.TestDepthWorkflows.test_depth_metadata_in_crawl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestDepthWorkflows.test_depth_metadata_in_crawl -``` - -```` - -````` - -`````{py:class} TestParserPluginWorkflows(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestParserPluginWorkflows - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows.__init__ -``` - -````{py:method} setUpClass() -:canonical: archivebox.cli.tests_piping.TestParserPluginWorkflows.setUpClass -:classmethod: - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows.setUpClass -``` - -```` - -````{py:method} tearDownClass() -:canonical: archivebox.cli.tests_piping.TestParserPluginWorkflows.tearDownClass -:classmethod: - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows.tearDownClass -``` - -```` - -````{py:method} test_html_parser_workflow() -:canonical: archivebox.cli.tests_piping.TestParserPluginWorkflows.test_html_parser_workflow - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows.test_html_parser_workflow -``` - -```` - -````{py:method} test_rss_parser_workflow() -:canonical: archivebox.cli.tests_piping.TestParserPluginWorkflows.test_rss_parser_workflow - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows.test_rss_parser_workflow -``` - -```` - -````{py:method} test_multiple_parsers_dedupe() -:canonical: archivebox.cli.tests_piping.TestParserPluginWorkflows.test_multiple_parsers_dedupe - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestParserPluginWorkflows.test_multiple_parsers_dedupe -``` - -```` - -````` - -`````{py:class} TestEdgeCases(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestEdgeCases - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases.__init__ -``` - -````{py:method} test_empty_input() -:canonical: archivebox.cli.tests_piping.TestEdgeCases.test_empty_input - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases.test_empty_input -``` - -```` - -````{py:method} test_malformed_jsonl() -:canonical: archivebox.cli.tests_piping.TestEdgeCases.test_malformed_jsonl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases.test_malformed_jsonl -``` - -```` - -````{py:method} test_mixed_input_formats() -:canonical: archivebox.cli.tests_piping.TestEdgeCases.test_mixed_input_formats - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases.test_mixed_input_formats -``` - -```` - -````{py:method} test_crawl_with_multiple_urls() -:canonical: archivebox.cli.tests_piping.TestEdgeCases.test_crawl_with_multiple_urls - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestEdgeCases.test_crawl_with_multiple_urls -``` - -```` - -````` - -`````{py:class} TestPassThroughBehavior(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestPassThroughBehavior - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior.__init__ -``` - -````{py:method} test_crawl_passes_through_other_types() -:canonical: archivebox.cli.tests_piping.TestPassThroughBehavior.test_crawl_passes_through_other_types - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior.test_crawl_passes_through_other_types -``` - -```` - -````{py:method} test_snapshot_passes_through_crawl() -:canonical: archivebox.cli.tests_piping.TestPassThroughBehavior.test_snapshot_passes_through_crawl - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior.test_snapshot_passes_through_crawl -``` - -```` - -````{py:method} test_archiveresult_passes_through_snapshot() -:canonical: archivebox.cli.tests_piping.TestPassThroughBehavior.test_archiveresult_passes_through_snapshot - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior.test_archiveresult_passes_through_snapshot -``` - -```` - -````{py:method} test_run_passes_through_unknown_types() -:canonical: archivebox.cli.tests_piping.TestPassThroughBehavior.test_run_passes_through_unknown_types - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPassThroughBehavior.test_run_passes_through_unknown_types -``` - -```` - -````` - -`````{py:class} TestPipelineAccumulation(methodName='runTest') -:canonical: archivebox.cli.tests_piping.TestPipelineAccumulation - -Bases: {py:obj}`unittest.TestCase` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipelineAccumulation -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipelineAccumulation.__init__ -``` - -````{py:method} test_full_pipeline_output_types() -:canonical: archivebox.cli.tests_piping.TestPipelineAccumulation.test_full_pipeline_output_types - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipelineAccumulation.test_full_pipeline_output_types -``` - -```` - -````{py:method} test_pipeline_preserves_ids() -:canonical: archivebox.cli.tests_piping.TestPipelineAccumulation.test_pipeline_preserves_ids - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipelineAccumulation.test_pipeline_preserves_ids -``` - -```` - -````{py:method} test_jq_transform_pattern() -:canonical: archivebox.cli.tests_piping.TestPipelineAccumulation.test_jq_transform_pattern - -```{autodoc2-docstring} archivebox.cli.tests_piping.TestPipelineAccumulation.test_jq_transform_pattern -``` - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.config.collection.md b/docs/apidocs/archivebox/archivebox.config.collection.md index a4bd9e16..b5b52596 100644 --- a/docs/apidocs/archivebox/archivebox.config.collection.md +++ b/docs/apidocs/archivebox/archivebox.config.collection.md @@ -9,141 +9,173 @@ ## Module Contents -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`PluginConfigSection ` - - ```{autodoc2-docstring} archivebox.config.collection.PluginConfigSection - :summary: - ``` -```` - ### Functions ````{list-table} :class: autosummary longtable :align: left -* - {py:obj}`get_real_name ` - - ```{autodoc2-docstring} archivebox.config.collection.get_real_name +* - {py:obj}`_coerce_to_str_dict ` + - ```{autodoc2-docstring} archivebox.config.collection._coerce_to_str_dict :summary: ``` -* - {py:obj}`load_config_val ` - - ```{autodoc2-docstring} archivebox.config.collection.load_config_val +* - {py:obj}`_load_file_config_dict ` + - ```{autodoc2-docstring} archivebox.config.collection._load_file_config_dict :summary: ``` -* - {py:obj}`load_config_file ` - - ```{autodoc2-docstring} archivebox.config.collection.load_config_file +* - {py:obj}`_resolve_section_for_key ` + - ```{autodoc2-docstring} archivebox.config.collection._resolve_section_for_key :summary: ``` -* - {py:obj}`section_for_key ` - - ```{autodoc2-docstring} archivebox.config.collection.section_for_key +* - {py:obj}`_render_config_file_content ` + - ```{autodoc2-docstring} archivebox.config.collection._render_config_file_content + :summary: + ``` +* - {py:obj}`_write_file_if_changed ` + - ```{autodoc2-docstring} archivebox.config.collection._write_file_if_changed + :summary: + ``` +* - {py:obj}`mirror_machine_config_to_file ` + - ```{autodoc2-docstring} archivebox.config.collection.mirror_machine_config_to_file + :summary: + ``` +* - {py:obj}`_coerce_from_str_dict ` + - ```{autodoc2-docstring} archivebox.config.collection._coerce_from_str_dict + :summary: + ``` +* - {py:obj}`_mirror_file_to_machine_config ` + - ```{autodoc2-docstring} archivebox.config.collection._mirror_file_to_machine_config + :summary: + ``` +* - {py:obj}`sync_machine_and_file ` + - ```{autodoc2-docstring} archivebox.config.collection.sync_machine_and_file :summary: ``` * - {py:obj}`write_config_file ` - ```{autodoc2-docstring} archivebox.config.collection.write_config_file :summary: ``` -* - {py:obj}`load_config ` - - ```{autodoc2-docstring} archivebox.config.collection.load_config +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`CONFIG_FILE_HEADER ` + - ```{autodoc2-docstring} archivebox.config.collection.CONFIG_FILE_HEADER :summary: ``` -* - {py:obj}`load_all_config ` - - ```{autodoc2-docstring} archivebox.config.collection.load_all_config +* - {py:obj}`_MIRROR_IN_PROGRESS ` + - ```{autodoc2-docstring} archivebox.config.collection._MIRROR_IN_PROGRESS + :summary: + ``` +* - {py:obj}`_INITIAL_SYNC_DONE ` + - ```{autodoc2-docstring} archivebox.config.collection._INITIAL_SYNC_DONE :summary: ``` ```` ### API -````{py:function} get_real_name(key: str) -> str -:canonical: archivebox.config.collection.get_real_name +````{py:data} CONFIG_FILE_HEADER +:canonical: archivebox.config.collection.CONFIG_FILE_HEADER +:value: -```{autodoc2-docstring} archivebox.config.collection.get_real_name +```{autodoc2-docstring} archivebox.config.collection.CONFIG_FILE_HEADER ``` + ```` -````{py:function} load_config_val(key: str, default: typing.Any = None, type: typing.Optional[typing.Type] = None, aliases: typing.Optional[typing.Tuple[str, ...]] = None, config: typing.Optional[benedict.benedict] = None, env_vars: typing.Optional[os._Environ] = None, config_file_vars: typing.Optional[typing.Dict[str, str]] = None) -> typing.Any -:canonical: archivebox.config.collection.load_config_val - -```{autodoc2-docstring} archivebox.config.collection.load_config_val -``` -```` - -````{py:function} load_config_file() -> typing.Optional[benedict.benedict] -:canonical: archivebox.config.collection.load_config_file - -```{autodoc2-docstring} archivebox.config.collection.load_config_file -``` -```` - -`````{py:class} PluginConfigSection(key: str) -:canonical: archivebox.config.collection.PluginConfigSection - -```{autodoc2-docstring} archivebox.config.collection.PluginConfigSection -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.config.collection.PluginConfigSection.__init__ -``` - -````{py:attribute} toml_section_header -:canonical: archivebox.config.collection.PluginConfigSection.toml_section_header +````{py:data} _MIRROR_IN_PROGRESS +:canonical: archivebox.config.collection._MIRROR_IN_PROGRESS +:type: bool :value: > - 'PLUGINS' + False -```{autodoc2-docstring} archivebox.config.collection.PluginConfigSection.toml_section_header +```{autodoc2-docstring} archivebox.config.collection._MIRROR_IN_PROGRESS ``` ```` -````{py:method} __getattr__(name: str) -> typing.Any -:canonical: archivebox.config.collection.PluginConfigSection.__getattr__ +````{py:data} _INITIAL_SYNC_DONE +:canonical: archivebox.config.collection._INITIAL_SYNC_DONE +:type: bool +:value: > + False -```{autodoc2-docstring} archivebox.config.collection.PluginConfigSection.__getattr__ +```{autodoc2-docstring} archivebox.config.collection._INITIAL_SYNC_DONE ``` ```` -````{py:method} update_in_place(warn: bool = True, persist: bool = False, **kwargs) -:canonical: archivebox.config.collection.PluginConfigSection.update_in_place +````{py:function} _coerce_to_str_dict(config: typing.Any) -> dict[str, str] +:canonical: archivebox.config.collection._coerce_to_str_dict -```{autodoc2-docstring} archivebox.config.collection.PluginConfigSection.update_in_place -``` - -```` - -````` - -````{py:function} section_for_key(key: str) -> typing.Any -:canonical: archivebox.config.collection.section_for_key - -```{autodoc2-docstring} archivebox.config.collection.section_for_key +```{autodoc2-docstring} archivebox.config.collection._coerce_to_str_dict ``` ```` -````{py:function} write_config_file(config: typing.Dict[str, str]) -> benedict.benedict +````{py:function} _load_file_config_dict() -> tuple[dict[str, str], float | None] +:canonical: archivebox.config.collection._load_file_config_dict + +```{autodoc2-docstring} archivebox.config.collection._load_file_config_dict +``` +```` + +````{py:function} _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str +:canonical: archivebox.config.collection._resolve_section_for_key + +```{autodoc2-docstring} archivebox.config.collection._resolve_section_for_key +``` +```` + +````{py:function} _render_config_file_content(config: dict[str, str]) -> str +:canonical: archivebox.config.collection._render_config_file_content + +```{autodoc2-docstring} archivebox.config.collection._render_config_file_content +``` +```` + +````{py:function} _write_file_if_changed(content: str) -> bool +:canonical: archivebox.config.collection._write_file_if_changed + +```{autodoc2-docstring} archivebox.config.collection._write_file_if_changed +``` +```` + +````{py:function} mirror_machine_config_to_file(config: typing.Any) -> None +:canonical: archivebox.config.collection.mirror_machine_config_to_file + +```{autodoc2-docstring} archivebox.config.collection.mirror_machine_config_to_file +``` +```` + +````{py:function} _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, typing.Any] +:canonical: archivebox.config.collection._coerce_from_str_dict + +```{autodoc2-docstring} archivebox.config.collection._coerce_from_str_dict +``` +```` + +````{py:function} _mirror_file_to_machine_config(file_config: dict[str, str]) -> None +:canonical: archivebox.config.collection._mirror_file_to_machine_config + +```{autodoc2-docstring} archivebox.config.collection._mirror_file_to_machine_config +``` +```` + +````{py:function} sync_machine_and_file(machine: typing.Any = None) -> None +:canonical: archivebox.config.collection.sync_machine_and_file + +```{autodoc2-docstring} archivebox.config.collection.sync_machine_and_file +``` +```` + +````{py:function} write_config_file(config: dict[str, str]) -> archivebox.misc.logging.AttrDict :canonical: archivebox.config.collection.write_config_file ```{autodoc2-docstring} archivebox.config.collection.write_config_file ``` ```` - -````{py:function} load_config(defaults: typing.Dict[str, typing.Any], config: typing.Optional[benedict.benedict] = None, out_dir: typing.Optional[str] = None, env_vars: typing.Optional[os._Environ] = None, config_file_vars: typing.Optional[typing.Dict[str, str]] = None) -> benedict.benedict -:canonical: archivebox.config.collection.load_config - -```{autodoc2-docstring} archivebox.config.collection.load_config -``` -```` - -````{py:function} load_all_config() -:canonical: archivebox.config.collection.load_all_config - -```{autodoc2-docstring} archivebox.config.collection.load_all_config -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.config.common.md b/docs/apidocs/archivebox/archivebox.config.common.md index e93e9776..305cc571 100644 --- a/docs/apidocs/archivebox/archivebox.config.common.md +++ b/docs/apidocs/archivebox/archivebox.config.common.md @@ -23,10 +23,84 @@ - * - {py:obj}`ServerConfig ` - +* - {py:obj}`DatabaseConfig ` + - * - {py:obj}`ArchivingConfig ` - * - {py:obj}`SearchBackendConfig ` - +* - {py:obj}`ArchiveBoxBaseConfig ` + - ```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig + :summary: + ``` +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_legacy_bool ` + - ```{autodoc2-docstring} archivebox.config.common._legacy_bool + :summary: + ``` +* - {py:obj}`permissions_from_legacy_public_flags ` + - ```{autodoc2-docstring} archivebox.config.common.permissions_from_legacy_public_flags + :summary: + ``` +* - {py:obj}`is_sensitive_config_key ` + - ```{autodoc2-docstring} archivebox.config.common.is_sensitive_config_key + :summary: + ``` +* - {py:obj}`redact_sensitive_config ` + - ```{autodoc2-docstring} archivebox.config.common.redact_sensitive_config + :summary: + ``` +* - {py:obj}`rprint ` + - ```{autodoc2-docstring} archivebox.config.common.rprint + :summary: + ``` +* - {py:obj}`parse_delete_after ` + - ```{autodoc2-docstring} archivebox.config.common.parse_delete_after + :summary: + ``` +* - {py:obj}`_plugin_user_config_value ` + - ```{autodoc2-docstring} archivebox.config.common._plugin_user_config_value + :summary: + ``` +* - {py:obj}`_plugin_user_config ` + - ```{autodoc2-docstring} archivebox.config.common._plugin_user_config + :summary: + ``` +* - {py:obj}`_discover_plugin_config_schemas ` + - ```{autodoc2-docstring} archivebox.config.common._discover_plugin_config_schemas + :summary: + ``` +* - {py:obj}`_plugin_config_properties ` + - ```{autodoc2-docstring} archivebox.config.common._plugin_config_properties + :summary: + ``` +* - {py:obj}`_plugin_config_model ` + - ```{autodoc2-docstring} archivebox.config.common._plugin_config_model + :summary: + ``` +* - {py:obj}`_archivebox_config_input_names ` + - ```{autodoc2-docstring} archivebox.config.common._archivebox_config_input_names + :summary: + ``` +* - {py:obj}`_build_archivebox_config_model ` + - ```{autodoc2-docstring} archivebox.config.common._build_archivebox_config_model + :summary: + ``` +* - {py:obj}`get_config ` + - ```{autodoc2-docstring} archivebox.config.common.get_config + :summary: + ``` +* - {py:obj}`get_all_configs ` + - ```{autodoc2-docstring} archivebox.config.common.get_all_configs + :summary: + ``` ```` ### Data @@ -35,34 +109,166 @@ :class: autosummary longtable :align: left -* - {py:obj}`SHELL_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.common.SHELL_CONFIG +* - {py:obj}`ConfigOverrides ` + - ```{autodoc2-docstring} archivebox.config.common.ConfigOverrides :summary: ``` -* - {py:obj}`STORAGE_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.common.STORAGE_CONFIG +* - {py:obj}`ConfigPayload ` + - ```{autodoc2-docstring} archivebox.config.common.ConfigPayload :summary: ``` -* - {py:obj}`GENERAL_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.common.GENERAL_CONFIG +* - {py:obj}`PluginSchemaDocuments ` + - ```{autodoc2-docstring} archivebox.config.common.PluginSchemaDocuments :summary: ``` -* - {py:obj}`SERVER_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.common.SERVER_CONFIG +* - {py:obj}`_STDOUT_CONSOLE ` + - ```{autodoc2-docstring} archivebox.config.common._STDOUT_CONSOLE :summary: ``` -* - {py:obj}`ARCHIVING_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.common.ARCHIVING_CONFIG +* - {py:obj}`_STDERR_CONSOLE ` + - ```{autodoc2-docstring} archivebox.config.common._STDERR_CONSOLE :summary: ``` -* - {py:obj}`SEARCH_BACKEND_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.common.SEARCH_BACKEND_CONFIG +* - {py:obj}`_WARNED_ARCHIVING_CONFIGS ` + - ```{autodoc2-docstring} archivebox.config.common._WARNED_ARCHIVING_CONFIGS + :summary: + ``` +* - {py:obj}`_SENSITIVE_CONFIG_KEY_NEEDLES ` + - ```{autodoc2-docstring} archivebox.config.common._SENSITIVE_CONFIG_KEY_NEEDLES + :summary: + ``` +* - {py:obj}`SENSITIVE_CONFIG_VALUE_REDACTED ` + - ```{autodoc2-docstring} archivebox.config.common.SENSITIVE_CONFIG_VALUE_REDACTED + :summary: + ``` +* - {py:obj}`PLUGIN_CONFIG_SCHEMAS ` + - ```{autodoc2-docstring} archivebox.config.common.PLUGIN_CONFIG_SCHEMAS + :summary: + ``` +* - {py:obj}`ArchiveBoxConfig ` + - ```{autodoc2-docstring} archivebox.config.common.ArchiveBoxConfig :summary: ``` ```` ### API +````{py:data} ConfigOverrides +:canonical: archivebox.config.common.ConfigOverrides +:value: > + None + +```{autodoc2-docstring} archivebox.config.common.ConfigOverrides +``` + +```` + +````{py:data} ConfigPayload +:canonical: archivebox.config.common.ConfigPayload +:value: > + None + +```{autodoc2-docstring} archivebox.config.common.ConfigPayload +``` + +```` + +````{py:data} PluginSchemaDocuments +:canonical: archivebox.config.common.PluginSchemaDocuments +:value: > + None + +```{autodoc2-docstring} archivebox.config.common.PluginSchemaDocuments +``` + +```` + +````{py:data} _STDOUT_CONSOLE +:canonical: archivebox.config.common._STDOUT_CONSOLE +:value: > + 'Console(...)' + +```{autodoc2-docstring} archivebox.config.common._STDOUT_CONSOLE +``` + +```` + +````{py:data} _STDERR_CONSOLE +:canonical: archivebox.config.common._STDERR_CONSOLE +:value: > + 'Console(...)' + +```{autodoc2-docstring} archivebox.config.common._STDERR_CONSOLE +``` + +```` + +````{py:data} _WARNED_ARCHIVING_CONFIGS +:canonical: archivebox.config.common._WARNED_ARCHIVING_CONFIGS +:type: set[tuple[int, bool]] +:value: > + 'set(...)' + +```{autodoc2-docstring} archivebox.config.common._WARNED_ARCHIVING_CONFIGS +``` + +```` + +````{py:function} _legacy_bool(value: object) -> bool | None +:canonical: archivebox.config.common._legacy_bool + +```{autodoc2-docstring} archivebox.config.common._legacy_bool +``` +```` + +````{py:function} permissions_from_legacy_public_flags(raw_config: collections.abc.Mapping[str, object]) -> str | None +:canonical: archivebox.config.common.permissions_from_legacy_public_flags + +```{autodoc2-docstring} archivebox.config.common.permissions_from_legacy_public_flags +``` +```` + +````{py:data} _SENSITIVE_CONFIG_KEY_NEEDLES +:canonical: archivebox.config.common._SENSITIVE_CONFIG_KEY_NEEDLES +:value: > + ('TOKEN', 'SECRET', 'API_KEY', 'APIKEY', 'PASSWORD') + +```{autodoc2-docstring} archivebox.config.common._SENSITIVE_CONFIG_KEY_NEEDLES +``` + +```` + +````{py:data} SENSITIVE_CONFIG_VALUE_REDACTED +:canonical: archivebox.config.common.SENSITIVE_CONFIG_VALUE_REDACTED +:value: > + '********' + +```{autodoc2-docstring} archivebox.config.common.SENSITIVE_CONFIG_VALUE_REDACTED +``` + +```` + +````{py:function} is_sensitive_config_key(key: str) -> bool +:canonical: archivebox.config.common.is_sensitive_config_key + +```{autodoc2-docstring} archivebox.config.common.is_sensitive_config_key +``` +```` + +````{py:function} redact_sensitive_config(config: collections.abc.Mapping[str, typing.Any] | None) -> dict[str, typing.Any] +:canonical: archivebox.config.common.redact_sensitive_config + +```{autodoc2-docstring} archivebox.config.common.redact_sensitive_config +``` +```` + +````{py:function} rprint(*args, file=None, **kwargs) +:canonical: archivebox.config.common.rprint + +```{autodoc2-docstring} archivebox.config.common.rprint +``` +```` + `````{py:class} ShellConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) :canonical: archivebox.config.common.ShellConfig @@ -147,7 +353,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} ANSI :canonical: archivebox.config.common.ShellConfig.ANSI -:type: typing.Dict[str, str] +:type: dict[str, str] :value: > 'Field(...)' @@ -167,7 +373,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:property} COMMIT_HASH :canonical: archivebox.config.common.ShellConfig.COMMIT_HASH -:type: typing.Optional[str] +:type: str | None ```{autodoc2-docstring} archivebox.config.common.ShellConfig.COMMIT_HASH ``` @@ -185,16 +391,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````` -````{py:data} SHELL_CONFIG -:canonical: archivebox.config.common.SHELL_CONFIG -:value: > - 'ShellConfig(...)' - -```{autodoc2-docstring} archivebox.config.common.SHELL_CONFIG -``` - -```` - `````{py:class} StorageConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) :canonical: archivebox.config.common.StorageConfig @@ -211,6 +407,39 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` +````{py:attribute} ARCHIVE_DIR +:canonical: archivebox.config.common.StorageConfig.ARCHIVE_DIR +:type: pathlib.Path +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.StorageConfig.ARCHIVE_DIR +``` + +```` + +````{py:attribute} USERS_DIR +:canonical: archivebox.config.common.StorageConfig.USERS_DIR +:type: pathlib.Path +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.StorageConfig.USERS_DIR +``` + +```` + +````{py:attribute} PERSONAS_DIR +:canonical: archivebox.config.common.StorageConfig.PERSONAS_DIR +:type: pathlib.Path +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.StorageConfig.PERSONAS_DIR +``` + +```` + ````{py:attribute} TMP_DIR :canonical: archivebox.config.common.StorageConfig.TMP_DIR :type: pathlib.Path @@ -266,17 +495,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} RESTRICT_FILE_NAMES -:canonical: archivebox.config.common.StorageConfig.RESTRICT_FILE_NAMES -:type: str -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.StorageConfig.RESTRICT_FILE_NAMES -``` - -```` - ````{py:attribute} ENFORCE_ATOMIC_WRITES :canonical: archivebox.config.common.StorageConfig.ENFORCE_ATOMIC_WRITES :type: bool @@ -288,29 +506,19 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} DIR_OUTPUT_PERMISSIONS -:canonical: archivebox.config.common.StorageConfig.DIR_OUTPUT_PERMISSIONS -:type: str +````{py:attribute} ALLOW_NO_UNIX_SOCKETS +:canonical: archivebox.config.common.StorageConfig.ALLOW_NO_UNIX_SOCKETS +:type: bool :value: > 'Field(...)' -```{autodoc2-docstring} archivebox.config.common.StorageConfig.DIR_OUTPUT_PERMISSIONS +```{autodoc2-docstring} archivebox.config.common.StorageConfig.ALLOW_NO_UNIX_SOCKETS ``` ```` ````` -````{py:data} STORAGE_CONFIG -:canonical: archivebox.config.common.STORAGE_CONFIG -:value: > - 'StorageConfig(...)' - -```{autodoc2-docstring} archivebox.config.common.STORAGE_CONFIG -``` - -```` - `````{py:class} GeneralConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) :canonical: archivebox.config.common.GeneralConfig @@ -340,16 +548,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````` -````{py:data} GENERAL_CONFIG -:canonical: archivebox.config.common.GENERAL_CONFIG -:value: > - 'GeneralConfig(...)' - -```{autodoc2-docstring} archivebox.config.common.GENERAL_CONFIG -``` - -```` - `````{py:class} ServerConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) :canonical: archivebox.config.common.ServerConfig @@ -366,6 +564,17 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` +````{py:attribute} SERVER_SECURITY_MODES +:canonical: archivebox.config.common.ServerConfig.SERVER_SECURITY_MODES +:type: typing.ClassVar[tuple[str, ...]] +:value: > + ('safe-subdomains-fullreplay', 'safe-onedomain-nojsreplay', 'unsafe-onedomain-noadmin', 'danger-oned... + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.SERVER_SECURITY_MODES +``` + +```` + ````{py:attribute} SECRET_KEY :canonical: archivebox.config.common.ServerConfig.SECRET_KEY :type: str @@ -388,35 +597,13 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} LISTEN_HOST -:canonical: archivebox.config.common.ServerConfig.LISTEN_HOST +````{py:attribute} BASE_URL +:canonical: archivebox.config.common.ServerConfig.BASE_URL :type: str :value: > 'Field(...)' -```{autodoc2-docstring} archivebox.config.common.ServerConfig.LISTEN_HOST -``` - -```` - -````{py:attribute} ADMIN_BASE_URL -:canonical: archivebox.config.common.ServerConfig.ADMIN_BASE_URL -:type: str -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ServerConfig.ADMIN_BASE_URL -``` - -```` - -````{py:attribute} ARCHIVE_BASE_URL -:canonical: archivebox.config.common.ServerConfig.ARCHIVE_BASE_URL -:type: str -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ServerConfig.ARCHIVE_BASE_URL +```{autodoc2-docstring} archivebox.config.common.ServerConfig.BASE_URL ``` ```` @@ -443,6 +630,17 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` +````{py:attribute} SERVER_SECURITY_MODE +:canonical: archivebox.config.common.ServerConfig.SERVER_SECURITY_MODE +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.SERVER_SECURITY_MODE +``` + +```` + ````{py:attribute} SNAPSHOTS_PER_PAGE :canonical: archivebox.config.common.ServerConfig.SNAPSHOTS_PER_PAGE :type: int @@ -454,17 +652,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} PREVIEW_ORIGINALS -:canonical: archivebox.config.common.ServerConfig.PREVIEW_ORIGINALS -:type: bool -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ServerConfig.PREVIEW_ORIGINALS -``` - -```` - ````{py:attribute} FOOTER_INFO :canonical: archivebox.config.common.ServerConfig.FOOTER_INFO :type: str @@ -487,17 +674,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} PUBLIC_SNAPSHOTS -:canonical: archivebox.config.common.ServerConfig.PUBLIC_SNAPSHOTS -:type: bool -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ServerConfig.PUBLIC_SNAPSHOTS -``` - -```` - ````{py:attribute} PUBLIC_ADD_VIEW :canonical: archivebox.config.common.ServerConfig.PUBLIC_ADD_VIEW :type: bool @@ -511,7 +687,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} ADMIN_USERNAME :canonical: archivebox.config.common.ServerConfig.ADMIN_USERNAME -:type: typing.Optional[str] +:type: str | None :value: > 'Field(...)' @@ -522,7 +698,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} ADMIN_PASSWORD :canonical: archivebox.config.common.ServerConfig.ADMIN_PASSWORD -:type: typing.Optional[str] +:type: str | None :value: > 'Field(...)' @@ -564,18 +740,172 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````` +````{py:method} validate_server_security_mode(v: str) -> str +:canonical: archivebox.config.common.ServerConfig.validate_server_security_mode -````{py:data} SERVER_CONFIG -:canonical: archivebox.config.common.SERVER_CONFIG -:value: > - 'ServerConfig(...)' - -```{autodoc2-docstring} archivebox.config.common.SERVER_CONFIG +```{autodoc2-docstring} archivebox.config.common.ServerConfig.validate_server_security_mode ``` ```` +````{py:property} USES_SUBDOMAIN_ROUTING +:canonical: archivebox.config.common.ServerConfig.USES_SUBDOMAIN_ROUTING +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.USES_SUBDOMAIN_ROUTING +``` + +```` + +````{py:property} ENABLES_FULL_JS_REPLAY +:canonical: archivebox.config.common.ServerConfig.ENABLES_FULL_JS_REPLAY +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.ENABLES_FULL_JS_REPLAY +``` + +```` + +````{py:property} CONTROL_PLANE_ENABLED +:canonical: archivebox.config.common.ServerConfig.CONTROL_PLANE_ENABLED +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.CONTROL_PLANE_ENABLED +``` + +```` + +````{py:property} BLOCK_UNSAFE_METHODS +:canonical: archivebox.config.common.ServerConfig.BLOCK_UNSAFE_METHODS +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.BLOCK_UNSAFE_METHODS +``` + +```` + +````{py:property} SHOULD_NEUTER_RISKY_REPLAY +:canonical: archivebox.config.common.ServerConfig.SHOULD_NEUTER_RISKY_REPLAY +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.SHOULD_NEUTER_RISKY_REPLAY +``` + +```` + +````{py:property} IS_UNSAFE_MODE +:canonical: archivebox.config.common.ServerConfig.IS_UNSAFE_MODE +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.IS_UNSAFE_MODE +``` + +```` + +````{py:property} IS_DANGEROUS_MODE +:canonical: archivebox.config.common.ServerConfig.IS_DANGEROUS_MODE +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.IS_DANGEROUS_MODE +``` + +```` + +````{py:property} IS_LOWER_SECURITY_MODE +:canonical: archivebox.config.common.ServerConfig.IS_LOWER_SECURITY_MODE +:type: bool + +```{autodoc2-docstring} archivebox.config.common.ServerConfig.IS_LOWER_SECURITY_MODE +``` + +```` + +````` + +`````{py:class} DatabaseConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) +:canonical: archivebox.config.common.DatabaseConfig + +Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` + +````{py:attribute} toml_section_header +:canonical: archivebox.config.common.DatabaseConfig.toml_section_header +:type: str +:value: > + 'DATABASE_CONFIG' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.toml_section_header +``` + +```` + +````{py:attribute} DATABASE_NAME +:canonical: archivebox.config.common.DatabaseConfig.DATABASE_NAME +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.DATABASE_NAME +``` + +```` + +````{py:attribute} SQLITE_JOURNAL_MODE +:canonical: archivebox.config.common.DatabaseConfig.SQLITE_JOURNAL_MODE +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.SQLITE_JOURNAL_MODE +``` + +```` + +````{py:attribute} SQLITE_MMAP_SIZE +:canonical: archivebox.config.common.DatabaseConfig.SQLITE_MMAP_SIZE +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.SQLITE_MMAP_SIZE +``` + +```` + +````{py:attribute} SQLITE_BUSY_TIMEOUT +:canonical: archivebox.config.common.DatabaseConfig.SQLITE_BUSY_TIMEOUT +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.SQLITE_BUSY_TIMEOUT +``` + +```` + +````{py:attribute} SQLITE_LOCK_RETRY_TIMEOUT +:canonical: archivebox.config.common.DatabaseConfig.SQLITE_LOCK_RETRY_TIMEOUT +:type: float +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.SQLITE_LOCK_RETRY_TIMEOUT +``` + +```` + +````{py:attribute} SQLITE_LOCK_RETRY_INTERVAL +:canonical: archivebox.config.common.DatabaseConfig.SQLITE_LOCK_RETRY_INTERVAL +:type: float +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.DatabaseConfig.SQLITE_LOCK_RETRY_INTERVAL +``` + +```` + +````` + `````{py:class} ArchivingConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) :canonical: archivebox.config.common.ArchivingConfig @@ -592,6 +922,17 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` +````{py:attribute} PLUGINS +:canonical: archivebox.config.common.ArchivingConfig.PLUGINS +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.PLUGINS +``` + +```` + ````{py:attribute} ONLY_NEW :canonical: archivebox.config.common.ArchivingConfig.ONLY_NEW :type: bool @@ -603,17 +944,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} OVERWRITE -:canonical: archivebox.config.common.ArchivingConfig.OVERWRITE -:type: bool -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.OVERWRITE -``` - -```` - ````{py:attribute} TIMEOUT :canonical: archivebox.config.common.ArchivingConfig.TIMEOUT :type: int @@ -625,13 +955,57 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} MAX_URL_ATTEMPTS -:canonical: archivebox.config.common.ArchivingConfig.MAX_URL_ATTEMPTS +````{py:attribute} CRAWL_MAX_URLS +:canonical: archivebox.config.common.ArchivingConfig.CRAWL_MAX_URLS :type: int :value: > 'Field(...)' -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.MAX_URL_ATTEMPTS +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.CRAWL_MAX_URLS +``` + +```` + +````{py:attribute} CRAWL_MAX_SIZE +:canonical: archivebox.config.common.ArchivingConfig.CRAWL_MAX_SIZE +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.CRAWL_MAX_SIZE +``` + +```` + +````{py:attribute} CRAWL_TIMEOUT +:canonical: archivebox.config.common.ArchivingConfig.CRAWL_TIMEOUT +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.CRAWL_TIMEOUT +``` + +```` + +````{py:attribute} CRAWL_MAX_CONCURRENT_SNAPSHOTS +:canonical: archivebox.config.common.ArchivingConfig.CRAWL_MAX_CONCURRENT_SNAPSHOTS +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.CRAWL_MAX_CONCURRENT_SNAPSHOTS +``` + +```` + +````{py:attribute} SNAPSHOT_MAX_SIZE +:canonical: archivebox.config.common.ArchivingConfig.SNAPSHOT_MAX_SIZE +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.SNAPSHOT_MAX_SIZE ``` ```` @@ -702,28 +1076,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} SAVE_ALLOWLIST -:canonical: archivebox.config.common.ArchivingConfig.SAVE_ALLOWLIST -:type: typing.Dict[str, typing.List[str]] -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.SAVE_ALLOWLIST -``` - -```` - -````{py:attribute} SAVE_DENYLIST -:canonical: archivebox.config.common.ArchivingConfig.SAVE_DENYLIST -:type: typing.Dict[str, typing.List[str]] -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.SAVE_DENYLIST -``` - -```` - ````{py:attribute} DEFAULT_PERSONA :canonical: archivebox.config.common.ArchivingConfig.DEFAULT_PERSONA :type: str @@ -735,10 +1087,32 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:method} validate() -:canonical: archivebox.config.common.ArchivingConfig.validate +````{py:attribute} PERMISSIONS +:canonical: archivebox.config.common.ArchivingConfig.PERMISSIONS +:type: str +:value: > + 'Field(...)' -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.validate +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.PERMISSIONS +``` + +```` + +````{py:attribute} DELETE_AFTER +:canonical: archivebox.config.common.ArchivingConfig.DELETE_AFTER +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.DELETE_AFTER +``` + +```` + +````{py:method} warn_if_invalid() -> None +:canonical: archivebox.config.common.ArchivingConfig.warn_if_invalid + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.warn_if_invalid ``` ```` @@ -751,6 +1125,24 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` +````{py:method} validate_delete_after(value) +:canonical: archivebox.config.common.ArchivingConfig.validate_delete_after +:classmethod: + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.validate_delete_after +``` + +```` + +````{py:method} validate_permissions(value) +:canonical: archivebox.config.common.ArchivingConfig.validate_permissions +:classmethod: + +```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.validate_permissions +``` + +```` + ````{py:property} URL_ALLOWLIST_PTN :canonical: archivebox.config.common.ArchivingConfig.URL_ALLOWLIST_PTN :type: re.Pattern | None @@ -769,34 +1161,13 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:property} SAVE_ALLOWLIST_PTNS -:canonical: archivebox.config.common.ArchivingConfig.SAVE_ALLOWLIST_PTNS -:type: typing.Dict[re.Pattern, typing.List[str]] - -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.SAVE_ALLOWLIST_PTNS -``` - -```` - -````{py:property} SAVE_DENYLIST_PTNS -:canonical: archivebox.config.common.ArchivingConfig.SAVE_DENYLIST_PTNS -:type: typing.Dict[re.Pattern, typing.List[str]] - -```{autodoc2-docstring} archivebox.config.common.ArchivingConfig.SAVE_DENYLIST_PTNS -``` - -```` - ````` -````{py:data} ARCHIVING_CONFIG -:canonical: archivebox.config.common.ARCHIVING_CONFIG -:value: > - 'ArchivingConfig(...)' +````{py:function} parse_delete_after(value) -> datetime.timedelta | None +:canonical: archivebox.config.common.parse_delete_after -```{autodoc2-docstring} archivebox.config.common.ARCHIVING_CONFIG +```{autodoc2-docstring} archivebox.config.common.parse_delete_after ``` - ```` `````{py:class} SearchBackendConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) @@ -815,28 +1186,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} USE_INDEXING_BACKEND -:canonical: archivebox.config.common.SearchBackendConfig.USE_INDEXING_BACKEND -:type: bool -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.SearchBackendConfig.USE_INDEXING_BACKEND -``` - -```` - -````{py:attribute} USE_SEARCHING_BACKEND -:canonical: archivebox.config.common.SearchBackendConfig.USE_SEARCHING_BACKEND -:type: bool -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.config.common.SearchBackendConfig.USE_SEARCHING_BACKEND -``` - -```` - ````{py:attribute} SEARCH_BACKEND_ENGINE :canonical: archivebox.config.common.SearchBackendConfig.SEARCH_BACKEND_ENGINE :type: str @@ -848,25 +1197,176 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` -````{py:attribute} SEARCH_PROCESS_HTML -:canonical: archivebox.config.common.SearchBackendConfig.SEARCH_PROCESS_HTML -:type: bool +````` + +````{py:function} _plugin_user_config_value(value: typing.Any) -> str +:canonical: archivebox.config.common._plugin_user_config_value + +```{autodoc2-docstring} archivebox.config.common._plugin_user_config_value +``` +```` + +````{py:function} _plugin_user_config(config: collections.abc.Mapping[str, object]) -> dict[str, str] +:canonical: archivebox.config.common._plugin_user_config + +```{autodoc2-docstring} archivebox.config.common._plugin_user_config +``` +```` + +````{py:function} _discover_plugin_config_schemas() -> archivebox.config.common.PluginSchemaDocuments +:canonical: archivebox.config.common._discover_plugin_config_schemas + +```{autodoc2-docstring} archivebox.config.common._discover_plugin_config_schemas +``` +```` + +````{py:function} _plugin_config_properties(plugin_schemas: archivebox.config.common.PluginSchemaDocuments) -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.config.common._plugin_config_properties + +```{autodoc2-docstring} archivebox.config.common._plugin_config_properties +``` +```` + +````{py:function} _plugin_config_model(plugin_schemas: archivebox.config.common.PluginSchemaDocuments) -> type[pydantic.BaseModel] +:canonical: archivebox.config.common._plugin_config_model + +```{autodoc2-docstring} archivebox.config.common._plugin_config_model +``` +```` + +````{py:function} _archivebox_config_input_names() -> set[str] +:canonical: archivebox.config.common._archivebox_config_input_names + +```{autodoc2-docstring} archivebox.config.common._archivebox_config_input_names +``` +```` + +`````{py:class} ArchiveBoxBaseConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) +:canonical: archivebox.config.common.ArchiveBoxBaseConfig + +Bases: {py:obj}`archivebox.config.common.ShellConfig`, {py:obj}`archivebox.config.common.StorageConfig`, {py:obj}`archivebox.config.common.GeneralConfig`, {py:obj}`archivebox.config.common.ServerConfig`, {py:obj}`archivebox.config.common.DatabaseConfig`, {py:obj}`archivebox.config.common.ArchivingConfig`, {py:obj}`archivebox.config.common.SearchBackendConfig`, {py:obj}`archivebox.config.ldap.LDAPConfig` + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.__init__ +``` + +````{py:attribute} model_config +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.model_config +:value: > + 'SettingsConfigDict(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.model_config +``` + +```` + +````{py:attribute} DATA_DIR +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.DATA_DIR +:type: pathlib.Path :value: > 'Field(...)' -```{autodoc2-docstring} archivebox.config.common.SearchBackendConfig.SEARCH_PROCESS_HTML +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.DATA_DIR +``` + +```` + +````{py:attribute} ABX_RUNTIME +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.ABX_RUNTIME +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.ABX_RUNTIME +``` + +```` + +````{py:attribute} CRAWL_DIR +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.CRAWL_DIR +:type: pathlib.Path | None +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.CRAWL_DIR +``` + +```` + +````{py:attribute} SNAP_DIR +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.SNAP_DIR +:type: pathlib.Path | None +:value: > + 'Field(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.SNAP_DIR +``` + +```` + +````{py:attribute} computed_config_keys +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.computed_config_keys +:type: typing.ClassVar[tuple[str, ...]] +:value: > + None + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.computed_config_keys +``` + +```` + +````{py:method} resolve_runtime_paths() +:canonical: archivebox.config.common.ArchiveBoxBaseConfig.resolve_runtime_paths + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxBaseConfig.resolve_runtime_paths ``` ```` ````` -````{py:data} SEARCH_BACKEND_CONFIG -:canonical: archivebox.config.common.SEARCH_BACKEND_CONFIG -:value: > - 'SearchBackendConfig(...)' +````{py:function} _build_archivebox_config_model(plugin_schemas: archivebox.config.common.PluginSchemaDocuments) -> type[archivebox.config.common.ArchiveBoxBaseConfig] +:canonical: archivebox.config.common._build_archivebox_config_model -```{autodoc2-docstring} archivebox.config.common.SEARCH_BACKEND_CONFIG +```{autodoc2-docstring} archivebox.config.common._build_archivebox_config_model +``` +```` + +````{py:data} PLUGIN_CONFIG_SCHEMAS +:canonical: archivebox.config.common.PLUGIN_CONFIG_SCHEMAS +:value: > + '_discover_plugin_config_schemas(...)' + +```{autodoc2-docstring} archivebox.config.common.PLUGIN_CONFIG_SCHEMAS ``` ```` + +````{py:data} ArchiveBoxConfig +:canonical: archivebox.config.common.ArchiveBoxConfig +:value: > + '_build_archivebox_config_model(...)' + +```{autodoc2-docstring} archivebox.config.common.ArchiveBoxConfig +``` + +```` + +````{py:function} get_config(defaults: archivebox.config.common.ConfigOverrides | None = None, overrides: archivebox.config.common.ConfigOverrides | None = None, base_config: archivebox.config.common.ArchiveBoxBaseConfig | collections.abc.Mapping[str, object] | None = None, persona: typing.Any = None, user: typing.Any = None, crawl: typing.Any = None, snapshot: typing.Any = None, archiveresult: typing.Any = None, machine: typing.Any = None, include_machine: bool = True, resolve_plugins: bool = True) -> archivebox.config.common.ArchiveBoxBaseConfig +:canonical: archivebox.config.common.get_config + +```{autodoc2-docstring} archivebox.config.common.get_config +``` +```` + +````{py:function} get_all_configs() -> dict[str, archivebox.config.configset.BaseConfigSet] +:canonical: archivebox.config.common.get_all_configs + +```{autodoc2-docstring} archivebox.config.common.get_all_configs +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.config.configset.md b/docs/apidocs/archivebox/archivebox.config.configset.md index d2f797e4..fa53df45 100644 --- a/docs/apidocs/archivebox/archivebox.config.configset.md +++ b/docs/apidocs/archivebox/archivebox.config.configset.md @@ -15,6 +15,8 @@ :class: autosummary longtable :align: left +* - {py:obj}`CaseConfigParser ` + - * - {py:obj}`IniConfigSettingsSource ` - ```{autodoc2-docstring} archivebox.config.configset.IniConfigSettingsSource :summary: @@ -31,24 +33,8 @@ :class: autosummary longtable :align: left -* - {py:obj}`get_config ` - - ```{autodoc2-docstring} archivebox.config.configset.get_config - :summary: - ``` -* - {py:obj}`get_flat_config ` - - ```{autodoc2-docstring} archivebox.config.configset.get_flat_config - :summary: - ``` -* - {py:obj}`get_all_configs ` - - ```{autodoc2-docstring} archivebox.config.configset.get_all_configs - :summary: - ``` -* - {py:obj}`_parse_env_value ` - - ```{autodoc2-docstring} archivebox.config.configset._parse_env_value - :summary: - ``` -* - {py:obj}`get_worker_concurrency ` - - ```{autodoc2-docstring} archivebox.config.configset.get_worker_concurrency +* - {py:obj}`_read_ini_config_cached ` + - ```{autodoc2-docstring} archivebox.config.configset._read_ini_config_cached :summary: ``` ```` @@ -59,14 +45,61 @@ :class: autosummary longtable :align: left -* - {py:obj}`DEFAULT_WORKER_CONCURRENCY ` - - ```{autodoc2-docstring} archivebox.config.configset.DEFAULT_WORKER_CONCURRENCY +* - {py:obj}`COMPUTED_CONFIG_KEYS ` + - ```{autodoc2-docstring} archivebox.config.configset.COMPUTED_CONFIG_KEYS + :summary: + ``` +* - {py:obj}`_INI_CACHE ` + - ```{autodoc2-docstring} archivebox.config.configset._INI_CACHE :summary: ``` ```` ### API +````{py:data} COMPUTED_CONFIG_KEYS +:canonical: archivebox.config.configset.COMPUTED_CONFIG_KEYS +:value: > + ('TERM_WIDTH', 'COMMIT_HASH', 'BUILD_TIME', 'USES_SUBDOMAIN_ROUTING', 'ENABLES_FULL_JS_REPLAY', 'CON... + +```{autodoc2-docstring} archivebox.config.configset.COMPUTED_CONFIG_KEYS +``` + +```` + +`````{py:class} CaseConfigParser(defaults=None, dict_type=_default_dict, allow_no_value=False, *, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, interpolation=_UNSET, converters=_UNSET, allow_unnamed_section=False) +:canonical: archivebox.config.configset.CaseConfigParser + +Bases: {py:obj}`configparser.ConfigParser` + +````{py:method} optionxform(optionstr: str) -> str +:canonical: archivebox.config.configset.CaseConfigParser.optionxform + +```{autodoc2-docstring} archivebox.config.configset.CaseConfigParser.optionxform +``` + +```` + +````` + +````{py:data} _INI_CACHE +:canonical: archivebox.config.configset._INI_CACHE +:type: dict[tuple[str, float], dict[str, typing.Any]] +:value: > + None + +```{autodoc2-docstring} archivebox.config.configset._INI_CACHE +``` + +```` + +````{py:function} _read_ini_config_cached(config_path_str: str) -> dict[str, typing.Any] +:canonical: archivebox.config.configset._read_ini_config_cached + +```{autodoc2-docstring} archivebox.config.configset._read_ini_config_cached +``` +```` + `````{py:class} IniConfigSettingsSource(settings_cls: type[pydantic_settings.main.BaseSettings]) :canonical: archivebox.config.configset.IniConfigSettingsSource @@ -81,12 +114,12 @@ Bases: {py:obj}`pydantic_settings.PydanticBaseSettingsSource` ```{autodoc2-docstring} archivebox.config.configset.IniConfigSettingsSource.__init__ ``` -````{py:method} get_field_value(field: typing.Any, field_name: str) -> typing.Tuple[typing.Any, str, bool] +````{py:method} get_field_value(field: typing.Any, field_name: str) -> tuple[typing.Any, str, bool] :canonical: archivebox.config.configset.IniConfigSettingsSource.get_field_value ```` -````{py:method} __call__() -> typing.Dict[str, typing.Any] +````{py:method} __call__() -> dict[str, typing.Any] :canonical: archivebox.config.configset.IniConfigSettingsSource.__call__ ```{autodoc2-docstring} archivebox.config.configset.IniConfigSettingsSource.__call__ @@ -94,7 +127,7 @@ Bases: {py:obj}`pydantic_settings.PydanticBaseSettingsSource` ```` -````{py:method} _load_config_file() -> typing.Dict[str, typing.Any] +````{py:method} _load_config_file() -> dict[str, typing.Any] :canonical: archivebox.config.configset.IniConfigSettingsSource._load_config_file ```{autodoc2-docstring} archivebox.config.configset.IniConfigSettingsSource._load_config_file @@ -121,14 +154,25 @@ Bases: {py:obj}`pydantic_settings.BaseSettings` ````{py:attribute} model_config :canonical: archivebox.config.configset.BaseConfigSet.model_config :value: > - 'ConfigDict(...)' + 'SettingsConfigDict(...)' ```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.model_config ``` ```` -````{py:method} settings_customise_sources(settings_cls: typing.Type[pydantic_settings.BaseSettings], init_settings: pydantic_settings.PydanticBaseSettingsSource, env_settings: pydantic_settings.PydanticBaseSettingsSource, dotenv_settings: pydantic_settings.PydanticBaseSettingsSource, file_secret_settings: pydantic_settings.PydanticBaseSettingsSource) -> typing.Tuple[pydantic_settings.PydanticBaseSettingsSource, ...] +````{py:attribute} computed_config_keys +:canonical: archivebox.config.configset.BaseConfigSet.computed_config_keys +:type: typing.ClassVar[tuple[str, ...]] +:value: > + () + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.computed_config_keys +``` + +```` + +````{py:method} settings_customise_sources(settings_cls: type[pydantic_settings.BaseSettings], init_settings: pydantic_settings.PydanticBaseSettingsSource, env_settings: pydantic_settings.PydanticBaseSettingsSource, dotenv_settings: pydantic_settings.PydanticBaseSettingsSource, file_secret_settings: pydantic_settings.PydanticBaseSettingsSource) -> tuple[pydantic_settings.PydanticBaseSettingsSource, ...] :canonical: archivebox.config.configset.BaseConfigSet.settings_customise_sources :classmethod: @@ -137,7 +181,7 @@ Bases: {py:obj}`pydantic_settings.BaseSettings` ```` -````{py:method} load_from_file(config_path: pathlib.Path) -> typing.Dict[str, str] +````{py:method} load_from_file(config_path: pathlib.Path) -> dict[str, str] :canonical: archivebox.config.configset.BaseConfigSet.load_from_file :classmethod: @@ -146,57 +190,76 @@ Bases: {py:obj}`pydantic_settings.BaseSettings` ```` -````{py:method} update_in_place(warn: bool = True, persist: bool = False, **kwargs) -> None -:canonical: archivebox.config.configset.BaseConfigSet.update_in_place +````{py:method} __getitem__(key: str) -> typing.Any +:canonical: archivebox.config.configset.BaseConfigSet.__getitem__ -```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.update_in_place +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.__getitem__ +``` + +```` + +````{py:method} __setitem__(key: str, value: typing.Any) -> None +:canonical: archivebox.config.configset.BaseConfigSet.__setitem__ + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.__setitem__ +``` + +```` + +````{py:method} update(*args, **kwargs) -> None +:canonical: archivebox.config.configset.BaseConfigSet.update + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.update +``` + +```` + +````{py:method} __contains__(key: str) -> bool +:canonical: archivebox.config.configset.BaseConfigSet.__contains__ + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.__contains__ +``` + +```` + +````{py:method} get(key: str, default: typing.Any = None) -> typing.Any +:canonical: archivebox.config.configset.BaseConfigSet.get + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.get +``` + +```` + +````{py:method} as_dict() -> dict[str, typing.Any] +:canonical: archivebox.config.configset.BaseConfigSet.as_dict + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.as_dict +``` + +```` + +````{py:method} items() +:canonical: archivebox.config.configset.BaseConfigSet.items + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.items +``` + +```` + +````{py:method} keys() +:canonical: archivebox.config.configset.BaseConfigSet.keys + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.keys +``` + +```` + +````{py:method} values() +:canonical: archivebox.config.configset.BaseConfigSet.values + +```{autodoc2-docstring} archivebox.config.configset.BaseConfigSet.values ``` ```` ````` - -````{py:function} get_config(defaults: typing.Optional[typing.Dict] = None, persona: typing.Any = None, user: typing.Any = None, crawl: typing.Any = None, snapshot: typing.Any = None, archiveresult: typing.Any = None, machine: typing.Any = None) -> typing.Dict[str, typing.Any] -:canonical: archivebox.config.configset.get_config - -```{autodoc2-docstring} archivebox.config.configset.get_config -``` -```` - -````{py:function} get_flat_config() -> typing.Dict[str, typing.Any] -:canonical: archivebox.config.configset.get_flat_config - -```{autodoc2-docstring} archivebox.config.configset.get_flat_config -``` -```` - -````{py:function} get_all_configs() -> typing.Dict[str, archivebox.config.configset.BaseConfigSet] -:canonical: archivebox.config.configset.get_all_configs - -```{autodoc2-docstring} archivebox.config.configset.get_all_configs -``` -```` - -````{py:function} _parse_env_value(value: str, default: typing.Any = None) -> typing.Any -:canonical: archivebox.config.configset._parse_env_value - -```{autodoc2-docstring} archivebox.config.configset._parse_env_value -``` -```` - -````{py:data} DEFAULT_WORKER_CONCURRENCY -:canonical: archivebox.config.configset.DEFAULT_WORKER_CONCURRENCY -:value: > - None - -```{autodoc2-docstring} archivebox.config.configset.DEFAULT_WORKER_CONCURRENCY -``` - -```` - -````{py:function} get_worker_concurrency() -> typing.Dict[str, int] -:canonical: archivebox.config.configset.get_worker_concurrency - -```{autodoc2-docstring} archivebox.config.configset.get_worker_concurrency -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.config.constants.md b/docs/apidocs/archivebox/archivebox.config.constants.md index 427cc027..100543c8 100644 --- a/docs/apidocs/archivebox/archivebox.config.constants.md +++ b/docs/apidocs/archivebox/archivebox.config.constants.md @@ -16,7 +16,9 @@ :align: left * - {py:obj}`ConstantsDict ` - - + - ```{autodoc2-docstring} archivebox.config.constants.ConstantsDict + :summary: + ``` ```` ### Data @@ -40,7 +42,8 @@ `````{py:class} ConstantsDict :canonical: archivebox.config.constants.ConstantsDict -Bases: {py:obj}`collections.abc.Mapping` +```{autodoc2-docstring} archivebox.config.constants.ConstantsDict +``` ````{py:attribute} PACKAGE_DIR :canonical: archivebox.config.constants.ConstantsDict.PACKAGE_DIR @@ -75,6 +78,17 @@ Bases: {py:obj}`collections.abc.Mapping` ```` +````{py:attribute} USERS_DIR +:canonical: archivebox.config.constants.ConstantsDict.USERS_DIR +:type: pathlib.Path +:value: > + None + +```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.USERS_DIR +``` + +```` + ````{py:attribute} MACHINE_TYPE :canonical: archivebox.config.constants.ConstantsDict.MACHINE_TYPE :type: str @@ -284,6 +298,39 @@ Bases: {py:obj}`collections.abc.Mapping` ```` +````{py:attribute} USERS_DIR_NAME +:canonical: archivebox.config.constants.ConstantsDict.USERS_DIR_NAME +:type: str +:value: > + 'users' + +```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.USERS_DIR_NAME +``` + +```` + +````{py:attribute} SNAPSHOTS_DIR_NAME +:canonical: archivebox.config.constants.ConstantsDict.SNAPSHOTS_DIR_NAME +:type: str +:value: > + 'snapshots' + +```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.SNAPSHOTS_DIR_NAME +``` + +```` + +````{py:attribute} CRAWLS_DIR_NAME +:canonical: archivebox.config.constants.ConstantsDict.CRAWLS_DIR_NAME +:type: str +:value: > + 'crawls' + +```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.CRAWLS_DIR_NAME +``` + +```` + ````{py:attribute} SOURCES_DIR_NAME :canonical: archivebox.config.constants.ConstantsDict.SOURCES_DIR_NAME :type: str @@ -552,7 +599,7 @@ Bases: {py:obj}`collections.abc.Mapping` :canonical: archivebox.config.constants.ConstantsDict.DEFAULT_LIB_DIR :type: pathlib.Path :value: > - None + '_env_path(...)' ```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.DEFAULT_LIB_DIR ``` @@ -563,13 +610,24 @@ Bases: {py:obj}`collections.abc.Mapping` :canonical: archivebox.config.constants.ConstantsDict.DEFAULT_LIB_BIN_DIR :type: pathlib.Path :value: > - None + '_env_path(...)' ```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.DEFAULT_LIB_BIN_DIR ``` ```` +````{py:attribute} RESERVED_ARCHIVE_DIR_NAMES +:canonical: archivebox.config.constants.ConstantsDict.RESERVED_ARCHIVE_DIR_NAMES +:type: frozenset[str] +:value: > + 'frozenset(...)' + +```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.RESERVED_ARCHIVE_DIR_NAMES +``` + +```` + ````{py:attribute} TIMEZONE :canonical: archivebox.config.constants.ConstantsDict.TIMEZONE :type: str @@ -583,7 +641,7 @@ Bases: {py:obj}`collections.abc.Mapping` ````{py:attribute} DEFAULT_CLI_COLORS :canonical: archivebox.config.constants.ConstantsDict.DEFAULT_CLI_COLORS -:type: typing.Dict[str, str] +:type: dict[str, str] :value: > None @@ -594,9 +652,9 @@ Bases: {py:obj}`collections.abc.Mapping` ````{py:attribute} DISABLED_CLI_COLORS :canonical: archivebox.config.constants.ConstantsDict.DISABLED_CLI_COLORS -:type: typing.Dict[str, str] +:type: dict[str, str] :value: > - 'benedict(...)' + 'AttrDict(...)' ```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.DISABLED_CLI_COLORS ``` @@ -689,39 +747,12 @@ Bases: {py:obj}`collections.abc.Mapping` ```` -````{py:method} __benedict__() -:canonical: archivebox.config.constants.ConstantsDict.__benedict__ -:classmethod: - -```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.__benedict__ -``` - -```` - -````{py:method} __len__() -:canonical: archivebox.config.constants.ConstantsDict.__len__ -:classmethod: - -```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.__len__ -``` - -```` - -````{py:method} __iter__() -:canonical: archivebox.config.constants.ConstantsDict.__iter__ -:classmethod: - -```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.__iter__ -``` - -```` - ````` ````{py:data} CONSTANTS :canonical: archivebox.config.constants.CONSTANTS :value: > - 'ConstantsDict(...)' + None ```{autodoc2-docstring} archivebox.config.constants.CONSTANTS ``` @@ -731,7 +762,7 @@ Bases: {py:obj}`collections.abc.Mapping` ````{py:data} CONSTANTS_CONFIG :canonical: archivebox.config.constants.CONSTANTS_CONFIG :value: > - '__benedict__(...)' + 'AttrDict(...)' ```{autodoc2-docstring} archivebox.config.constants.CONSTANTS_CONFIG ``` diff --git a/docs/apidocs/archivebox/archivebox.config.django.md b/docs/apidocs/archivebox/archivebox.config.django.md index f2a96e22..e6563608 100644 --- a/docs/apidocs/archivebox/archivebox.config.django.md +++ b/docs/apidocs/archivebox/archivebox.config.django.md @@ -15,10 +15,6 @@ :class: autosummary longtable :align: left -* - {py:obj}`setup_django_minimal ` - - ```{autodoc2-docstring} archivebox.config.django.setup_django_minimal - :summary: - ``` * - {py:obj}`setup_django ` - ```{autodoc2-docstring} archivebox.config.django.setup_django :summary: @@ -31,6 +27,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`CONFIG ` + - ```{autodoc2-docstring} archivebox.config.django.CONFIG + :summary: + ``` * - {py:obj}`STDERR ` - ```{autodoc2-docstring} archivebox.config.django.STDERR :summary: @@ -43,6 +43,16 @@ ### API +````{py:data} CONFIG +:canonical: archivebox.config.django.CONFIG +:value: > + 'get_config(...)' + +```{autodoc2-docstring} archivebox.config.django.CONFIG +``` + +```` + ````{py:data} STDERR :canonical: archivebox.config.django.STDERR :value: > @@ -53,13 +63,6 @@ ```` -````{py:function} setup_django_minimal() -:canonical: archivebox.config.django.setup_django_minimal - -```{autodoc2-docstring} archivebox.config.django.setup_django_minimal -``` -```` - ````{py:data} DJANGO_SET_UP :canonical: archivebox.config.django.DJANGO_SET_UP :value: > diff --git a/docs/apidocs/archivebox/archivebox.config.ldap.md b/docs/apidocs/archivebox/archivebox.config.ldap.md index 7eab851e..cfc2478d 100644 --- a/docs/apidocs/archivebox/archivebox.config.ldap.md +++ b/docs/apidocs/archivebox/archivebox.config.ldap.md @@ -21,18 +21,6 @@ ``` ```` -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`LDAP_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.ldap.LDAP_CONFIG - :summary: - ``` -```` - ### API `````{py:class} LDAPConfig(_case_sensitive: bool | None = None, _nested_model_default_partial_update: bool | None = None, _env_prefix: str | None = None, _env_prefix_target: pydantic_settings.sources.EnvPrefixTarget | None = None, _env_file: pydantic_settings.sources.DotenvType | None = ENV_FILE_SENTINEL, _env_file_encoding: str | None = None, _env_ignore_empty: bool | None = None, _env_nested_delimiter: str | None = None, _env_nested_max_split: int | None = None, _env_parse_none_str: str | None = None, _env_parse_enums: bool | None = None, _cli_prog_name: str | None = None, _cli_parse_args: bool | list[str] | tuple[str, ...] | None = None, _cli_settings_source: pydantic_settings.sources.CliSettingsSource[typing.Any] | None = None, _cli_parse_none_str: str | None = None, _cli_hide_none_type: bool | None = None, _cli_avoid_json: bool | None = None, _cli_enforce_required: bool | None = None, _cli_use_class_docs_for_groups: bool | None = None, _cli_exit_on_error: bool | None = None, _cli_prefix: str | None = None, _cli_flag_prefix_char: str | None = None, _cli_implicit_flags: bool | typing.Literal[dual, toggle] | None = None, _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | typing.Literal[all, no_enums] | None = None, _cli_shortcuts: collections.abc.Mapping[str, str | list[str]] | None = None, _secrets_dir: pydantic_settings.sources.PathType | None = None, _build_sources: tuple[tuple[pydantic_settings.sources.PydanticBaseSettingsSource, ...], dict[str, typing.Any]] | None = None, **values: typing.Any) @@ -73,7 +61,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} LDAP_SERVER_URI :canonical: archivebox.config.ldap.LDAPConfig.LDAP_SERVER_URI -:type: typing.Optional[str] +:type: str | None :value: > 'Field(...)' @@ -84,7 +72,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} LDAP_BIND_DN :canonical: archivebox.config.ldap.LDAPConfig.LDAP_BIND_DN -:type: typing.Optional[str] +:type: str | None :value: > 'Field(...)' @@ -95,7 +83,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} LDAP_BIND_PASSWORD :canonical: archivebox.config.ldap.LDAPConfig.LDAP_BIND_PASSWORD -:type: typing.Optional[str] +:type: str | None :value: > 'Field(...)' @@ -106,7 +94,7 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ````{py:attribute} LDAP_USER_BASE :canonical: archivebox.config.ldap.LDAPConfig.LDAP_USER_BASE -:type: typing.Optional[str] +:type: str | None :value: > 'Field(...)' @@ -190,13 +178,3 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet` ```` ````` - -````{py:data} LDAP_CONFIG -:canonical: archivebox.config.ldap.LDAP_CONFIG -:value: > - 'LDAPConfig(...)' - -```{autodoc2-docstring} archivebox.config.ldap.LDAP_CONFIG -``` - -```` diff --git a/docs/apidocs/archivebox/archivebox.config.md b/docs/apidocs/archivebox/archivebox.config.md index 66b6a681..47fd1fbd 100644 --- a/docs/apidocs/archivebox/archivebox.config.md +++ b/docs/apidocs/archivebox/archivebox.config.md @@ -33,18 +33,10 @@ archivebox.config.views :class: autosummary longtable :align: left -* - {py:obj}`_get_config ` - - ```{autodoc2-docstring} archivebox.config._get_config - :summary: - ``` * - {py:obj}`__getattr__ ` - ```{autodoc2-docstring} archivebox.config.__getattr__ :summary: ``` -* - {py:obj}`get_CONFIG ` - - ```{autodoc2-docstring} archivebox.config.get_CONFIG - :summary: - ``` ```` ### Data @@ -57,6 +49,10 @@ archivebox.config.views - ```{autodoc2-docstring} archivebox.config.__order__ :summary: ``` +* - {py:obj}`__all__ ` + - ```{autodoc2-docstring} archivebox.config.__all__ + :summary: + ``` ```` ### API @@ -71,13 +67,6 @@ archivebox.config.views ```` -````{py:function} _get_config() -:canonical: archivebox.config._get_config - -```{autodoc2-docstring} archivebox.config._get_config -``` -```` - ````{py:function} __getattr__(name: str) :canonical: archivebox.config.__getattr__ @@ -85,9 +74,12 @@ archivebox.config.views ``` ```` -````{py:function} get_CONFIG() -:canonical: archivebox.config.get_CONFIG +````{py:data} __all__ +:canonical: archivebox.config.__all__ +:value: > + ('CONSTANTS', 'CONSTANTS_CONFIG', 'PACKAGE_DIR', 'DATA_DIR', 'VERSION') -```{autodoc2-docstring} archivebox.config.get_CONFIG +```{autodoc2-docstring} archivebox.config.__all__ ``` + ```` diff --git a/docs/apidocs/archivebox/archivebox.config.paths.md b/docs/apidocs/archivebox/archivebox.config.paths.md index 2c6d417a..8204e61b 100644 --- a/docs/apidocs/archivebox/archivebox.config.paths.md +++ b/docs/apidocs/archivebox/archivebox.config.paths.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_env_path ` + - ```{autodoc2-docstring} archivebox.config.paths._env_path + :summary: + ``` * - {py:obj}`_get_collection_id ` - ```{autodoc2-docstring} archivebox.config.paths._get_collection_id :summary: @@ -83,6 +87,10 @@ - ```{autodoc2-docstring} archivebox.config.paths.ARCHIVE_DIR :summary: ``` +* - {py:obj}`USERS_DIR ` + - ```{autodoc2-docstring} archivebox.config.paths.USERS_DIR + :summary: + ``` * - {py:obj}`IN_DOCKER ` - ```{autodoc2-docstring} archivebox.config.paths.IN_DOCKER :summary: @@ -117,17 +125,35 @@ ```` +````{py:function} _env_path(key: str, default: pathlib.Path) -> pathlib.Path +:canonical: archivebox.config.paths._env_path + +```{autodoc2-docstring} archivebox.config.paths._env_path +``` +```` + ````{py:data} ARCHIVE_DIR :canonical: archivebox.config.paths.ARCHIVE_DIR :type: pathlib.Path :value: > - None + '_env_path(...)' ```{autodoc2-docstring} archivebox.config.paths.ARCHIVE_DIR ``` ```` +````{py:data} USERS_DIR +:canonical: archivebox.config.paths.USERS_DIR +:type: pathlib.Path +:value: > + '_env_path(...)' + +```{autodoc2-docstring} archivebox.config.paths.USERS_DIR +``` + +```` + ````{py:data} IN_DOCKER :canonical: archivebox.config.paths.IN_DOCKER :value: > @@ -204,28 +230,28 @@ ``` ```` -````{py:function} get_or_create_working_tmp_dir(autofix=True, quiet=True) +````{py:function} get_or_create_working_tmp_dir(autofix=True, quiet=True, config: ArchiveBoxConfig | None = None, **config_kwargs) :canonical: archivebox.config.paths.get_or_create_working_tmp_dir ```{autodoc2-docstring} archivebox.config.paths.get_or_create_working_tmp_dir ``` ```` -````{py:function} get_or_create_working_lib_dir(autofix=True, quiet=False) +````{py:function} get_or_create_working_lib_dir(autofix=True, quiet=False, config: ArchiveBoxConfig | None = None, **config_kwargs) :canonical: archivebox.config.paths.get_or_create_working_lib_dir ```{autodoc2-docstring} archivebox.config.paths.get_or_create_working_lib_dir ``` ```` -````{py:function} get_data_locations() +````{py:function} get_data_locations(config: ArchiveBoxConfig | None = None, **config_kwargs) :canonical: archivebox.config.paths.get_data_locations ```{autodoc2-docstring} archivebox.config.paths.get_data_locations ``` ```` -````{py:function} get_code_locations() +````{py:function} get_code_locations(config: ArchiveBoxConfig | None = None, **config_kwargs) :canonical: archivebox.config.paths.get_code_locations ```{autodoc2-docstring} archivebox.config.paths.get_code_locations diff --git a/docs/apidocs/archivebox/archivebox.config.permissions.md b/docs/apidocs/archivebox/archivebox.config.permissions.md index 97906a6a..bd42c5c0 100644 --- a/docs/apidocs/archivebox/archivebox.config.permissions.md +++ b/docs/apidocs/archivebox/archivebox.config.permissions.md @@ -212,7 +212,7 @@ :canonical: archivebox.config.permissions.HOSTNAME :type: str :value: > - 'max(...)' + 'cast(...)' ```{autodoc2-docstring} archivebox.config.permissions.HOSTNAME ``` diff --git a/docs/apidocs/archivebox/archivebox.config.version.md b/docs/apidocs/archivebox/archivebox.config.version.md index 98a87df7..d5109064 100644 --- a/docs/apidocs/archivebox/archivebox.config.version.md +++ b/docs/apidocs/archivebox/archivebox.config.version.md @@ -43,14 +43,6 @@ - ```{autodoc2-docstring} archivebox.config.version.PACKAGE_DIR :summary: ``` -* - {py:obj}`DATA_DIR ` - - ```{autodoc2-docstring} archivebox.config.version.DATA_DIR - :summary: - ``` -* - {py:obj}`ARCHIVE_DIR ` - - ```{autodoc2-docstring} archivebox.config.version.ARCHIVE_DIR - :summary: - ``` * - {py:obj}`VERSION ` - ```{autodoc2-docstring} archivebox.config.version.VERSION :summary: @@ -80,28 +72,6 @@ ```` -````{py:data} DATA_DIR -:canonical: archivebox.config.version.DATA_DIR -:type: pathlib.Path -:value: > - 'resolve(...)' - -```{autodoc2-docstring} archivebox.config.version.DATA_DIR -``` - -```` - -````{py:data} ARCHIVE_DIR -:canonical: archivebox.config.version.ARCHIVE_DIR -:type: pathlib.Path -:value: > - None - -```{autodoc2-docstring} archivebox.config.version.ARCHIVE_DIR -``` - -```` - ````{py:function} detect_installed_version(PACKAGE_DIR: pathlib.Path = PACKAGE_DIR) :canonical: archivebox.config.version.detect_installed_version @@ -109,7 +79,7 @@ ``` ```` -````{py:function} get_COMMIT_HASH() -> typing.Optional[str] +````{py:function} get_COMMIT_HASH() -> str | None :canonical: archivebox.config.version.get_COMMIT_HASH ```{autodoc2-docstring} archivebox.config.version.get_COMMIT_HASH diff --git a/docs/apidocs/archivebox/archivebox.config.views.md b/docs/apidocs/archivebox/archivebox.config.views.md index 60b0c159..64cca826 100644 --- a/docs/apidocs/archivebox/archivebox.config.views.md +++ b/docs/apidocs/archivebox/archivebox.config.views.md @@ -15,12 +15,84 @@ :class: autosummary longtable :align: left +* - {py:obj}`is_superuser ` + - ```{autodoc2-docstring} archivebox.config.views.is_superuser + :summary: + ``` +* - {py:obj}`format_parsed_datetime ` + - ```{autodoc2-docstring} archivebox.config.views.format_parsed_datetime + :summary: + ``` +* - {py:obj}`render_code_block ` + - ```{autodoc2-docstring} archivebox.config.views.render_code_block + :summary: + ``` +* - {py:obj}`render_highlighted_json_block ` + - ```{autodoc2-docstring} archivebox.config.views.render_highlighted_json_block + :summary: + ``` +* - {py:obj}`get_plugin_docs_url ` + - ```{autodoc2-docstring} archivebox.config.views.get_plugin_docs_url + :summary: + ``` +* - {py:obj}`get_plugin_hook_source_url ` + - ```{autodoc2-docstring} archivebox.config.views.get_plugin_hook_source_url + :summary: + ``` +* - {py:obj}`get_live_config_url ` + - ```{autodoc2-docstring} archivebox.config.views.get_live_config_url + :summary: + ``` +* - {py:obj}`get_environment_binary_url ` + - ```{autodoc2-docstring} archivebox.config.views.get_environment_binary_url + :summary: + ``` +* - {py:obj}`get_installed_binary_change_url ` + - ```{autodoc2-docstring} archivebox.config.views.get_installed_binary_change_url + :summary: + ``` +* - {py:obj}`get_machine_admin_url ` + - ```{autodoc2-docstring} archivebox.config.views.get_machine_admin_url + :summary: + ``` +* - {py:obj}`render_code_tag_list ` + - ```{autodoc2-docstring} archivebox.config.views.render_code_tag_list + :summary: + ``` +* - {py:obj}`render_plugin_metadata_html ` + - ```{autodoc2-docstring} archivebox.config.views.render_plugin_metadata_html + :summary: + ``` +* - {py:obj}`render_link_tag_list ` + - ```{autodoc2-docstring} archivebox.config.views.render_link_tag_list + :summary: + ``` +* - {py:obj}`render_property_links ` + - ```{autodoc2-docstring} archivebox.config.views.render_property_links + :summary: + ``` +* - {py:obj}`render_config_properties_html ` + - ```{autodoc2-docstring} archivebox.config.views.render_config_properties_html + :summary: + ``` +* - {py:obj}`render_hook_links_html ` + - ```{autodoc2-docstring} archivebox.config.views.render_hook_links_html + :summary: + ``` +* - {py:obj}`render_binary_detail_description ` + - ```{autodoc2-docstring} archivebox.config.views.render_binary_detail_description + :summary: + ``` * - {py:obj}`obj_to_yaml ` - ```{autodoc2-docstring} archivebox.config.views.obj_to_yaml :summary: ``` -* - {py:obj}`get_detected_binaries ` - - ```{autodoc2-docstring} archivebox.config.views.get_detected_binaries +* - {py:obj}`_binary_sort_key ` + - ```{autodoc2-docstring} archivebox.config.views._binary_sort_key + :summary: + ``` +* - {py:obj}`get_db_binaries_by_name ` + - ```{autodoc2-docstring} archivebox.config.views.get_db_binaries_by_name :summary: ``` * - {py:obj}`get_filesystem_plugins ` @@ -67,24 +139,213 @@ :class: autosummary longtable :align: left -* - {py:obj}`KNOWN_BINARIES ` - - ```{autodoc2-docstring} archivebox.config.views.KNOWN_BINARIES +* - {py:obj}`ABX_PLUGINS_DOCS_BASE_URL ` + - ```{autodoc2-docstring} archivebox.config.views.ABX_PLUGINS_DOCS_BASE_URL + :summary: + ``` +* - {py:obj}`ABX_PLUGINS_GITHUB_BASE_URL ` + - ```{autodoc2-docstring} archivebox.config.views.ABX_PLUGINS_GITHUB_BASE_URL + :summary: + ``` +* - {py:obj}`LIVE_CONFIG_BASE_URL ` + - ```{autodoc2-docstring} archivebox.config.views.LIVE_CONFIG_BASE_URL + :summary: + ``` +* - {py:obj}`ENVIRONMENT_BINARIES_BASE_URL ` + - ```{autodoc2-docstring} archivebox.config.views.ENVIRONMENT_BINARIES_BASE_URL + :summary: + ``` +* - {py:obj}`INSTALLED_BINARIES_BASE_URL ` + - ```{autodoc2-docstring} archivebox.config.views.INSTALLED_BINARIES_BASE_URL + :summary: + ``` +* - {py:obj}`JSON_TOKEN_RE ` + - ```{autodoc2-docstring} archivebox.config.views.JSON_TOKEN_RE :summary: ``` ```` ### API -````{py:data} KNOWN_BINARIES -:canonical: archivebox.config.views.KNOWN_BINARIES +````{py:data} ABX_PLUGINS_DOCS_BASE_URL +:canonical: archivebox.config.views.ABX_PLUGINS_DOCS_BASE_URL :value: > - ['wget', 'curl', 'chromium', 'chrome', 'google-chrome', 'google-chrome-stable', 'node', 'npm', 'npx'... + 'https://archivebox.github.io/abx-plugins/' -```{autodoc2-docstring} archivebox.config.views.KNOWN_BINARIES +```{autodoc2-docstring} archivebox.config.views.ABX_PLUGINS_DOCS_BASE_URL ``` ```` +````{py:data} ABX_PLUGINS_GITHUB_BASE_URL +:canonical: archivebox.config.views.ABX_PLUGINS_GITHUB_BASE_URL +:value: > + 'https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/' + +```{autodoc2-docstring} archivebox.config.views.ABX_PLUGINS_GITHUB_BASE_URL +``` + +```` + +````{py:data} LIVE_CONFIG_BASE_URL +:canonical: archivebox.config.views.LIVE_CONFIG_BASE_URL +:value: > + '/admin/environment/config/' + +```{autodoc2-docstring} archivebox.config.views.LIVE_CONFIG_BASE_URL +``` + +```` + +````{py:data} ENVIRONMENT_BINARIES_BASE_URL +:canonical: archivebox.config.views.ENVIRONMENT_BINARIES_BASE_URL +:value: > + '/admin/environment/binaries/' + +```{autodoc2-docstring} archivebox.config.views.ENVIRONMENT_BINARIES_BASE_URL +``` + +```` + +````{py:data} INSTALLED_BINARIES_BASE_URL +:canonical: archivebox.config.views.INSTALLED_BINARIES_BASE_URL +:value: > + '/admin/machine/binary/' + +```{autodoc2-docstring} archivebox.config.views.INSTALLED_BINARIES_BASE_URL +``` + +```` + +````{py:function} is_superuser(request: django.http.HttpRequest) -> bool +:canonical: archivebox.config.views.is_superuser + +```{autodoc2-docstring} archivebox.config.views.is_superuser +``` +```` + +````{py:function} format_parsed_datetime(value: object) -> str +:canonical: archivebox.config.views.format_parsed_datetime + +```{autodoc2-docstring} archivebox.config.views.format_parsed_datetime +``` +```` + +````{py:data} JSON_TOKEN_RE +:canonical: archivebox.config.views.JSON_TOKEN_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.config.views.JSON_TOKEN_RE +``` + +```` + +````{py:function} render_code_block(text: str, *, highlighted: bool = False) -> str +:canonical: archivebox.config.views.render_code_block + +```{autodoc2-docstring} archivebox.config.views.render_code_block +``` +```` + +````{py:function} render_highlighted_json_block(value: typing.Any) -> str +:canonical: archivebox.config.views.render_highlighted_json_block + +```{autodoc2-docstring} archivebox.config.views.render_highlighted_json_block +``` +```` + +````{py:function} get_plugin_docs_url(plugin_name: str) -> str +:canonical: archivebox.config.views.get_plugin_docs_url + +```{autodoc2-docstring} archivebox.config.views.get_plugin_docs_url +``` +```` + +````{py:function} get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str +:canonical: archivebox.config.views.get_plugin_hook_source_url + +```{autodoc2-docstring} archivebox.config.views.get_plugin_hook_source_url +``` +```` + +````{py:function} get_live_config_url(key: str) -> str +:canonical: archivebox.config.views.get_live_config_url + +```{autodoc2-docstring} archivebox.config.views.get_live_config_url +``` +```` + +````{py:function} get_environment_binary_url(name: str) -> str +:canonical: archivebox.config.views.get_environment_binary_url + +```{autodoc2-docstring} archivebox.config.views.get_environment_binary_url +``` +```` + +````{py:function} get_installed_binary_change_url(name: str, binary: archivebox.machine.models.Binary | None) -> str | None +:canonical: archivebox.config.views.get_installed_binary_change_url + +```{autodoc2-docstring} archivebox.config.views.get_installed_binary_change_url +``` +```` + +````{py:function} get_machine_admin_url() -> str | None +:canonical: archivebox.config.views.get_machine_admin_url + +```{autodoc2-docstring} archivebox.config.views.get_machine_admin_url +``` +```` + +````{py:function} render_code_tag_list(values: list[str]) -> str +:canonical: archivebox.config.views.render_code_tag_list + +```{autodoc2-docstring} archivebox.config.views.render_code_tag_list +``` +```` + +````{py:function} render_plugin_metadata_html(config: dict[str, typing.Any]) -> str +:canonical: archivebox.config.views.render_plugin_metadata_html + +```{autodoc2-docstring} archivebox.config.views.render_plugin_metadata_html +``` +```` + +````{py:function} render_link_tag_list(values: list[str], url_resolver: collections.abc.Callable[[str], str] | None = None) -> str +:canonical: archivebox.config.views.render_link_tag_list + +```{autodoc2-docstring} archivebox.config.views.render_link_tag_list +``` +```` + +````{py:function} render_property_links(prop_name: str, prop_info: dict[str, typing.Any], machine_admin_url: str | None) -> str +:canonical: archivebox.config.views.render_property_links + +```{autodoc2-docstring} archivebox.config.views.render_property_links +``` +```` + +````{py:function} render_config_properties_html(properties: dict[str, typing.Any], machine_admin_url: str | None) -> str +:canonical: archivebox.config.views.render_config_properties_html + +```{autodoc2-docstring} archivebox.config.views.render_config_properties_html +``` +```` + +````{py:function} render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str +:canonical: archivebox.config.views.render_hook_links_html + +```{autodoc2-docstring} archivebox.config.views.render_hook_links_html +``` +```` + +````{py:function} render_binary_detail_description(name: str, merged: dict[str, typing.Any], db_binary: typing.Any) -> str +:canonical: archivebox.config.views.render_binary_detail_description + +```{autodoc2-docstring} archivebox.config.views.render_binary_detail_description +``` +```` + ````{py:function} obj_to_yaml(obj: typing.Any, indent: int = 0) -> str :canonical: archivebox.config.views.obj_to_yaml @@ -92,14 +353,21 @@ ``` ```` -````{py:function} get_detected_binaries() -> typing.Dict[str, typing.Dict[str, typing.Any]] -:canonical: archivebox.config.views.get_detected_binaries +````{py:function} _binary_sort_key(binary: archivebox.machine.models.Binary) -> tuple[int, int, int, typing.Any] +:canonical: archivebox.config.views._binary_sort_key -```{autodoc2-docstring} archivebox.config.views.get_detected_binaries +```{autodoc2-docstring} archivebox.config.views._binary_sort_key ``` ```` -````{py:function} get_filesystem_plugins() -> typing.Dict[str, typing.Dict[str, typing.Any]] +````{py:function} get_db_binaries_by_name() -> dict[str, archivebox.machine.models.Binary] +:canonical: archivebox.config.views.get_db_binaries_by_name + +```{autodoc2-docstring} archivebox.config.views.get_db_binaries_by_name +``` +```` + +````{py:function} get_filesystem_plugins() -> dict[str, dict[str, typing.Any]] :canonical: archivebox.config.views.get_filesystem_plugins ```{autodoc2-docstring} archivebox.config.views.get_filesystem_plugins diff --git a/docs/apidocs/archivebox/archivebox.core.actors.md b/docs/apidocs/archivebox/archivebox.core.actors.md deleted file mode 100644 index 6672f9f4..00000000 --- a/docs/apidocs/archivebox/archivebox.core.actors.md +++ /dev/null @@ -1,8 +0,0 @@ -# {py:mod}`archivebox.core.actors` - -```{py:module} archivebox.core.actors -``` - -```{autodoc2-docstring} archivebox.core.actors -:allowtitles: -``` diff --git a/docs/apidocs/archivebox/archivebox.core.admin_archiveresults.md b/docs/apidocs/archivebox/archivebox.core.admin_archiveresults.md index b68829f9..11074cca 100644 --- a/docs/apidocs/archivebox/archivebox.core.admin_archiveresults.md +++ b/docs/apidocs/archivebox/archivebox.core.admin_archiveresults.md @@ -27,6 +27,22 @@ :class: autosummary longtable :align: left +* - {py:obj}`_get_replay_source_url ` + - ```{autodoc2-docstring} archivebox.core.admin_archiveresults._get_replay_source_url + :summary: + ``` +* - {py:obj}`build_abx_dl_display_command ` + - ```{autodoc2-docstring} archivebox.core.admin_archiveresults.build_abx_dl_display_command + :summary: + ``` +* - {py:obj}`build_abx_dl_replay_command ` + - ```{autodoc2-docstring} archivebox.core.admin_archiveresults.build_abx_dl_replay_command + :summary: + ``` +* - {py:obj}`get_plugin_admin_url ` + - ```{autodoc2-docstring} archivebox.core.admin_archiveresults.get_plugin_admin_url + :summary: + ``` * - {py:obj}`render_archiveresults_list ` - ```{autodoc2-docstring} archivebox.core.admin_archiveresults.render_archiveresults_list :summary: @@ -39,7 +55,35 @@ ### API -````{py:function} render_archiveresults_list(archiveresults_qs, limit=50) +````{py:function} _get_replay_source_url(result: archivebox.core.models.ArchiveResult) -> str +:canonical: archivebox.core.admin_archiveresults._get_replay_source_url + +```{autodoc2-docstring} archivebox.core.admin_archiveresults._get_replay_source_url +``` +```` + +````{py:function} build_abx_dl_display_command(result: archivebox.core.models.ArchiveResult) -> str +:canonical: archivebox.core.admin_archiveresults.build_abx_dl_display_command + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.build_abx_dl_display_command +``` +```` + +````{py:function} build_abx_dl_replay_command(result: archivebox.core.models.ArchiveResult, config=None) -> str +:canonical: archivebox.core.admin_archiveresults.build_abx_dl_replay_command + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.build_abx_dl_replay_command +``` +```` + +````{py:function} get_plugin_admin_url(plugin_name: str) -> str +:canonical: archivebox.core.admin_archiveresults.get_plugin_admin_url + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.get_plugin_admin_url +``` +```` + +````{py:function} render_archiveresults_list(archiveresults_qs, limit=50, config=None) :canonical: archivebox.core.admin_archiveresults.render_archiveresults_list ```{autodoc2-docstring} archivebox.core.admin_archiveresults.render_archiveresults_list @@ -198,16 +242,36 @@ Bases: {py:obj}`django.contrib.admin.TabularInline` Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` +````{py:attribute} list_select_related +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_select_related +:value: > + () + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_select_related +``` + +```` + ````{py:attribute} list_display :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_display :value: > - ('id', 'created_at', 'snapshot_info', 'tags_str', 'status', 'plugin_with_icon', 'cmd_str', 'output_s... + ('details_link', 'zip_link', 'created_at', 'snapshot_info', 'tags_inline', 'status_badge', 'plugin_w... ```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_display ``` ```` +````{py:attribute} list_display_links +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_display_links +:value: > + None + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_display_links +``` + +```` + ````{py:attribute} sort_fields :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.sort_fields :value: > @@ -221,7 +285,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} readonly_fields :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.readonly_fields :value: > - ('cmd', 'cmd_version', 'pwd', 'cmd_str', 'snapshot_info', 'tags_str', 'created_at', 'modified_at', '... + ('admin_actions', 'cmd', 'cmd_version', 'pwd', 'cmd_str', 'snapshot_info', 'tags_str', 'created_at',... ```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.readonly_fields ``` @@ -231,7 +295,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} search_fields :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.search_fields :value: > - ('id', 'snapshot__url', 'plugin', 'output_str', 'cmd_version', 'cmd', 'snapshot__timestamp') + ('snapshot__id', 'snapshot__url', 'snapshot__tags__name', 'snapshot__crawl_id', 'plugin', 'hook_name... ```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.search_fields ``` @@ -251,7 +315,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} fieldsets :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.fieldsets :value: > - (('Snapshot',), ('Plugin',), ('Timing',), ('Command',), ('Output',)) + (('Actions',), ('Snapshot',), ('Plugin',), ('Timing',), ('Command',), ('Output',)) ```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.fieldsets ``` @@ -281,7 +345,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} list_per_page :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_per_page :value: > - None + 50 ```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.list_per_page ``` @@ -308,6 +372,16 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:attribute} show_full_result_count +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.show_full_result_count +:value: > + False + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.show_full_result_count +``` + +```` + ````{py:attribute} actions :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.actions :value: > @@ -354,6 +428,69 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} changelist_view(request, extra_context=None) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.changelist_view + +```` + +````{py:method} get_queryset(request) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_queryset + +```` + +````{py:method} get_search_results(request, queryset, search_term) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_search_results + +```` + +````{py:method} get_snapshot_view_url(result: archivebox.core.models.ArchiveResult) -> str +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_snapshot_view_url + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_snapshot_view_url +``` + +```` + +````{py:method} get_output_view_url(result: archivebox.core.models.ArchiveResult) -> str +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_output_view_url + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_output_view_url +``` + +```` + +````{py:method} get_output_files_url(result: archivebox.core.models.ArchiveResult) -> str +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_output_files_url + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_output_files_url +``` + +```` + +````{py:method} get_output_zip_url(result: archivebox.core.models.ArchiveResult) -> str +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_output_zip_url + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.get_output_zip_url +``` + +```` + +````{py:method} details_link(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.details_link + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.details_link +``` + +```` + +````{py:method} zip_link(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.zip_link + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.zip_link +``` + +```` + ````{py:method} snapshot_info(result) :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.snapshot_info @@ -370,6 +507,22 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} tags_inline(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.tags_inline + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.tags_inline +``` + +```` + +````{py:method} status_badge(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.status_badge + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.status_badge +``` + +```` + ````{py:method} plugin_with_icon(result) :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.plugin_with_icon @@ -378,6 +531,22 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} process_link(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.process_link + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.process_link +``` + +```` + +````{py:method} machine_link(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.machine_link + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.machine_link +``` + +```` + ````{py:method} cmd_str(result) :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.cmd_str @@ -394,6 +563,22 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} output_str_display(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.output_str_display + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.output_str_display +``` + +```` + +````{py:method} admin_actions(result) +:canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.admin_actions + +```{autodoc2-docstring} archivebox.core.admin_archiveresults.ArchiveResultAdmin.admin_actions +``` + +```` + ````{py:method} output_summary(result) :canonical: archivebox.core.admin_archiveresults.ArchiveResultAdmin.output_summary diff --git a/docs/apidocs/archivebox/archivebox.core.admin_site.md b/docs/apidocs/archivebox/archivebox.core.admin_site.md index a1364823..c77309dd 100644 --- a/docs/apidocs/archivebox/archivebox.core.admin_site.md +++ b/docs/apidocs/archivebox/archivebox.core.admin_site.md @@ -90,6 +90,62 @@ Bases: {py:obj}`django.contrib.admin.AdminSite` ```` +````{py:method} each_context(request: django.http.HttpRequest) -> dict[str, typing.Any] +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin.each_context + +```` + +````{py:method} _format_object_count(count: int) -> tuple[int, str, str] +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin._format_object_count +:staticmethod: + +```{autodoc2-docstring} archivebox.core.admin_site.ArchiveBoxAdmin._format_object_count +``` + +```` + +````{py:method} _set_model_object_count(models_by_table: dict[str, list[dict[str, typing.Any]]], table: str, count: int, title: str | None = None) -> None +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin._set_model_object_count + +```{autodoc2-docstring} archivebox.core.admin_site.ArchiveBoxAdmin._set_model_object_count +``` + +```` + +````{py:method} get_app_list(request: django.http.HttpRequest, app_label: str | None = None) -> list[admin_data_views.typing.AppDict] +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin.get_app_list + +```` + +````{py:method} admin_data_index_view(request: django.http.HttpRequest, **kwargs: typing.Any) -> django.template.response.TemplateResponse +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin.admin_data_index_view + +```{autodoc2-docstring} archivebox.core.admin_site.ArchiveBoxAdmin.admin_data_index_view +``` + +```` + +````{py:method} index(request: django.http.HttpRequest, extra_context: dict[str, typing.Any] | None = None) -> django.template.response.TemplateResponse +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin.index + +```` + +````{py:method} get_admin_data_urls() -> list[URLResolver | URLPattern] +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin.get_admin_data_urls + +```{autodoc2-docstring} archivebox.core.admin_site.ArchiveBoxAdmin.get_admin_data_urls +``` + +```` + +````{py:method} get_urls() -> list[URLResolver | URLPattern] +:canonical: archivebox.core.admin_site.ArchiveBoxAdmin.get_urls + +```{autodoc2-docstring} archivebox.core.admin_site.ArchiveBoxAdmin.get_urls +``` + +```` + ````` ````{py:data} archivebox_admin diff --git a/docs/apidocs/archivebox/archivebox.core.admin_snapshots.md b/docs/apidocs/archivebox/archivebox.core.admin_snapshots.md index 0cb68b95..bdf496ee 100644 --- a/docs/apidocs/archivebox/archivebox.core.admin_snapshots.md +++ b/docs/apidocs/archivebox/archivebox.core.admin_snapshots.md @@ -21,6 +21,42 @@ - ```{autodoc2-docstring} archivebox.core.admin_snapshots.TagNameListFilter :summary: ``` +* - {py:obj}`SnapshotPermissionsListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotPermissionsListFilter + :summary: + ``` +* - {py:obj}`SnapshotStatusListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotStatusListFilter + :summary: + ``` +* - {py:obj}`SnapshotDepthListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotDepthListFilter + :summary: + ``` +* - {py:obj}`SnapshotRelationListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRelationListFilter + :summary: + ``` +* - {py:obj}`SnapshotArchiveStateListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter + :summary: + ``` +* - {py:obj}`SnapshotSizeListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotSizeListFilter + :summary: + ``` +* - {py:obj}`SnapshotRetryListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRetryListFilter + :summary: + ``` +* - {py:obj}`SnapshotResultHealthListFilter ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotResultHealthListFilter + :summary: + ``` +* - {py:obj}`SnapshotChangeList ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotChangeList + :summary: + ``` * - {py:obj}`SnapshotAdminForm ` - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdminForm :summary: @@ -29,6 +65,18 @@ - ```` +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_plugin_sort_order ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots._plugin_sort_order + :summary: + ``` +```` + ### Data ````{list-table} @@ -39,6 +87,10 @@ - ```{autodoc2-docstring} archivebox.core.admin_snapshots.GLOBAL_CONTEXT :summary: ``` +* - {py:obj}`SNAPSHOT_PERMISSION_META ` + - ```{autodoc2-docstring} archivebox.core.admin_snapshots.SNAPSHOT_PERMISSION_META + :summary: + ``` ```` ### API @@ -53,6 +105,23 @@ ```` +````{py:data} SNAPSHOT_PERMISSION_META +:canonical: archivebox.core.admin_snapshots.SNAPSHOT_PERMISSION_META +:value: > + None + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SNAPSHOT_PERMISSION_META +``` + +```` + +````{py:function} _plugin_sort_order() -> dict[str, int] +:canonical: archivebox.core.admin_snapshots._plugin_sort_order + +```{autodoc2-docstring} archivebox.core.admin_snapshots._plugin_sort_order +``` +```` + `````{py:class} SnapshotActionForm(*args, **kwargs) :canonical: archivebox.core.admin_snapshots.SnapshotActionForm @@ -114,6 +183,398 @@ Bases: {py:obj}`django.contrib.admin.SimpleListFilter` ````` +`````{py:class} SnapshotPermissionsListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotPermissionsListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotPermissionsListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.title +:value: > + 'permission' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.parameter_name +:value: > + 'permissions' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotPermissionsListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotStatusListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotStatusListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotStatusListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotStatusListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotStatusListFilter.title +:value: > + 'snapshot status' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotStatusListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotStatusListFilter.parameter_name +:value: > + 'snapshot_status' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotStatusListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotStatusListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotStatusListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotDepthListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotDepthListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotDepthListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotDepthListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotDepthListFilter.title +:value: > + 'depth' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotDepthListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotDepthListFilter.parameter_name +:value: > + 'depth_bucket' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotDepthListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotDepthListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotDepthListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotRelationListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotRelationListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRelationListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRelationListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotRelationListFilter.title +:value: > + 'crawl position' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRelationListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotRelationListFilter.parameter_name +:value: > + 'position' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRelationListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotRelationListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotRelationListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotArchiveStateListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.title +:value: > + 'archive state' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.parameter_name +:value: > + 'archive_state' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotArchiveStateListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotSizeListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotSizeListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotSizeListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotSizeListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotSizeListFilter.title +:value: > + 'size' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotSizeListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotSizeListFilter.parameter_name +:value: > + 'size' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotSizeListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotSizeListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotSizeListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotRetryListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotRetryListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRetryListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRetryListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotRetryListFilter.title +:value: > + 'retry' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRetryListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotRetryListFilter.parameter_name +:value: > + 'retry' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotRetryListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotRetryListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotRetryListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotResultHealthListFilter(request, params, model, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotResultHealthListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotResultHealthListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.title +:value: > + 'ArchiveResult status' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.parameter_name +:value: > + 'archiveresult_status' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotResultHealthListFilter.queryset + +```` + +````` + +`````{py:class} SnapshotChangeList(request, *args, **kwargs) +:canonical: archivebox.core.admin_snapshots.SnapshotChangeList + +Bases: {py:obj}`archivebox.search.admin.SearchResultsChangeList` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotChangeList +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotChangeList.__init__ +``` + +````{py:method} get_results(request) +:canonical: archivebox.core.admin_snapshots.SnapshotChangeList.get_results + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotChangeList.get_results +``` + +```` + +````` + ``````{py:class} SnapshotAdminForm(*args, **kwargs) :canonical: archivebox.core.admin_snapshots.SnapshotAdminForm @@ -138,6 +599,16 @@ Bases: {py:obj}`django.forms.ModelForm` ```` +````{py:attribute} permissions_config +:canonical: archivebox.core.admin_snapshots.SnapshotAdminForm.permissions_config +:value: > + 'ChoiceField(...)' + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdminForm.permissions_config +``` + +```` + `````{py:class} Meta :canonical: archivebox.core.admin_snapshots.SnapshotAdminForm.Meta @@ -188,16 +659,46 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ```` +````{py:attribute} raw_id_fields +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.raw_id_fields +:value: > + ('crawl', 'parent_snapshot') + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.raw_id_fields +``` + +```` + +````{py:attribute} list_select_related +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.list_select_related +:value: > + () + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.list_select_related +``` + +```` + ````{py:attribute} list_display :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.list_display :value: > - ('created_at', 'preview_icon', 'title_str', 'tags_inline', 'status_with_progress', 'files', 'size_wi... + ('permissions_badge', 'created_at', 'preview_icon', 'title_str', 'tags_inline', 'status_with_progres... ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.list_display ``` ```` +````{py:attribute} list_display_links +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.list_display_links +:value: > + ('created_at',) + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.list_display_links +``` + +```` + ````{py:attribute} sort_fields :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.sort_fields :value: > @@ -211,7 +712,7 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ````{py:attribute} readonly_fields :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.readonly_fields :value: > - ('admin_actions', 'status_info', 'imported_timestamp', 'created_at', 'modified_at', 'downloaded_at',... + ('admin_actions', 'snapshot_summary', 'url_favicon', 'tags_badges', 'imported_timestamp', 'created_a... ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.readonly_fields ``` @@ -231,7 +732,7 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ````{py:attribute} list_filter :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.list_filter :value: > - ('created_at', 'downloaded_at', 'archiveresult__status', 'crawl__created_by') + () ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.list_filter ``` @@ -241,7 +742,7 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ````{py:attribute} fieldsets :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.fieldsets :value: > - (('Actions',), ('URL',), ('Tags',), ('Status',), ('Timestamps',), ('Relations',), ('Config',), ('Fil... + (('Actions',), ('Snapshot',), ('URL',), ('Tags',), ('Status',), ('Timestamps',), ('Relations',), ('C... ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.fieldsets ``` @@ -251,7 +752,7 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ````{py:attribute} ordering :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.ordering :value: > - ['-created_at'] + ['-timestamp'] ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.ordering ``` @@ -281,7 +782,7 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ````{py:attribute} list_per_page :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.list_per_page :value: > - 'min(...)' + 50 ```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.list_per_page ``` @@ -328,6 +829,24 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ```` +````{py:method} get_changelist(request, **kwargs) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_changelist + +```` + +````{py:method} get_ordering(request) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_ordering + +```` + +````{py:method} change_view(request, object_id, form_url='', extra_context=None) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.change_view + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.change_view +``` + +```` + ````{py:method} changelist_view(request, extra_context=None) :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.changelist_view @@ -338,16 +857,96 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ```` +````{py:method} lookup_allowed(lookup, value, request=None) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.lookup_allowed + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.lookup_allowed +``` + +```` + +````{py:method} get_snapshot_view_url(obj: archivebox.core.models.Snapshot) -> str +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_snapshot_view_url + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.get_snapshot_view_url +``` + +```` + +````{py:method} get_snapshot_files_url(obj: archivebox.core.models.Snapshot) -> str +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_snapshot_files_url + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.get_snapshot_files_url +``` + +```` + +````{py:method} get_snapshot_zip_url(obj: archivebox.core.models.Snapshot) -> str +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_snapshot_zip_url + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.get_snapshot_zip_url +``` + +```` + ````{py:method} get_urls() :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_urls ```` +````{py:method} search_stream_view(request) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.search_stream_view + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.search_stream_view +``` + +```` + +````{py:method} iter_meta_search_ids(query, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.iter_meta_search_ids + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.iter_meta_search_ids +``` + +```` + +````{py:method} iter_backend_search_ids(iterator, queryset) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.iter_backend_search_ids + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.iter_backend_search_ids +``` + +```` + +````{py:method} set_permissions_view(request, object_id) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.set_permissions_view + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.set_permissions_view +``` + +```` + +````{py:method} redo_failed_view(request, object_id) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.redo_failed_view + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.redo_failed_view +``` + +```` + ````{py:method} get_queryset(request) :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.get_queryset ```` +````{py:method} permissions_badge(obj) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.permissions_badge + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.permissions_badge +``` + +```` + ````{py:method} imported_timestamp(obj) :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.imported_timestamp @@ -396,6 +995,30 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ```` +````{py:method} tags_badges(obj) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.tags_badges + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.tags_badges +``` + +```` + +````{py:method} _get_preview_data(obj) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin._get_preview_data + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin._get_preview_data +``` + +```` + +````{py:method} url_favicon(obj) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.url_favicon + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.url_favicon +``` + +```` + ````{py:method} preview_icon(obj) :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.preview_icon @@ -404,6 +1027,14 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ```` +````{py:method} snapshot_summary(obj) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin.snapshot_summary + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin.snapshot_summary +``` + +```` + ````{py:method} files(obj) :canonical: archivebox.core.admin_snapshots.SnapshotAdmin.files @@ -452,6 +1083,22 @@ Bases: {py:obj}`archivebox.search.admin.SearchResultsAdminMixin`, {py:obj}`archi ```` +````{py:method} _get_expected_hook_total(obj) -> int +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin._get_expected_hook_total + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin._get_expected_hook_total +``` + +```` + +````{py:method} _get_prefetched_tags(obj) +:canonical: archivebox.core.admin_snapshots.SnapshotAdmin._get_prefetched_tags + +```{autodoc2-docstring} archivebox.core.admin_snapshots.SnapshotAdmin._get_prefetched_tags +``` + +```` + ````{py:method} _get_ordering_fields(request) :canonical: archivebox.core.admin_snapshots.SnapshotAdmin._get_ordering_fields diff --git a/docs/apidocs/archivebox/archivebox.core.admin_tags.md b/docs/apidocs/archivebox/archivebox.core.admin_tags.md index acbf7eba..8bb3df50 100644 --- a/docs/apidocs/archivebox/archivebox.core.admin_tags.md +++ b/docs/apidocs/archivebox/archivebox.core.admin_tags.md @@ -17,6 +17,8 @@ * - {py:obj}`TagInline ` - +* - {py:obj}`TagAdminForm ` + - * - {py:obj}`TagAdmin ` - ```` @@ -92,15 +94,98 @@ Bases: {py:obj}`django.contrib.admin.TabularInline` ````` +``````{py:class} TagAdminForm(data=None, files=None, auto_id='id_%s', prefix=None, initial=None, error_class=ErrorList, label_suffix=None, empty_permitted=False, instance=None, use_required_attribute=None, renderer=None) +:canonical: archivebox.core.admin_tags.TagAdminForm + +Bases: {py:obj}`django.forms.ModelForm` + +`````{py:class} Meta +:canonical: archivebox.core.admin_tags.TagAdminForm.Meta + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdminForm.Meta +``` + +````{py:attribute} model +:canonical: archivebox.core.admin_tags.TagAdminForm.Meta.model +:value: > + None + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdminForm.Meta.model +``` + +```` + +````{py:attribute} fields +:canonical: archivebox.core.admin_tags.TagAdminForm.Meta.fields +:value: > + '__all__' + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdminForm.Meta.fields +``` + +```` + +````{py:attribute} widgets +:canonical: archivebox.core.admin_tags.TagAdminForm.Meta.widgets +:value: > + None + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdminForm.Meta.widgets +``` + +```` + +````` + +````{py:method} clean_name() +:canonical: archivebox.core.admin_tags.TagAdminForm.clean_name + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdminForm.clean_name +``` + +```` + +`````` + `````{py:class} TagAdmin(model, admin_site) :canonical: archivebox.core.admin_tags.TagAdmin Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` +````{py:attribute} form +:canonical: archivebox.core.admin_tags.TagAdmin.form +:value: > + None + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.form +``` + +```` + +````{py:attribute} change_list_template +:canonical: archivebox.core.admin_tags.TagAdmin.change_list_template +:value: > + 'admin/core/tag/change_list.html' + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.change_list_template +``` + +```` + +````{py:attribute} change_form_template +:canonical: archivebox.core.admin_tags.TagAdmin.change_form_template +:value: > + 'admin/core/tag/change_form.html' + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.change_form_template +``` + +```` + ````{py:attribute} list_display :canonical: archivebox.core.admin_tags.TagAdmin.list_display :value: > - ('created_at', 'created_by', 'id', 'name', 'num_snapshots', 'snapshots') + ('name', 'num_snapshots', 'created_at', 'created_by') ```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.list_display ``` @@ -117,12 +202,12 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` -````{py:attribute} sort_fields -:canonical: archivebox.core.admin_tags.TagAdmin.sort_fields +````{py:attribute} search_fields +:canonical: archivebox.core.admin_tags.TagAdmin.search_fields :value: > - ('name', 'slug', 'id', 'created_by', 'created_at') + ('id', 'name') -```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.sort_fields +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.search_fields ``` ```` @@ -130,27 +215,17 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} readonly_fields :canonical: archivebox.core.admin_tags.TagAdmin.readonly_fields :value: > - ('slug', 'id', 'created_at', 'modified_at', 'snapshots') + ('id', 'created_at', 'modified_at', 'snapshots') ```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.readonly_fields ``` ```` -````{py:attribute} search_fields -:canonical: archivebox.core.admin_tags.TagAdmin.search_fields -:value: > - ('id', 'name', 'slug') - -```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.search_fields -``` - -```` - ````{py:attribute} actions :canonical: archivebox.core.admin_tags.TagAdmin.actions :value: > - ['delete_selected', 'merge_tags'] + ['delete_selected'] ```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.actions ``` @@ -160,7 +235,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} ordering :canonical: archivebox.core.admin_tags.TagAdmin.ordering :value: > - ['-created_at'] + ['name', 'id'] ```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.ordering ``` @@ -170,32 +245,60 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} fieldsets :canonical: archivebox.core.admin_tags.TagAdmin.fieldsets :value: > - (('Tag Info',), ('Metadata',), ('Snapshots',)) + (('Tag',), ('Metadata',), ('Recent Snapshots',)) ```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.fieldsets ``` ```` -````{py:attribute} paginator -:canonical: archivebox.core.admin_tags.TagAdmin.paginator +````{py:attribute} add_fieldsets +:canonical: archivebox.core.admin_tags.TagAdmin.add_fieldsets :value: > - None + (('Tag',), ('Metadata',)) -```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.paginator +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.add_fieldsets ``` ```` -````{py:method} num_snapshots(tag) -:canonical: archivebox.core.admin_tags.TagAdmin.num_snapshots +````{py:method} get_fieldsets(request: django.http.HttpRequest, obj: archivebox.core.models.Tag | None = None) +:canonical: archivebox.core.admin_tags.TagAdmin.get_fieldsets -```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.num_snapshots +```` + +````{py:method} changelist_view(request: django.http.HttpRequest, extra_context=None) +:canonical: archivebox.core.admin_tags.TagAdmin.changelist_view + +```` + +````{py:method} render_change_form(request, context, add=False, change=False, form_url='', obj=None) +:canonical: archivebox.core.admin_tags.TagAdmin.render_change_form + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.render_change_form ``` ```` -````{py:method} snapshots(tag) +````{py:method} response_add(request: django.http.HttpRequest, obj: archivebox.core.models.Tag, post_url_continue=None) +:canonical: archivebox.core.admin_tags.TagAdmin.response_add + +```` + +````{py:method} response_change(request: django.http.HttpRequest, obj: archivebox.core.models.Tag) +:canonical: archivebox.core.admin_tags.TagAdmin.response_change + +```` + +````{py:method} _redirect_to_changelist(query: str = '') -> django.http.HttpResponseRedirect +:canonical: archivebox.core.admin_tags.TagAdmin._redirect_to_changelist + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin._redirect_to_changelist +``` + +```` + +````{py:method} snapshots(tag: archivebox.core.models.Tag) :canonical: archivebox.core.admin_tags.TagAdmin.snapshots ```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.snapshots @@ -203,6 +306,14 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} num_snapshots(tag: archivebox.core.models.Tag) +:canonical: archivebox.core.admin_tags.TagAdmin.num_snapshots + +```{autodoc2-docstring} archivebox.core.admin_tags.TagAdmin.num_snapshots +``` + +```` + ````` ````{py:function} register_admin(admin_site) diff --git a/docs/apidocs/archivebox/archivebox.core.admin_users.md b/docs/apidocs/archivebox/archivebox.core.admin_users.md index 75fd2fd3..4b83e318 100644 --- a/docs/apidocs/archivebox/archivebox.core.admin_users.md +++ b/docs/apidocs/archivebox/archivebox.core.admin_users.md @@ -68,6 +68,16 @@ Bases: {py:obj}`django.contrib.auth.admin.UserAdmin` ```` +````{py:attribute} change_form_template +:canonical: archivebox.core.admin_users.CustomUserAdmin.change_form_template +:value: > + 'admin/auth/user/change_form.html' + +```{autodoc2-docstring} archivebox.core.admin_users.CustomUserAdmin.change_form_template +``` + +```` + ````{py:attribute} add_fieldsets :canonical: archivebox.core.admin_users.CustomUserAdmin.add_fieldsets :value: > @@ -88,6 +98,40 @@ Bases: {py:obj}`django.contrib.auth.admin.UserAdmin` ```` +````{py:method} get_queryset(request) +:canonical: archivebox.core.admin_users.CustomUserAdmin.get_queryset + +```` + +````{py:method} snapshot_rss_badge(obj, api_token: str = '') +:canonical: archivebox.core.admin_users.CustomUserAdmin.snapshot_rss_badge + +```{autodoc2-docstring} archivebox.core.admin_users.CustomUserAdmin.snapshot_rss_badge +``` + +```` + +````{py:method} snapshot_count_badge(obj) +:canonical: archivebox.core.admin_users.CustomUserAdmin.snapshot_count_badge + +```{autodoc2-docstring} archivebox.core.admin_users.CustomUserAdmin.snapshot_count_badge +``` + +```` + +````{py:method} snapshot_count_column(obj) +:canonical: archivebox.core.admin_users.CustomUserAdmin.snapshot_count_column + +```{autodoc2-docstring} archivebox.core.admin_users.CustomUserAdmin.snapshot_count_column +``` + +```` + +````{py:method} get_list_display(request) +:canonical: archivebox.core.admin_users.CustomUserAdmin.get_list_display + +```` + ````{py:method} snapshot_set(obj) :canonical: archivebox.core.admin_users.CustomUserAdmin.snapshot_set diff --git a/docs/apidocs/archivebox/archivebox.core.apps.md b/docs/apidocs/archivebox/archivebox.core.apps.md index cfa3f40c..525376aa 100644 --- a/docs/apidocs/archivebox/archivebox.core.apps.md +++ b/docs/apidocs/archivebox/archivebox.core.apps.md @@ -19,30 +19,8 @@ - ```` -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_ORCHESTRATOR_BOOTSTRAPPED ` - - ```{autodoc2-docstring} archivebox.core.apps._ORCHESTRATOR_BOOTSTRAPPED - :summary: - ``` -```` - ### API -````{py:data} _ORCHESTRATOR_BOOTSTRAPPED -:canonical: archivebox.core.apps._ORCHESTRATOR_BOOTSTRAPPED -:value: > - False - -```{autodoc2-docstring} archivebox.core.apps._ORCHESTRATOR_BOOTSTRAPPED -``` - -```` - `````{py:class} CoreConfig(app_name, app_module) :canonical: archivebox.core.apps.CoreConfig diff --git a/docs/apidocs/archivebox/archivebox.core.asgi.md b/docs/apidocs/archivebox/archivebox.core.asgi.md index a0bdefb0..5358a364 100644 --- a/docs/apidocs/archivebox/archivebox.core.asgi.md +++ b/docs/apidocs/archivebox/archivebox.core.asgi.md @@ -9,6 +9,18 @@ ## Module Contents +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_patch_thread_sensitive_context_shutdown ` + - ```{autodoc2-docstring} archivebox.core.asgi._patch_thread_sensitive_context_shutdown + :summary: + ``` +```` + ### Data ````{list-table} @@ -23,6 +35,13 @@ ### API +````{py:function} _patch_thread_sensitive_context_shutdown() -> None +:canonical: archivebox.core.asgi._patch_thread_sensitive_context_shutdown + +```{autodoc2-docstring} archivebox.core.asgi._patch_thread_sensitive_context_shutdown +``` +```` + ````{py:data} application :canonical: archivebox.core.asgi.application :value: > diff --git a/docs/apidocs/archivebox/archivebox.core.context_processors.md b/docs/apidocs/archivebox/archivebox.core.context_processors.md new file mode 100644 index 00000000..e2256638 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.context_processors.md @@ -0,0 +1,31 @@ +# {py:mod}`archivebox.core.context_processors` + +```{py:module} archivebox.core.context_processors +``` + +```{autodoc2-docstring} archivebox.core.context_processors +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`archivebox_globals ` + - ```{autodoc2-docstring} archivebox.core.context_processors.archivebox_globals + :summary: + ``` +```` + +### API + +````{py:function} archivebox_globals(request) +:canonical: archivebox.core.context_processors.archivebox_globals + +```{autodoc2-docstring} archivebox.core.context_processors.archivebox_globals +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.forms.md b/docs/apidocs/archivebox/archivebox.core.forms.md index 8b6eeda0..ee7c7fab 100644 --- a/docs/apidocs/archivebox/archivebox.core.forms.md +++ b/docs/apidocs/archivebox/archivebox.core.forms.md @@ -15,12 +15,12 @@ :class: autosummary longtable :align: left -* - {py:obj}`AddLinkForm ` - - -* - {py:obj}`TagWidgetMixin ` - - ```{autodoc2-docstring} archivebox.core.forms.TagWidgetMixin +* - {py:obj}`PluginConfigFormMixin ` + - ```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin :summary: ``` +* - {py:obj}`AddLinkForm ` + - * - {py:obj}`TagWidget ` - * - {py:obj}`TagField ` @@ -39,6 +39,54 @@ - ```{autodoc2-docstring} archivebox.core.forms.get_plugin_choices :summary: ``` +* - {py:obj}`get_plugin_choice_label ` + - ```{autodoc2-docstring} archivebox.core.forms.get_plugin_choice_label + :summary: + ``` +* - {py:obj}`get_choice_field ` + - ```{autodoc2-docstring} archivebox.core.forms.get_choice_field + :summary: + ``` +* - {py:obj}`_plugin_config_input_name ` + - ```{autodoc2-docstring} archivebox.core.forms._plugin_config_input_name + :summary: + ``` +* - {py:obj}`_schema_types ` + - ```{autodoc2-docstring} archivebox.core.forms._schema_types + :summary: + ``` +* - {py:obj}`_jsonish ` + - ```{autodoc2-docstring} archivebox.core.forms._jsonish + :summary: + ``` +* - {py:obj}`_same_config_value ` + - ```{autodoc2-docstring} archivebox.core.forms._same_config_value + :summary: + ``` +* - {py:obj}`_coerce_plugin_config_value ` + - ```{autodoc2-docstring} archivebox.core.forms._coerce_plugin_config_value + :summary: + ``` +* - {py:obj}`_resolve_required_binary_name ` + - ```{autodoc2-docstring} archivebox.core.forms._resolve_required_binary_name + :summary: + ``` +* - {py:obj}`_iter_required_binary_names ` + - ```{autodoc2-docstring} archivebox.core.forms._iter_required_binary_names + :summary: + ``` +* - {py:obj}`_build_required_binary_url_lookup ` + - ```{autodoc2-docstring} archivebox.core.forms._build_required_binary_url_lookup + :summary: + ``` +* - {py:obj}`_build_required_binary_links ` + - ```{autodoc2-docstring} archivebox.core.forms._build_required_binary_links + :summary: + ``` +* - {py:obj}`get_plugin_config_binary_urls ` + - ```{autodoc2-docstring} archivebox.core.forms.get_plugin_config_binary_urls + :summary: + ``` ```` ### Data @@ -51,6 +99,26 @@ - ```{autodoc2-docstring} archivebox.core.forms.DEPTH_CHOICES :summary: ``` +* - {py:obj}`PLUGIN_CONFIG_FIELD_PREFIX ` + - ```{autodoc2-docstring} archivebox.core.forms.PLUGIN_CONFIG_FIELD_PREFIX + :summary: + ``` +* - {py:obj}`PLUGIN_GROUP_DEFINITIONS ` + - ```{autodoc2-docstring} archivebox.core.forms.PLUGIN_GROUP_DEFINITIONS + :summary: + ``` +* - {py:obj}`HIDDEN_PLUGIN_CONFIG_UI_PLUGINS ` + - ```{autodoc2-docstring} archivebox.core.forms.HIDDEN_PLUGIN_CONFIG_UI_PLUGINS + :summary: + ``` +* - {py:obj}`TIMEOUT_INPUT_PATTERN ` + - ```{autodoc2-docstring} archivebox.core.forms.TIMEOUT_INPUT_PATTERN + :summary: + ``` +* - {py:obj}`_BINARY_TEMPLATE_PATTERN ` + - ```{autodoc2-docstring} archivebox.core.forms._BINARY_TEMPLATE_PATTERN + :summary: + ``` ```` ### API @@ -65,6 +133,46 @@ ```` +````{py:data} PLUGIN_CONFIG_FIELD_PREFIX +:canonical: archivebox.core.forms.PLUGIN_CONFIG_FIELD_PREFIX +:value: > + 'plugin_config__' + +```{autodoc2-docstring} archivebox.core.forms.PLUGIN_CONFIG_FIELD_PREFIX +``` + +```` + +````{py:data} PLUGIN_GROUP_DEFINITIONS +:canonical: archivebox.core.forms.PLUGIN_GROUP_DEFINITIONS +:value: > + (('main_plugins', 'Main', '', '', '', ('dom', 'screenshot', 'pdf', 'singlefile', 'wget', 'archivedot... + +```{autodoc2-docstring} archivebox.core.forms.PLUGIN_GROUP_DEFINITIONS +``` + +```` + +````{py:data} HIDDEN_PLUGIN_CONFIG_UI_PLUGINS +:canonical: archivebox.core.forms.HIDDEN_PLUGIN_CONFIG_UI_PLUGINS +:value: > + None + +```{autodoc2-docstring} archivebox.core.forms.HIDDEN_PLUGIN_CONFIG_UI_PLUGINS +``` + +```` + +````{py:data} TIMEOUT_INPUT_PATTERN +:canonical: archivebox.core.forms.TIMEOUT_INPUT_PATTERN +:value: > + '(0|[1-9][0-9]*|[0-9]+(?:\\.[0-9]+)?\\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs...' + +```{autodoc2-docstring} archivebox.core.forms.TIMEOUT_INPUT_PATTERN +``` + +```` + ````{py:function} get_plugin_choices() :canonical: archivebox.core.forms.get_plugin_choices @@ -72,15 +180,168 @@ ``` ```` +````{py:function} get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str +:canonical: archivebox.core.forms.get_plugin_choice_label + +```{autodoc2-docstring} archivebox.core.forms.get_plugin_choice_label +``` +```` + +````{py:function} get_choice_field(form: django.forms.Form, name: str) -> django.forms.ChoiceField +:canonical: archivebox.core.forms.get_choice_field + +```{autodoc2-docstring} archivebox.core.forms.get_choice_field +``` +```` + +````{py:function} _plugin_config_input_name(plugin_name: str, config_key: str) -> str +:canonical: archivebox.core.forms._plugin_config_input_name + +```{autodoc2-docstring} archivebox.core.forms._plugin_config_input_name +``` +```` + +````{py:function} _schema_types(schema: collections.abc.Mapping[str, typing.Any]) -> list[str] +:canonical: archivebox.core.forms._schema_types + +```{autodoc2-docstring} archivebox.core.forms._schema_types +``` +```` + +````{py:function} _jsonish(value: typing.Any) -> str +:canonical: archivebox.core.forms._jsonish + +```{autodoc2-docstring} archivebox.core.forms._jsonish +``` +```` + +````{py:function} _same_config_value(left: typing.Any, right: typing.Any) -> bool +:canonical: archivebox.core.forms._same_config_value + +```{autodoc2-docstring} archivebox.core.forms._same_config_value +``` +```` + +````{py:function} _coerce_plugin_config_value(raw_value: typing.Any, schema: collections.abc.Mapping[str, typing.Any]) -> typing.Any +:canonical: archivebox.core.forms._coerce_plugin_config_value + +```{autodoc2-docstring} archivebox.core.forms._coerce_plugin_config_value +``` +```` + +`````{py:class} PluginConfigFormMixin +:canonical: archivebox.core.forms.PluginConfigFormMixin + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin +``` + +````{py:attribute} plugin_groups +:canonical: archivebox.core.forms.PluginConfigFormMixin.plugin_groups +:type: list[dict[str, typing.Any]] +:value: > + None + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin.plugin_groups +``` + +```` + +````{py:method} build_plugin_groups(runtime_config: collections.abc.Mapping[str, typing.Any] | None = None) -> None +:canonical: archivebox.core.forms.PluginConfigFormMixin.build_plugin_groups + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin.build_plugin_groups +``` + +```` + +````{py:method} _build_plugin_cards(field_name: str, plugin_names: collections.abc.Iterable[str], plugin_configs: dict[str, dict[str, typing.Any]], runtime_config: collections.abc.Mapping[str, typing.Any], binary_url_lookup: collections.abc.Mapping[str, str] | None = None) -> list[dict[str, typing.Any]] +:canonical: archivebox.core.forms.PluginConfigFormMixin._build_plugin_cards + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin._build_plugin_cards +``` + +```` + +````{py:method} _build_plugin_config_field(plugin_name: str, config_key: str, prop_schema: collections.abc.Mapping[str, typing.Any], runtime_config: collections.abc.Mapping[str, typing.Any]) -> dict[str, typing.Any] +:canonical: archivebox.core.forms.PluginConfigFormMixin._build_plugin_config_field + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin._build_plugin_config_field +``` + +```` + +````{py:method} clean_plugin_config_overrides(effective_config: collections.abc.Mapping[str, typing.Any] | None = None) -> dict[str, typing.Any] +:canonical: archivebox.core.forms.PluginConfigFormMixin.clean_plugin_config_overrides + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin.clean_plugin_config_overrides +``` + +```` + +````{py:method} plugin_config_keys() -> set[str] +:canonical: archivebox.core.forms.PluginConfigFormMixin.plugin_config_keys + +```{autodoc2-docstring} archivebox.core.forms.PluginConfigFormMixin.plugin_config_keys +``` + +```` + +````` + +````{py:data} _BINARY_TEMPLATE_PATTERN +:canonical: archivebox.core.forms._BINARY_TEMPLATE_PATTERN +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.core.forms._BINARY_TEMPLATE_PATTERN +``` + +```` + +````{py:function} _resolve_required_binary_name(template_name: str, runtime_config: collections.abc.Mapping[str, typing.Any]) -> str +:canonical: archivebox.core.forms._resolve_required_binary_name + +```{autodoc2-docstring} archivebox.core.forms._resolve_required_binary_name +``` +```` + +````{py:function} _iter_required_binary_names(required_binaries: collections.abc.Iterable[typing.Any], runtime_config: collections.abc.Mapping[str, typing.Any]) -> collections.abc.Iterable[str] +:canonical: archivebox.core.forms._iter_required_binary_names + +```{autodoc2-docstring} archivebox.core.forms._iter_required_binary_names +``` +```` + +````{py:function} _build_required_binary_url_lookup(plugin_configs: collections.abc.Mapping[str, dict[str, typing.Any]], runtime_config: collections.abc.Mapping[str, typing.Any]) -> dict[str, str] +:canonical: archivebox.core.forms._build_required_binary_url_lookup + +```{autodoc2-docstring} archivebox.core.forms._build_required_binary_url_lookup +``` +```` + +````{py:function} _build_required_binary_links(required_binaries: list[dict[str, typing.Any]], runtime_config: collections.abc.Mapping[str, typing.Any], binary_url_lookup: collections.abc.Mapping[str, str] | None = None) -> list[dict[str, str]] +:canonical: archivebox.core.forms._build_required_binary_links + +```{autodoc2-docstring} archivebox.core.forms._build_required_binary_links +``` +```` + +````{py:function} get_plugin_config_binary_urls(runtime_config: collections.abc.Mapping[str, typing.Any]) -> dict[str, str] +:canonical: archivebox.core.forms.get_plugin_config_binary_urls + +```{autodoc2-docstring} archivebox.core.forms.get_plugin_config_binary_urls +``` +```` + `````{py:class} AddLinkForm(*args, **kwargs) :canonical: archivebox.core.forms.AddLinkForm -Bases: {py:obj}`django.forms.Form` +Bases: {py:obj}`archivebox.core.forms.PluginConfigFormMixin`, {py:obj}`django.forms.Form` ````{py:attribute} url :canonical: archivebox.core.forms.AddLinkForm.url :value: > - 'RegexField(...)' + 'CharField(...)' ```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.url ``` @@ -107,6 +368,76 @@ Bases: {py:obj}`django.forms.Form` ```` +````{py:attribute} max_urls +:canonical: archivebox.core.forms.AddLinkForm.max_urls +:value: > + 'IntegerField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.max_urls +``` + +```` + +````{py:attribute} crawl_max_size +:canonical: archivebox.core.forms.AddLinkForm.crawl_max_size +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.crawl_max_size +``` + +```` + +````{py:attribute} crawl_timeout +:canonical: archivebox.core.forms.AddLinkForm.crawl_timeout +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.crawl_timeout +``` + +```` + +````{py:attribute} timeout +:canonical: archivebox.core.forms.AddLinkForm.timeout +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.timeout +``` + +```` + +````{py:attribute} snapshot_max_size +:canonical: archivebox.core.forms.AddLinkForm.snapshot_max_size +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.snapshot_max_size +``` + +```` + +````{py:attribute} delete_after +:canonical: archivebox.core.forms.AddLinkForm.delete_after +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.delete_after +``` + +```` + +````{py:attribute} crawl_max_concurrent_snapshots +:canonical: archivebox.core.forms.AddLinkForm.crawl_max_concurrent_snapshots +:value: > + 'IntegerField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.crawl_max_concurrent_snapshots +``` + +```` + ````{py:attribute} notes :canonical: archivebox.core.forms.AddLinkForm.notes :value: > @@ -117,62 +448,82 @@ Bases: {py:obj}`django.forms.Form` ```` -````{py:attribute} chrome_plugins -:canonical: archivebox.core.forms.AddLinkForm.chrome_plugins +````{py:attribute} url_filters +:canonical: archivebox.core.forms.AddLinkForm.url_filters :value: > - 'MultipleChoiceField(...)' + 'Field(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.chrome_plugins +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.url_filters ``` ```` -````{py:attribute} archiving_plugins -:canonical: archivebox.core.forms.AddLinkForm.archiving_plugins +````{py:attribute} main_plugins +:canonical: archivebox.core.forms.AddLinkForm.main_plugins :value: > 'MultipleChoiceField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.archiving_plugins +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.main_plugins ``` ```` -````{py:attribute} parsing_plugins -:canonical: archivebox.core.forms.AddLinkForm.parsing_plugins +````{py:attribute} page_setup_plugins +:canonical: archivebox.core.forms.AddLinkForm.page_setup_plugins :value: > 'MultipleChoiceField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.parsing_plugins +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.page_setup_plugins ``` ```` -````{py:attribute} search_plugins -:canonical: archivebox.core.forms.AddLinkForm.search_plugins +````{py:attribute} media_plugins +:canonical: archivebox.core.forms.AddLinkForm.media_plugins :value: > 'MultipleChoiceField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.search_plugins +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.media_plugins ``` ```` -````{py:attribute} binary_plugins -:canonical: archivebox.core.forms.AddLinkForm.binary_plugins +````{py:attribute} text_plugins +:canonical: archivebox.core.forms.AddLinkForm.text_plugins :value: > 'MultipleChoiceField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.binary_plugins +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.text_plugins ``` ```` -````{py:attribute} extension_plugins -:canonical: archivebox.core.forms.AddLinkForm.extension_plugins +````{py:attribute} metadata_plugins +:canonical: archivebox.core.forms.AddLinkForm.metadata_plugins :value: > 'MultipleChoiceField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.extension_plugins +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.metadata_plugins +``` + +```` + +````{py:attribute} postprocessing_plugins +:canonical: archivebox.core.forms.AddLinkForm.postprocessing_plugins +:value: > + 'MultipleChoiceField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.postprocessing_plugins +``` + +```` + +````{py:attribute} other_plugins +:canonical: archivebox.core.forms.AddLinkForm.other_plugins +:value: > + 'MultipleChoiceField(...)' + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.other_plugins ``` ```` @@ -190,39 +541,29 @@ Bases: {py:obj}`django.forms.Form` ````{py:attribute} persona :canonical: archivebox.core.forms.AddLinkForm.persona :value: > - 'CharField(...)' + 'ModelChoiceField(...)' ```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.persona ``` ```` -````{py:attribute} overwrite -:canonical: archivebox.core.forms.AddLinkForm.overwrite +````{py:attribute} permissions +:canonical: archivebox.core.forms.AddLinkForm.permissions :value: > - 'BooleanField(...)' + 'ChoiceField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.overwrite +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.permissions ``` ```` -````{py:attribute} update -:canonical: archivebox.core.forms.AddLinkForm.update +````{py:attribute} start_paused +:canonical: archivebox.core.forms.AddLinkForm.start_paused :value: > 'BooleanField(...)' -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.update -``` - -```` - -````{py:attribute} index_only -:canonical: archivebox.core.forms.AddLinkForm.index_only -:value: > - 'BooleanField(...)' - -```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.index_only +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.start_paused ``` ```` @@ -242,6 +583,86 @@ Bases: {py:obj}`django.forms.Form` ```` +````{py:method} clean_url() +:canonical: archivebox.core.forms.AddLinkForm.clean_url + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_url +``` + +```` + +````{py:method} clean_url_filters() +:canonical: archivebox.core.forms.AddLinkForm.clean_url_filters + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_url_filters +``` + +```` + +````{py:method} clean_max_urls() +:canonical: archivebox.core.forms.AddLinkForm.clean_max_urls + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_max_urls +``` + +```` + +````{py:method} clean_crawl_max_size() +:canonical: archivebox.core.forms.AddLinkForm.clean_crawl_max_size + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_crawl_max_size +``` + +```` + +````{py:method} clean_crawl_timeout() +:canonical: archivebox.core.forms.AddLinkForm.clean_crawl_timeout + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_crawl_timeout +``` + +```` + +````{py:method} clean_timeout() +:canonical: archivebox.core.forms.AddLinkForm.clean_timeout + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_timeout +``` + +```` + +````{py:method} _clean_timeout_seconds(raw_value, field_label: str, *, blank_value) +:canonical: archivebox.core.forms.AddLinkForm._clean_timeout_seconds + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm._clean_timeout_seconds +``` + +```` + +````{py:method} clean_snapshot_max_size() +:canonical: archivebox.core.forms.AddLinkForm.clean_snapshot_max_size + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_snapshot_max_size +``` + +```` + +````{py:method} clean_delete_after() +:canonical: archivebox.core.forms.AddLinkForm.clean_delete_after + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_delete_after +``` + +```` + +````{py:method} clean_crawl_max_concurrent_snapshots() +:canonical: archivebox.core.forms.AddLinkForm.clean_crawl_max_concurrent_snapshots + +```{autodoc2-docstring} archivebox.core.forms.AddLinkForm.clean_crawl_max_concurrent_snapshots +``` + +```` + ````{py:method} clean_schedule() :canonical: archivebox.core.forms.AddLinkForm.clean_schedule @@ -252,29 +673,18 @@ Bases: {py:obj}`django.forms.Form` ````` -`````{py:class} TagWidgetMixin -:canonical: archivebox.core.forms.TagWidgetMixin +`````{py:class} TagWidget(attrs=None) +:canonical: archivebox.core.forms.TagWidget -```{autodoc2-docstring} archivebox.core.forms.TagWidgetMixin -``` +Bases: {py:obj}`django.forms.TextInput` ````{py:method} format_value(value) -:canonical: archivebox.core.forms.TagWidgetMixin.format_value - -```{autodoc2-docstring} archivebox.core.forms.TagWidgetMixin.format_value -``` +:canonical: archivebox.core.forms.TagWidget.format_value ```` ````` -```{py:class} TagWidget(attrs=None) -:canonical: archivebox.core.forms.TagWidget - -Bases: {py:obj}`archivebox.core.forms.TagWidgetMixin`, {py:obj}`django.forms.TextInput` - -``` - `````{py:class} TagField(*, max_length=None, min_length=None, strip=True, empty_value='', **kwargs) :canonical: archivebox.core.forms.TagField @@ -304,7 +714,7 @@ Bases: {py:obj}`django.forms.CharField` ```` -````{py:method} has_changed(initial_value, data_value) +````{py:method} has_changed(initial, data) :canonical: archivebox.core.forms.TagField.has_changed ```` diff --git a/docs/apidocs/archivebox/archivebox.core.host_util.md b/docs/apidocs/archivebox/archivebox.core.host_util.md new file mode 100644 index 00000000..2f111134 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.host_util.md @@ -0,0 +1,510 @@ +# {py:mod}`archivebox.core.host_util` + +```{py:module} archivebox.core.host_util +``` + +```{autodoc2-docstring} archivebox.core.host_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`split_host_port ` + - ```{autodoc2-docstring} archivebox.core.host_util.split_host_port + :summary: + ``` +* - {py:obj}`_normalize_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util._normalize_base_url + :summary: + ``` +* - {py:obj}`normalize_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.normalize_base_url + :summary: + ``` +* - {py:obj}`_csrf_trusted_origins ` + - ```{autodoc2-docstring} archivebox.core.host_util._csrf_trusted_origins + :summary: + ``` +* - {py:obj}`_allowed_hosts ` + - ```{autodoc2-docstring} archivebox.core.host_util._allowed_hosts + :summary: + ``` +* - {py:obj}`derive_base_url_from_csrf ` + - ```{autodoc2-docstring} archivebox.core.host_util.derive_base_url_from_csrf + :summary: + ``` +* - {py:obj}`get_listen_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_listen_host + :summary: + ``` +* - {py:obj}`get_listen_parts ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_listen_parts + :summary: + ``` +* - {py:obj}`_with_port ` + - ```{autodoc2-docstring} archivebox.core.host_util._with_port + :summary: + ``` +* - {py:obj}`strip_role_subdomain ` + - ```{autodoc2-docstring} archivebox.core.host_util.strip_role_subdomain + :summary: + ``` +* - {py:obj}`_is_local_bind_host ` + - ```{autodoc2-docstring} archivebox.core.host_util._is_local_bind_host + :summary: + ``` +* - {py:obj}`canonical_base_host_for_request ` + - ```{autodoc2-docstring} archivebox.core.host_util.canonical_base_host_for_request + :summary: + ``` +* - {py:obj}`_root_host_from_listen ` + - ```{autodoc2-docstring} archivebox.core.host_util._root_host_from_listen + :summary: + ``` +* - {py:obj}`get_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_base_url + :summary: + ``` +* - {py:obj}`get_base_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_base_host + :summary: + ``` +* - {py:obj}`_build_base_host ` + - ```{autodoc2-docstring} archivebox.core.host_util._build_base_host + :summary: + ``` +* - {py:obj}`get_admin_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_admin_host + :summary: + ``` +* - {py:obj}`get_web_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_web_host + :summary: + ``` +* - {py:obj}`get_api_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_api_host + :summary: + ``` +* - {py:obj}`get_public_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_public_host + :summary: + ``` +* - {py:obj}`get_snapshot_subdomain ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_subdomain + :summary: + ``` +* - {py:obj}`get_snapshot_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_host + :summary: + ``` +* - {py:obj}`get_original_host ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_original_host + :summary: + ``` +* - {py:obj}`is_snapshot_subdomain ` + - ```{autodoc2-docstring} archivebox.core.host_util.is_snapshot_subdomain + :summary: + ``` +* - {py:obj}`get_snapshot_lookup_key ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_lookup_key + :summary: + ``` +* - {py:obj}`get_listen_subdomain ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_listen_subdomain + :summary: + ``` +* - {py:obj}`host_matches ` + - ```{autodoc2-docstring} archivebox.core.host_util.host_matches + :summary: + ``` +* - {py:obj}`_scheme_from_request ` + - ```{autodoc2-docstring} archivebox.core.host_util._scheme_from_request + :summary: + ``` +* - {py:obj}`_build_base_url_for_host ` + - ```{autodoc2-docstring} archivebox.core.host_util._build_base_url_for_host + :summary: + ``` +* - {py:obj}`get_admin_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_admin_base_url + :summary: + ``` +* - {py:obj}`get_web_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_web_base_url + :summary: + ``` +* - {py:obj}`get_api_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_api_base_url + :summary: + ``` +* - {py:obj}`get_public_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_public_base_url + :summary: + ``` +* - {py:obj}`get_snapshot_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_base_url + :summary: + ``` +* - {py:obj}`get_original_base_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.get_original_base_url + :summary: + ``` +* - {py:obj}`build_admin_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.build_admin_url + :summary: + ``` +* - {py:obj}`build_web_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.build_web_url + :summary: + ``` +* - {py:obj}`build_snapshot_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.build_snapshot_url + :summary: + ``` +* - {py:obj}`build_original_url ` + - ```{autodoc2-docstring} archivebox.core.host_util.build_original_url + :summary: + ``` +* - {py:obj}`_build_url ` + - ```{autodoc2-docstring} archivebox.core.host_util._build_url + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_SNAPSHOT_ID_RE ` + - ```{autodoc2-docstring} archivebox.core.host_util._SNAPSHOT_ID_RE + :summary: + ``` +* - {py:obj}`_SNAPSHOT_SUBDOMAIN_RE ` + - ```{autodoc2-docstring} archivebox.core.host_util._SNAPSHOT_SUBDOMAIN_RE + :summary: + ``` +* - {py:obj}`_ROLE_SUBDOMAIN_LABELS ` + - ```{autodoc2-docstring} archivebox.core.host_util._ROLE_SUBDOMAIN_LABELS + :summary: + ``` +```` + +### API + +````{py:data} _SNAPSHOT_ID_RE +:canonical: archivebox.core.host_util._SNAPSHOT_ID_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.core.host_util._SNAPSHOT_ID_RE +``` + +```` + +````{py:data} _SNAPSHOT_SUBDOMAIN_RE +:canonical: archivebox.core.host_util._SNAPSHOT_SUBDOMAIN_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.core.host_util._SNAPSHOT_SUBDOMAIN_RE +``` + +```` + +````{py:data} _ROLE_SUBDOMAIN_LABELS +:canonical: archivebox.core.host_util._ROLE_SUBDOMAIN_LABELS +:value: > + ('admin', 'web', 'api', 'public') + +```{autodoc2-docstring} archivebox.core.host_util._ROLE_SUBDOMAIN_LABELS +``` + +```` + +````{py:function} split_host_port(host: str) -> tuple[str, str | None] +:canonical: archivebox.core.host_util.split_host_port + +```{autodoc2-docstring} archivebox.core.host_util.split_host_port +``` +```` + +````{py:function} _normalize_base_url(value: str | None) -> str +:canonical: archivebox.core.host_util._normalize_base_url + +```{autodoc2-docstring} archivebox.core.host_util._normalize_base_url +``` +```` + +````{py:function} normalize_base_url(value: str | None) -> str +:canonical: archivebox.core.host_util.normalize_base_url + +```{autodoc2-docstring} archivebox.core.host_util.normalize_base_url +``` +```` + +````{py:function} _csrf_trusted_origins(config) -> list[str] +:canonical: archivebox.core.host_util._csrf_trusted_origins + +```{autodoc2-docstring} archivebox.core.host_util._csrf_trusted_origins +``` +```` + +````{py:function} _allowed_hosts(config) -> set[str] +:canonical: archivebox.core.host_util._allowed_hosts + +```{autodoc2-docstring} archivebox.core.host_util._allowed_hosts +``` +```` + +````{py:function} derive_base_url_from_csrf(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.derive_base_url_from_csrf + +```{autodoc2-docstring} archivebox.core.host_util.derive_base_url_from_csrf +``` +```` + +````{py:function} get_listen_host(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_listen_host + +```{autodoc2-docstring} archivebox.core.host_util.get_listen_host +``` +```` + +````{py:function} get_listen_parts(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> tuple[str, str | None] +:canonical: archivebox.core.host_util.get_listen_parts + +```{autodoc2-docstring} archivebox.core.host_util.get_listen_parts +``` +```` + +````{py:function} _with_port(host: str, port: str | None) -> str +:canonical: archivebox.core.host_util._with_port + +```{autodoc2-docstring} archivebox.core.host_util._with_port +``` +```` + +````{py:function} strip_role_subdomain(host: str) -> str +:canonical: archivebox.core.host_util.strip_role_subdomain + +```{autodoc2-docstring} archivebox.core.host_util.strip_role_subdomain +``` +```` + +````{py:function} _is_local_bind_host(host: str) -> bool +:canonical: archivebox.core.host_util._is_local_bind_host + +```{autodoc2-docstring} archivebox.core.host_util._is_local_bind_host +``` +```` + +````{py:function} canonical_base_host_for_request(request_host: str) -> str +:canonical: archivebox.core.host_util.canonical_base_host_for_request + +```{autodoc2-docstring} archivebox.core.host_util.canonical_base_host_for_request +``` +```` + +````{py:function} _root_host_from_listen(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util._root_host_from_listen + +```{autodoc2-docstring} archivebox.core.host_util._root_host_from_listen +``` +```` + +````{py:function} get_base_url(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_base_url +``` +```` + +````{py:function} get_base_host(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_base_host + +```{autodoc2-docstring} archivebox.core.host_util.get_base_host +``` +```` + +````{py:function} _build_base_host(subdomain: str | None, request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util._build_base_host + +```{autodoc2-docstring} archivebox.core.host_util._build_base_host +``` +```` + +````{py:function} get_admin_host(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_admin_host + +```{autodoc2-docstring} archivebox.core.host_util.get_admin_host +``` +```` + +````{py:function} get_web_host(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_web_host + +```{autodoc2-docstring} archivebox.core.host_util.get_web_host +``` +```` + +````{py:function} get_api_host(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_api_host + +```{autodoc2-docstring} archivebox.core.host_util.get_api_host +``` +```` + +````{py:function} get_public_host(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_public_host + +```{autodoc2-docstring} archivebox.core.host_util.get_public_host +``` +```` + +````{py:function} get_snapshot_subdomain(snapshot_id: str) -> str +:canonical: archivebox.core.host_util.get_snapshot_subdomain + +```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_subdomain +``` +```` + +````{py:function} get_snapshot_host(snapshot_id: str, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_snapshot_host + +```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_host +``` +```` + +````{py:function} get_original_host(domain: str, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_original_host + +```{autodoc2-docstring} archivebox.core.host_util.get_original_host +``` +```` + +````{py:function} is_snapshot_subdomain(subdomain: str) -> bool +:canonical: archivebox.core.host_util.is_snapshot_subdomain + +```{autodoc2-docstring} archivebox.core.host_util.is_snapshot_subdomain +``` +```` + +````{py:function} get_snapshot_lookup_key(snapshot_ref: str) -> str +:canonical: archivebox.core.host_util.get_snapshot_lookup_key + +```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_lookup_key +``` +```` + +````{py:function} get_listen_subdomain(request_host: str, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_listen_subdomain + +```{autodoc2-docstring} archivebox.core.host_util.get_listen_subdomain +``` +```` + +````{py:function} host_matches(request_host: str, target_host: str) -> bool +:canonical: archivebox.core.host_util.host_matches + +```{autodoc2-docstring} archivebox.core.host_util.host_matches +``` +```` + +````{py:function} _scheme_from_request(request=None, config: dict[str, typing.Any] | None = None) -> str +:canonical: archivebox.core.host_util._scheme_from_request + +```{autodoc2-docstring} archivebox.core.host_util._scheme_from_request +``` +```` + +````{py:function} _build_base_url_for_host(host: str, request=None, config: dict[str, typing.Any] | None = None) -> str +:canonical: archivebox.core.host_util._build_base_url_for_host + +```{autodoc2-docstring} archivebox.core.host_util._build_base_url_for_host +``` +```` + +````{py:function} get_admin_base_url(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_admin_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_admin_base_url +``` +```` + +````{py:function} get_web_base_url(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_web_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_web_base_url +``` +```` + +````{py:function} get_api_base_url(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_api_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_api_base_url +``` +```` + +````{py:function} get_public_base_url(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_public_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_public_base_url +``` +```` + +````{py:function} get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_snapshot_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_base_url +``` +```` + +````{py:function} get_original_base_url(domain: str, request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.get_original_base_url + +```{autodoc2-docstring} archivebox.core.host_util.get_original_base_url +``` +```` + +````{py:function} build_admin_url(path: str = '', request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.build_admin_url + +```{autodoc2-docstring} archivebox.core.host_util.build_admin_url +``` +```` + +````{py:function} build_web_url(path: str = '', request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.build_web_url + +```{autodoc2-docstring} archivebox.core.host_util.build_web_url +``` +```` + +````{py:function} build_snapshot_url(snapshot_id: str, path: str = '', request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.build_snapshot_url + +```{autodoc2-docstring} archivebox.core.host_util.build_snapshot_url +``` +```` + +````{py:function} build_original_url(domain: str, path: str = '', request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.core.host_util.build_original_url + +```{autodoc2-docstring} archivebox.core.host_util.build_original_url +``` +```` + +````{py:function} _build_url(base_url: str, path: str) -> str +:canonical: archivebox.core.host_util._build_url + +```{autodoc2-docstring} archivebox.core.host_util._build_url +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.host_utils.md b/docs/apidocs/archivebox/archivebox.core.host_utils.md deleted file mode 100644 index 70b36869..00000000 --- a/docs/apidocs/archivebox/archivebox.core.host_utils.md +++ /dev/null @@ -1,372 +0,0 @@ -# {py:mod}`archivebox.core.host_utils` - -```{py:module} archivebox.core.host_utils -``` - -```{autodoc2-docstring} archivebox.core.host_utils -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`split_host_port ` - - ```{autodoc2-docstring} archivebox.core.host_utils.split_host_port - :summary: - ``` -* - {py:obj}`_normalize_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils._normalize_base_url - :summary: - ``` -* - {py:obj}`normalize_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.normalize_base_url - :summary: - ``` -* - {py:obj}`get_listen_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_listen_host - :summary: - ``` -* - {py:obj}`get_listen_parts ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_listen_parts - :summary: - ``` -* - {py:obj}`_build_listen_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils._build_listen_host - :summary: - ``` -* - {py:obj}`get_admin_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_admin_host - :summary: - ``` -* - {py:obj}`get_web_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_web_host - :summary: - ``` -* - {py:obj}`get_api_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_api_host - :summary: - ``` -* - {py:obj}`get_public_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_public_host - :summary: - ``` -* - {py:obj}`get_snapshot_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_snapshot_host - :summary: - ``` -* - {py:obj}`get_original_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_original_host - :summary: - ``` -* - {py:obj}`is_snapshot_subdomain ` - - ```{autodoc2-docstring} archivebox.core.host_utils.is_snapshot_subdomain - :summary: - ``` -* - {py:obj}`get_listen_subdomain ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_listen_subdomain - :summary: - ``` -* - {py:obj}`host_matches ` - - ```{autodoc2-docstring} archivebox.core.host_utils.host_matches - :summary: - ``` -* - {py:obj}`_scheme_from_request ` - - ```{autodoc2-docstring} archivebox.core.host_utils._scheme_from_request - :summary: - ``` -* - {py:obj}`_build_base_url_for_host ` - - ```{autodoc2-docstring} archivebox.core.host_utils._build_base_url_for_host - :summary: - ``` -* - {py:obj}`get_admin_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_admin_base_url - :summary: - ``` -* - {py:obj}`get_web_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_web_base_url - :summary: - ``` -* - {py:obj}`get_api_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_api_base_url - :summary: - ``` -* - {py:obj}`get_archive_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_archive_base_url - :summary: - ``` -* - {py:obj}`get_snapshot_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_snapshot_base_url - :summary: - ``` -* - {py:obj}`get_original_base_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.get_original_base_url - :summary: - ``` -* - {py:obj}`build_admin_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.build_admin_url - :summary: - ``` -* - {py:obj}`build_web_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.build_web_url - :summary: - ``` -* - {py:obj}`build_api_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.build_api_url - :summary: - ``` -* - {py:obj}`build_archive_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.build_archive_url - :summary: - ``` -* - {py:obj}`build_snapshot_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.build_snapshot_url - :summary: - ``` -* - {py:obj}`build_original_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils.build_original_url - :summary: - ``` -* - {py:obj}`_build_url ` - - ```{autodoc2-docstring} archivebox.core.host_utils._build_url - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_SNAPSHOT_ID_RE ` - - ```{autodoc2-docstring} archivebox.core.host_utils._SNAPSHOT_ID_RE - :summary: - ``` -```` - -### API - -````{py:data} _SNAPSHOT_ID_RE -:canonical: archivebox.core.host_utils._SNAPSHOT_ID_RE -:value: > - 'compile(...)' - -```{autodoc2-docstring} archivebox.core.host_utils._SNAPSHOT_ID_RE -``` - -```` - -````{py:function} split_host_port(host: str) -> tuple[str, str | None] -:canonical: archivebox.core.host_utils.split_host_port - -```{autodoc2-docstring} archivebox.core.host_utils.split_host_port -``` -```` - -````{py:function} _normalize_base_url(value: str | None) -> str -:canonical: archivebox.core.host_utils._normalize_base_url - -```{autodoc2-docstring} archivebox.core.host_utils._normalize_base_url -``` -```` - -````{py:function} normalize_base_url(value: str | None) -> str -:canonical: archivebox.core.host_utils.normalize_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.normalize_base_url -``` -```` - -````{py:function} get_listen_host() -> str -:canonical: archivebox.core.host_utils.get_listen_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_listen_host -``` -```` - -````{py:function} get_listen_parts() -> tuple[str, str | None] -:canonical: archivebox.core.host_utils.get_listen_parts - -```{autodoc2-docstring} archivebox.core.host_utils.get_listen_parts -``` -```` - -````{py:function} _build_listen_host(subdomain: str | None) -> str -:canonical: archivebox.core.host_utils._build_listen_host - -```{autodoc2-docstring} archivebox.core.host_utils._build_listen_host -``` -```` - -````{py:function} get_admin_host() -> str -:canonical: archivebox.core.host_utils.get_admin_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_admin_host -``` -```` - -````{py:function} get_web_host() -> str -:canonical: archivebox.core.host_utils.get_web_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_web_host -``` -```` - -````{py:function} get_api_host() -> str -:canonical: archivebox.core.host_utils.get_api_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_api_host -``` -```` - -````{py:function} get_public_host() -> str -:canonical: archivebox.core.host_utils.get_public_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_public_host -``` -```` - -````{py:function} get_snapshot_host(snapshot_id: str) -> str -:canonical: archivebox.core.host_utils.get_snapshot_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_snapshot_host -``` -```` - -````{py:function} get_original_host(domain: str) -> str -:canonical: archivebox.core.host_utils.get_original_host - -```{autodoc2-docstring} archivebox.core.host_utils.get_original_host -``` -```` - -````{py:function} is_snapshot_subdomain(subdomain: str) -> bool -:canonical: archivebox.core.host_utils.is_snapshot_subdomain - -```{autodoc2-docstring} archivebox.core.host_utils.is_snapshot_subdomain -``` -```` - -````{py:function} get_listen_subdomain(request_host: str) -> str -:canonical: archivebox.core.host_utils.get_listen_subdomain - -```{autodoc2-docstring} archivebox.core.host_utils.get_listen_subdomain -``` -```` - -````{py:function} host_matches(request_host: str, target_host: str) -> bool -:canonical: archivebox.core.host_utils.host_matches - -```{autodoc2-docstring} archivebox.core.host_utils.host_matches -``` -```` - -````{py:function} _scheme_from_request(request=None) -> str -:canonical: archivebox.core.host_utils._scheme_from_request - -```{autodoc2-docstring} archivebox.core.host_utils._scheme_from_request -``` -```` - -````{py:function} _build_base_url_for_host(host: str, request=None) -> str -:canonical: archivebox.core.host_utils._build_base_url_for_host - -```{autodoc2-docstring} archivebox.core.host_utils._build_base_url_for_host -``` -```` - -````{py:function} get_admin_base_url(request=None) -> str -:canonical: archivebox.core.host_utils.get_admin_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.get_admin_base_url -``` -```` - -````{py:function} get_web_base_url(request=None) -> str -:canonical: archivebox.core.host_utils.get_web_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.get_web_base_url -``` -```` - -````{py:function} get_api_base_url(request=None) -> str -:canonical: archivebox.core.host_utils.get_api_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.get_api_base_url -``` -```` - -````{py:function} get_archive_base_url(request=None) -> str -:canonical: archivebox.core.host_utils.get_archive_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.get_archive_base_url -``` -```` - -````{py:function} get_snapshot_base_url(snapshot_id: str, request=None) -> str -:canonical: archivebox.core.host_utils.get_snapshot_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.get_snapshot_base_url -``` -```` - -````{py:function} get_original_base_url(domain: str, request=None) -> str -:canonical: archivebox.core.host_utils.get_original_base_url - -```{autodoc2-docstring} archivebox.core.host_utils.get_original_base_url -``` -```` - -````{py:function} build_admin_url(path: str = '', request=None) -> str -:canonical: archivebox.core.host_utils.build_admin_url - -```{autodoc2-docstring} archivebox.core.host_utils.build_admin_url -``` -```` - -````{py:function} build_web_url(path: str = '', request=None) -> str -:canonical: archivebox.core.host_utils.build_web_url - -```{autodoc2-docstring} archivebox.core.host_utils.build_web_url -``` -```` - -````{py:function} build_api_url(path: str = '', request=None) -> str -:canonical: archivebox.core.host_utils.build_api_url - -```{autodoc2-docstring} archivebox.core.host_utils.build_api_url -``` -```` - -````{py:function} build_archive_url(path: str = '', request=None) -> str -:canonical: archivebox.core.host_utils.build_archive_url - -```{autodoc2-docstring} archivebox.core.host_utils.build_archive_url -``` -```` - -````{py:function} build_snapshot_url(snapshot_id: str, path: str = '', request=None) -> str -:canonical: archivebox.core.host_utils.build_snapshot_url - -```{autodoc2-docstring} archivebox.core.host_utils.build_snapshot_url -``` -```` - -````{py:function} build_original_url(domain: str, path: str = '', request=None) -> str -:canonical: archivebox.core.host_utils.build_original_url - -```{autodoc2-docstring} archivebox.core.host_utils.build_original_url -``` -```` - -````{py:function} _build_url(base_url: str, path: str) -> str -:canonical: archivebox.core.host_utils._build_url - -```{autodoc2-docstring} archivebox.core.host_utils._build_url -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.core.management.commands.archivebox.md b/docs/apidocs/archivebox/archivebox.core.management.commands.archivebox.md new file mode 100644 index 00000000..e93f3c6a --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.management.commands.archivebox.md @@ -0,0 +1,49 @@ +# {py:mod}`archivebox.core.management.commands.archivebox` + +```{py:module} archivebox.core.management.commands.archivebox +``` + +```{autodoc2-docstring} archivebox.core.management.commands.archivebox +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`Command ` + - +```` + +### API + +`````{py:class} Command(stdout=None, stderr=None, no_color=False, force_color=False) +:canonical: archivebox.core.management.commands.archivebox.Command + +Bases: {py:obj}`django.core.management.base.BaseCommand` + +````{py:attribute} help +:canonical: archivebox.core.management.commands.archivebox.Command.help +:value: > + 'Run an ArchiveBox CLI subcommand (e.g. add, remove, list, etc)' + +```{autodoc2-docstring} archivebox.core.management.commands.archivebox.Command.help +``` + +```` + +````{py:method} add_arguments(parser) +:canonical: archivebox.core.management.commands.archivebox.Command.add_arguments + +```` + +````{py:method} handle(*args, **kwargs) +:canonical: archivebox.core.management.commands.archivebox.Command.handle + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.core.management.commands.md b/docs/apidocs/archivebox/archivebox.core.management.commands.md new file mode 100644 index 00000000..201417d7 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.management.commands.md @@ -0,0 +1,17 @@ +# {py:mod}`archivebox.core.management.commands` + +```{py:module} archivebox.core.management.commands +``` + +```{autodoc2-docstring} archivebox.core.management.commands +:allowtitles: +``` + +## Submodules + +```{toctree} +:titlesonly: +:maxdepth: 1 + +archivebox.core.management.commands.archivebox +``` diff --git a/docs/apidocs/archivebox/archivebox.core.management.md b/docs/apidocs/archivebox/archivebox.core.management.md new file mode 100644 index 00000000..26ddb6c5 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.management.md @@ -0,0 +1,17 @@ +# {py:mod}`archivebox.core.management` + +```{py:module} archivebox.core.management +``` + +```{autodoc2-docstring} archivebox.core.management +:allowtitles: +``` + +## Subpackages + +```{toctree} +:titlesonly: +:maxdepth: 3 + +archivebox.core.management.commands +``` diff --git a/docs/apidocs/archivebox/archivebox.core.md b/docs/apidocs/archivebox/archivebox.core.md index aa43be5d..2e340b21 100644 --- a/docs/apidocs/archivebox/archivebox.core.md +++ b/docs/apidocs/archivebox/archivebox.core.md @@ -14,6 +14,8 @@ :maxdepth: 3 archivebox.core.templatetags +archivebox.core.management +archivebox.core.sqlite_backend ``` ## Submodules @@ -22,21 +24,26 @@ archivebox.core.templatetags :titlesonly: :maxdepth: 1 -archivebox.core.host_utils archivebox.core.admin_archiveresults archivebox.core.admin_tags archivebox.core.models archivebox.core.admin_users +archivebox.core.host_util archivebox.core.asgi archivebox.core.settings_logging archivebox.core.apps archivebox.core.widgets archivebox.core.admin_snapshots archivebox.core.forms +archivebox.core.context_processors +archivebox.core.tag_util +archivebox.core.shutdown_util archivebox.core.admin archivebox.core.admin_site archivebox.core.settings -archivebox.core.actors +archivebox.core.permissions +archivebox.core.recovery_util +archivebox.core.takeover_util archivebox.core.urls archivebox.core.middleware archivebox.core.views @@ -55,10 +62,6 @@ archivebox.core.wsgi - ```{autodoc2-docstring} archivebox.core.register_admin :summary: ``` -* - {py:obj}`get_CONFIG ` - - ```{autodoc2-docstring} archivebox.core.get_CONFIG - :summary: - ``` ```` ### Data @@ -91,10 +94,3 @@ archivebox.core.wsgi ```{autodoc2-docstring} archivebox.core.register_admin ``` ```` - -````{py:function} get_CONFIG() -:canonical: archivebox.core.get_CONFIG - -```{autodoc2-docstring} archivebox.core.get_CONFIG -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.core.middleware.md b/docs/apidocs/archivebox/archivebox.core.middleware.md index 95ff7086..4904fbf4 100644 --- a/docs/apidocs/archivebox/archivebox.core.middleware.md +++ b/docs/apidocs/archivebox/archivebox.core.middleware.md @@ -25,6 +25,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_admin_login_hint_cookie_domain ` + - ```{autodoc2-docstring} archivebox.core.middleware._admin_login_hint_cookie_domain + :summary: + ``` * - {py:obj}`detect_timezone ` - ```{autodoc2-docstring} archivebox.core.middleware.detect_timezone :summary: @@ -37,14 +41,47 @@ - ```{autodoc2-docstring} archivebox.core.middleware.CacheControlMiddleware :summary: ``` +* - {py:obj}`ServerSecurityModeMiddleware ` + - ```{autodoc2-docstring} archivebox.core.middleware.ServerSecurityModeMiddleware + :summary: + ``` * - {py:obj}`HostRoutingMiddleware ` - ```{autodoc2-docstring} archivebox.core.middleware.HostRoutingMiddleware :summary: ``` ```` +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`ADMIN_LOGIN_HINT_COOKIE ` + - ```{autodoc2-docstring} archivebox.core.middleware.ADMIN_LOGIN_HINT_COOKIE + :summary: + ``` +```` + ### API +````{py:data} ADMIN_LOGIN_HINT_COOKIE +:canonical: archivebox.core.middleware.ADMIN_LOGIN_HINT_COOKIE +:value: > + 'archivebox_admin_logged_in' + +```{autodoc2-docstring} archivebox.core.middleware.ADMIN_LOGIN_HINT_COOKIE +``` + +```` + +````{py:function} _admin_login_hint_cookie_domain(config) -> str | None +:canonical: archivebox.core.middleware._admin_login_hint_cookie_domain + +```{autodoc2-docstring} archivebox.core.middleware._admin_login_hint_cookie_domain +``` +```` + ````{py:function} detect_timezone(request, activate: bool = True) :canonical: archivebox.core.middleware.detect_timezone @@ -66,6 +103,13 @@ ``` ```` +````{py:function} ServerSecurityModeMiddleware(get_response) +:canonical: archivebox.core.middleware.ServerSecurityModeMiddleware + +```{autodoc2-docstring} archivebox.core.middleware.ServerSecurityModeMiddleware +``` +```` + ````{py:function} HostRoutingMiddleware(get_response) :canonical: archivebox.core.middleware.HostRoutingMiddleware @@ -81,7 +125,7 @@ Bases: {py:obj}`django.contrib.auth.middleware.RemoteUserMiddleware` ````{py:attribute} header :canonical: archivebox.core.middleware.ReverseProxyAuthMiddleware.header :value: > - 'format(...)' + 'HTTP_REMOTE_USER' ```{autodoc2-docstring} archivebox.core.middleware.ReverseProxyAuthMiddleware.header ``` diff --git a/docs/apidocs/archivebox/archivebox.core.models.md b/docs/apidocs/archivebox/archivebox.core.models.md index 4708c1d0..eb7560a6 100644 --- a/docs/apidocs/archivebox/archivebox.core.models.md +++ b/docs/apidocs/archivebox/archivebox.core.models.md @@ -35,10 +35,6 @@ ``` * - {py:obj}`ArchiveResult ` - -* - {py:obj}`ArchiveResultMachine ` - - ```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine - :summary: - ``` ```` ### API @@ -98,16 +94,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID` ```` -````{py:attribute} slug -:canonical: archivebox.core.models.Tag.slug -:value: > - 'SlugField(...)' - -```{autodoc2-docstring} archivebox.core.models.Tag.slug -``` - -```` - ````{py:attribute} snapshot_set :canonical: archivebox.core.models.Tag.snapshot_set :type: django.db.models.Manager[Snapshot] @@ -122,7 +108,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID` `````{py:class} Meta :canonical: archivebox.core.models.Tag.Meta -Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID.Meta` ````{py:attribute} app_label :canonical: archivebox.core.models.Tag.Meta.app_label @@ -161,8 +147,12 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} save(*args, **kwargs) -:canonical: archivebox.core.models.Tag.save +````{py:property} slug +:canonical: archivebox.core.models.Tag.slug +:type: str + +```{autodoc2-docstring} archivebox.core.models.Tag.slug +``` ```` @@ -183,7 +173,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} from_json(record: typing.Dict[str, typing.Any], overrides: typing.Dict[str, typing.Any] = None) +````{py:method} from_json(record: dict[str, typing.Any], overrides: dict[str, typing.Any] | None = None) :canonical: archivebox.core.models.Tag.from_json :staticmethod: @@ -283,6 +273,14 @@ Bases: {py:obj}`django.db.models.QuerySet` ```{autodoc2-docstring} archivebox.core.models.SnapshotQuerySet.__init__ ``` +````{py:method} paged_iterator(chunk_size: int = 500) +:canonical: archivebox.core.models.SnapshotQuerySet.paged_iterator + +```{autodoc2-docstring} archivebox.core.models.SnapshotQuerySet.paged_iterator +``` + +```` + ````{py:attribute} FILTER_TYPES :canonical: archivebox.core.models.SnapshotQuerySet.FILTER_TYPES :value: > @@ -293,7 +291,7 @@ Bases: {py:obj}`django.db.models.QuerySet` ```` -````{py:method} filter_by_patterns(patterns: typing.List[str], filter_type: str = 'exact') -> archivebox.core.models.SnapshotQuerySet +````{py:method} filter_by_patterns(patterns: list[str], filter_type: str = 'exact') -> archivebox.core.models.SnapshotQuerySet :canonical: archivebox.core.models.SnapshotQuerySet.filter_by_patterns ```{autodoc2-docstring} archivebox.core.models.SnapshotQuerySet.filter_by_patterns @@ -301,7 +299,7 @@ Bases: {py:obj}`django.db.models.QuerySet` ```` -````{py:method} search(patterns: typing.List[str]) -> archivebox.core.models.SnapshotQuerySet +````{py:method} search(patterns: list[str]) -> archivebox.core.models.SnapshotQuerySet :canonical: archivebox.core.models.SnapshotQuerySet.search ```{autodoc2-docstring} archivebox.core.models.SnapshotQuerySet.search @@ -317,7 +315,7 @@ Bases: {py:obj}`django.db.models.QuerySet` ```` -````{py:method} to_csv(cols: typing.Optional[typing.List[str]] = None, header: bool = True, separator: str = ',', ljust: int = 0) -> str +````{py:method} to_csv(cols: list[str] | None = None, header: bool = True, separator: str = ',', ljust: int = 0) -> str :canonical: archivebox.core.models.SnapshotQuerySet.to_csv ```{autodoc2-docstring} archivebox.core.models.SnapshotQuerySet.to_csv @@ -372,7 +370,7 @@ Bases: {py:obj}`models.Manager.from_queryset`\({py:obj}`SnapshotQuerySet`\) ``````{py:class} Snapshot(*args, **kwargs) :canonical: archivebox.core.models.Snapshot -Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` ````{py:attribute} id :canonical: archivebox.core.models.Snapshot.id @@ -407,7 +405,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch ````{py:attribute} url :canonical: archivebox.core.models.Snapshot.url :value: > - 'URLField(...)' + 'CharField(...)' ```{autodoc2-docstring} archivebox.core.models.Snapshot.url ``` @@ -535,6 +533,26 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch ```` +````{py:attribute} permissions +:canonical: archivebox.core.models.Snapshot.permissions +:value: > + 'GeneratedField(...)' + +```{autodoc2-docstring} archivebox.core.models.Snapshot.permissions +``` + +```` + +````{py:attribute} output_size +:canonical: archivebox.core.models.Snapshot.output_size +:value: > + 'BigIntegerField(...)' + +```{autodoc2-docstring} archivebox.core.models.Snapshot.output_size +``` + +```` + ````{py:attribute} notes :canonical: archivebox.core.models.Snapshot.notes :value: > @@ -605,6 +623,49 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch ```` +````{py:attribute} delete_after_final_statuses +:canonical: archivebox.core.models.Snapshot.delete_after_final_statuses +:value: > + () + +```{autodoc2-docstring} archivebox.core.models.Snapshot.delete_after_final_statuses +``` + +```` + +````{py:attribute} crawl_id +:canonical: archivebox.core.models.Snapshot.crawl_id +:type: uuid.UUID +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot.crawl_id +``` + +```` + +````{py:attribute} parent_snapshot_id +:canonical: archivebox.core.models.Snapshot.parent_snapshot_id +:type: uuid.UUID | None +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot.parent_snapshot_id +``` + +```` + +````{py:attribute} _prefetched_objects_cache +:canonical: archivebox.core.models.Snapshot._prefetched_objects_cache +:type: dict[str, typing.Any] +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot._prefetched_objects_cache +``` + +```` + ````{py:attribute} objects :canonical: archivebox.core.models.Snapshot.objects :value: > @@ -629,7 +690,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch `````{py:class} Meta :canonical: archivebox.core.models.Snapshot.Meta -Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta` ````{py:attribute} app_label :canonical: archivebox.core.models.Snapshot.Meta.app_label @@ -661,6 +722,16 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:attribute} indexes +:canonical: archivebox.core.models.Snapshot.Meta.indexes +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.Snapshot.Meta.indexes +``` + +```` + ````{py:attribute} constraints :canonical: archivebox.core.models.Snapshot.Meta.constraints :value: > @@ -678,6 +749,104 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} update_and_requeue(**kwargs) -> bool +:canonical: archivebox.core.models.Snapshot.update_and_requeue + +```{autodoc2-docstring} archivebox.core.models.Snapshot.update_and_requeue +``` + +```` + +````{py:method} queue_for_extraction(*, when=None) -> bool +:canonical: archivebox.core.models.Snapshot.queue_for_extraction + +```{autodoc2-docstring} archivebox.core.models.Snapshot.queue_for_extraction +``` + +```` + +````{py:method} pause(*, save: bool = True) -> bool +:canonical: archivebox.core.models.Snapshot.pause + +```{autodoc2-docstring} archivebox.core.models.Snapshot.pause +``` + +```` + +````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool +:canonical: archivebox.core.models.Snapshot.resume + +```{autodoc2-docstring} archivebox.core.models.Snapshot.resume +``` + +```` + +````{py:method} restore_paused_scheduler_marker() -> None +:canonical: archivebox.core.models.Snapshot.restore_paused_scheduler_marker + +```{autodoc2-docstring} archivebox.core.models.Snapshot.restore_paused_scheduler_marker +``` + +```` + +````{py:method} reconcile_parent_lifecycle(*, lock_seconds: int = 60) -> bool | None +:canonical: archivebox.core.models.Snapshot.reconcile_parent_lifecycle + +```{autodoc2-docstring} archivebox.core.models.Snapshot.reconcile_parent_lifecycle +``` + +```` + +````{py:method} finalize_completed_upload_results() -> int +:canonical: archivebox.core.models.Snapshot.finalize_completed_upload_results + +```{autodoc2-docstring} archivebox.core.models.Snapshot.finalize_completed_upload_results +``` + +```` + +````{py:method} reset_abandoned_results() -> tuple[int, int] +:canonical: archivebox.core.models.Snapshot.reset_abandoned_results + +```{autodoc2-docstring} archivebox.core.models.Snapshot.reset_abandoned_results +``` + +```` + +````{py:method} cancel() -> None +:canonical: archivebox.core.models.Snapshot.cancel + +```{autodoc2-docstring} archivebox.core.models.Snapshot.cancel +``` + +```` + +````{py:method} get_delete_after_config_value() +:canonical: archivebox.core.models.Snapshot.get_delete_after_config_value + +```{autodoc2-docstring} archivebox.core.models.Snapshot.get_delete_after_config_value +``` + +```` + +````{py:method} missing_delete_at_candidates() +:canonical: archivebox.core.models.Snapshot.missing_delete_at_candidates +:classmethod: + +```{autodoc2-docstring} archivebox.core.models.Snapshot.missing_delete_at_candidates +``` + +```` + +````{py:method} is_archivebox_internal_url(url: str) -> bool +:canonical: archivebox.core.models.Snapshot.is_archivebox_internal_url +:classmethod: + +```{autodoc2-docstring} archivebox.core.models.Snapshot.is_archivebox_internal_url +``` + +```` + ````{py:property} created_by :canonical: archivebox.core.models.Snapshot.created_by @@ -733,7 +902,32 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _fs_migrate_from_0_8_0_to_0_9_0() +````{py:method} is_legacy_archive_dir(path: pathlib.Path) -> bool +:canonical: archivebox.core.models.Snapshot.is_legacy_archive_dir +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.Snapshot.is_legacy_archive_dir +``` + +```` + +````{py:method} migrate_filesystem_to_current_version(source_dir: pathlib.Path | None = None, config: ArchiveBoxBaseConfig | None = None) -> None +:canonical: archivebox.core.models.Snapshot.migrate_filesystem_to_current_version + +```{autodoc2-docstring} archivebox.core.models.Snapshot.migrate_filesystem_to_current_version +``` + +```` + +````{py:method} _fs_migrate_from_0_7_0_to_0_9_0(source_dir: pathlib.Path | None = None, config: ArchiveBoxBaseConfig | None = None) +:canonical: archivebox.core.models.Snapshot._fs_migrate_from_0_7_0_to_0_9_0 + +```{autodoc2-docstring} archivebox.core.models.Snapshot._fs_migrate_from_0_7_0_to_0_9_0 +``` + +```` + +````{py:method} _fs_migrate_from_0_8_0_to_0_9_0(source_dir: pathlib.Path | None = None, config: ArchiveBoxBaseConfig | None = None) :canonical: archivebox.core.models.Snapshot._fs_migrate_from_0_8_0_to_0_9_0 ```{autodoc2-docstring} archivebox.core.models.Snapshot._fs_migrate_from_0_8_0_to_0_9_0 @@ -741,6 +935,14 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} _fs_migrate_legacy_to_0_9_0(source_dir: pathlib.Path | None = None, target_dir: pathlib.Path | None = None, config: ArchiveBoxBaseConfig | None = None) +:canonical: archivebox.core.models.Snapshot._fs_migrate_legacy_to_0_9_0 + +```{autodoc2-docstring} archivebox.core.models.Snapshot._fs_migrate_legacy_to_0_9_0 +``` + +```` + ````{py:method} _cleanup_old_migration_dir(old_dir: pathlib.Path, new_dir: pathlib.Path) :canonical: archivebox.core.models.Snapshot._cleanup_old_migration_dir @@ -758,7 +960,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} get_storage_path_for_version(version: str) -> pathlib.Path +````{py:method} get_storage_path_for_version(version: str, config: ArchiveBoxBaseConfig | None = None) -> pathlib.Path :canonical: archivebox.core.models.Snapshot.get_storage_path_for_version ```{autodoc2-docstring} archivebox.core.models.Snapshot.get_storage_path_for_version @@ -784,7 +986,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _select_best_timestamp(index_timestamp: str, folder_name: str) -> typing.Optional[str] +````{py:method} _select_best_timestamp(index_timestamp: object | None, folder_name: str) -> str | None :canonical: archivebox.core.models.Snapshot._select_best_timestamp :staticmethod: @@ -811,7 +1013,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} reconcile_with_index() +````{py:method} reconcile_with_index(output_dir: pathlib.Path | None = None, update_existing_archive_results: bool = True) :canonical: archivebox.core.models.Snapshot.reconcile_with_index ```{autodoc2-docstring} archivebox.core.models.Snapshot.reconcile_with_index @@ -819,7 +1021,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} reconcile_with_index_json() +````{py:method} reconcile_with_index_json(output_dir: pathlib.Path | None = None, update_existing_archive_results: bool = True) :canonical: archivebox.core.models.Snapshot.reconcile_with_index_json ```{autodoc2-docstring} archivebox.core.models.Snapshot.reconcile_with_index_json @@ -843,7 +1045,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _merge_archive_results_from_index(index_data: dict) +````{py:method} _merge_archive_results_from_index(index_data: dict, update_existing: bool = True) :canonical: archivebox.core.models.Snapshot._merge_archive_results_from_index ```{autodoc2-docstring} archivebox.core.models.Snapshot._merge_archive_results_from_index @@ -851,7 +1053,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _create_archive_result_if_missing(result_data: dict, existing: dict) +````{py:method} _create_archive_result_if_missing(result_data: dict, existing: dict, update_existing: bool = True) :canonical: archivebox.core.models.Snapshot._create_archive_result_if_missing ```{autodoc2-docstring} archivebox.core.models.Snapshot._create_archive_result_if_missing @@ -867,7 +1069,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} write_index_jsonl() +````{py:method} write_index_jsonl(output_dir: pathlib.Path | None = None) :canonical: archivebox.core.models.Snapshot.write_index_jsonl ```{autodoc2-docstring} archivebox.core.models.Snapshot.write_index_jsonl @@ -875,7 +1077,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} read_index_jsonl() -> dict +````{py:method} read_index_jsonl(output_dir: pathlib.Path | None = None) -> dict :canonical: archivebox.core.models.Snapshot.read_index_jsonl ```{autodoc2-docstring} archivebox.core.models.Snapshot.read_index_jsonl @@ -883,7 +1085,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} convert_index_json_to_jsonl() -> bool +````{py:method} convert_index_json_to_jsonl(output_dir: pathlib.Path | None = None) -> bool :canonical: archivebox.core.models.Snapshot.convert_index_json_to_jsonl ```{autodoc2-docstring} archivebox.core.models.Snapshot.convert_index_json_to_jsonl @@ -909,7 +1111,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _merge_snapshots(snapshots: list[archivebox.core.models.Snapshot]) +````{py:method} _merge_snapshots(snapshots: collections.abc.Sequence[archivebox.core.models.Snapshot]) :canonical: archivebox.core.models.Snapshot._merge_snapshots :classmethod: @@ -952,7 +1154,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} icons(path: typing.Optional[str] = None) -> str +````{py:method} icons(path: str | None = None) -> str :canonical: archivebox.core.models.Snapshot.icons ```{autodoc2-docstring} archivebox.core.models.Snapshot.icons @@ -985,8 +1187,44 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:property} title_stripped +:canonical: archivebox.core.models.Snapshot.title_stripped +:type: str + +```{autodoc2-docstring} archivebox.core.models.Snapshot.title_stripped +``` + +```` + +````{py:method} _normalize_title_candidate(candidate: str | None, *, snapshot_url: str) -> str +:canonical: archivebox.core.models.Snapshot._normalize_title_candidate +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.Snapshot._normalize_title_candidate +``` + +```` + +````{py:property} resolved_title +:canonical: archivebox.core.models.Snapshot.resolved_title +:type: str + +```{autodoc2-docstring} archivebox.core.models.Snapshot.resolved_title +``` + +```` + +````{py:method} hashes_index() -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.core.models.Snapshot.hashes_index + +```{autodoc2-docstring} archivebox.core.models.Snapshot.hashes_index +``` + +```` + ````{py:property} output_dir :canonical: archivebox.core.models.Snapshot.output_dir +:type: pathlib.Path ```{autodoc2-docstring} archivebox.core.models.Snapshot.output_dir ``` @@ -1001,7 +1239,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} ensure_crawl_symlink() -> None +````{py:method} ensure_crawl_symlink(*, crawl_dir: pathlib.Path | None = None, snapshot_dir: pathlib.Path | None = None) -> None :canonical: archivebox.core.models.Snapshot.ensure_crawl_symlink ```{autodoc2-docstring} archivebox.core.models.Snapshot.ensure_crawl_symlink @@ -1049,7 +1287,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} save_tags(tags: typing.Iterable[str] = ()) -> None +````{py:method} save_tags(tags: collections.abc.Iterable[str] = ()) -> None :canonical: archivebox.core.models.Snapshot.save_tags ```{autodoc2-docstring} archivebox.core.models.Snapshot.save_tags @@ -1089,7 +1327,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} from_json(record: typing.Dict[str, typing.Any], overrides: typing.Dict[str, typing.Any] = None, queue_for_extraction: bool = True) +````{py:method} from_json(record: dict[str, typing.Any], overrides: dict[str, typing.Any] | None = None, queue_for_extraction: bool = True) :canonical: archivebox.core.models.Snapshot.from_json :staticmethod: @@ -1122,7 +1360,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} retry_failed_archiveresults(retry_at: typing.Optional[django.utils.timezone.datetime] = None) -> int +````{py:method} retry_failed_archiveresults() -> int :canonical: archivebox.core.models.Snapshot.retry_failed_archiveresults ```{autodoc2-docstring} archivebox.core.models.Snapshot.retry_failed_archiveresults @@ -1194,7 +1432,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} bookmarked_date() -> typing.Optional[str] +````{py:method} bookmarked_date() -> str | None :canonical: archivebox.core.models.Snapshot.bookmarked_date ```{autodoc2-docstring} archivebox.core.models.Snapshot.bookmarked_date @@ -1202,7 +1440,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} downloaded_datestr() -> typing.Optional[str] +````{py:method} downloaded_datestr() -> str | None :canonical: archivebox.core.models.Snapshot.downloaded_datestr ```{autodoc2-docstring} archivebox.core.models.Snapshot.downloaded_datestr @@ -1210,7 +1448,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} archive_dates() -> typing.List[datetime.datetime] +````{py:method} archive_dates() -> list[datetime.datetime] :canonical: archivebox.core.models.Snapshot.archive_dates ```{autodoc2-docstring} archivebox.core.models.Snapshot.archive_dates @@ -1218,7 +1456,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} oldest_archive_date() -> typing.Optional[datetime.datetime] +````{py:method} oldest_archive_date() -> datetime.datetime | None :canonical: archivebox.core.models.Snapshot.oldest_archive_date ```{autodoc2-docstring} archivebox.core.models.Snapshot.oldest_archive_date @@ -1226,7 +1464,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} newest_archive_date() -> typing.Optional[datetime.datetime] +````{py:method} newest_archive_date() -> datetime.datetime | None :canonical: archivebox.core.models.Snapshot.newest_archive_date ```{autodoc2-docstring} archivebox.core.models.Snapshot.newest_archive_date @@ -1250,7 +1488,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} latest_outputs(status: typing.Optional[str] = None) -> typing.Dict[str, typing.Any] +````{py:method} latest_outputs(status: str | None = None) -> dict[str, typing.Any] :canonical: archivebox.core.models.Snapshot.latest_outputs ```{autodoc2-docstring} archivebox.core.models.Snapshot.latest_outputs @@ -1258,7 +1496,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} discover_outputs() -> list[dict] +````{py:method} discover_outputs(include_filesystem_fallback: bool = True) -> list[dict] :canonical: archivebox.core.models.Snapshot.discover_outputs ```{autodoc2-docstring} archivebox.core.models.Snapshot.discover_outputs @@ -1266,7 +1504,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} to_dict(extended: bool = False) -> typing.Dict[str, typing.Any] +````{py:method} to_dict(extended: bool = False) -> dict[str, typing.Any] :canonical: archivebox.core.models.Snapshot.to_dict ```{autodoc2-docstring} archivebox.core.models.Snapshot.to_dict @@ -1282,7 +1520,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} to_csv(cols: typing.Optional[typing.List[str]] = None, separator: str = ',', ljust: int = 0) -> str +````{py:method} to_csv(cols: list[str] | None = None, separator: str = ',', ljust: int = 0) -> str :canonical: archivebox.core.models.Snapshot.to_csv ```{autodoc2-docstring} archivebox.core.models.Snapshot.to_csv @@ -1290,7 +1528,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} write_json_details(out_dir: typing.Optional[str] = None) -> None +````{py:method} write_json_details(out_dir: pathlib.Path | str | None = None) -> None :canonical: archivebox.core.models.Snapshot.write_json_details ```{autodoc2-docstring} archivebox.core.models.Snapshot.write_json_details @@ -1298,7 +1536,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} write_html_details(out_dir: typing.Optional[str] = None) -> None +````{py:method} write_html_details(out_dir: pathlib.Path | str | None = None) -> None :canonical: archivebox.core.models.Snapshot.write_html_details ```{autodoc2-docstring} archivebox.core.models.Snapshot.write_html_details @@ -1306,7 +1544,15 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _ts_to_date_str(dt: typing.Optional[datetime.datetime]) -> typing.Optional[str] +````{py:method} get_detail_page_auxiliary_items(outputs: list[dict] | None = None, hidden_card_plugins: set[str] | None = None) -> tuple[list[dict[str, object]], list[dict[str, object]]] +:canonical: archivebox.core.models.Snapshot.get_detail_page_auxiliary_items + +```{autodoc2-docstring} archivebox.core.models.Snapshot.get_detail_page_auxiliary_items +``` + +```` + +````{py:method} _ts_to_date_str(dt: datetime.datetime | None) -> str | None :canonical: archivebox.core.models.Snapshot._ts_to_date_str :staticmethod: @@ -1361,6 +1607,16 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:attribute} paused +:canonical: archivebox.core.models.SnapshotMachine.paused +:value: > + 'State(...)' + +```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.paused +``` + +```` + ````{py:attribute} sealed :canonical: archivebox.core.models.SnapshotMachine.sealed :value: > @@ -1384,13 +1640,44 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ````{py:attribute} seal :canonical: archivebox.core.models.SnapshotMachine.seal :value: > - 'to(...)' + None ```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.seal ``` ```` +````{py:attribute} pause_requested +:canonical: archivebox.core.models.SnapshotMachine.pause_requested +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.pause_requested +``` + +```` + +````{py:attribute} resume_requested +:canonical: archivebox.core.models.SnapshotMachine.resume_requested +:value: > + 'to(...)' + +```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.resume_requested +``` + +```` + +````{py:attribute} snapshot +:canonical: archivebox.core.models.SnapshotMachine.snapshot +:type: archivebox.core.models.Snapshot +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.snapshot +``` + +```` + ````{py:method} can_start() -> bool :canonical: archivebox.core.models.SnapshotMachine.can_start @@ -1407,6 +1694,14 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:method} has_finished_archive_results() -> bool +:canonical: archivebox.core.models.SnapshotMachine.has_finished_archive_results + +```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.has_finished_archive_results +``` + +```` + ````{py:method} enter_queued() :canonical: archivebox.core.models.SnapshotMachine.enter_queued @@ -1415,6 +1710,14 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:method} enter_paused() +:canonical: archivebox.core.models.SnapshotMachine.enter_paused + +```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_paused +``` + +```` + ````{py:method} enter_started() :canonical: archivebox.core.models.SnapshotMachine.enter_started @@ -1436,7 +1739,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ``````{py:class} ArchiveResult(*args, **kwargs) :canonical: archivebox.core.models.ArchiveResult -Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes` `````{py:class} StatusChoices() :canonical: archivebox.core.models.ArchiveResult.StatusChoices @@ -1463,6 +1766,16 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} PAUSED +:canonical: archivebox.core.models.ArchiveResult.StatusChoices.PAUSED +:value: > + ('paused', 'Paused') + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.StatusChoices.PAUSED +``` + +```` + ````{py:attribute} BACKOFF :canonical: archivebox.core.models.ArchiveResult.StatusChoices.BACKOFF :value: > @@ -1503,8 +1816,86 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} NORESULTS +:canonical: archivebox.core.models.ArchiveResult.StatusChoices.NORESULTS +:value: > + ('noresults', 'No Results') + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.StatusChoices.NORESULTS +``` + +```` + ````` +````{py:attribute} INITIAL_STATE +:canonical: archivebox.core.models.ArchiveResult.INITIAL_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.INITIAL_STATE +``` + +```` + +````{py:attribute} ACTIVE_STATE +:canonical: archivebox.core.models.ArchiveResult.ACTIVE_STATE +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.ACTIVE_STATE +``` + +```` + +````{py:attribute} FINAL_STATES +:canonical: archivebox.core.models.ArchiveResult.FINAL_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.FINAL_STATES +``` + +```` + +````{py:attribute} FINAL_OR_ACTIVE_STATES +:canonical: archivebox.core.models.ArchiveResult.FINAL_OR_ACTIVE_STATES +:value: > + () + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.FINAL_OR_ACTIVE_STATES +``` + +```` + +````{py:attribute} delete_after_final_statuses +:canonical: archivebox.core.models.ArchiveResult.delete_after_final_statuses +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.delete_after_final_statuses +``` + +```` + +````{py:method} normalize_status(status: str | None) -> str +:canonical: archivebox.core.models.ArchiveResult.normalize_status +:classmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.normalize_status +``` + +```` + +````{py:method} output_files_upload_complete(output_files: dict[str, dict[str, typing.Any]]) -> bool +:canonical: archivebox.core.models.ArchiveResult.output_files_upload_complete +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.output_files_upload_complete +``` + +```` + ````{py:method} get_plugin_choices() :canonical: archivebox.core.models.ArchiveResult.get_plugin_choices :classmethod: @@ -1658,7 +2049,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ````{py:attribute} status :canonical: archivebox.core.models.ArchiveResult.status :value: > - 'StatusField(...)' + 'CharField(...)' ```{autodoc2-docstring} archivebox.core.models.ArchiveResult.status ``` @@ -1668,7 +2059,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ````{py:attribute} retry_at :canonical: archivebox.core.models.ArchiveResult.retry_at :value: > - 'RetryAtField(...)' + 'DateTimeField(...)' ```{autodoc2-docstring} archivebox.core.models.ArchiveResult.retry_at ``` @@ -1685,42 +2076,24 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:attribute} state_machine_name -:canonical: archivebox.core.models.ArchiveResult.state_machine_name -:value: > - 'archivebox.core.models.ArchiveResultMachine' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.state_machine_name -``` - -```` - -````{py:attribute} retry_at_field_name -:canonical: archivebox.core.models.ArchiveResult.retry_at_field_name -:value: > - 'retry_at' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.retry_at_field_name -``` - -```` - -````{py:attribute} state_field_name -:canonical: archivebox.core.models.ArchiveResult.state_field_name -:value: > - 'status' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.state_field_name -``` - -```` - -````{py:attribute} active_state -:canonical: archivebox.core.models.ArchiveResult.active_state +````{py:attribute} snapshot_id +:canonical: archivebox.core.models.ArchiveResult.snapshot_id +:type: uuid.UUID :value: > None -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.active_state +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.snapshot_id +``` + +```` + +````{py:attribute} process_id +:canonical: archivebox.core.models.ArchiveResult.process_id +:type: uuid.UUID | None +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.process_id ``` ```` @@ -1728,7 +2101,7 @@ Bases: {py:obj}`django.db.models.TextChoices` `````{py:class} Meta :canonical: archivebox.core.models.ArchiveResult.Meta -Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta` ````{py:attribute} app_label :canonical: archivebox.core.models.ArchiveResult.Meta.app_label @@ -1770,6 +2143,16 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:attribute} constraints +:canonical: archivebox.core.models.ArchiveResult.Meta.constraints +:value: > + None + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.Meta.constraints +``` + +```` + ````` ````{py:method} __str__() @@ -1777,6 +2160,40 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} _format_output_line_for_display(line: str) -> str +:canonical: archivebox.core.models.ArchiveResult._format_output_line_for_display +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult._format_output_line_for_display +``` + +```` + +````{py:method} output_str_for_display() -> str +:canonical: archivebox.core.models.ArchiveResult.output_str_for_display + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.output_str_for_display +``` + +```` + +````{py:method} get_delete_after_config_value() +:canonical: archivebox.core.models.ArchiveResult.get_delete_after_config_value + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.get_delete_after_config_value +``` + +```` + +````{py:method} missing_delete_at_candidates() +:canonical: archivebox.core.models.ArchiveResult.missing_delete_at_candidates +:classmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.missing_delete_at_candidates +``` + +```` + ````{py:property} created_by :canonical: archivebox.core.models.ArchiveResult.created_by @@ -1793,7 +2210,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} from_json(record: typing.Dict[str, typing.Any], overrides: typing.Dict[str, typing.Any] = None) +````{py:method} from_json(record: dict[str, typing.Any], overrides: dict[str, typing.Any] | None = None) :canonical: archivebox.core.models.ArchiveResult.from_json :staticmethod: @@ -1807,6 +2224,23 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} delete(*args, **kwargs) +:canonical: archivebox.core.models.ArchiveResult.delete + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.delete +``` + +```` + +````{py:method} refresh_snapshot_output_sizes(snapshot_ids) +:canonical: archivebox.core.models.ArchiveResult.refresh_snapshot_output_sizes +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.refresh_snapshot_output_sizes +``` + +```` + ````{py:method} snapshot_dir() :canonical: archivebox.core.models.ArchiveResult.snapshot_dir @@ -1840,6 +2274,57 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} reset_for_retry(*, save: bool = True) -> None +:canonical: archivebox.core.models.ArchiveResult.reset_for_retry + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.reset_for_retry +``` + +```` + +````{py:property} is_paused +:canonical: archivebox.core.models.ArchiveResult.is_paused +:type: bool + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.is_paused +``` + +```` + +````{py:method} pause_queryset(queryset) -> int +:canonical: archivebox.core.models.ArchiveResult.pause_queryset +:classmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.pause_queryset +``` + +```` + +````{py:method} resume_queryset(queryset, *, when: datetime.datetime | None = None) -> int +:canonical: archivebox.core.models.ArchiveResult.resume_queryset +:classmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.resume_queryset +``` + +```` + +````{py:method} pause(*, save: bool = True) -> bool +:canonical: archivebox.core.models.ArchiveResult.pause + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.pause +``` + +```` + +````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool +:canonical: archivebox.core.models.ArchiveResult.resume + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.resume +``` + +```` + ````{py:property} plugin_module :canonical: archivebox.core.models.ArchiveResult.plugin_module :type: typing.Any | None @@ -1849,6 +2334,64 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} _normalize_output_files(raw_output_files: typing.Any) -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.core.models.ArchiveResult._normalize_output_files +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult._normalize_output_files +``` + +```` + +````{py:method} _coerce_output_file_size(value: typing.Any) -> int +:canonical: archivebox.core.models.ArchiveResult._coerce_output_file_size +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult._coerce_output_file_size +``` + +```` + +````{py:method} output_file_map() -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.core.models.ArchiveResult.output_file_map + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.output_file_map +``` + +```` + +````{py:method} output_file_paths() -> list[str] +:canonical: archivebox.core.models.ArchiveResult.output_file_paths + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.output_file_paths +``` + +```` + +````{py:method} output_file_count() -> int +:canonical: archivebox.core.models.ArchiveResult.output_file_count + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.output_file_count +``` + +```` + +````{py:method} output_size_from_files() -> int +:canonical: archivebox.core.models.ArchiveResult.output_size_from_files + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.output_size_from_files +``` + +```` + +````{py:method} update_output_metadata_from_filesystem(snapshot_dir: pathlib.Path | None = None, save: bool = True) -> bool +:canonical: archivebox.core.models.ArchiveResult.update_output_metadata_from_filesystem + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.update_output_metadata_from_filesystem +``` + +```` + ````{py:method} output_exists() -> bool :canonical: archivebox.core.models.ArchiveResult.output_exists @@ -1857,7 +2400,33 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} _find_best_output_file(dir_path: pathlib.Path, plugin_name: str | None = None) -> typing.Optional[pathlib.Path] +````{py:method} _looks_like_output_path(raw_output: str | None, plugin_name: str | None = None) -> bool +:canonical: archivebox.core.models.ArchiveResult._looks_like_output_path +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult._looks_like_output_path +``` + +```` + +````{py:method} _existing_output_path(raw_output: str | None) -> str | None +:canonical: archivebox.core.models.ArchiveResult._existing_output_path + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult._existing_output_path +``` + +```` + +````{py:method} _fallback_output_file_path(output_file_paths: collections.abc.Sequence[str], plugin_name: str | None = None, output_file_map: dict[str, dict[str, typing.Any]] | None = None) -> str | None +:canonical: archivebox.core.models.ArchiveResult._fallback_output_file_path +:staticmethod: + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult._fallback_output_file_path +``` + +```` + +````{py:method} _find_best_output_file(dir_path: pathlib.Path, plugin_name: str | None = None) -> pathlib.Path | None :canonical: archivebox.core.models.ArchiveResult._find_best_output_file :staticmethod: @@ -1866,18 +2435,18 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} embed_path() -> typing.Optional[str] -:canonical: archivebox.core.models.ArchiveResult.embed_path +````{py:method} embed_path_db(output_file_map: dict[str, dict[str, typing.Any]] | None = None) -> str | None +:canonical: archivebox.core.models.ArchiveResult.embed_path_db -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.embed_path +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.embed_path_db ``` ```` -````{py:method} create_output_dir() -:canonical: archivebox.core.models.ArchiveResult.create_output_dir +````{py:method} embed_path() -> str | None +:canonical: archivebox.core.models.ArchiveResult.embed_path -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.create_output_dir +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.embed_path ``` ```` @@ -1900,6 +2469,14 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:property} process_record +:canonical: archivebox.core.models.ArchiveResult.process_record + +```{autodoc2-docstring} archivebox.core.models.ArchiveResult.process_record +``` + +```` + ````{py:property} pwd :canonical: archivebox.core.models.ArchiveResult.pwd :type: str @@ -1968,22 +2545,6 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} cascade_health_update(success: bool) -:canonical: archivebox.core.models.ArchiveResult.cascade_health_update - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.cascade_health_update -``` - -```` - -````{py:method} run() -:canonical: archivebox.core.models.ArchiveResult.run - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.run -``` - -```` - ````{py:method} update_from_output() :canonical: archivebox.core.models.ArchiveResult.update_from_output @@ -2017,220 +2578,4 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} is_background_hook() -> bool -:canonical: archivebox.core.models.ArchiveResult.is_background_hook - -```{autodoc2-docstring} archivebox.core.models.ArchiveResult.is_background_hook -``` - -```` - `````` - -`````{py:class} ArchiveResultMachine(obj, *args, **kwargs) -:canonical: archivebox.core.models.ArchiveResultMachine - -Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.__init__ -``` - -````{py:attribute} model_attr_name -:canonical: archivebox.core.models.ArchiveResultMachine.model_attr_name -:value: > - 'archiveresult' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.model_attr_name -``` - -```` - -````{py:attribute} queued -:canonical: archivebox.core.models.ArchiveResultMachine.queued -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.queued -``` - -```` - -````{py:attribute} started -:canonical: archivebox.core.models.ArchiveResultMachine.started -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.started -``` - -```` - -````{py:attribute} backoff -:canonical: archivebox.core.models.ArchiveResultMachine.backoff -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.backoff -``` - -```` - -````{py:attribute} succeeded -:canonical: archivebox.core.models.ArchiveResultMachine.succeeded -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.succeeded -``` - -```` - -````{py:attribute} failed -:canonical: archivebox.core.models.ArchiveResultMachine.failed -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.failed -``` - -```` - -````{py:attribute} skipped -:canonical: archivebox.core.models.ArchiveResultMachine.skipped -:value: > - 'State(...)' - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.skipped -``` - -```` - -````{py:attribute} tick -:canonical: archivebox.core.models.ArchiveResultMachine.tick -:value: > - None - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.tick -``` - -```` - -````{py:method} can_start() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.can_start - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.can_start -``` - -```` - -````{py:method} is_exceeded_max_attempts() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.is_exceeded_max_attempts - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.is_exceeded_max_attempts -``` - -```` - -````{py:method} is_succeeded() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.is_succeeded - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.is_succeeded -``` - -```` - -````{py:method} is_failed() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.is_failed - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.is_failed -``` - -```` - -````{py:method} is_skipped() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.is_skipped - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.is_skipped -``` - -```` - -````{py:method} is_backoff() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.is_backoff - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.is_backoff -``` - -```` - -````{py:method} is_finished() -> bool -:canonical: archivebox.core.models.ArchiveResultMachine.is_finished - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.is_finished -``` - -```` - -````{py:method} enter_queued() -:canonical: archivebox.core.models.ArchiveResultMachine.enter_queued - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.enter_queued -``` - -```` - -````{py:method} enter_started() -:canonical: archivebox.core.models.ArchiveResultMachine.enter_started - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.enter_started -``` - -```` - -````{py:method} enter_backoff() -:canonical: archivebox.core.models.ArchiveResultMachine.enter_backoff - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.enter_backoff -``` - -```` - -````{py:method} _check_and_seal_parent_snapshot() -:canonical: archivebox.core.models.ArchiveResultMachine._check_and_seal_parent_snapshot - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine._check_and_seal_parent_snapshot -``` - -```` - -````{py:method} enter_succeeded() -:canonical: archivebox.core.models.ArchiveResultMachine.enter_succeeded - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.enter_succeeded -``` - -```` - -````{py:method} enter_failed() -:canonical: archivebox.core.models.ArchiveResultMachine.enter_failed - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.enter_failed -``` - -```` - -````{py:method} enter_skipped() -:canonical: archivebox.core.models.ArchiveResultMachine.enter_skipped - -```{autodoc2-docstring} archivebox.core.models.ArchiveResultMachine.enter_skipped -``` - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.core.permissions.md b/docs/apidocs/archivebox/archivebox.core.permissions.md new file mode 100644 index 00000000..9521ba73 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.permissions.md @@ -0,0 +1,211 @@ +# {py:mod}`archivebox.core.permissions` + +```{py:module} archivebox.core.permissions +``` + +```{autodoc2-docstring} archivebox.core.permissions +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`normalize_permissions ` + - ```{autodoc2-docstring} archivebox.core.permissions.normalize_permissions + :summary: + ``` +* - {py:obj}`is_admin_user ` + - ```{autodoc2-docstring} archivebox.core.permissions.is_admin_user + :summary: + ``` +* - {py:obj}`get_snapshot_permissions ` + - ```{autodoc2-docstring} archivebox.core.permissions.get_snapshot_permissions + :summary: + ``` +* - {py:obj}`can_view_snapshot ` + - ```{autodoc2-docstring} archivebox.core.permissions.can_view_snapshot + :summary: + ``` +* - {py:obj}`_persona_ids_for_permissions ` + - ```{autodoc2-docstring} archivebox.core.permissions._persona_ids_for_permissions + :summary: + ``` +* - {py:obj}`filter_personas_by_permissions ` + - ```{autodoc2-docstring} archivebox.core.permissions.filter_personas_by_permissions + :summary: + ``` +* - {py:obj}`filter_snapshots_by_permissions ` + - ```{autodoc2-docstring} archivebox.core.permissions.filter_snapshots_by_permissions + :summary: + ``` +* - {py:obj}`public_snapshots_queryset ` + - ```{autodoc2-docstring} archivebox.core.permissions.public_snapshots_queryset + :summary: + ``` +* - {py:obj}`direct_snapshots_queryset ` + - ```{autodoc2-docstring} archivebox.core.permissions.direct_snapshots_queryset + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`PERMISSIONS_PUBLIC ` + - ```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_PUBLIC + :summary: + ``` +* - {py:obj}`PERMISSIONS_UNLISTED ` + - ```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_UNLISTED + :summary: + ``` +* - {py:obj}`PERMISSIONS_PRIVATE ` + - ```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_PRIVATE + :summary: + ``` +* - {py:obj}`PERMISSIONS_CHOICES ` + - ```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_CHOICES + :summary: + ``` +* - {py:obj}`PERMISSIONS_VALUES ` + - ```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_VALUES + :summary: + ``` +* - {py:obj}`PERMISSIONS_META ` + - ```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_META + :summary: + ``` +```` + +### API + +````{py:data} PERMISSIONS_PUBLIC +:canonical: archivebox.core.permissions.PERMISSIONS_PUBLIC +:value: > + 'public' + +```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_PUBLIC +``` + +```` + +````{py:data} PERMISSIONS_UNLISTED +:canonical: archivebox.core.permissions.PERMISSIONS_UNLISTED +:value: > + 'unlisted' + +```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_UNLISTED +``` + +```` + +````{py:data} PERMISSIONS_PRIVATE +:canonical: archivebox.core.permissions.PERMISSIONS_PRIVATE +:value: > + 'private' + +```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_PRIVATE +``` + +```` + +````{py:data} PERMISSIONS_CHOICES +:canonical: archivebox.core.permissions.PERMISSIONS_CHOICES +:value: > + ((), (), ()) + +```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_CHOICES +``` + +```` + +````{py:data} PERMISSIONS_VALUES +:canonical: archivebox.core.permissions.PERMISSIONS_VALUES +:value: > + None + +```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_VALUES +``` + +```` + +````{py:data} PERMISSIONS_META +:canonical: archivebox.core.permissions.PERMISSIONS_META +:value: > + None + +```{autodoc2-docstring} archivebox.core.permissions.PERMISSIONS_META +``` + +```` + +````{py:function} normalize_permissions(permissions: object, *, default: str = PERMISSIONS_PRIVATE) -> str +:canonical: archivebox.core.permissions.normalize_permissions + +```{autodoc2-docstring} archivebox.core.permissions.normalize_permissions +``` +```` + +````{py:function} is_admin_user(request: django.http.HttpRequest) -> bool +:canonical: archivebox.core.permissions.is_admin_user + +```{autodoc2-docstring} archivebox.core.permissions.is_admin_user +``` +```` + +````{py:function} get_snapshot_permissions(snapshot) -> str +:canonical: archivebox.core.permissions.get_snapshot_permissions + +```{autodoc2-docstring} archivebox.core.permissions.get_snapshot_permissions +``` +```` + +````{py:function} can_view_snapshot(request: django.http.HttpRequest, snapshot) -> bool +:canonical: archivebox.core.permissions.can_view_snapshot + +```{autodoc2-docstring} archivebox.core.permissions.can_view_snapshot +``` +```` + +````{py:function} _persona_ids_for_permissions(allowed_permissions: set[str]) -> list[str] +:canonical: archivebox.core.permissions._persona_ids_for_permissions + +```{autodoc2-docstring} archivebox.core.permissions._persona_ids_for_permissions +``` +```` + +````{py:function} filter_personas_by_permissions(queryset: django.db.models.QuerySet, allowed_permissions: set[str]) -> django.db.models.QuerySet +:canonical: archivebox.core.permissions.filter_personas_by_permissions + +```{autodoc2-docstring} archivebox.core.permissions.filter_personas_by_permissions +``` +```` + +````{py:function} filter_snapshots_by_permissions(queryset: django.db.models.QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> django.db.models.QuerySet +:canonical: archivebox.core.permissions.filter_snapshots_by_permissions + +```{autodoc2-docstring} archivebox.core.permissions.filter_snapshots_by_permissions +``` +```` + +````{py:function} public_snapshots_queryset(queryset: django.db.models.QuerySet) -> django.db.models.QuerySet +:canonical: archivebox.core.permissions.public_snapshots_queryset + +```{autodoc2-docstring} archivebox.core.permissions.public_snapshots_queryset +``` +```` + +````{py:function} direct_snapshots_queryset(request: django.http.HttpRequest, queryset: django.db.models.QuerySet) -> django.db.models.QuerySet +:canonical: archivebox.core.permissions.direct_snapshots_queryset + +```{autodoc2-docstring} archivebox.core.permissions.direct_snapshots_queryset +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.recovery_util.md b/docs/apidocs/archivebox/archivebox.core.recovery_util.md new file mode 100644 index 00000000..67d6195e --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.recovery_util.md @@ -0,0 +1,31 @@ +# {py:mod}`archivebox.core.recovery_util` + +```{py:module} archivebox.core.recovery_util +``` + +```{autodoc2-docstring} archivebox.core.recovery_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`recover_orchestrator_state ` + - ```{autodoc2-docstring} archivebox.core.recovery_util.recover_orchestrator_state + :summary: + ``` +```` + +### API + +````{py:function} recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int] +:canonical: archivebox.core.recovery_util.recover_orchestrator_state + +```{autodoc2-docstring} archivebox.core.recovery_util.recover_orchestrator_state +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.settings.md b/docs/apidocs/archivebox/archivebox.core.settings.md index f5e3e120..723c2055 100644 --- a/docs/apidocs/archivebox/archivebox.core.settings.md +++ b/docs/apidocs/archivebox/archivebox.core.settings.md @@ -31,6 +31,14 @@ - ```{autodoc2-docstring} archivebox.core.settings.IS_GETTING_VERSION_OR_HELP :summary: ``` +* - {py:obj}`CONFIG ` + - ```{autodoc2-docstring} archivebox.core.settings.CONFIG + :summary: + ``` +* - {py:obj}`PACKAGE_DIR ` + - ```{autodoc2-docstring} archivebox.core.settings.PACKAGE_DIR + :summary: + ``` * - {py:obj}`ALL_PLUGINS ` - ```{autodoc2-docstring} archivebox.core.settings.ALL_PLUGINS :summary: @@ -111,6 +119,14 @@ - ```{autodoc2-docstring} archivebox.core.settings.DATABASE_NAME :summary: ``` +* - {py:obj}`SQLITE_JOURNAL_MODE ` + - ```{autodoc2-docstring} archivebox.core.settings.SQLITE_JOURNAL_MODE + :summary: + ``` +* - {py:obj}`SQLITE_MMAP_SIZE ` + - ```{autodoc2-docstring} archivebox.core.settings.SQLITE_MMAP_SIZE + :summary: + ``` * - {py:obj}`SQLITE_CONNECTION_OPTIONS ` - ```{autodoc2-docstring} archivebox.core.settings.SQLITE_CONNECTION_OPTIONS :summary: @@ -335,6 +351,26 @@ ```` +````{py:data} CONFIG +:canonical: archivebox.core.settings.CONFIG +:value: > + 'get_config(...)' + +```{autodoc2-docstring} archivebox.core.settings.CONFIG +``` + +```` + +````{py:data} PACKAGE_DIR +:canonical: archivebox.core.settings.PACKAGE_DIR +:value: > + None + +```{autodoc2-docstring} archivebox.core.settings.PACKAGE_DIR +``` + +```` + ````{py:data} ALL_PLUGINS :canonical: archivebox.core.settings.ALL_PLUGINS :value: > @@ -398,7 +434,7 @@ ````{py:data} LOGOUT_REDIRECT_URL :canonical: archivebox.core.settings.LOGOUT_REDIRECT_URL :value: > - 'get(...)' + None ```{autodoc2-docstring} archivebox.core.settings.LOGOUT_REDIRECT_URL ``` @@ -528,13 +564,33 @@ ````{py:data} DATABASE_NAME :canonical: archivebox.core.settings.DATABASE_NAME :value: > - 'get(...)' + None ```{autodoc2-docstring} archivebox.core.settings.DATABASE_NAME ``` ```` +````{py:data} SQLITE_JOURNAL_MODE +:canonical: archivebox.core.settings.SQLITE_JOURNAL_MODE +:value: > + None + +```{autodoc2-docstring} archivebox.core.settings.SQLITE_JOURNAL_MODE +``` + +```` + +````{py:data} SQLITE_MMAP_SIZE +:canonical: archivebox.core.settings.SQLITE_MMAP_SIZE +:value: > + None + +```{autodoc2-docstring} archivebox.core.settings.SQLITE_MMAP_SIZE +``` + +```` + ````{py:data} SQLITE_CONNECTION_OPTIONS :canonical: archivebox.core.settings.SQLITE_CONNECTION_OPTIONS :value: > @@ -638,7 +694,7 @@ ````{py:data} ALLOWED_HOSTS :canonical: archivebox.core.settings.ALLOWED_HOSTS :value: > - 'split(...)' + None ```{autodoc2-docstring} archivebox.core.settings.ALLOWED_HOSTS ``` @@ -947,6 +1003,7 @@ ````{py:data} SIGNAL_WEBHOOKS :canonical: archivebox.core.settings.SIGNAL_WEBHOOKS +:type: dict[str, object] :value: > None diff --git a/docs/apidocs/archivebox/archivebox.core.settings_logging.md b/docs/apidocs/archivebox/archivebox.core.settings_logging.md index ab1be9ef..1ac3a7ee 100644 --- a/docs/apidocs/archivebox/archivebox.core.settings_logging.md +++ b/docs/apidocs/archivebox/archivebox.core.settings_logging.md @@ -17,6 +17,14 @@ * - {py:obj}`NoisyRequestsFilter ` - +* - {py:obj}`DaphneCloseTimeoutFilter ` + - ```{autodoc2-docstring} archivebox.core.settings_logging.DaphneCloseTimeoutFilter + :summary: + ``` +* - {py:obj}`AsyncioCancelledShieldFilter ` + - ```{autodoc2-docstring} archivebox.core.settings_logging.AsyncioCancelledShieldFilter + :summary: + ``` * - {py:obj}`CustomOutboundWebhookLogFormatter ` - * - {py:obj}`StripANSIColorCodesFilter ` @@ -79,6 +87,48 @@ Bases: {py:obj}`logging.Filter` ````` +`````{py:class} DaphneCloseTimeoutFilter(name='') +:canonical: archivebox.core.settings_logging.DaphneCloseTimeoutFilter + +Bases: {py:obj}`logging.Filter` + +```{autodoc2-docstring} archivebox.core.settings_logging.DaphneCloseTimeoutFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.settings_logging.DaphneCloseTimeoutFilter.__init__ +``` + +````{py:method} filter(record) -> bool +:canonical: archivebox.core.settings_logging.DaphneCloseTimeoutFilter.filter + +```` + +````` + +`````{py:class} AsyncioCancelledShieldFilter(name='') +:canonical: archivebox.core.settings_logging.AsyncioCancelledShieldFilter + +Bases: {py:obj}`logging.Filter` + +```{autodoc2-docstring} archivebox.core.settings_logging.AsyncioCancelledShieldFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.settings_logging.AsyncioCancelledShieldFilter.__init__ +``` + +````{py:method} filter(record) -> bool +:canonical: archivebox.core.settings_logging.AsyncioCancelledShieldFilter.filter + +```` + +````` + `````{py:class} CustomOutboundWebhookLogFormatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) :canonical: archivebox.core.settings_logging.CustomOutboundWebhookLogFormatter diff --git a/docs/apidocs/archivebox/archivebox.core.shutdown_util.md b/docs/apidocs/archivebox/archivebox.core.shutdown_util.md new file mode 100644 index 00000000..a41a3fd4 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.shutdown_util.md @@ -0,0 +1,151 @@ +# {py:mod}`archivebox.core.shutdown_util` + +```{py:module} archivebox.core.shutdown_util +``` + +```{autodoc2-docstring} archivebox.core.shutdown_util +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`ShutdownSignalState ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.ShutdownSignalState + :summary: + ``` +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`raise_if_shutdown_requested ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.raise_if_shutdown_requested + :summary: + ``` +* - {py:obj}`configured_stopwaitsecs ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.configured_stopwaitsecs + :summary: + ``` +* - {py:obj}`wait_popen_and_kill_children ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.wait_popen_and_kill_children + :summary: + ``` +* - {py:obj}`wait_psutil_and_kill_children ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.wait_psutil_and_kill_children + :summary: + ``` +* - {py:obj}`kill_remaining_processes ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.kill_remaining_processes + :summary: + ``` +* - {py:obj}`foreground_shutdown_signals ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.foreground_shutdown_signals + :summary: + ``` +* - {py:obj}`foreground_parent_watchdog ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util.foreground_parent_watchdog + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_active_shutdown_state ` + - ```{autodoc2-docstring} archivebox.core.shutdown_util._active_shutdown_state + :summary: + ``` +```` + +### API + +`````{py:class} ShutdownSignalState +:canonical: archivebox.core.shutdown_util.ShutdownSignalState + +```{autodoc2-docstring} archivebox.core.shutdown_util.ShutdownSignalState +``` + +````{py:attribute} signal_name +:canonical: archivebox.core.shutdown_util.ShutdownSignalState.signal_name +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.core.shutdown_util.ShutdownSignalState.signal_name +``` + +```` + +````` + +````{py:data} _active_shutdown_state +:canonical: archivebox.core.shutdown_util._active_shutdown_state +:type: archivebox.core.shutdown_util.ShutdownSignalState | None +:value: > + None + +```{autodoc2-docstring} archivebox.core.shutdown_util._active_shutdown_state +``` + +```` + +````{py:function} raise_if_shutdown_requested() -> None +:canonical: archivebox.core.shutdown_util.raise_if_shutdown_requested + +```{autodoc2-docstring} archivebox.core.shutdown_util.raise_if_shutdown_requested +``` +```` + +````{py:function} configured_stopwaitsecs(workers: list[dict[str, str]] | tuple[dict[str, str], ...], *, default: int = 5, buffer: int = 5) -> int +:canonical: archivebox.core.shutdown_util.configured_stopwaitsecs + +```{autodoc2-docstring} archivebox.core.shutdown_util.configured_stopwaitsecs +``` +```` + +````{py:function} wait_popen_and_kill_children(proc: subprocess.Popen, children: list[psutil.Process], *, timeout: float, kill_timeout: float = 2.0) -> None +:canonical: archivebox.core.shutdown_util.wait_popen_and_kill_children + +```{autodoc2-docstring} archivebox.core.shutdown_util.wait_popen_and_kill_children +``` +```` + +````{py:function} wait_psutil_and_kill_children(proc: psutil.Process, children: list[psutil.Process], *, timeout: float, kill_timeout: float = 2.0) -> None +:canonical: archivebox.core.shutdown_util.wait_psutil_and_kill_children + +```{autodoc2-docstring} archivebox.core.shutdown_util.wait_psutil_and_kill_children +``` +```` + +````{py:function} kill_remaining_processes(processes: list[psutil.Process], *, timeout: float = 2.0) -> None +:canonical: archivebox.core.shutdown_util.kill_remaining_processes + +```{autodoc2-docstring} archivebox.core.shutdown_util.kill_remaining_processes +``` +```` + +````{py:function} foreground_shutdown_signals(handled_signals: tuple[signal.Signals, ...] = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM), *, first_signal_message: str | None = '\n[🛑] Got {signal_name}, stopping gracefully...\n', on_signal: collections.abc.Callable[[signal.Signals], None] | None = None, raise_on_first_signal: bool = True) -> collections.abc.Iterator[archivebox.core.shutdown_util.ShutdownSignalState] +:canonical: archivebox.core.shutdown_util.foreground_shutdown_signals + +```{autodoc2-docstring} archivebox.core.shutdown_util.foreground_shutdown_signals +``` +```` + +````{py:function} foreground_parent_watchdog(*, enabled: bool = True, check_interval: float = 2.0, shutdown_signal: signal.Signals = signal.SIGTERM) -> collections.abc.Iterator[None] +:canonical: archivebox.core.shutdown_util.foreground_parent_watchdog + +```{autodoc2-docstring} archivebox.core.shutdown_util.foreground_parent_watchdog +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.tag_util.md b/docs/apidocs/archivebox/archivebox.core.tag_util.md new file mode 100644 index 00000000..d9b0e8cc --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.tag_util.md @@ -0,0 +1,290 @@ +# {py:mod}`archivebox.core.tag_util` + +```{py:module} archivebox.core.tag_util +``` + +```{autodoc2-docstring} archivebox.core.tag_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`normalize_tag_name ` + - ```{autodoc2-docstring} archivebox.core.tag_util.normalize_tag_name + :summary: + ``` +* - {py:obj}`normalize_tag_sort ` + - ```{autodoc2-docstring} archivebox.core.tag_util.normalize_tag_sort + :summary: + ``` +* - {py:obj}`normalize_has_snapshots_filter ` + - ```{autodoc2-docstring} archivebox.core.tag_util.normalize_has_snapshots_filter + :summary: + ``` +* - {py:obj}`normalize_created_by_filter ` + - ```{autodoc2-docstring} archivebox.core.tag_util.normalize_created_by_filter + :summary: + ``` +* - {py:obj}`normalize_created_year_filter ` + - ```{autodoc2-docstring} archivebox.core.tag_util.normalize_created_year_filter + :summary: + ``` +* - {py:obj}`get_matching_tags ` + - ```{autodoc2-docstring} archivebox.core.tag_util.get_matching_tags + :summary: + ``` +* - {py:obj}`add_snapshot_counts ` + - ```{autodoc2-docstring} archivebox.core.tag_util.add_snapshot_counts + :summary: + ``` +* - {py:obj}`get_tag_creator_choices ` + - ```{autodoc2-docstring} archivebox.core.tag_util.get_tag_creator_choices + :summary: + ``` +* - {py:obj}`get_tag_year_choices ` + - ```{autodoc2-docstring} archivebox.core.tag_util.get_tag_year_choices + :summary: + ``` +* - {py:obj}`get_tag_by_ref ` + - ```{autodoc2-docstring} archivebox.core.tag_util.get_tag_by_ref + :summary: + ``` +* - {py:obj}`get_or_create_tag ` + - ```{autodoc2-docstring} archivebox.core.tag_util.get_or_create_tag + :summary: + ``` +* - {py:obj}`rename_tag ` + - ```{autodoc2-docstring} archivebox.core.tag_util.rename_tag + :summary: + ``` +* - {py:obj}`delete_tag ` + - ```{autodoc2-docstring} archivebox.core.tag_util.delete_tag + :summary: + ``` +* - {py:obj}`export_tag_urls ` + - ```{autodoc2-docstring} archivebox.core.tag_util.export_tag_urls + :summary: + ``` +* - {py:obj}`export_tag_snapshots_jsonl ` + - ```{autodoc2-docstring} archivebox.core.tag_util.export_tag_snapshots_jsonl + :summary: + ``` +* - {py:obj}`_display_snapshot_title ` + - ```{autodoc2-docstring} archivebox.core.tag_util._display_snapshot_title + :summary: + ``` +* - {py:obj}`_build_snapshot_preview ` + - ```{autodoc2-docstring} archivebox.core.tag_util._build_snapshot_preview + :summary: + ``` +* - {py:obj}`_build_snapshot_preview_map ` + - ```{autodoc2-docstring} archivebox.core.tag_util._build_snapshot_preview_map + :summary: + ``` +* - {py:obj}`build_tag_card ` + - ```{autodoc2-docstring} archivebox.core.tag_util.build_tag_card + :summary: + ``` +* - {py:obj}`build_tag_cards ` + - ```{autodoc2-docstring} archivebox.core.tag_util.build_tag_cards + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`TAG_SNAPSHOT_PREVIEW_LIMIT ` + - ```{autodoc2-docstring} archivebox.core.tag_util.TAG_SNAPSHOT_PREVIEW_LIMIT + :summary: + ``` +* - {py:obj}`TAG_SORT_CHOICES ` + - ```{autodoc2-docstring} archivebox.core.tag_util.TAG_SORT_CHOICES + :summary: + ``` +* - {py:obj}`TAG_HAS_SNAPSHOTS_CHOICES ` + - ```{autodoc2-docstring} archivebox.core.tag_util.TAG_HAS_SNAPSHOTS_CHOICES + :summary: + ``` +```` + +### API + +````{py:data} TAG_SNAPSHOT_PREVIEW_LIMIT +:canonical: archivebox.core.tag_util.TAG_SNAPSHOT_PREVIEW_LIMIT +:value: > + 10 + +```{autodoc2-docstring} archivebox.core.tag_util.TAG_SNAPSHOT_PREVIEW_LIMIT +``` + +```` + +````{py:data} TAG_SORT_CHOICES +:canonical: archivebox.core.tag_util.TAG_SORT_CHOICES +:value: > + (('name_asc', 'Name A-Z'), ('name_desc', 'Name Z-A'), ('created_desc', 'Created newest'), ('created_... + +```{autodoc2-docstring} archivebox.core.tag_util.TAG_SORT_CHOICES +``` + +```` + +````{py:data} TAG_HAS_SNAPSHOTS_CHOICES +:canonical: archivebox.core.tag_util.TAG_HAS_SNAPSHOTS_CHOICES +:value: > + (('all', 'All'), ('yes', 'Has snapshots'), ('no', 'No snapshots')) + +```{autodoc2-docstring} archivebox.core.tag_util.TAG_HAS_SNAPSHOTS_CHOICES +``` + +```` + +````{py:function} normalize_tag_name(name: str) -> str +:canonical: archivebox.core.tag_util.normalize_tag_name + +```{autodoc2-docstring} archivebox.core.tag_util.normalize_tag_name +``` +```` + +````{py:function} normalize_tag_sort(sort: str = 'created_desc') -> str +:canonical: archivebox.core.tag_util.normalize_tag_sort + +```{autodoc2-docstring} archivebox.core.tag_util.normalize_tag_sort +``` +```` + +````{py:function} normalize_has_snapshots_filter(value: str = 'all') -> str +:canonical: archivebox.core.tag_util.normalize_has_snapshots_filter + +```{autodoc2-docstring} archivebox.core.tag_util.normalize_has_snapshots_filter +``` +```` + +````{py:function} normalize_created_by_filter(created_by: str = '') -> str +:canonical: archivebox.core.tag_util.normalize_created_by_filter + +```{autodoc2-docstring} archivebox.core.tag_util.normalize_created_by_filter +``` +```` + +````{py:function} normalize_created_year_filter(year: str = '') -> str +:canonical: archivebox.core.tag_util.normalize_created_year_filter + +```{autodoc2-docstring} archivebox.core.tag_util.normalize_created_year_filter +``` +```` + +````{py:function} get_matching_tags(query: str = '', sort: str = 'created_desc', created_by: str = '', year: str = '', has_snapshots: str = 'all', with_snapshot_counts: bool = True) -> django.db.models.QuerySet[archivebox.core.models.Tag] +:canonical: archivebox.core.tag_util.get_matching_tags + +```{autodoc2-docstring} archivebox.core.tag_util.get_matching_tags +``` +```` + +````{py:function} add_snapshot_counts(tags: list[archivebox.core.models.Tag], snapshot_queryset: django.db.models.QuerySet[archivebox.core.models.Snapshot] | None = None) -> None +:canonical: archivebox.core.tag_util.add_snapshot_counts + +```{autodoc2-docstring} archivebox.core.tag_util.add_snapshot_counts +``` +```` + +````{py:function} get_tag_creator_choices() -> list[tuple[str, str]] +:canonical: archivebox.core.tag_util.get_tag_creator_choices + +```{autodoc2-docstring} archivebox.core.tag_util.get_tag_creator_choices +``` +```` + +````{py:function} get_tag_year_choices() -> list[str] +:canonical: archivebox.core.tag_util.get_tag_year_choices + +```{autodoc2-docstring} archivebox.core.tag_util.get_tag_year_choices +``` +```` + +````{py:function} get_tag_by_ref(tag_ref: str | int) -> archivebox.core.models.Tag +:canonical: archivebox.core.tag_util.get_tag_by_ref + +```{autodoc2-docstring} archivebox.core.tag_util.get_tag_by_ref +``` +```` + +````{py:function} get_or_create_tag(name: str, created_by: django.contrib.auth.models.User | None = None) -> tuple[archivebox.core.models.Tag, bool] +:canonical: archivebox.core.tag_util.get_or_create_tag + +```{autodoc2-docstring} archivebox.core.tag_util.get_or_create_tag +``` +```` + +````{py:function} rename_tag(tag: archivebox.core.models.Tag, name: str) -> archivebox.core.models.Tag +:canonical: archivebox.core.tag_util.rename_tag + +```{autodoc2-docstring} archivebox.core.tag_util.rename_tag +``` +```` + +````{py:function} delete_tag(tag: archivebox.core.models.Tag) -> tuple[int, dict[str, int]] +:canonical: archivebox.core.tag_util.delete_tag + +```{autodoc2-docstring} archivebox.core.tag_util.delete_tag +``` +```` + +````{py:function} export_tag_urls(tag: archivebox.core.models.Tag) -> str +:canonical: archivebox.core.tag_util.export_tag_urls + +```{autodoc2-docstring} archivebox.core.tag_util.export_tag_urls +``` +```` + +````{py:function} export_tag_snapshots_jsonl(tag: archivebox.core.models.Tag) -> str +:canonical: archivebox.core.tag_util.export_tag_snapshots_jsonl + +```{autodoc2-docstring} archivebox.core.tag_util.export_tag_snapshots_jsonl +``` +```` + +````{py:function} _display_snapshot_title(snapshot: archivebox.core.models.Snapshot) -> str +:canonical: archivebox.core.tag_util._display_snapshot_title + +```{autodoc2-docstring} archivebox.core.tag_util._display_snapshot_title +``` +```` + +````{py:function} _build_snapshot_preview(snapshot: archivebox.core.models.Snapshot, request: django.http.HttpRequest | None = None, config: typing.Any | None = None) -> dict[str, typing.Any] +:canonical: archivebox.core.tag_util._build_snapshot_preview + +```{autodoc2-docstring} archivebox.core.tag_util._build_snapshot_preview +``` +```` + +````{py:function} _build_snapshot_preview_map(tags: list[archivebox.core.models.Tag], request: django.http.HttpRequest | None = None, preview_limit: int = TAG_SNAPSHOT_PREVIEW_LIMIT) -> dict[int, list[dict[str, typing.Any]]] +:canonical: archivebox.core.tag_util._build_snapshot_preview_map + +```{autodoc2-docstring} archivebox.core.tag_util._build_snapshot_preview_map +``` +```` + +````{py:function} build_tag_card(tag: archivebox.core.models.Tag, snapshot_previews: list[dict[str, typing.Any]] | None = None) -> dict[str, typing.Any] +:canonical: archivebox.core.tag_util.build_tag_card + +```{autodoc2-docstring} archivebox.core.tag_util.build_tag_card +``` +```` + +````{py:function} build_tag_cards(query: str = '', request: django.http.HttpRequest | None = None, limit: int | None = None, preview_limit: int = TAG_SNAPSHOT_PREVIEW_LIMIT, sort: str = 'created_desc', created_by: str = '', year: str = '', has_snapshots: str = 'all') -> list[dict[str, typing.Any]] +:canonical: archivebox.core.tag_util.build_tag_cards + +```{autodoc2-docstring} archivebox.core.tag_util.build_tag_cards +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.takeover_util.md b/docs/apidocs/archivebox/archivebox.core.takeover_util.md new file mode 100644 index 00000000..8aae8ef8 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.core.takeover_util.md @@ -0,0 +1,235 @@ +# {py:mod}`archivebox.core.takeover_util` + +```{py:module} archivebox.core.takeover_util +``` + +```{autodoc2-docstring} archivebox.core.takeover_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`runtime_stack_owner_types ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types + :summary: + ``` +* - {py:obj}`current_command ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.current_command + :summary: + ``` +* - {py:obj}`live_processes ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.live_processes + :summary: + ``` +* - {py:obj}`newest_live_process ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process + :summary: + ``` +* - {py:obj}`command_is_newest ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest + :summary: + ``` +* - {py:obj}`runtime_stack_owner ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner + :summary: + ``` +* - {py:obj}`command_owns_runtime_stack ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.command_owns_runtime_stack + :summary: + ``` +* - {py:obj}`runtime_stack_component_label ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label + :summary: + ``` +* - {py:obj}`ensure_daemon_stack ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.ensure_daemon_stack + :summary: + ``` +* - {py:obj}`healthy_orchestrator ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator + :summary: + ``` +* - {py:obj}`_runner_sort_key ` + - ```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key + :summary: + ``` +* - {py:obj}`live_runner_processes ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes + :summary: + ``` +* - {py:obj}`enter_single_runner_gate ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.enter_single_runner_gate + :summary: + ``` +* - {py:obj}`standby_until_leader_needed ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed + :summary: + ``` +* - {py:obj}`standby_until_runtime_stack_needed ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_runtime_stack_needed + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`RUNNER_ACTIVE_WORKER_TYPE ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.RUNNER_ACTIVE_WORKER_TYPE + :summary: + ``` +* - {py:obj}`RUNNER_WAITING_WORKER_TYPE ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.RUNNER_WAITING_WORKER_TYPE + :summary: + ``` +* - {py:obj}`RUNNER_GATE_WORKER_TYPES ` + - ```{autodoc2-docstring} archivebox.core.takeover_util.RUNNER_GATE_WORKER_TYPES + :summary: + ``` +```` + +### API + +````{py:data} RUNNER_ACTIVE_WORKER_TYPE +:canonical: archivebox.core.takeover_util.RUNNER_ACTIVE_WORKER_TYPE +:value: > + 'worker_runner' + +```{autodoc2-docstring} archivebox.core.takeover_util.RUNNER_ACTIVE_WORKER_TYPE +``` + +```` + +````{py:data} RUNNER_WAITING_WORKER_TYPE +:canonical: archivebox.core.takeover_util.RUNNER_WAITING_WORKER_TYPE +:value: > + 'runner_waiting' + +```{autodoc2-docstring} archivebox.core.takeover_util.RUNNER_WAITING_WORKER_TYPE +``` + +```` + +````{py:data} RUNNER_GATE_WORKER_TYPES +:canonical: archivebox.core.takeover_util.RUNNER_GATE_WORKER_TYPES +:value: > + () + +```{autodoc2-docstring} archivebox.core.takeover_util.RUNNER_GATE_WORKER_TYPES +``` + +```` + +````{py:function} runtime_stack_owner_types() +:canonical: archivebox.core.takeover_util.runtime_stack_owner_types + +```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types +``` +```` + +````{py:function} current_command(process_type: str, *, data_dir: str | pathlib.Path, url: str | None = None) +:canonical: archivebox.core.takeover_util.current_command + +```{autodoc2-docstring} archivebox.core.takeover_util.current_command +``` +```` + +````{py:function} live_processes(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None) +:canonical: archivebox.core.takeover_util.live_processes + +```{autodoc2-docstring} archivebox.core.takeover_util.live_processes +``` +```` + +````{py:function} newest_live_process(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None) +:canonical: archivebox.core.takeover_util.newest_live_process + +```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process +``` +```` + +````{py:function} command_is_newest(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None) -> bool +:canonical: archivebox.core.takeover_util.command_is_newest + +```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest +``` +```` + +````{py:function} runtime_stack_owner(*, data_dir: str | pathlib.Path, exclude_id=None) +:canonical: archivebox.core.takeover_util.runtime_stack_owner + +```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner +``` +```` + +````{py:function} command_owns_runtime_stack(command, *, data_dir: str | pathlib.Path) -> bool +:canonical: archivebox.core.takeover_util.command_owns_runtime_stack + +```{autodoc2-docstring} archivebox.core.takeover_util.command_owns_runtime_stack +``` +```` + +````{py:function} runtime_stack_component_label(*, owner=None, data_dir: str | pathlib.Path) -> str +:canonical: archivebox.core.takeover_util.runtime_stack_component_label + +```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label +``` +```` + +````{py:function} ensure_daemon_stack(*, reason: str = '') +:canonical: archivebox.core.takeover_util.ensure_daemon_stack + +```{autodoc2-docstring} archivebox.core.takeover_util.ensure_daemon_stack +``` +```` + +````{py:function} healthy_orchestrator(*, data_dir: str | pathlib.Path) +:canonical: archivebox.core.takeover_util.healthy_orchestrator + +```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator +``` +```` + +````{py:function} _runner_sort_key(process) +:canonical: archivebox.core.takeover_util._runner_sort_key + +```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key +``` +```` + +````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path, exclude_id=None) +:canonical: archivebox.core.takeover_util.live_runner_processes + +```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes +``` +```` + +````{py:function} enter_single_runner_gate(command, *, data_dir: str | pathlib.Path, graceful_timeout: float = 5.0) -> bool +:canonical: archivebox.core.takeover_util.enter_single_runner_gate + +```{autodoc2-docstring} archivebox.core.takeover_util.enter_single_runner_gate +``` +```` + +````{py:function} standby_until_leader_needed(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None, interval: float = 2.0) -> None +:canonical: archivebox.core.takeover_util.standby_until_leader_needed + +```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed +``` +```` + +````{py:function} standby_until_runtime_stack_needed(command, *, data_dir: str | pathlib.Path, interval: float = 2.0) -> dict[str, object] +:canonical: archivebox.core.takeover_util.standby_until_runtime_stack_needed + +```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_runtime_stack_needed +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.templatetags.config_tags.md b/docs/apidocs/archivebox/archivebox.core.templatetags.config_tags.md deleted file mode 100644 index e239bd35..00000000 --- a/docs/apidocs/archivebox/archivebox.core.templatetags.config_tags.md +++ /dev/null @@ -1,53 +0,0 @@ -# {py:mod}`archivebox.core.templatetags.config_tags` - -```{py:module} archivebox.core.templatetags.config_tags -``` - -```{autodoc2-docstring} archivebox.core.templatetags.config_tags -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`get_config ` - - ```{autodoc2-docstring} archivebox.core.templatetags.config_tags.get_config - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`register ` - - ```{autodoc2-docstring} archivebox.core.templatetags.config_tags.register - :summary: - ``` -```` - -### API - -````{py:data} register -:canonical: archivebox.core.templatetags.config_tags.register -:value: > - 'Library(...)' - -```{autodoc2-docstring} archivebox.core.templatetags.config_tags.register -``` - -```` - -````{py:function} get_config(key: str) -> any -:canonical: archivebox.core.templatetags.config_tags.get_config - -```{autodoc2-docstring} archivebox.core.templatetags.config_tags.get_config -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.core.templatetags.core_tags.md b/docs/apidocs/archivebox/archivebox.core.templatetags.core_tags.md index 651cc5b9..ca5116aa 100644 --- a/docs/apidocs/archivebox/archivebox.core.templatetags.core_tags.md +++ b/docs/apidocs/archivebox/archivebox.core.templatetags.core_tags.md @@ -15,6 +15,14 @@ :class: autosummary longtable :align: left +* - {py:obj}`_normalize_output_files ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._normalize_output_files + :summary: + ``` +* - {py:obj}`_coerce_output_file_size ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._coerce_output_file_size + :summary: + ``` * - {py:obj}`_count_media_files ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._count_media_files :summary: @@ -23,14 +31,58 @@ - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._list_media_files :summary: ``` +* - {py:obj}`_resolve_snapshot_output_file ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._resolve_snapshot_output_file + :summary: + ``` +* - {py:obj}`_is_text_preview_path ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._is_text_preview_path + :summary: + ``` +* - {py:obj}`_is_image_preview_path ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._is_image_preview_path + :summary: + ``` +* - {py:obj}`_is_root_snapshot_output_path ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._is_root_snapshot_output_path + :summary: + ``` +* - {py:obj}`_build_snapshot_files_url ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._build_snapshot_files_url + :summary: + ``` +* - {py:obj}`_build_snapshot_preview_url ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._build_snapshot_preview_url + :summary: + ``` +* - {py:obj}`_render_text_preview ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._render_text_preview + :summary: + ``` +* - {py:obj}`_render_fallback_card ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._render_fallback_card + :summary: + ``` +* - {py:obj}`_render_text_file_preview ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._render_text_file_preview + :summary: + ``` * - {py:obj}`split ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.split :summary: ``` +* - {py:obj}`index ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.index + :summary: + ``` * - {py:obj}`file_size ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.file_size :summary: ``` +* - {py:obj}`intcomma ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.intcomma + :summary: + ``` * - {py:obj}`result_list ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.result_list :summary: @@ -39,10 +91,26 @@ - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.result_list_tag :summary: ``` +* - {py:obj}`_machine_health_stats ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._machine_health_stats + :summary: + ``` +* - {py:obj}`system_warnings_banner ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.system_warnings_banner + :summary: + ``` +* - {py:obj}`_unconfigured_banner_context ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._unconfigured_banner_context + :summary: + ``` * - {py:obj}`url_replace ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.url_replace :summary: ``` +* - {py:obj}`has_real_admin_users ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.has_real_admin_users + :summary: + ``` * - {py:obj}`admin_base_url ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.admin_base_url :summary: @@ -51,6 +119,10 @@ - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.web_base_url :summary: ``` +* - {py:obj}`public_base_url ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.public_base_url + :summary: + ``` * - {py:obj}`snapshot_base_url ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.snapshot_base_url :summary: @@ -59,6 +131,10 @@ - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.snapshot_url :summary: ``` +* - {py:obj}`snapshot_preview_url ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.snapshot_preview_url + :summary: + ``` * - {py:obj}`plugin_icon ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.plugin_icon :summary: @@ -67,6 +143,10 @@ - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.plugin_card :summary: ``` +* - {py:obj}`output_card ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.output_card + :summary: + ``` * - {py:obj}`plugin_full ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.plugin_full :summary: @@ -91,10 +171,46 @@ - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.register :summary: ``` +* - {py:obj}`_TEXT_PREVIEW_EXTS ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._TEXT_PREVIEW_EXTS + :summary: + ``` +* - {py:obj}`_IMAGE_PREVIEW_EXTS ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._IMAGE_PREVIEW_EXTS + :summary: + ``` +* - {py:obj}`_MHTML_PREVIEW_EXTS ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._MHTML_PREVIEW_EXTS + :summary: + ``` +* - {py:obj}`_WACZ_PREVIEW_EXTS ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._WACZ_PREVIEW_EXTS + :summary: + ``` * - {py:obj}`_MEDIA_FILE_EXTS ` - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._MEDIA_FILE_EXTS :summary: ``` +* - {py:obj}`_LOW_DISK_THRESHOLD_GB ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB + :summary: + ``` +* - {py:obj}`_HIGH_MEMORY_THRESHOLD_PCT ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT + :summary: + ``` +* - {py:obj}`_HIGH_LOAD_MULTIPLE ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE + :summary: + ``` +* - {py:obj}`_HEALTH_CHECK_INTERVAL_SECONDS ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS + :summary: + ``` +* - {py:obj}`_health_cache ` + - ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._health_cache + :summary: + ``` ```` ### API @@ -109,6 +225,46 @@ ```` +````{py:data} _TEXT_PREVIEW_EXTS +:canonical: archivebox.core.templatetags.core_tags._TEXT_PREVIEW_EXTS +:value: > + ('.json', '.jsonl', '.txt', '.csv', '.tsv', '.xml', '.yml', '.yaml', '.md', '.log') + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._TEXT_PREVIEW_EXTS +``` + +```` + +````{py:data} _IMAGE_PREVIEW_EXTS +:canonical: archivebox.core.templatetags.core_tags._IMAGE_PREVIEW_EXTS +:value: > + ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico', '.avif') + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._IMAGE_PREVIEW_EXTS +``` + +```` + +````{py:data} _MHTML_PREVIEW_EXTS +:canonical: archivebox.core.templatetags.core_tags._MHTML_PREVIEW_EXTS +:value: > + ('.mhtml', '.mht') + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._MHTML_PREVIEW_EXTS +``` + +```` + +````{py:data} _WACZ_PREVIEW_EXTS +:canonical: archivebox.core.templatetags.core_tags._WACZ_PREVIEW_EXTS +:value: > + ('.wacz', '.warc', '.warc.gz') + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._WACZ_PREVIEW_EXTS +``` + +```` + ````{py:data} _MEDIA_FILE_EXTS :canonical: archivebox.core.templatetags.core_tags._MEDIA_FILE_EXTS :value: > @@ -119,6 +275,20 @@ ```` +````{py:function} _normalize_output_files(output_files: typing.Any) -> dict[str, dict[str, typing.Any]] +:canonical: archivebox.core.templatetags.core_tags._normalize_output_files + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._normalize_output_files +``` +```` + +````{py:function} _coerce_output_file_size(value: typing.Any) -> int | None +:canonical: archivebox.core.templatetags.core_tags._coerce_output_file_size + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._coerce_output_file_size +``` +```` + ````{py:function} _count_media_files(result) -> int :canonical: archivebox.core.templatetags.core_tags._count_media_files @@ -133,6 +303,69 @@ ``` ```` +````{py:function} _resolve_snapshot_output_file(snapshot_dir: str | pathlib.Path | None, raw_output_path: str | None) -> pathlib.Path | None +:canonical: archivebox.core.templatetags.core_tags._resolve_snapshot_output_file + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._resolve_snapshot_output_file +``` +```` + +````{py:function} _is_text_preview_path(raw_output_path: str | None) -> bool +:canonical: archivebox.core.templatetags.core_tags._is_text_preview_path + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._is_text_preview_path +``` +```` + +````{py:function} _is_image_preview_path(raw_output_path: str | None) -> bool +:canonical: archivebox.core.templatetags.core_tags._is_image_preview_path + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._is_image_preview_path +``` +```` + +````{py:function} _is_root_snapshot_output_path(raw_output_path: str | None) -> bool +:canonical: archivebox.core.templatetags.core_tags._is_root_snapshot_output_path + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._is_root_snapshot_output_path +``` +```` + +````{py:function} _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> str +:canonical: archivebox.core.templatetags.core_tags._build_snapshot_files_url + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._build_snapshot_files_url +``` +```` + +````{py:function} _build_snapshot_preview_url(snapshot_id: str, path: str = '', request=None, config=None) -> str +:canonical: archivebox.core.templatetags.core_tags._build_snapshot_preview_url + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._build_snapshot_preview_url +``` +```` + +````{py:function} _render_text_preview(plugin: str, icon_html: str, snippet: str) -> str +:canonical: archivebox.core.templatetags.core_tags._render_text_preview + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._render_text_preview +``` +```` + +````{py:function} _render_fallback_card(plugin: str, icon_html: str, fallback_label: str) -> str +:canonical: archivebox.core.templatetags.core_tags._render_fallback_card + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._render_fallback_card +``` +```` + +````{py:function} _render_text_file_preview(snapshot_dir: str | pathlib.Path | None, raw_output_path: str | None, plugin: str, icon_html: str) -> str | None +:canonical: archivebox.core.templatetags.core_tags._render_text_file_preview + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._render_text_file_preview +``` +```` + ````{py:function} split(value, separator: str = ',') :canonical: archivebox.core.templatetags.core_tags.split @@ -140,14 +373,28 @@ ``` ```` -````{py:function} file_size(num_bytes: typing.Union[int, float]) -> str +````{py:function} index(value, position) +:canonical: archivebox.core.templatetags.core_tags.index + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.index +``` +```` + +````{py:function} file_size(num_bytes: int | float) -> str :canonical: archivebox.core.templatetags.core_tags.file_size ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.file_size ``` ```` -````{py:function} result_list(cl) +````{py:function} intcomma(value: int | str | None) -> str +:canonical: archivebox.core.templatetags.core_tags.intcomma + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.intcomma +``` +```` + +````{py:function} result_list(context, cl) :canonical: archivebox.core.templatetags.core_tags.result_list ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.result_list @@ -161,6 +408,78 @@ ``` ```` +````{py:data} _LOW_DISK_THRESHOLD_GB +:canonical: archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB +:value: > + 1.0 + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB +``` + +```` + +````{py:data} _HIGH_MEMORY_THRESHOLD_PCT +:canonical: archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT +:value: > + 95.0 + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT +``` + +```` + +````{py:data} _HIGH_LOAD_MULTIPLE +:canonical: archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE +:value: > + 3 + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE +``` + +```` + +````{py:data} _HEALTH_CHECK_INTERVAL_SECONDS +:canonical: archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS +:value: > + 30 + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS +``` + +```` + +````{py:data} _health_cache +:canonical: archivebox.core.templatetags.core_tags._health_cache +:type: dict +:value: > + None + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._health_cache +``` + +```` + +````{py:function} _machine_health_stats() -> dict +:canonical: archivebox.core.templatetags.core_tags._machine_health_stats + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._machine_health_stats +``` +```` + +````{py:function} system_warnings_banner(context) +:canonical: archivebox.core.templatetags.core_tags.system_warnings_banner + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.system_warnings_banner +``` +```` + +````{py:function} _unconfigured_banner_context(request) -> dict +:canonical: archivebox.core.templatetags.core_tags._unconfigured_banner_context + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags._unconfigured_banner_context +``` +```` + ````{py:function} url_replace(context, **kwargs) :canonical: archivebox.core.templatetags.core_tags.url_replace @@ -168,6 +487,13 @@ ``` ```` +````{py:function} has_real_admin_users() -> bool +:canonical: archivebox.core.templatetags.core_tags.has_real_admin_users + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.has_real_admin_users +``` +```` + ````{py:function} admin_base_url(context) -> str :canonical: archivebox.core.templatetags.core_tags.admin_base_url @@ -182,6 +508,13 @@ ``` ```` +````{py:function} public_base_url(context) -> str +:canonical: archivebox.core.templatetags.core_tags.public_base_url + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.public_base_url +``` +```` + ````{py:function} snapshot_base_url(context, snapshot) -> str :canonical: archivebox.core.templatetags.core_tags.snapshot_base_url @@ -196,6 +529,13 @@ ``` ```` +````{py:function} snapshot_preview_url(context, snapshot, path: str = '') -> str +:canonical: archivebox.core.templatetags.core_tags.snapshot_preview_url + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.snapshot_preview_url +``` +```` + ````{py:function} plugin_icon(plugin: str) -> str :canonical: archivebox.core.templatetags.core_tags.plugin_icon @@ -210,6 +550,13 @@ ``` ```` +````{py:function} output_card(snapshot, output_path: str, plugin: str) -> str +:canonical: archivebox.core.templatetags.core_tags.output_card + +```{autodoc2-docstring} archivebox.core.templatetags.core_tags.output_card +``` +```` + ````{py:function} plugin_full(context, result) -> str :canonical: archivebox.core.templatetags.core_tags.plugin_full diff --git a/docs/apidocs/archivebox/archivebox.core.templatetags.md b/docs/apidocs/archivebox/archivebox.core.templatetags.md index 2852fddc..634fe7cc 100644 --- a/docs/apidocs/archivebox/archivebox.core.templatetags.md +++ b/docs/apidocs/archivebox/archivebox.core.templatetags.md @@ -14,5 +14,4 @@ :maxdepth: 1 archivebox.core.templatetags.core_tags -archivebox.core.templatetags.config_tags ``` diff --git a/docs/apidocs/archivebox/archivebox.core.urls.md b/docs/apidocs/archivebox/archivebox.core.urls.md index a49b4ed3..d59c15c2 100644 --- a/docs/apidocs/archivebox/archivebox.core.urls.md +++ b/docs/apidocs/archivebox/archivebox.core.urls.md @@ -9,12 +9,32 @@ ## Module Contents +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_raise_test_error ` + - ```{autodoc2-docstring} archivebox.core.urls._raise_test_error + :summary: + ``` +```` + ### Data ````{list-table} :class: autosummary longtable :align: left +* - {py:obj}`CONFIG ` + - ```{autodoc2-docstring} archivebox.core.urls.CONFIG + :summary: + ``` +* - {py:obj}`DEBUG ` + - ```{autodoc2-docstring} archivebox.core.urls.DEBUG + :summary: + ``` * - {py:obj}`urlpatterns ` - ```{autodoc2-docstring} archivebox.core.urls.urlpatterns :summary: @@ -23,6 +43,26 @@ ### API +````{py:data} CONFIG +:canonical: archivebox.core.urls.CONFIG +:value: > + 'get_config(...)' + +```{autodoc2-docstring} archivebox.core.urls.CONFIG +``` + +```` + +````{py:data} DEBUG +:canonical: archivebox.core.urls.DEBUG +:value: > + None + +```{autodoc2-docstring} archivebox.core.urls.DEBUG +``` + +```` + ````{py:data} urlpatterns :canonical: archivebox.core.urls.urlpatterns :value: > @@ -32,3 +72,10 @@ ``` ```` + +````{py:function} _raise_test_error(_request: django.http.HttpRequest) +:canonical: archivebox.core.urls._raise_test_error + +```{autodoc2-docstring} archivebox.core.urls._raise_test_error +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.core.views.md b/docs/apidocs/archivebox/archivebox.core.views.md index 46e6c9e6..b3f30ada 100644 --- a/docs/apidocs/archivebox/archivebox.core.views.md +++ b/docs/apidocs/archivebox/archivebox.core.views.md @@ -27,10 +27,18 @@ - ```{autodoc2-docstring} archivebox.core.views.SnapshotHostView :summary: ``` +* - {py:obj}`SnapshotReplayView ` + - ```{autodoc2-docstring} archivebox.core.views.SnapshotReplayView + :summary: + ``` * - {py:obj}`OriginalDomainHostView ` - ```{autodoc2-docstring} archivebox.core.views.OriginalDomainHostView :summary: ``` +* - {py:obj}`OriginalDomainReplayView ` + - ```{autodoc2-docstring} archivebox.core.views.OriginalDomainReplayView + :summary: + ``` * - {py:obj}`PublicIndexView ` - * - {py:obj}`AddView ` @@ -49,14 +57,46 @@ :class: autosummary longtable :align: left +* - {py:obj}`_live_progress_plugin_names ` + - ```{autodoc2-docstring} archivebox.core.views._live_progress_plugin_names + :summary: + ``` +* - {py:obj}`_get_request_config ` + - ```{autodoc2-docstring} archivebox.core.views._get_request_config + :summary: + ``` * - {py:obj}`_files_index_target ` - ```{autodoc2-docstring} archivebox.core.views._files_index_target :summary: ``` +* - {py:obj}`_find_snapshot_by_ref ` + - ```{autodoc2-docstring} archivebox.core.views._find_snapshot_by_ref + :summary: + ``` +* - {py:obj}`_admin_login_redirect_or_forbidden ` + - ```{autodoc2-docstring} archivebox.core.views._admin_login_redirect_or_forbidden + :summary: + ``` * - {py:obj}`_safe_archive_relpath ` - ```{autodoc2-docstring} archivebox.core.views._safe_archive_relpath :summary: ``` +* - {py:obj}`_coerce_sort_timestamp ` + - ```{autodoc2-docstring} archivebox.core.views._coerce_sort_timestamp + :summary: + ``` +* - {py:obj}`_snapshot_sort_key ` + - ```{autodoc2-docstring} archivebox.core.views._snapshot_sort_key + :summary: + ``` +* - {py:obj}`_snapshot_id_from_replay_path ` + - ```{autodoc2-docstring} archivebox.core.views._snapshot_id_from_replay_path + :summary: + ``` +* - {py:obj}`_replay_path_visible ` + - ```{autodoc2-docstring} archivebox.core.views._replay_path_visible + :summary: + ``` * - {py:obj}`_latest_response_match ` - ```{autodoc2-docstring} archivebox.core.views._latest_response_match :summary: @@ -65,10 +105,26 @@ - ```{autodoc2-docstring} archivebox.core.views._latest_responses_root :summary: ``` +* - {py:obj}`_latest_snapshot_for_domain ` + - ```{autodoc2-docstring} archivebox.core.views._latest_snapshot_for_domain + :summary: + ``` +* - {py:obj}`_original_request_url ` + - ```{autodoc2-docstring} archivebox.core.views._original_request_url + :summary: + ``` * - {py:obj}`_serve_responses_path ` - ```{autodoc2-docstring} archivebox.core.views._serve_responses_path :summary: ``` +* - {py:obj}`_serve_snapshot_replay ` + - ```{autodoc2-docstring} archivebox.core.views._serve_snapshot_replay + :summary: + ``` +* - {py:obj}`_serve_original_domain_replay ` + - ```{autodoc2-docstring} archivebox.core.views._serve_original_domain_replay + :summary: + ``` * - {py:obj}`live_progress_view ` - ```{autodoc2-docstring} archivebox.core.views.live_progress_view :summary: @@ -93,6 +149,14 @@ - ```{autodoc2-docstring} archivebox.core.views.find_config_source :summary: ``` +* - {py:obj}`find_plugin_for_config_key ` + - ```{autodoc2-docstring} archivebox.core.views.find_plugin_for_config_key + :summary: + ``` +* - {py:obj}`get_config_definition_link ` + - ```{autodoc2-docstring} archivebox.core.views.get_config_definition_link + :summary: + ``` * - {py:obj}`live_config_list_view ` - ```{autodoc2-docstring} archivebox.core.views.live_config_list_view :summary: @@ -103,8 +167,58 @@ ``` ```` +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`ABX_PLUGINS_GITHUB_BASE_URL ` + - ```{autodoc2-docstring} archivebox.core.views.ABX_PLUGINS_GITHUB_BASE_URL + :summary: + ``` +* - {py:obj}`LIVE_PLUGIN_BASE_URL ` + - ```{autodoc2-docstring} archivebox.core.views.LIVE_PLUGIN_BASE_URL + :summary: + ``` +```` + ### API +````{py:data} ABX_PLUGINS_GITHUB_BASE_URL +:canonical: archivebox.core.views.ABX_PLUGINS_GITHUB_BASE_URL +:value: > + 'https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/' + +```{autodoc2-docstring} archivebox.core.views.ABX_PLUGINS_GITHUB_BASE_URL +``` + +```` + +````{py:data} LIVE_PLUGIN_BASE_URL +:canonical: archivebox.core.views.LIVE_PLUGIN_BASE_URL +:value: > + '/admin/environment/plugins/' + +```{autodoc2-docstring} archivebox.core.views.LIVE_PLUGIN_BASE_URL +``` + +```` + +````{py:function} _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]] +:canonical: archivebox.core.views._live_progress_plugin_names + +```{autodoc2-docstring} archivebox.core.views._live_progress_plugin_names +``` +```` + +````{py:function} _get_request_config(request: django.http.HttpRequest, *, resolve_plugins: bool = False) +:canonical: archivebox.core.views._get_request_config + +```{autodoc2-docstring} archivebox.core.views._get_request_config +``` +```` + ````{py:function} _files_index_target(snapshot: archivebox.core.models.Snapshot, archivefile: str | None) -> str :canonical: archivebox.core.views._files_index_target @@ -112,6 +226,20 @@ ``` ```` +````{py:function} _find_snapshot_by_ref(snapshot_ref: str) -> archivebox.core.models.Snapshot | None +:canonical: archivebox.core.views._find_snapshot_by_ref + +```{autodoc2-docstring} archivebox.core.views._find_snapshot_by_ref +``` +```` + +````{py:function} _admin_login_redirect_or_forbidden(request: django.http.HttpRequest) +:canonical: archivebox.core.views._admin_login_redirect_or_forbidden + +```{autodoc2-docstring} archivebox.core.views._admin_login_redirect_or_forbidden +``` +```` + `````{py:class} HomepageView(**kwargs) :canonical: archivebox.core.views.HomepageView @@ -141,6 +269,15 @@ Bases: {py:obj}`django.views.View` ```` +````{py:method} find_snapshots_for_id(slug: str) +:canonical: archivebox.core.views.SnapshotView.find_snapshots_for_id +:staticmethod: + +```{autodoc2-docstring} archivebox.core.views.SnapshotView.find_snapshots_for_id +``` + +```` + ````{py:method} render_live_index(request, snapshot) :canonical: archivebox.core.views.SnapshotView.render_live_index :staticmethod: @@ -191,20 +328,62 @@ Bases: {py:obj}`django.views.View` ``` ```` -````{py:function} _latest_response_match(domain: str, rel_path: str) -> tuple[pathlib.Path, pathlib.Path] | None +````{py:function} _coerce_sort_timestamp(value: str | float | None) -> float +:canonical: archivebox.core.views._coerce_sort_timestamp + +```{autodoc2-docstring} archivebox.core.views._coerce_sort_timestamp +``` +```` + +````{py:function} _snapshot_sort_key(match_path: str, cache: dict[str, float]) -> tuple[float, str] +:canonical: archivebox.core.views._snapshot_sort_key + +```{autodoc2-docstring} archivebox.core.views._snapshot_sort_key +``` +```` + +````{py:function} _snapshot_id_from_replay_path(path: pathlib.Path) -> str | None +:canonical: archivebox.core.views._snapshot_id_from_replay_path + +```{autodoc2-docstring} archivebox.core.views._snapshot_id_from_replay_path +``` +```` + +````{py:function} _replay_path_visible(request: django.http.HttpRequest, path: pathlib.Path) -> bool +:canonical: archivebox.core.views._replay_path_visible + +```{autodoc2-docstring} archivebox.core.views._replay_path_visible +``` +```` + +````{py:function} _latest_response_match(request: django.http.HttpRequest, domain: str, rel_path: str, *, data_root: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path] | None :canonical: archivebox.core.views._latest_response_match ```{autodoc2-docstring} archivebox.core.views._latest_response_match ``` ```` -````{py:function} _latest_responses_root(domain: str) -> pathlib.Path | None +````{py:function} _latest_responses_root(request: django.http.HttpRequest, domain: str, *, data_root: pathlib.Path) -> pathlib.Path | None :canonical: archivebox.core.views._latest_responses_root ```{autodoc2-docstring} archivebox.core.views._latest_responses_root ``` ```` +````{py:function} _latest_snapshot_for_domain(request: django.http.HttpRequest, domain: str) -> archivebox.core.models.Snapshot | None +:canonical: archivebox.core.views._latest_snapshot_for_domain + +```{autodoc2-docstring} archivebox.core.views._latest_snapshot_for_domain +``` +```` + +````{py:function} _original_request_url(domain: str, path: str = '', query_string: str = '') -> str +:canonical: archivebox.core.views._original_request_url + +```{autodoc2-docstring} archivebox.core.views._original_request_url +``` +```` + ````{py:function} _serve_responses_path(request, responses_root: pathlib.Path, rel_path: str, show_indexes: bool) :canonical: archivebox.core.views._serve_responses_path @@ -212,6 +391,20 @@ Bases: {py:obj}`django.views.View` ``` ```` +````{py:function} _serve_snapshot_replay(request: django.http.HttpRequest, snapshot: archivebox.core.models.Snapshot, path: str = '') +:canonical: archivebox.core.views._serve_snapshot_replay + +```{autodoc2-docstring} archivebox.core.views._serve_snapshot_replay +``` +```` + +````{py:function} _serve_original_domain_replay(request: django.http.HttpRequest, domain: str, path: str = '') +:canonical: archivebox.core.views._serve_original_domain_replay + +```{autodoc2-docstring} archivebox.core.views._serve_original_domain_replay +``` +```` + `````{py:class} SnapshotHostView(**kwargs) :canonical: archivebox.core.views.SnapshotHostView @@ -236,6 +429,30 @@ Bases: {py:obj}`django.views.View` ````` +`````{py:class} SnapshotReplayView(**kwargs) +:canonical: archivebox.core.views.SnapshotReplayView + +Bases: {py:obj}`django.views.View` + +```{autodoc2-docstring} archivebox.core.views.SnapshotReplayView +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.views.SnapshotReplayView.__init__ +``` + +````{py:method} get(request, snapshot_id: str, path: str = '') +:canonical: archivebox.core.views.SnapshotReplayView.get + +```{autodoc2-docstring} archivebox.core.views.SnapshotReplayView.get +``` + +```` + +````` + `````{py:class} OriginalDomainHostView(**kwargs) :canonical: archivebox.core.views.OriginalDomainHostView @@ -260,6 +477,30 @@ Bases: {py:obj}`django.views.View` ````` +`````{py:class} OriginalDomainReplayView(**kwargs) +:canonical: archivebox.core.views.OriginalDomainReplayView + +Bases: {py:obj}`django.views.View` + +```{autodoc2-docstring} archivebox.core.views.OriginalDomainReplayView +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.views.OriginalDomainReplayView.__init__ +``` + +````{py:method} get(request, domain: str, path: str = '') +:canonical: archivebox.core.views.OriginalDomainReplayView.get + +```{autodoc2-docstring} archivebox.core.views.OriginalDomainReplayView.get +``` + +```` + +````` + `````{py:class} PublicIndexView(**kwargs) :canonical: archivebox.core.views.PublicIndexView @@ -285,16 +526,6 @@ Bases: {py:obj}`django.views.generic.list.ListView` ```` -````{py:attribute} paginate_by -:canonical: archivebox.core.views.PublicIndexView.paginate_by -:value: > - None - -```{autodoc2-docstring} archivebox.core.views.PublicIndexView.paginate_by -``` - -```` - ````{py:attribute} ordering :canonical: archivebox.core.views.PublicIndexView.ordering :value: > @@ -305,6 +536,21 @@ Bases: {py:obj}`django.views.generic.list.ListView` ```` +````{py:attribute} paginator_class +:canonical: archivebox.core.views.PublicIndexView.paginator_class +:value: > + None + +```{autodoc2-docstring} archivebox.core.views.PublicIndexView.paginator_class +``` + +```` + +````{py:method} get_paginate_by(queryset) +:canonical: archivebox.core.views.PublicIndexView.get_paginate_by + +```` + ````{py:method} get_context_data(**kwargs) :canonical: archivebox.core.views.PublicIndexView.get_context_data @@ -358,6 +604,11 @@ Bases: {py:obj}`django.contrib.auth.mixins.UserPassesTestMixin`, {py:obj}`django ```` +````{py:method} get_form_kwargs() +:canonical: archivebox.core.views.AddView.get_form_kwargs + +```` + ````{py:method} test_func() :canonical: archivebox.core.views.AddView.test_func @@ -431,7 +682,7 @@ Bases: {py:obj}`archivebox.core.views.AddView` ```` -````{py:method} get(request, url: str) +````{py:method} get(request: django.http.HttpRequest, *args: object, **kwargs: object) :canonical: archivebox.core.views.WebAddView.get ```` @@ -504,6 +755,20 @@ Bases: {py:obj}`django.views.View` ``` ```` +````{py:function} find_plugin_for_config_key(key: str) -> str | None +:canonical: archivebox.core.views.find_plugin_for_config_key + +```{autodoc2-docstring} archivebox.core.views.find_plugin_for_config_key +``` +```` + +````{py:function} get_config_definition_link(key: str) -> tuple[str, str] +:canonical: archivebox.core.views.get_config_definition_link + +```{autodoc2-docstring} archivebox.core.views.get_config_definition_link +``` +```` + ````{py:function} live_config_list_view(request: django.http.HttpRequest, **kwargs) -> admin_data_views.typing.TableContext :canonical: archivebox.core.views.live_config_list_view diff --git a/docs/apidocs/archivebox/archivebox.core.widgets.md b/docs/apidocs/archivebox/archivebox.core.widgets.md index 52d79364..ff013e87 100644 --- a/docs/apidocs/archivebox/archivebox.core.widgets.md +++ b/docs/apidocs/archivebox/archivebox.core.widgets.md @@ -19,6 +19,10 @@ - ```{autodoc2-docstring} archivebox.core.widgets.TagEditorWidget :summary: ``` +* - {py:obj}`URLFiltersWidget ` + - ```{autodoc2-docstring} archivebox.core.widgets.URLFiltersWidget + :summary: + ``` * - {py:obj}`InlineTagEditorWidget ` - ```{autodoc2-docstring} archivebox.core.widgets.InlineTagEditorWidget :summary: @@ -43,8 +47,7 @@ Bases: {py:obj}`django.forms.Widget` ````{py:attribute} template_name :canonical: archivebox.core.widgets.TagEditorWidget.template_name -:value: > - None +:value: ```{autodoc2-docstring} archivebox.core.widgets.TagEditorWidget.template_name ``` @@ -113,7 +116,42 @@ Bases: {py:obj}`django.forms.Widget` `````` -`````{py:class} InlineTagEditorWidget(attrs=None, snapshot_id=None) +`````{py:class} URLFiltersWidget(attrs=None, *, source_selector='textarea[name="url"]') +:canonical: archivebox.core.widgets.URLFiltersWidget + +Bases: {py:obj}`django.forms.Widget` + +```{autodoc2-docstring} archivebox.core.widgets.URLFiltersWidget +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.core.widgets.URLFiltersWidget.__init__ +``` + +````{py:attribute} template_name +:canonical: archivebox.core.widgets.URLFiltersWidget.template_name +:value: + +```{autodoc2-docstring} archivebox.core.widgets.URLFiltersWidget.template_name +``` + +```` + +````{py:method} render(name, value, attrs=None, renderer=None) +:canonical: archivebox.core.widgets.URLFiltersWidget.render + +```` + +````{py:method} value_from_datadict(data, files, name) +:canonical: archivebox.core.widgets.URLFiltersWidget.value_from_datadict + +```` + +````` + +`````{py:class} InlineTagEditorWidget(attrs=None, snapshot_id=None, editable=True) :canonical: archivebox.core.widgets.InlineTagEditorWidget Bases: {py:obj}`archivebox.core.widgets.TagEditorWidget` diff --git a/docs/apidocs/archivebox/archivebox.crawls.admin.md b/docs/apidocs/archivebox/archivebox.crawls.admin.md index 699ecdf9..ac91978c 100644 --- a/docs/apidocs/archivebox/archivebox.crawls.admin.md +++ b/docs/apidocs/archivebox/archivebox.crawls.admin.md @@ -15,6 +15,14 @@ :class: autosummary longtable :align: left +* - {py:obj}`MaxDepthListFilter ` + - ```{autodoc2-docstring} archivebox.crawls.admin.MaxDepthListFilter + :summary: + ``` +* - {py:obj}`URLFiltersField ` + - ```{autodoc2-docstring} archivebox.crawls.admin.URLFiltersField + :summary: + ``` * - {py:obj}`CrawlAdminForm ` - ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdminForm :summary: @@ -43,14 +51,94 @@ ### API -````{py:function} render_snapshots_list(snapshots_qs, limit=20) +`````{py:class} MaxDepthListFilter(request, params, model, model_admin) +:canonical: archivebox.crawls.admin.MaxDepthListFilter + +Bases: {py:obj}`django.contrib.admin.SimpleListFilter` + +```{autodoc2-docstring} archivebox.crawls.admin.MaxDepthListFilter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.crawls.admin.MaxDepthListFilter.__init__ +``` + +````{py:attribute} title +:canonical: archivebox.crawls.admin.MaxDepthListFilter.title +:value: > + 'max depth' + +```{autodoc2-docstring} archivebox.crawls.admin.MaxDepthListFilter.title +``` + +```` + +````{py:attribute} parameter_name +:canonical: archivebox.crawls.admin.MaxDepthListFilter.parameter_name +:value: > + 'max_depth' + +```{autodoc2-docstring} archivebox.crawls.admin.MaxDepthListFilter.parameter_name +``` + +```` + +````{py:method} lookups(request, model_admin) +:canonical: archivebox.crawls.admin.MaxDepthListFilter.lookups + +```` + +````{py:method} queryset(request, queryset) +:canonical: archivebox.crawls.admin.MaxDepthListFilter.queryset + +```` + +````` + +````{py:function} render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, prefix='snapshots') :canonical: archivebox.crawls.admin.render_snapshots_list ```{autodoc2-docstring} archivebox.crawls.admin.render_snapshots_list ``` ```` -``````{py:class} CrawlAdminForm(data=None, files=None, auto_id='id_%s', prefix=None, initial=None, error_class=ErrorList, label_suffix=None, empty_permitted=False, instance=None, use_required_attribute=None, renderer=None) +`````{py:class} URLFiltersField(*, required=True, widget=None, label=None, initial=None, help_text='', error_messages=None, show_hidden_initial=False, validators=(), localize=False, disabled=False, label_suffix=None, template_name=None, bound_field_class=None) +:canonical: archivebox.crawls.admin.URLFiltersField + +Bases: {py:obj}`django.forms.Field` + +```{autodoc2-docstring} archivebox.crawls.admin.URLFiltersField +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.crawls.admin.URLFiltersField.__init__ +``` + +````{py:attribute} widget +:canonical: archivebox.crawls.admin.URLFiltersField.widget +:value: > + 'URLFiltersWidget(...)' + +```{autodoc2-docstring} archivebox.crawls.admin.URLFiltersField.widget +``` + +```` + +````{py:method} to_python(value) +:canonical: archivebox.crawls.admin.URLFiltersField.to_python + +```{autodoc2-docstring} archivebox.crawls.admin.URLFiltersField.to_python +``` + +```` + +````` + +``````{py:class} CrawlAdminForm(*args, **kwargs) :canonical: archivebox.crawls.admin.CrawlAdminForm Bases: {py:obj}`django.forms.ModelForm` @@ -64,6 +152,26 @@ Bases: {py:obj}`django.forms.ModelForm` ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdminForm.__init__ ``` +````{py:attribute} tags_editor +:canonical: archivebox.crawls.admin.CrawlAdminForm.tags_editor +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdminForm.tags_editor +``` + +```` + +````{py:attribute} url_filters +:canonical: archivebox.crawls.admin.CrawlAdminForm.url_filters +:value: > + 'URLFiltersField(...)' + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdminForm.url_filters +``` + +```` + `````{py:class} Meta :canonical: archivebox.crawls.admin.CrawlAdminForm.Meta @@ -102,9 +210,30 @@ Bases: {py:obj}`django.forms.ModelForm` ````` +````{py:method} clean_tags_editor() +:canonical: archivebox.crawls.admin.CrawlAdminForm.clean_tags_editor + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdminForm.clean_tags_editor +``` + +```` + +````{py:method} clean_url_filters() +:canonical: archivebox.crawls.admin.CrawlAdminForm.clean_url_filters + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdminForm.clean_url_filters +``` + +```` + +````{py:method} save(commit=True) +:canonical: archivebox.crawls.admin.CrawlAdminForm.save + +```` + `````` -`````{py:class} CrawlAdmin(model, admin_site) +``````{py:class} CrawlAdmin(model, admin_site) :canonical: archivebox.crawls.admin.CrawlAdmin Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archivebox.base_models.admin.BaseModelAdmin` @@ -119,10 +248,50 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` +````{py:attribute} change_form_template +:canonical: archivebox.crawls.admin.CrawlAdmin.change_form_template +:value: > + 'admin/crawls/crawl/change_form.html' + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.change_form_template +``` + +```` + +````{py:attribute} list_select_related +:canonical: archivebox.crawls.admin.CrawlAdmin.list_select_related +:value: > + () + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.list_select_related +``` + +```` + +````{py:attribute} paginator +:canonical: archivebox.crawls.admin.CrawlAdmin.paginator +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.paginator +``` + +```` + +````{py:attribute} show_full_result_count +:canonical: archivebox.crawls.admin.CrawlAdmin.show_full_result_count +:value: > + False + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.show_full_result_count +``` + +```` + ````{py:attribute} list_display :canonical: archivebox.crawls.admin.CrawlAdmin.list_display :value: > - ('id', 'created_at', 'created_by', 'max_depth', 'label', 'notes', 'urls_preview', 'schedule_str', 's... + ('short_id', 'permissions_badge', 'created_at', 'owner', 'depth', 'status_with_stop_reason', 'pause_... ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.list_display ``` @@ -152,7 +321,7 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ````{py:attribute} readonly_fields :canonical: archivebox.crawls.admin.CrawlAdmin.readonly_fields :value: > - ('created_at', 'modified_at', 'snapshots') + ('created_at', 'modified_at', 'stop_reason_display') ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.readonly_fields ``` @@ -162,17 +331,27 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ````{py:attribute} fieldsets :canonical: archivebox.crawls.admin.CrawlAdmin.fieldsets :value: > - (('URLs',), ('Info',), ('Settings',), ('Status',), ('Relations',), ('Timestamps',), ('Snapshots',)) + (('URLs',), ('Overview',), ('Config',)) ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.fieldsets ``` ```` +````{py:attribute} add_fieldsets +:canonical: archivebox.crawls.admin.CrawlAdmin.add_fieldsets +:value: > + (('URLs',), ('Overview',), ('Config',)) + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.add_fieldsets +``` + +```` + ````{py:attribute} list_filter :canonical: archivebox.crawls.admin.CrawlAdmin.list_filter :value: > - ('max_depth', 'schedule', 'created_by', 'status', 'retry_at') + () ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.list_filter ``` @@ -192,7 +371,7 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ````{py:attribute} list_per_page :canonical: archivebox.crawls.admin.CrawlAdmin.list_per_page :value: > - 100 + 50 ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.list_per_page ``` @@ -202,7 +381,7 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ````{py:attribute} actions :canonical: archivebox.crawls.admin.CrawlAdmin.actions :value: > - ['delete_selected_batched'] + ['pause_selected_crawls', 'resume_selected_crawls', 'seal_selected_crawls', 'delete_selected_batched... ```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.actions ``` @@ -219,6 +398,55 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` +`````{py:class} Media +:canonical: archivebox.crawls.admin.CrawlAdmin.Media + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.Media +``` + +````{py:attribute} css +:canonical: archivebox.crawls.admin.CrawlAdmin.Media.css +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.Media.css +``` + +```` + +````{py:attribute} js +:canonical: archivebox.crawls.admin.CrawlAdmin.Media.js +:value: > + ('admin/crawls/crawl_admin.js',) + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.Media.js +``` + +```` + +````` + +````{py:method} changelist_view(request, extra_context=None) +:canonical: archivebox.crawls.admin.CrawlAdmin.changelist_view + +```` + +````{py:method} should_annotate_snapshot_counts(request) +:canonical: archivebox.crawls.admin.CrawlAdmin.should_annotate_snapshot_counts + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.should_annotate_snapshot_counts +``` + +```` + +````{py:method} hydrate_visible_snapshot_counts(crawls) +:canonical: archivebox.crawls.admin.CrawlAdmin.hydrate_visible_snapshot_counts + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.hydrate_visible_snapshot_counts +``` + +```` + ````{py:method} get_queryset(request) :canonical: archivebox.crawls.admin.CrawlAdmin.get_queryset @@ -227,6 +455,37 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` +````{py:method} change_view(request, object_id, form_url='', extra_context=None) +:canonical: archivebox.crawls.admin.CrawlAdmin.change_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.change_view +``` + +```` + +````{py:method} add_view(request, form_url='', extra_context=None) +:canonical: archivebox.crawls.admin.CrawlAdmin.add_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.add_view +``` + +```` + +````{py:method} get_fieldsets(request, obj=None) +:canonical: archivebox.crawls.admin.CrawlAdmin.get_fieldsets + +```` + +````{py:method} get_urls() +:canonical: archivebox.crawls.admin.CrawlAdmin.get_urls + +```` + +````{py:method} get_actions(request) +:canonical: archivebox.crawls.admin.CrawlAdmin.get_actions + +```` + ````{py:method} delete_selected_batched(request, queryset) :canonical: archivebox.crawls.admin.CrawlAdmin.delete_selected_batched @@ -235,6 +494,46 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` +````{py:method} pause_selected_crawls(request, queryset) +:canonical: archivebox.crawls.admin.CrawlAdmin.pause_selected_crawls + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.pause_selected_crawls +``` + +```` + +````{py:method} resume_selected_crawls(request, queryset) +:canonical: archivebox.crawls.admin.CrawlAdmin.resume_selected_crawls + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.resume_selected_crawls +``` + +```` + +````{py:method} seal_selected_crawls(request, queryset) +:canonical: archivebox.crawls.admin.CrawlAdmin.seal_selected_crawls + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.seal_selected_crawls +``` + +```` + +````{py:method} set_crawl_permissions(request, queryset) +:canonical: archivebox.crawls.admin.CrawlAdmin.set_crawl_permissions + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.set_crawl_permissions +``` + +```` + +````{py:method} update_crawl_permissions(queryset, permissions) +:canonical: archivebox.crawls.admin.CrawlAdmin.update_crawl_permissions + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.update_crawl_permissions +``` + +```` + ````{py:method} recrawl(request, obj) :canonical: archivebox.crawls.admin.CrawlAdmin.recrawl @@ -243,18 +542,122 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` -````{py:method} num_snapshots(obj) -:canonical: archivebox.crawls.admin.CrawlAdmin.num_snapshots +````{py:method} stop_reason_display(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.stop_reason_display -```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.num_snapshots +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.stop_reason_display ``` ```` -````{py:method} snapshots(obj) -:canonical: archivebox.crawls.admin.CrawlAdmin.snapshots +````{py:method} stop_reason_for_crawl(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.stop_reason_for_crawl -```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.snapshots +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.stop_reason_for_crawl +``` + +```` + +````{py:method} limit_config_for_crawl(obj, output_dir) +:canonical: archivebox.crawls.admin.CrawlAdmin.limit_config_for_crawl + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.limit_config_for_crawl +``` + +```` + +````{py:method} status_with_stop_reason(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.status_with_stop_reason + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.status_with_stop_reason +``` + +```` + +````{py:method} short_id(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.short_id + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.short_id +``` + +```` + +````{py:method} owner(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.owner + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.owner +``` + +```` + +````{py:method} depth(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.depth + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.depth +``` + +```` + +````{py:method} permissions_badge(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.permissions_badge + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.permissions_badge +``` + +```` + +````{py:method} pause_resume_control(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.pause_resume_control + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.pause_resume_control +``` + +```` + +````{py:method} num_archived_snapshots(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.num_archived_snapshots + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.num_archived_snapshots +``` + +```` + +````{py:method} num_total_snapshots(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.num_total_snapshots + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.num_total_snapshots +``` + +```` + +````{py:method} snapshots_changelist(obj) +:canonical: archivebox.crawls.admin.CrawlAdmin.snapshots_changelist + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.snapshots_changelist +``` + +```` + +````{py:method} delete_snapshot_view(request: django.http.HttpRequest, object_id: str, snapshot_id: str) +:canonical: archivebox.crawls.admin.CrawlAdmin.delete_snapshot_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.delete_snapshot_view +``` + +```` + +````{py:method} exclude_domain_view(request: django.http.HttpRequest, object_id: str, snapshot_id: str) +:canonical: archivebox.crawls.admin.CrawlAdmin.exclude_domain_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.exclude_domain_view +``` + +```` + +````{py:method} set_permissions_view(request: django.http.HttpRequest, object_id: str) +:canonical: archivebox.crawls.admin.CrawlAdmin.set_permissions_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.set_permissions_view ``` ```` @@ -275,14 +678,6 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` -````{py:method} health_display(obj) -:canonical: archivebox.crawls.admin.CrawlAdmin.health_display - -```{autodoc2-docstring} archivebox.crawls.admin.CrawlAdmin.health_display -``` - -```` - ````{py:method} urls_editor(obj) :canonical: archivebox.crawls.admin.CrawlAdmin.urls_editor @@ -291,7 +686,7 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` -````` +`````` `````{py:class} CrawlScheduleAdmin(model, admin_site) :canonical: archivebox.crawls.admin.CrawlScheduleAdmin @@ -388,6 +783,37 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} get_queryset(request) +:canonical: archivebox.crawls.admin.CrawlScheduleAdmin.get_queryset + +```` + +````{py:method} change_view(request, object_id, form_url='', extra_context=None) +:canonical: archivebox.crawls.admin.CrawlScheduleAdmin.change_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlScheduleAdmin.change_view +``` + +```` + +````{py:method} add_view(request, form_url='', extra_context=None) +:canonical: archivebox.crawls.admin.CrawlScheduleAdmin.add_view + +```{autodoc2-docstring} archivebox.crawls.admin.CrawlScheduleAdmin.add_view +``` + +```` + +````{py:method} get_fieldsets(request, obj=None) +:canonical: archivebox.crawls.admin.CrawlScheduleAdmin.get_fieldsets + +```` + +````{py:method} save_model(request, obj, form, change) +:canonical: archivebox.crawls.admin.CrawlScheduleAdmin.save_model + +```` + ````{py:method} template_str(obj) :canonical: archivebox.crawls.admin.CrawlScheduleAdmin.template_str diff --git a/docs/apidocs/archivebox/archivebox.crawls.md b/docs/apidocs/archivebox/archivebox.crawls.md index 512be7f9..733002b5 100644 --- a/docs/apidocs/archivebox/archivebox.crawls.md +++ b/docs/apidocs/archivebox/archivebox.crawls.md @@ -13,7 +13,7 @@ :titlesonly: :maxdepth: 1 -archivebox.crawls.schedule_utils +archivebox.crawls.schedule_util archivebox.crawls.models archivebox.crawls.apps archivebox.crawls.admin diff --git a/docs/apidocs/archivebox/archivebox.crawls.models.md b/docs/apidocs/archivebox/archivebox.crawls.models.md index f367b367..b1fe8bdc 100644 --- a/docs/apidocs/archivebox/archivebox.crawls.models.md +++ b/docs/apidocs/archivebox/archivebox.crawls.models.md @@ -20,9 +20,7 @@ * - {py:obj}`Crawl ` - * - {py:obj}`CrawlMachine ` - - ```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine - :summary: - ``` + - ```` ### API @@ -137,7 +135,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID`, {py:obj}`archivebo `````{py:class} Meta :canonical: archivebox.crawls.models.CrawlSchedule.Meta -Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta` ````{py:attribute} app_label :canonical: archivebox.crawls.models.CrawlSchedule.Meta.app_label @@ -227,7 +225,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ``````{py:class} Crawl(*args, **kwargs) :canonical: archivebox.crawls.models.Crawl -Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine` ````{py:attribute} id :canonical: archivebox.crawls.models.Crawl.id @@ -289,6 +287,16 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch ```` +````{py:attribute} permissions +:canonical: archivebox.crawls.models.Crawl.permissions +:value: > + 'GeneratedField(...)' + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.permissions +``` + +```` + ````{py:attribute} max_depth :canonical: archivebox.crawls.models.Crawl.max_depth :value: > @@ -309,12 +317,12 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch ```` -````{py:attribute} persona_id -:canonical: archivebox.crawls.models.Crawl.persona_id +````{py:attribute} persona +:canonical: archivebox.crawls.models.Crawl.persona :value: > - 'UUIDField(...)' + 'ForeignKey(...)' -```{autodoc2-docstring} archivebox.crawls.models.Crawl.persona_id +```{autodoc2-docstring} archivebox.crawls.models.Crawl.persona ``` ```` @@ -419,6 +427,38 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch ```` +````{py:attribute} delete_after_final_statuses +:canonical: archivebox.crawls.models.Crawl.delete_after_final_statuses +:value: > + () + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.delete_after_final_statuses +``` + +```` + +````{py:attribute} schedule_id +:canonical: archivebox.crawls.models.Crawl.schedule_id +:type: uuid.UUID | None +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.schedule_id +``` + +```` + +````{py:attribute} sm +:canonical: archivebox.crawls.models.Crawl.sm +:type: CrawlMachine +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.sm +``` + +```` + ````{py:attribute} snapshot_set :canonical: archivebox.crawls.models.Crawl.snapshot_set :type: django.db.models.Manager[archivebox.core.models.Snapshot] @@ -433,7 +473,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`arch `````{py:class} Meta :canonical: archivebox.crawls.models.Crawl.Meta -Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta` ````{py:attribute} app_label :canonical: archivebox.crawls.models.Crawl.Meta.app_label @@ -465,6 +505,16 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:attribute} indexes +:canonical: archivebox.crawls.models.Crawl.Meta.indexes +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.Meta.indexes +``` + +```` + ````` ````{py:method} __str__() @@ -472,6 +522,63 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} get_delete_after_config_value() +:canonical: archivebox.crawls.models.Crawl.get_delete_after_config_value + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.get_delete_after_config_value +``` + +```` + +````{py:method} pause(*, save: bool = True) -> bool +:canonical: archivebox.crawls.models.Crawl.pause + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.pause +``` + +```` + +````{py:method} resume(*, when=None, save: bool = True) -> bool +:canonical: archivebox.crawls.models.Crawl.resume + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.resume +``` + +```` + +````{py:method} cancel() -> None +:canonical: archivebox.crawls.models.Crawl.cancel + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.cancel +``` + +```` + +````{py:method} schedule_child_snapshots_for_sealing() -> int +:canonical: archivebox.crawls.models.Crawl.schedule_child_snapshots_for_sealing + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.schedule_child_snapshots_for_sealing +``` + +```` + +````{py:method} schedule_child_snapshots_for_pause() -> int +:canonical: archivebox.crawls.models.Crawl.schedule_child_snapshots_for_pause + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.schedule_child_snapshots_for_pause +``` + +```` + +````{py:method} missing_delete_at_candidates() +:canonical: archivebox.crawls.models.Crawl.missing_delete_at_candidates +:classmethod: + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.missing_delete_at_candidates +``` + +```` + ````{py:method} save(*args, **kwargs) :canonical: archivebox.crawls.models.Crawl.save @@ -486,6 +593,31 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} parse_tag_names(tags: collections.abc.Iterable[str] | str, *, pattern: str = ',') -> list[str] +:canonical: archivebox.crawls.models.Crawl.parse_tag_names +:staticmethod: + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.parse_tag_names +``` + +```` + +````{py:method} current_tag_names() -> list[str] +:canonical: archivebox.crawls.models.Crawl.current_tag_names + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.current_tag_names +``` + +```` + +````{py:method} apply_snapshot_tag_diff(*, added_tag_names: collections.abc.Iterable[str], removed_tag_names: collections.abc.Iterable[str]) -> None +:canonical: archivebox.crawls.models.Crawl.apply_snapshot_tag_diff + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.apply_snapshot_tag_diff +``` + +```` + ````{py:method} to_json() -> dict :canonical: archivebox.crawls.models.Crawl.to_json @@ -494,7 +626,7 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` -````{py:method} from_json(record: dict, overrides: dict = None) +````{py:method} from_json(record: dict, overrides: dict | None = None) :canonical: archivebox.crawls.models.Crawl.from_json :staticmethod: @@ -503,6 +635,14 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} output_dir_for_config(runtime_config: collections.abc.Mapping[str, typing.Any] | typing.Any) -> pathlib.Path +:canonical: archivebox.crawls.models.Crawl.output_dir_for_config + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.output_dir_for_config +``` + +```` + ````{py:property} output_dir :canonical: archivebox.crawls.models.Crawl.output_dir :type: pathlib.Path @@ -520,6 +660,153 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} normalize_domain(value: str) -> str +:canonical: archivebox.crawls.models.Crawl.normalize_domain +:staticmethod: + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.normalize_domain +``` + +```` + +````{py:method} split_filter_patterns(value) -> list[str] +:canonical: archivebox.crawls.models.Crawl.split_filter_patterns +:staticmethod: + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.split_filter_patterns +``` + +```` + +````{py:method} _pattern_matches_url(url: str, pattern: str) -> bool +:canonical: archivebox.crawls.models.Crawl._pattern_matches_url +:classmethod: + +```{autodoc2-docstring} archivebox.crawls.models.Crawl._pattern_matches_url +``` + +```` + +````{py:method} get_url_allowlist(*, use_effective_config: bool = False, snapshot=None) -> list[str] +:canonical: archivebox.crawls.models.Crawl.get_url_allowlist + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.get_url_allowlist +``` + +```` + +````{py:method} get_url_denylist(*, use_effective_config: bool = False, snapshot=None) -> list[str] +:canonical: archivebox.crawls.models.Crawl.get_url_denylist + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.get_url_denylist +``` + +```` + +````{py:method} url_passes_filters(url: str, *, snapshot=None, use_effective_config: bool = True) -> bool +:canonical: archivebox.crawls.models.Crawl.url_passes_filters + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.url_passes_filters +``` + +```` + +````{py:method} url_passes_compiled_filters(url: str, *, allowlist: list[str], denylist: list[str]) -> bool +:canonical: archivebox.crawls.models.Crawl.url_passes_compiled_filters + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.url_passes_compiled_filters +``` + +```` + +````{py:method} set_url_filters(allowlist, denylist) -> None +:canonical: archivebox.crawls.models.Crawl.set_url_filters + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.set_url_filters +``` + +```` + +````{py:method} apply_crawl_config_filters() -> dict[str, int] +:canonical: archivebox.crawls.models.Crawl.apply_crawl_config_filters + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.apply_crawl_config_filters +``` + +```` + +````{py:method} _iter_url_lines() -> list[tuple[str, str]] +:canonical: archivebox.crawls.models.Crawl._iter_url_lines + +```{autodoc2-docstring} archivebox.crawls.models.Crawl._iter_url_lines +``` + +```` + +````{py:method} count_urls_for_limit() -> int +:canonical: archivebox.crawls.models.Crawl.count_urls_for_limit + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.count_urls_for_limit +``` + +```` + +````{py:method} remaining_url_capacity() -> int | None +:canonical: archivebox.crawls.models.Crawl.remaining_url_capacity + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.remaining_url_capacity +``` + +```` + +````{py:method} has_remaining_url_capacity() -> bool +:canonical: archivebox.crawls.models.Crawl.has_remaining_url_capacity + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.has_remaining_url_capacity +``` + +```` + +````{py:method} remaining_snapshot_capacity() -> int | None +:canonical: archivebox.crawls.models.Crawl.remaining_snapshot_capacity + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.remaining_snapshot_capacity +``` + +```` + +````{py:method} has_remaining_snapshot_capacity() -> bool +:canonical: archivebox.crawls.models.Crawl.has_remaining_snapshot_capacity + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.has_remaining_snapshot_capacity +``` + +```` + +````{py:method} prune_urls(predicate) -> list[str] +:canonical: archivebox.crawls.models.Crawl.prune_urls + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.prune_urls +``` + +```` + +````{py:method} prune_url(url: str) -> int +:canonical: archivebox.crawls.models.Crawl.prune_url + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.prune_url +``` + +```` + +````{py:method} exclude_domain(domain: str) -> dict[str, int | str | bool] +:canonical: archivebox.crawls.models.Crawl.exclude_domain + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.exclude_domain +``` + +```` + ````{py:method} get_system_task() -> str | None :canonical: archivebox.crawls.models.Crawl.get_system_task @@ -528,6 +815,22 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} resolve_persona() +:canonical: archivebox.crawls.models.Crawl.resolve_persona + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.resolve_persona +``` + +```` + +````{py:method} limit_stop_reason() -> str +:canonical: archivebox.crawls.models.Crawl.limit_stop_reason + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.limit_stop_reason +``` + +```` + ````{py:method} add_url(entry: dict) -> bool :canonical: archivebox.crawls.models.Crawl.add_url @@ -544,6 +847,30 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ```` +````{py:method} create_discovered_snapshot(parent_snapshot, *, url: str, depth: int, title: str = '', tags: str = '', created_by_id: int | None = None) +:canonical: archivebox.crawls.models.Crawl.create_discovered_snapshot + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.create_discovered_snapshot +``` + +```` + +````{py:method} create_discovered_snapshots(parent_snapshot, records: collections.abc.Iterable[collections.abc.Mapping[str, typing.Any]], *, depth: int, created_by_id: int | None = None) -> list[archivebox.core.models.Snapshot] +:canonical: archivebox.crawls.models.Crawl.create_discovered_snapshots + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.create_discovered_snapshots +``` + +```` + +````{py:method} install_declared_binaries(binary_names: set[str], machine=None) -> None +:canonical: archivebox.crawls.models.Crawl.install_declared_binaries + +```{autodoc2-docstring} archivebox.crawls.models.Crawl.install_declared_binaries +``` + +```` + ````{py:method} run() -> Snapshot | None :canonical: archivebox.crawls.models.Crawl.run @@ -575,14 +902,16 @@ Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine +````{py:attribute} crawl +:canonical: archivebox.crawls.models.CrawlMachine.crawl +:type: archivebox.crawls.models.Crawl +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.crawl ``` -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.__init__ -``` +```` ````{py:attribute} model_attr_name :canonical: archivebox.crawls.models.CrawlMachine.model_attr_name @@ -614,6 +943,16 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:attribute} paused +:canonical: archivebox.crawls.models.CrawlMachine.paused +:value: > + 'State(...)' + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.paused +``` + +```` + ````{py:attribute} sealed :canonical: archivebox.crawls.models.CrawlMachine.sealed :value: > @@ -637,13 +976,33 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ````{py:attribute} seal :canonical: archivebox.crawls.models.CrawlMachine.seal :value: > - 'to(...)' + None ```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.seal ``` ```` +````{py:attribute} pause_requested +:canonical: archivebox.crawls.models.CrawlMachine.pause_requested +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.pause_requested +``` + +```` + +````{py:attribute} resume_requested +:canonical: archivebox.crawls.models.CrawlMachine.resume_requested +:value: > + 'to(...)' + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.resume_requested +``` + +```` + ````{py:method} can_start() -> bool :canonical: archivebox.crawls.models.CrawlMachine.can_start @@ -660,6 +1019,22 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:method} has_finished_snapshots() -> bool +:canonical: archivebox.crawls.models.CrawlMachine.has_finished_snapshots + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.has_finished_snapshots +``` + +```` + +````{py:method} enter_queued() +:canonical: archivebox.crawls.models.CrawlMachine.enter_queued + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_queued +``` + +```` + ````{py:method} enter_started() :canonical: archivebox.crawls.models.CrawlMachine.enter_started @@ -668,6 +1043,14 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:method} enter_paused() +:canonical: archivebox.crawls.models.CrawlMachine.enter_paused + +```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_paused +``` + +```` + ````{py:method} enter_sealed() :canonical: archivebox.crawls.models.CrawlMachine.enter_sealed diff --git a/docs/apidocs/archivebox/archivebox.crawls.schedule_util.md b/docs/apidocs/archivebox/archivebox.crawls.schedule_util.md new file mode 100644 index 00000000..fd6a93f0 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.crawls.schedule_util.md @@ -0,0 +1,76 @@ +# {py:mod}`archivebox.crawls.schedule_util` + +```{py:module} archivebox.crawls.schedule_util +``` + +```{autodoc2-docstring} archivebox.crawls.schedule_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`normalize_schedule ` + - ```{autodoc2-docstring} archivebox.crawls.schedule_util.normalize_schedule + :summary: + ``` +* - {py:obj}`validate_schedule ` + - ```{autodoc2-docstring} archivebox.crawls.schedule_util.validate_schedule + :summary: + ``` +* - {py:obj}`next_run_for_schedule ` + - ```{autodoc2-docstring} archivebox.crawls.schedule_util.next_run_for_schedule + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`SCHEDULE_ALIASES ` + - ```{autodoc2-docstring} archivebox.crawls.schedule_util.SCHEDULE_ALIASES + :summary: + ``` +```` + +### API + +````{py:data} SCHEDULE_ALIASES +:canonical: archivebox.crawls.schedule_util.SCHEDULE_ALIASES +:type: dict[str, str] +:value: > + None + +```{autodoc2-docstring} archivebox.crawls.schedule_util.SCHEDULE_ALIASES +``` + +```` + +````{py:function} normalize_schedule(schedule: str) -> str +:canonical: archivebox.crawls.schedule_util.normalize_schedule + +```{autodoc2-docstring} archivebox.crawls.schedule_util.normalize_schedule +``` +```` + +````{py:function} validate_schedule(schedule: str) -> str +:canonical: archivebox.crawls.schedule_util.validate_schedule + +```{autodoc2-docstring} archivebox.crawls.schedule_util.validate_schedule +``` +```` + +````{py:function} next_run_for_schedule(schedule: str, after: datetime.datetime) -> datetime.datetime +:canonical: archivebox.crawls.schedule_util.next_run_for_schedule + +```{autodoc2-docstring} archivebox.crawls.schedule_util.next_run_for_schedule +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.crawls.schedule_utils.md b/docs/apidocs/archivebox/archivebox.crawls.schedule_utils.md deleted file mode 100644 index 7eb084b0..00000000 --- a/docs/apidocs/archivebox/archivebox.crawls.schedule_utils.md +++ /dev/null @@ -1,76 +0,0 @@ -# {py:mod}`archivebox.crawls.schedule_utils` - -```{py:module} archivebox.crawls.schedule_utils -``` - -```{autodoc2-docstring} archivebox.crawls.schedule_utils -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`normalize_schedule ` - - ```{autodoc2-docstring} archivebox.crawls.schedule_utils.normalize_schedule - :summary: - ``` -* - {py:obj}`validate_schedule ` - - ```{autodoc2-docstring} archivebox.crawls.schedule_utils.validate_schedule - :summary: - ``` -* - {py:obj}`next_run_for_schedule ` - - ```{autodoc2-docstring} archivebox.crawls.schedule_utils.next_run_for_schedule - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`SCHEDULE_ALIASES ` - - ```{autodoc2-docstring} archivebox.crawls.schedule_utils.SCHEDULE_ALIASES - :summary: - ``` -```` - -### API - -````{py:data} SCHEDULE_ALIASES -:canonical: archivebox.crawls.schedule_utils.SCHEDULE_ALIASES -:type: dict[str, str] -:value: > - None - -```{autodoc2-docstring} archivebox.crawls.schedule_utils.SCHEDULE_ALIASES -``` - -```` - -````{py:function} normalize_schedule(schedule: str) -> str -:canonical: archivebox.crawls.schedule_utils.normalize_schedule - -```{autodoc2-docstring} archivebox.crawls.schedule_utils.normalize_schedule -``` -```` - -````{py:function} validate_schedule(schedule: str) -> str -:canonical: archivebox.crawls.schedule_utils.validate_schedule - -```{autodoc2-docstring} archivebox.crawls.schedule_utils.validate_schedule -``` -```` - -````{py:function} next_run_for_schedule(schedule: str, after: datetime.datetime) -> datetime.datetime -:canonical: archivebox.crawls.schedule_utils.next_run_for_schedule - -```{autodoc2-docstring} archivebox.crawls.schedule_utils.next_run_for_schedule -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.hooks.md b/docs/apidocs/archivebox/archivebox.hooks.md index 0509cfed..79166ac1 100644 --- a/docs/apidocs/archivebox/archivebox.hooks.md +++ b/docs/apidocs/archivebox/archivebox.hooks.md @@ -15,10 +15,12 @@ :class: autosummary longtable :align: left -* - {py:obj}`HookResult ` - - ```{autodoc2-docstring} archivebox.hooks.HookResult - :summary: - ``` +* - {py:obj}`ConfigLookup ` + - +* - {py:obj}`PluginSpecialConfig ` + - +* - {py:obj}`ConfigDump ` + - ```` ### Functions @@ -27,6 +29,14 @@ :class: autosummary longtable :align: left +* - {py:obj}`_has_config_dump ` + - ```{autodoc2-docstring} archivebox.hooks._has_config_dump + :summary: + ``` +* - {py:obj}`_config_to_overrides ` + - ```{autodoc2-docstring} archivebox.hooks._config_to_overrides + :summary: + ``` * - {py:obj}`is_background_hook ` - ```{autodoc2-docstring} archivebox.hooks.is_background_hook :summary: @@ -39,6 +49,14 @@ - ```{autodoc2-docstring} archivebox.hooks.iter_plugin_dirs :summary: ``` +* - {py:obj}`normalize_hook_event_name ` + - ```{autodoc2-docstring} archivebox.hooks.normalize_hook_event_name + :summary: + ``` +* - {py:obj}`_model_output_dir_from_child_path ` + - ```{autodoc2-docstring} archivebox.hooks._model_output_dir_from_child_path + :summary: + ``` * - {py:obj}`discover_hooks ` - ```{autodoc2-docstring} archivebox.hooks.discover_hooks :summary: @@ -79,10 +97,6 @@ - ```{autodoc2-docstring} archivebox.hooks.discover_plugin_configs :summary: ``` -* - {py:obj}`get_config_defaults_from_plugins ` - - ```{autodoc2-docstring} archivebox.hooks.get_config_defaults_from_plugins - :summary: - ``` * - {py:obj}`get_plugin_special_config ` - ```{autodoc2-docstring} archivebox.hooks.get_plugin_special_config :summary: @@ -123,6 +137,98 @@ ### API +`````{py:class} ConfigLookup +:canonical: archivebox.hooks.ConfigLookup + +Bases: {py:obj}`typing.Protocol` + +````{py:method} get(key: str, default: typing.Any = None) -> typing.Any +:canonical: archivebox.hooks.ConfigLookup.get + +```{autodoc2-docstring} archivebox.hooks.ConfigLookup.get +``` + +```` + +````{py:method} items() -> collections.abc.Iterable[tuple[str, typing.Any]] +:canonical: archivebox.hooks.ConfigLookup.items + +```{autodoc2-docstring} archivebox.hooks.ConfigLookup.items +``` + +```` + +````` + +`````{py:class} PluginSpecialConfig() +:canonical: archivebox.hooks.PluginSpecialConfig + +Bases: {py:obj}`typing.TypedDict` + +````{py:attribute} enabled +:canonical: archivebox.hooks.PluginSpecialConfig.enabled +:type: bool +:value: > + None + +```{autodoc2-docstring} archivebox.hooks.PluginSpecialConfig.enabled +``` + +```` + +````{py:attribute} timeout +:canonical: archivebox.hooks.PluginSpecialConfig.timeout +:type: int +:value: > + None + +```{autodoc2-docstring} archivebox.hooks.PluginSpecialConfig.timeout +``` + +```` + +````{py:attribute} binary +:canonical: archivebox.hooks.PluginSpecialConfig.binary +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.hooks.PluginSpecialConfig.binary +``` + +```` + +````` + +`````{py:class} ConfigDump +:canonical: archivebox.hooks.ConfigDump + +Bases: {py:obj}`typing.Protocol` + +````{py:method} as_dict() -> dict[str, typing.Any] +:canonical: archivebox.hooks.ConfigDump.as_dict + +```{autodoc2-docstring} archivebox.hooks.ConfigDump.as_dict +``` + +```` + +````` + +````{py:function} _has_config_dump(config: object) -> typing.TypeGuard[archivebox.hooks.ConfigDump] +:canonical: archivebox.hooks._has_config_dump + +```{autodoc2-docstring} archivebox.hooks._has_config_dump +``` +```` + +````{py:function} _config_to_overrides(config: archivebox.hooks.ConfigLookup | collections.abc.Mapping[str, typing.Any] | None) -> dict[str, typing.Any] +:canonical: archivebox.hooks._config_to_overrides + +```{autodoc2-docstring} archivebox.hooks._config_to_overrides +``` +```` + ````{py:data} BUILTIN_PLUGINS_DIR :canonical: archivebox.hooks.BUILTIN_PLUGINS_DIR :value: > @@ -157,168 +263,56 @@ ``` ```` -````{py:function} iter_plugin_dirs() -> typing.List[pathlib.Path] +````{py:function} iter_plugin_dirs() -> list[pathlib.Path] :canonical: archivebox.hooks.iter_plugin_dirs ```{autodoc2-docstring} archivebox.hooks.iter_plugin_dirs ``` ```` -`````{py:class} HookResult() -:canonical: archivebox.hooks.HookResult +````{py:function} normalize_hook_event_name(event_name: str) -> str | None +:canonical: archivebox.hooks.normalize_hook_event_name -Bases: {py:obj}`typing.TypedDict` - -```{autodoc2-docstring} archivebox.hooks.HookResult +```{autodoc2-docstring} archivebox.hooks.normalize_hook_event_name ``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.hooks.HookResult.__init__ -``` - -````{py:attribute} returncode -:canonical: archivebox.hooks.HookResult.returncode -:type: int -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.returncode -``` - ```` -````{py:attribute} stdout -:canonical: archivebox.hooks.HookResult.stdout -:type: str -:value: > - None +````{py:function} _model_output_dir_from_child_path(path: pathlib.Path, marker: str) -> pathlib.Path | None +:canonical: archivebox.hooks._model_output_dir_from_child_path -```{autodoc2-docstring} archivebox.hooks.HookResult.stdout +```{autodoc2-docstring} archivebox.hooks._model_output_dir_from_child_path ``` - ```` -````{py:attribute} stderr -:canonical: archivebox.hooks.HookResult.stderr -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.stderr -``` - -```` - -````{py:attribute} output_json -:canonical: archivebox.hooks.HookResult.output_json -:type: typing.Optional[typing.Dict[str, typing.Any]] -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.output_json -``` - -```` - -````{py:attribute} output_files -:canonical: archivebox.hooks.HookResult.output_files -:type: typing.List[str] -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.output_files -``` - -```` - -````{py:attribute} duration_ms -:canonical: archivebox.hooks.HookResult.duration_ms -:type: int -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.duration_ms -``` - -```` - -````{py:attribute} hook -:canonical: archivebox.hooks.HookResult.hook -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.hook -``` - -```` - -````{py:attribute} plugin -:canonical: archivebox.hooks.HookResult.plugin -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.plugin -``` - -```` - -````{py:attribute} hook_name -:canonical: archivebox.hooks.HookResult.hook_name -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.hook_name -``` - -```` - -````{py:attribute} records -:canonical: archivebox.hooks.HookResult.records -:type: typing.List[typing.Dict[str, typing.Any]] -:value: > - None - -```{autodoc2-docstring} archivebox.hooks.HookResult.records -``` - -```` - -````` - -````{py:function} discover_hooks(event_name: str, filter_disabled: bool = True, config: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.List[pathlib.Path] +````{py:function} discover_hooks(event_name: str, filter_disabled: bool = True, config: archivebox.hooks.ConfigLookup | None = None, **config_kwargs: typing.Any) -> list[pathlib.Path] :canonical: archivebox.hooks.discover_hooks ```{autodoc2-docstring} archivebox.hooks.discover_hooks ``` ```` -````{py:function} run_hook(script: pathlib.Path, output_dir: pathlib.Path, config: typing.Dict[str, typing.Any], timeout: typing.Optional[int] = None, parent: typing.Optional[archivebox.machine.models.Process] = None, **kwargs: typing.Any) -> archivebox.machine.models.Process +````{py:function} run_hook(script: pathlib.Path, output_dir: pathlib.Path, config: archivebox.hooks.ConfigLookup | collections.abc.Mapping[str, typing.Any] | None = None, timeout: int | None = None, parent: typing.Optional[archivebox.machine.models.Process] = None, **kwargs: typing.Any) -> archivebox.machine.models.Process :canonical: archivebox.hooks.run_hook ```{autodoc2-docstring} archivebox.hooks.run_hook ``` ```` -````{py:function} extract_records_from_process(process: archivebox.machine.models.Process) -> typing.List[typing.Dict[str, typing.Any]] +````{py:function} extract_records_from_process(process: archivebox.machine.models.Process) -> list[dict[str, typing.Any]] :canonical: archivebox.hooks.extract_records_from_process ```{autodoc2-docstring} archivebox.hooks.extract_records_from_process ``` ```` -````{py:function} collect_urls_from_plugins(snapshot_dir: pathlib.Path) -> typing.List[typing.Dict[str, typing.Any]] +````{py:function} collect_urls_from_plugins(snapshot_dir: pathlib.Path) -> list[dict[str, typing.Any]] :canonical: archivebox.hooks.collect_urls_from_plugins ```{autodoc2-docstring} archivebox.hooks.collect_urls_from_plugins ``` ```` -````{py:function} get_plugins() -> typing.List[str] +````{py:function} get_plugins() -> list[str] :canonical: archivebox.hooks.get_plugins ```{autodoc2-docstring} archivebox.hooks.get_plugins @@ -332,42 +326,35 @@ Bases: {py:obj}`typing.TypedDict` ``` ```` -````{py:function} get_enabled_plugins(config: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.List[str] +````{py:function} get_enabled_plugins(config: archivebox.hooks.ConfigLookup | None = None, **config_kwargs: typing.Any) -> list[str] :canonical: archivebox.hooks.get_enabled_plugins ```{autodoc2-docstring} archivebox.hooks.get_enabled_plugins ``` ```` -````{py:function} discover_plugins_that_provide_interface(module_name: str, required_attrs: typing.List[str], plugin_prefix: typing.Optional[str] = None) -> typing.Dict[str, typing.Any] +````{py:function} discover_plugins_that_provide_interface(module_name: str, required_attrs: list[str], plugin_prefix: str | None = None) -> dict[str, typing.Any] :canonical: archivebox.hooks.discover_plugins_that_provide_interface ```{autodoc2-docstring} archivebox.hooks.discover_plugins_that_provide_interface ``` ```` -````{py:function} get_search_backends() -> typing.Dict[str, typing.Any] +````{py:function} get_search_backends() -> dict[str, typing.Any] :canonical: archivebox.hooks.get_search_backends ```{autodoc2-docstring} archivebox.hooks.get_search_backends ``` ```` -````{py:function} discover_plugin_configs() -> typing.Dict[str, typing.Dict[str, typing.Any]] +````{py:function} discover_plugin_configs() -> dict[str, dict[str, typing.Any]] :canonical: archivebox.hooks.discover_plugin_configs ```{autodoc2-docstring} archivebox.hooks.discover_plugin_configs ``` ```` -````{py:function} get_config_defaults_from_plugins() -> typing.Dict[str, typing.Any] -:canonical: archivebox.hooks.get_config_defaults_from_plugins - -```{autodoc2-docstring} archivebox.hooks.get_config_defaults_from_plugins -``` -```` - -````{py:function} get_plugin_special_config(plugin_name: str, config: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any] +````{py:function} get_plugin_special_config(plugin_name: str, config: archivebox.hooks.ConfigLookup, _visited: set[str] | None = None) -> archivebox.hooks.PluginSpecialConfig :canonical: archivebox.hooks.get_plugin_special_config ```{autodoc2-docstring} archivebox.hooks.get_plugin_special_config @@ -384,7 +371,7 @@ Bases: {py:obj}`typing.TypedDict` ```` -````{py:function} get_plugin_template(plugin: str, template_name: str, fallback: bool = True) -> typing.Optional[str] +````{py:function} get_plugin_template(plugin: str, template_name: str, fallback: bool = True) -> str | None :canonical: archivebox.hooks.get_plugin_template ```{autodoc2-docstring} archivebox.hooks.get_plugin_template @@ -398,7 +385,7 @@ Bases: {py:obj}`typing.TypedDict` ``` ```` -````{py:function} process_hook_records(records: typing.List[typing.Dict[str, typing.Any]], overrides: typing.Dict[str, typing.Any] = None) -> typing.Dict[str, int] +````{py:function} process_hook_records(records: list[dict[str, typing.Any]], overrides: dict[str, typing.Any] | None = None) -> dict[str, int] :canonical: archivebox.hooks.process_hook_records ```{autodoc2-docstring} archivebox.hooks.process_hook_records diff --git a/docs/apidocs/archivebox/archivebox.ideas.md b/docs/apidocs/archivebox/archivebox.ideas.md deleted file mode 100644 index 4a3fcfac..00000000 --- a/docs/apidocs/archivebox/archivebox.ideas.md +++ /dev/null @@ -1,17 +0,0 @@ -# {py:mod}`archivebox.ideas` - -```{py:module} archivebox.ideas -``` - -```{autodoc2-docstring} archivebox.ideas -:allowtitles: -``` - -## Submodules - -```{toctree} -:titlesonly: -:maxdepth: 1 - -archivebox.ideas.process_plugin -``` diff --git a/docs/apidocs/archivebox/archivebox.ideas.process_plugin.md b/docs/apidocs/archivebox/archivebox.ideas.process_plugin.md deleted file mode 100644 index 76b93fb7..00000000 --- a/docs/apidocs/archivebox/archivebox.ideas.process_plugin.md +++ /dev/null @@ -1,633 +0,0 @@ -# {py:mod}`archivebox.ideas.process_plugin` - -```{py:module} archivebox.ideas.process_plugin -``` - -```{autodoc2-docstring} archivebox.ideas.process_plugin -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`ProcessRecord ` - - -* - {py:obj}`ProcessLaunch ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch - :summary: - ``` -* - {py:obj}`ProcessStarted ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessStarted - :summary: - ``` -* - {py:obj}`ProcessExited ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessExited - :summary: - ``` -* - {py:obj}`ProcessKill ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessKill - :summary: - ``` -* - {py:obj}`_RunningProcess ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess - :summary: - ``` -* - {py:obj}`ProcessPlugin ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin - :summary: - ``` -```` - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_utcnow ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin._utcnow - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`JsonEventAdapter ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.JsonEventAdapter - :summary: - ``` -* - {py:obj}`__all__ ` - - ```{autodoc2-docstring} archivebox.ideas.process_plugin.__all__ - :summary: - ``` -```` - -### API - -````{py:function} _utcnow() -> datetime.datetime -:canonical: archivebox.ideas.process_plugin._utcnow - -```{autodoc2-docstring} archivebox.ideas.process_plugin._utcnow -``` -```` - -`````{py:class} ProcessRecord(/, **data: typing.Any) -:canonical: archivebox.ideas.process_plugin.ProcessRecord - -Bases: {py:obj}`pydantic.BaseModel` - -````{py:attribute} id -:canonical: archivebox.ideas.process_plugin.ProcessRecord.id -:type: str -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.id -``` - -```` - -````{py:attribute} cmd -:canonical: archivebox.ideas.process_plugin.ProcessRecord.cmd -:type: list[str] -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.cmd -``` - -```` - -````{py:attribute} cwd -:canonical: archivebox.ideas.process_plugin.ProcessRecord.cwd -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.cwd -``` - -```` - -````{py:attribute} env -:canonical: archivebox.ideas.process_plugin.ProcessRecord.env -:type: dict[str, str] -:value: > - 'Field(...)' - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.env -``` - -```` - -````{py:attribute} pid -:canonical: archivebox.ideas.process_plugin.ProcessRecord.pid -:type: int | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.pid -``` - -```` - -````{py:attribute} started_at -:canonical: archivebox.ideas.process_plugin.ProcessRecord.started_at -:type: datetime.datetime | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.started_at -``` - -```` - -````{py:attribute} ended_at -:canonical: archivebox.ideas.process_plugin.ProcessRecord.ended_at -:type: datetime.datetime | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.ended_at -``` - -```` - -````{py:attribute} exit_code -:canonical: archivebox.ideas.process_plugin.ProcessRecord.exit_code -:type: int | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.exit_code -``` - -```` - -````{py:attribute} stdout_path -:canonical: archivebox.ideas.process_plugin.ProcessRecord.stdout_path -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.stdout_path -``` - -```` - -````{py:attribute} stderr_path -:canonical: archivebox.ideas.process_plugin.ProcessRecord.stderr_path -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.stderr_path -``` - -```` - -````{py:attribute} cmd_path -:canonical: archivebox.ideas.process_plugin.ProcessRecord.cmd_path -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.cmd_path -``` - -```` - -````{py:attribute} pid_path -:canonical: archivebox.ideas.process_plugin.ProcessRecord.pid_path -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.pid_path -``` - -```` - -````{py:attribute} is_background -:canonical: archivebox.ideas.process_plugin.ProcessRecord.is_background -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.is_background -``` - -```` - -````{py:attribute} parent_process_id -:canonical: archivebox.ideas.process_plugin.ProcessRecord.parent_process_id -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessRecord.parent_process_id -``` - -```` - -````` - -`````{py:class} ProcessLaunch -:canonical: archivebox.ideas.process_plugin.ProcessLaunch - -Bases: {py:obj}`bubus.BaseEvent`\[{py:obj}`archivebox.ideas.process_plugin.ProcessRecord`\] - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch -``` - -````{py:attribute} cmd -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.cmd -:type: list[str] -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.cmd -``` - -```` - -````{py:attribute} cwd -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.cwd -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.cwd -``` - -```` - -````{py:attribute} env -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.env -:type: dict[str, str] | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.env -``` - -```` - -````{py:attribute} timeout -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.timeout -:type: float | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.timeout -``` - -```` - -````{py:attribute} output_dir -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.output_dir -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.output_dir -``` - -```` - -````{py:attribute} log_prefix -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.log_prefix -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.log_prefix -``` - -```` - -````{py:attribute} is_background -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.is_background -:type: bool -:value: > - False - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.is_background -``` - -```` - -````{py:attribute} parent_process_id -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.parent_process_id -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.parent_process_id -``` - -```` - -````{py:attribute} parse_stdout_events -:canonical: archivebox.ideas.process_plugin.ProcessLaunch.parse_stdout_events -:type: bool -:value: > - True - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessLaunch.parse_stdout_events -``` - -```` - -````` - -`````{py:class} ProcessStarted -:canonical: archivebox.ideas.process_plugin.ProcessStarted - -Bases: {py:obj}`bubus.BaseEvent`\[{py:obj}`None`\] - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessStarted -``` - -````{py:attribute} process -:canonical: archivebox.ideas.process_plugin.ProcessStarted.process -:type: archivebox.ideas.process_plugin.ProcessRecord -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessStarted.process -``` - -```` - -````` - -`````{py:class} ProcessExited -:canonical: archivebox.ideas.process_plugin.ProcessExited - -Bases: {py:obj}`bubus.BaseEvent`\[{py:obj}`None`\] - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessExited -``` - -````{py:attribute} process -:canonical: archivebox.ideas.process_plugin.ProcessExited.process -:type: archivebox.ideas.process_plugin.ProcessRecord -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessExited.process -``` - -```` - -````` - -`````{py:class} ProcessKill -:canonical: archivebox.ideas.process_plugin.ProcessKill - -Bases: {py:obj}`bubus.BaseEvent`\[{py:obj}`archivebox.ideas.process_plugin.ProcessRecord`\] - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessKill -``` - -````{py:attribute} process_id -:canonical: archivebox.ideas.process_plugin.ProcessKill.process_id -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessKill.process_id -``` - -```` - -````{py:attribute} signal -:canonical: archivebox.ideas.process_plugin.ProcessKill.signal -:type: int -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessKill.signal -``` - -```` - -````{py:attribute} timeout -:canonical: archivebox.ideas.process_plugin.ProcessKill.timeout -:type: float | None -:value: > - 10.0 - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessKill.timeout -``` - -```` - -````` - -`````{py:class} _RunningProcess -:canonical: archivebox.ideas.process_plugin._RunningProcess - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess -``` - -````{py:attribute} process -:canonical: archivebox.ideas.process_plugin._RunningProcess.process -:type: asyncio.subprocess.Process -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess.process -``` - -```` - -````{py:attribute} record -:canonical: archivebox.ideas.process_plugin._RunningProcess.record -:type: archivebox.ideas.process_plugin.ProcessRecord -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess.record -``` - -```` - -````{py:attribute} stdout_task -:canonical: archivebox.ideas.process_plugin._RunningProcess.stdout_task -:type: asyncio.Task[None] | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess.stdout_task -``` - -```` - -````{py:attribute} stderr_task -:canonical: archivebox.ideas.process_plugin._RunningProcess.stderr_task -:type: asyncio.Task[None] | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess.stderr_task -``` - -```` - -````{py:attribute} watcher_task -:canonical: archivebox.ideas.process_plugin._RunningProcess.watcher_task -:type: asyncio.Task[None] | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess.watcher_task -``` - -```` - -````{py:attribute} parent_event_id -:canonical: archivebox.ideas.process_plugin._RunningProcess.parent_event_id -:type: str | None -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin._RunningProcess.parent_event_id -``` - -```` - -````` - -````{py:data} JsonEventAdapter -:canonical: archivebox.ideas.process_plugin.JsonEventAdapter -:value: > - None - -```{autodoc2-docstring} archivebox.ideas.process_plugin.JsonEventAdapter -``` - -```` - -`````{py:class} ProcessPlugin(bus: bubus.EventBus, *, env: typing.Mapping[str, str] | None = None, json_event_adapter: archivebox.ideas.process_plugin.JsonEventAdapter | None = None) -:canonical: archivebox.ideas.process_plugin.ProcessPlugin - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin.__init__ -``` - -````{py:method} register_event_handlers() -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin.register_event_handlers - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin.register_event_handlers -``` - -```` - -````{py:method} on_ProcessLaunch(event: archivebox.ideas.process_plugin.ProcessLaunch) -> archivebox.ideas.process_plugin.ProcessRecord -:canonical: archivebox.ideas.process_plugin.ProcessPlugin.on_ProcessLaunch -:async: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin.on_ProcessLaunch -``` - -```` - -````{py:method} on_ProcessKill(event: archivebox.ideas.process_plugin.ProcessKill) -> archivebox.ideas.process_plugin.ProcessRecord -:canonical: archivebox.ideas.process_plugin.ProcessPlugin.on_ProcessKill -:async: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin.on_ProcessKill -``` - -```` - -````{py:method} _watch_process(process_id: str, timeout: float | None) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._watch_process -:async: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._watch_process -``` - -```` - -````{py:method} _finalize_process(process_id: str) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._finalize_process -:async: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._finalize_process -``` - -```` - -````{py:method} _consume_stream(stream: asyncio.StreamReader | None, path: pathlib.Path, parent_event_id: str | None, parse_events: bool) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._consume_stream -:async: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._consume_stream -``` - -```` - -````{py:method} _maybe_dispatch_json_event(line: str, parent_event_id: str | None) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._maybe_dispatch_json_event -:async: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._maybe_dispatch_json_event -``` - -```` - -````{py:method} _write_cmd_file(path: pathlib.Path, cmd: list[str]) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._write_cmd_file -:staticmethod: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._write_cmd_file -``` - -```` - -````{py:method} _write_pid_file(path: pathlib.Path, pid: int) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._write_pid_file -:staticmethod: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._write_pid_file -``` - -```` - -````{py:method} _terminate_process(proc: asyncio.subprocess.Process, sig: int) -> None -:canonical: archivebox.ideas.process_plugin.ProcessPlugin._terminate_process -:staticmethod: - -```{autodoc2-docstring} archivebox.ideas.process_plugin.ProcessPlugin._terminate_process -``` - -```` - -````` - -````{py:data} __all__ -:canonical: archivebox.ideas.process_plugin.__all__ -:value: > - ['ProcessRecord', 'ProcessLaunch', 'ProcessStarted', 'ProcessExited', 'ProcessKill', 'ProcessPlugin'... - -```{autodoc2-docstring} archivebox.ideas.process_plugin.__all__ -``` - -```` diff --git a/docs/apidocs/archivebox/archivebox.ldap.auth.md b/docs/apidocs/archivebox/archivebox.ldap.auth.md index ed14bc8e..25029ecf 100644 --- a/docs/apidocs/archivebox/archivebox.ldap.auth.md +++ b/docs/apidocs/archivebox/archivebox.ldap.auth.md @@ -26,7 +26,7 @@ `````{py:class} ArchiveBoxLDAPBackend :canonical: archivebox.ldap.auth.ArchiveBoxLDAPBackend -Bases: {py:obj}`django_auth_ldap.backend.LDAPBackend` +Bases: {py:obj}`archivebox.ldap.auth.BaseLDAPBackend` ```{autodoc2-docstring} archivebox.ldap.auth.ArchiveBoxLDAPBackend ``` diff --git a/docs/apidocs/archivebox/archivebox.machine.admin.md b/docs/apidocs/archivebox/archivebox.machine.admin.md index 3f5b6498..1f1199e2 100644 --- a/docs/apidocs/archivebox/archivebox.machine.admin.md +++ b/docs/apidocs/archivebox/archivebox.machine.admin.md @@ -31,6 +31,14 @@ :class: autosummary longtable :align: left +* - {py:obj}`_render_copy_block ` + - ```{autodoc2-docstring} archivebox.machine.admin._render_copy_block + :summary: + ``` +* - {py:obj}`_format_process_duration_seconds ` + - ```{autodoc2-docstring} archivebox.machine.admin._format_process_duration_seconds + :summary: + ``` * - {py:obj}`register_admin ` - ```{autodoc2-docstring} archivebox.machine.admin.register_admin :summary: @@ -39,6 +47,20 @@ ### API +````{py:function} _render_copy_block(text: str, *, multiline: bool = False) +:canonical: archivebox.machine.admin._render_copy_block + +```{autodoc2-docstring} archivebox.machine.admin._render_copy_block +``` +```` + +````{py:function} _format_process_duration_seconds(started_at, ended_at) -> str +:canonical: archivebox.machine.admin._format_process_duration_seconds + +```{autodoc2-docstring} archivebox.machine.admin._format_process_duration_seconds +``` +```` + `````{py:class} MachineAdmin(model, admin_site) :canonical: archivebox.machine.admin.MachineAdmin @@ -47,7 +69,7 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ````{py:attribute} list_display :canonical: archivebox.machine.admin.MachineAdmin.list_display :value: > - ('id', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manufacturer'... + ('id_display', 'created_at', 'hostname', 'ips', 'os_platform', 'hw_in_docker', 'hw_in_vm', 'hw_manuf... ```{autodoc2-docstring} archivebox.machine.admin.MachineAdmin.list_display ``` @@ -140,6 +162,14 @@ Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archiv ```` +````{py:method} id_display(machine) +:canonical: archivebox.machine.admin.MachineAdmin.id_display + +```{autodoc2-docstring} archivebox.machine.admin.MachineAdmin.id_display +``` + +```` + ````` `````{py:class} NetworkInterfaceAdmin(model, admin_site) @@ -293,7 +323,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} readonly_fields :canonical: archivebox.machine.admin.BinaryAdmin.readonly_fields :value: > - ('created_at', 'modified_at') + ('created_at', 'modified_at', 'output_dir') ```{autodoc2-docstring} archivebox.machine.admin.BinaryAdmin.readonly_fields ``` @@ -376,7 +406,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} list_display :canonical: archivebox.machine.admin.ProcessAdmin.list_display :value: > - ('id', 'created_at', 'machine_info', 'archiveresult_link', 'cmd_str', 'status', 'exit_code', 'pid', ... + ('id', 'created_at', 'machine_info', 'archiveresult_link', 'snapshot_link', 'crawl_link', 'cmd_str',... ```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.list_display ``` @@ -386,7 +416,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} sort_fields :canonical: archivebox.machine.admin.ProcessAdmin.sort_fields :value: > - ('id', 'created_at', 'status', 'exit_code', 'pid') + ('id', 'created_at', 'machine_info', 'archiveresult_link', 'snapshot_link', 'crawl_link', 'cmd_str',... ```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.sort_fields ``` @@ -406,7 +436,7 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} readonly_fields :canonical: archivebox.machine.admin.ProcessAdmin.readonly_fields :value: > - ('created_at', 'modified_at', 'machine', 'binary', 'iface', 'archiveresult_link') + ('created_at', 'modified_at', 'machine', 'binary_link', 'iface_link', 'archiveresult_link', 'snapsho... ```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.readonly_fields ``` @@ -456,13 +486,52 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ````{py:attribute} actions :canonical: archivebox.machine.admin.ProcessAdmin.actions :value: > - ['delete_selected'] + ['kill_processes', 'delete_selected'] ```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.actions ``` ```` +````{py:attribute} change_actions +:canonical: archivebox.machine.admin.ProcessAdmin.change_actions +:value: > + ['kill_process'] + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.change_actions +``` + +```` + +````{py:method} get_queryset(request) +:canonical: archivebox.machine.admin.ProcessAdmin.get_queryset + +```` + +````{py:method} _terminate_processes(request, processes) +:canonical: archivebox.machine.admin.ProcessAdmin._terminate_processes + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin._terminate_processes +``` + +```` + +````{py:method} kill_processes(request, queryset) +:canonical: archivebox.machine.admin.ProcessAdmin.kill_processes + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.kill_processes +``` + +```` + +````{py:method} kill_process(request, obj) +:canonical: archivebox.machine.admin.ProcessAdmin.kill_process + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.kill_process +``` + +```` + ````{py:method} machine_info(process) :canonical: archivebox.machine.admin.ProcessAdmin.machine_info @@ -479,6 +548,22 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} binary_link(process) +:canonical: archivebox.machine.admin.ProcessAdmin.binary_link + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.binary_link +``` + +```` + +````{py:method} iface_link(process) +:canonical: archivebox.machine.admin.ProcessAdmin.iface_link + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.iface_link +``` + +```` + ````{py:method} archiveresult_link(process) :canonical: archivebox.machine.admin.ProcessAdmin.archiveresult_link @@ -487,6 +572,22 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} snapshot_link(process) +:canonical: archivebox.machine.admin.ProcessAdmin.snapshot_link + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.snapshot_link +``` + +```` + +````{py:method} crawl_link(process) +:canonical: archivebox.machine.admin.ProcessAdmin.crawl_link + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.crawl_link +``` + +```` + ````{py:method} cmd_str(process) :canonical: archivebox.machine.admin.ProcessAdmin.cmd_str @@ -495,6 +596,46 @@ Bases: {py:obj}`archivebox.base_models.admin.BaseModelAdmin` ```` +````{py:method} status_badge(process) +:canonical: archivebox.machine.admin.ProcessAdmin.status_badge + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.status_badge +``` + +```` + +````{py:method} duration_display(process) +:canonical: archivebox.machine.admin.ProcessAdmin.duration_display + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.duration_display +``` + +```` + +````{py:method} output_summary(process) +:canonical: archivebox.machine.admin.ProcessAdmin.output_summary + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.output_summary +``` + +```` + +````{py:method} cmd_display(process) +:canonical: archivebox.machine.admin.ProcessAdmin.cmd_display + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.cmd_display +``` + +```` + +````{py:method} env_display(process) +:canonical: archivebox.machine.admin.ProcessAdmin.env_display + +```{autodoc2-docstring} archivebox.machine.admin.ProcessAdmin.env_display +``` + +```` + ````` ````{py:function} register_admin(admin_site) diff --git a/docs/apidocs/archivebox/archivebox.machine.detect.md b/docs/apidocs/archivebox/archivebox.machine.detect.md index d95e439e..6ccd2d95 100644 --- a/docs/apidocs/archivebox/archivebox.machine.detect.md +++ b/docs/apidocs/archivebox/archivebox.machine.detect.md @@ -43,10 +43,6 @@ - ```{autodoc2-docstring} archivebox.machine.detect.get_host_stats :summary: ``` -* - {py:obj}`get_host_immutable_info ` - - ```{autodoc2-docstring} archivebox.machine.detect.get_host_immutable_info - :summary: - ``` * - {py:obj}`get_host_guid ` - ```{autodoc2-docstring} archivebox.machine.detect.get_host_guid :summary: @@ -147,34 +143,27 @@ ``` ```` -````{py:function} get_host_network() -> typing.Dict[str, typing.Any] +````{py:function} get_host_network() -> dict[str, typing.Any] :canonical: archivebox.machine.detect.get_host_network ```{autodoc2-docstring} archivebox.machine.detect.get_host_network ``` ```` -````{py:function} get_os_info() -> typing.Dict[str, typing.Any] +````{py:function} get_os_info() -> dict[str, typing.Any] :canonical: archivebox.machine.detect.get_os_info ```{autodoc2-docstring} archivebox.machine.detect.get_os_info ``` ```` -````{py:function} get_host_stats() -> typing.Dict[str, typing.Any] +````{py:function} get_host_stats() -> dict[str, typing.Any] :canonical: archivebox.machine.detect.get_host_stats ```{autodoc2-docstring} archivebox.machine.detect.get_host_stats ``` ```` -````{py:function} get_host_immutable_info(host_info: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any] -:canonical: archivebox.machine.detect.get_host_immutable_info - -```{autodoc2-docstring} archivebox.machine.detect.get_host_immutable_info -``` -```` - ````{py:function} get_host_guid() -> str :canonical: archivebox.machine.detect.get_host_guid diff --git a/docs/apidocs/archivebox/archivebox.machine.env_util.md b/docs/apidocs/archivebox/archivebox.machine.env_util.md new file mode 100644 index 00000000..aebf1f12 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.machine.env_util.md @@ -0,0 +1,111 @@ +# {py:mod}`archivebox.machine.env_util` + +```{py:module} archivebox.machine.env_util +``` + +```{autodoc2-docstring} archivebox.machine.env_util +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`stringify_env_value ` + - ```{autodoc2-docstring} archivebox.machine.env_util.stringify_env_value + :summary: + ``` +* - {py:obj}`is_redacted_env_key ` + - ```{autodoc2-docstring} archivebox.machine.env_util.is_redacted_env_key + :summary: + ``` +* - {py:obj}`redact_env ` + - ```{autodoc2-docstring} archivebox.machine.env_util.redact_env + :summary: + ``` +* - {py:obj}`env_to_dotenv_text ` + - ```{autodoc2-docstring} archivebox.machine.env_util.env_to_dotenv_text + :summary: + ``` +* - {py:obj}`env_to_shell_exports ` + - ```{autodoc2-docstring} archivebox.machine.env_util.env_to_shell_exports + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`SENSITIVE_ENV_KEY_PARTS ` + - ```{autodoc2-docstring} archivebox.machine.env_util.SENSITIVE_ENV_KEY_PARTS + :summary: + ``` +* - {py:obj}`SHELL_ENV_KEY_RE ` + - ```{autodoc2-docstring} archivebox.machine.env_util.SHELL_ENV_KEY_RE + :summary: + ``` +```` + +### API + +````{py:data} SENSITIVE_ENV_KEY_PARTS +:canonical: archivebox.machine.env_util.SENSITIVE_ENV_KEY_PARTS +:value: > + ('KEY', 'TOKEN', 'SECRET') + +```{autodoc2-docstring} archivebox.machine.env_util.SENSITIVE_ENV_KEY_PARTS +``` + +```` + +````{py:data} SHELL_ENV_KEY_RE +:canonical: archivebox.machine.env_util.SHELL_ENV_KEY_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.machine.env_util.SHELL_ENV_KEY_RE +``` + +```` + +````{py:function} stringify_env_value(value: typing.Any) -> str +:canonical: archivebox.machine.env_util.stringify_env_value + +```{autodoc2-docstring} archivebox.machine.env_util.stringify_env_value +``` +```` + +````{py:function} is_redacted_env_key(key: str) -> bool +:canonical: archivebox.machine.env_util.is_redacted_env_key + +```{autodoc2-docstring} archivebox.machine.env_util.is_redacted_env_key +``` +```` + +````{py:function} redact_env(env: dict[str, typing.Any] | None) -> dict[str, typing.Any] +:canonical: archivebox.machine.env_util.redact_env + +```{autodoc2-docstring} archivebox.machine.env_util.redact_env +``` +```` + +````{py:function} env_to_dotenv_text(env: dict[str, typing.Any] | None) -> str +:canonical: archivebox.machine.env_util.env_to_dotenv_text + +```{autodoc2-docstring} archivebox.machine.env_util.env_to_dotenv_text +``` +```` + +````{py:function} env_to_shell_exports(env: dict[str, typing.Any] | None) -> str +:canonical: archivebox.machine.env_util.env_to_shell_exports + +```{autodoc2-docstring} archivebox.machine.env_util.env_to_shell_exports +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.machine.md b/docs/apidocs/archivebox/archivebox.machine.md index 341631d8..14525e02 100644 --- a/docs/apidocs/archivebox/archivebox.machine.md +++ b/docs/apidocs/archivebox/archivebox.machine.md @@ -14,6 +14,7 @@ :maxdepth: 1 archivebox.machine.models +archivebox.machine.env_util archivebox.machine.apps archivebox.machine.admin archivebox.machine.detect diff --git a/docs/apidocs/archivebox/archivebox.machine.models.md b/docs/apidocs/archivebox/archivebox.machine.models.md index eb257e1f..53e5bb6b 100644 --- a/docs/apidocs/archivebox/archivebox.machine.models.md +++ b/docs/apidocs/archivebox/archivebox.machine.models.md @@ -53,12 +53,40 @@ ``` ```` +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_find_existing_binary_for_reference ` + - ```{autodoc2-docstring} archivebox.machine.models._find_existing_binary_for_reference + :summary: + ``` +* - {py:obj}`_canonical_binary_name ` + - ```{autodoc2-docstring} archivebox.machine.models._canonical_binary_name + :summary: + ``` +* - {py:obj}`_get_process_binary_env_keys ` + - ```{autodoc2-docstring} archivebox.machine.models._get_process_binary_env_keys + :summary: + ``` +* - {py:obj}`_sanitize_machine_config ` + - ```{autodoc2-docstring} archivebox.machine.models._sanitize_machine_config + :summary: + ``` +```` + ### Data ````{list-table} :class: autosummary longtable :align: left +* - {py:obj}`_psutil ` + - ```{autodoc2-docstring} archivebox.machine.models._psutil + :summary: + ``` * - {py:obj}`_CURRENT_MACHINE ` - ```{autodoc2-docstring} archivebox.machine.models._CURRENT_MACHINE :summary: @@ -95,6 +123,10 @@ - ```{autodoc2-docstring} archivebox.machine.models.PID_REUSE_WINDOW :summary: ``` +* - {py:obj}`PROCESS_TIMEOUT_GRACE ` + - ```{autodoc2-docstring} archivebox.machine.models.PROCESS_TIMEOUT_GRACE + :summary: + ``` * - {py:obj}`START_TIME_TOLERANCE ` - ```{autodoc2-docstring} archivebox.machine.models.START_TIME_TOLERANCE :summary: @@ -103,8 +135,20 @@ ### API +````{py:data} _psutil +:canonical: archivebox.machine.models._psutil +:type: typing.Any | None +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models._psutil +``` + +```` + ````{py:data} _CURRENT_MACHINE :canonical: archivebox.machine.models._CURRENT_MACHINE +:type: archivebox.machine.models.Machine | None :value: > None @@ -115,6 +159,7 @@ ````{py:data} _CURRENT_INTERFACE :canonical: archivebox.machine.models._CURRENT_INTERFACE +:type: archivebox.machine.models.NetworkInterface | None :value: > None @@ -125,6 +170,7 @@ ````{py:data} _CURRENT_BINARIES :canonical: archivebox.machine.models._CURRENT_BINARIES +:type: dict[str, archivebox.machine.models.Binary] :value: > None @@ -135,6 +181,7 @@ ````{py:data} _CURRENT_PROCESS :canonical: archivebox.machine.models._CURRENT_PROCESS +:type: archivebox.machine.models.Process | None :value: > None @@ -193,6 +240,16 @@ ```` +````{py:data} PROCESS_TIMEOUT_GRACE +:canonical: archivebox.machine.models.PROCESS_TIMEOUT_GRACE +:value: > + 'timedelta(...)' + +```{autodoc2-docstring} archivebox.machine.models.PROCESS_TIMEOUT_GRACE +``` + +```` + ````{py:data} START_TIME_TOLERANCE :canonical: archivebox.machine.models.START_TIME_TOLERANCE :value: > @@ -203,6 +260,34 @@ ```` +````{py:function} _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None +:canonical: archivebox.machine.models._find_existing_binary_for_reference + +```{autodoc2-docstring} archivebox.machine.models._find_existing_binary_for_reference +``` +```` + +````{py:function} _canonical_binary_name(name: typing.Any) -> str +:canonical: archivebox.machine.models._canonical_binary_name + +```{autodoc2-docstring} archivebox.machine.models._canonical_binary_name +``` +```` + +````{py:function} _get_process_binary_env_keys(plugin_name: str, hook_path: str, env: dict[str, typing.Any] | None) -> list[str] +:canonical: archivebox.machine.models._get_process_binary_env_keys + +```{autodoc2-docstring} archivebox.machine.models._get_process_binary_env_keys +``` +```` + +````{py:function} _sanitize_machine_config(config: dict[str, typing.Any] | None, *, lib_dir: str | pathlib.Path | None = None) -> dict[str, typing.Any] +:canonical: archivebox.machine.models._sanitize_machine_config + +```{autodoc2-docstring} archivebox.machine.models._sanitize_machine_config +``` +```` + `````{py:class} MachineManager :canonical: archivebox.machine.models.MachineManager @@ -418,7 +503,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ````{py:attribute} objects :canonical: archivebox.machine.models.Machine.objects -:type: archivebox.machine.models.MachineManager :value: > 'MachineManager(...)' @@ -441,8 +525,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` `````{py:class} Meta :canonical: archivebox.machine.models.Machine.Meta -```{autodoc2-docstring} archivebox.machine.models.Machine.Meta -``` +Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta` ````{py:attribute} app_label :canonical: archivebox.machine.models.Machine.Meta.app_label @@ -456,7 +539,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ````` -````{py:method} current() -> archivebox.machine.models.Machine +````{py:method} current(refresh: bool = False) -> archivebox.machine.models.Machine :canonical: archivebox.machine.models.Machine.current :classmethod: @@ -465,11 +548,11 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ```` -````{py:method} _hydrate_config_from_sibling(machine: archivebox.machine.models.Machine) -> archivebox.machine.models.Machine -:canonical: archivebox.machine.models.Machine._hydrate_config_from_sibling +````{py:method} _sanitize_config(machine: archivebox.machine.models.Machine) -> archivebox.machine.models.Machine +:canonical: archivebox.machine.models.Machine._sanitize_config :classmethod: -```{autodoc2-docstring} archivebox.machine.models.Machine._hydrate_config_from_sibling +```{autodoc2-docstring} archivebox.machine.models.Machine._sanitize_config ``` ```` @@ -482,7 +565,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ```` -````{py:method} from_json(record: dict, overrides: dict = None) +````{py:method} from_json(record: dict[str, typing.Any], overrides: dict[str, typing.Any] | None = None) :canonical: archivebox.machine.models.Machine.from_json :staticmethod: @@ -491,6 +574,11 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ```` +````{py:method} save(*args, **kwargs) +:canonical: archivebox.machine.models.Machine.save + +```` + `````` `````{py:class} NetworkInterfaceManager @@ -658,7 +746,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ````{py:attribute} objects :canonical: archivebox.machine.models.NetworkInterface.objects -:type: archivebox.machine.models.NetworkInterfaceManager :value: > 'NetworkInterfaceManager(...)' @@ -667,11 +754,21 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ```` +````{py:attribute} machine_id +:canonical: archivebox.machine.models.NetworkInterface.machine_id +:type: uuid.UUID +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.NetworkInterface.machine_id +``` + +```` + `````{py:class} Meta :canonical: archivebox.machine.models.NetworkInterface.Meta -```{autodoc2-docstring} archivebox.machine.models.NetworkInterface.Meta -``` +Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta` ````{py:attribute} app_label :canonical: archivebox.machine.models.NetworkInterface.Meta.app_label @@ -693,9 +790,19 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats` ```` +````{py:attribute} constraints +:canonical: archivebox.machine.models.NetworkInterface.Meta.constraints +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.NetworkInterface.Meta.constraints +``` + +```` + ````` -````{py:method} current() -> archivebox.machine.models.NetworkInterface +````{py:method} current(refresh: bool = False) -> archivebox.machine.models.NetworkInterface :canonical: archivebox.machine.models.NetworkInterface.current :classmethod: @@ -722,7 +829,7 @@ Bases: {py:obj}`django.db.models.Manager` ```` -````{py:method} get_valid_binary(name: str, machine: Machine | None = None) -> Binary | None +````{py:method} get_valid_binary(name: str, machine: archivebox.machine.models.Machine | None = None) -> archivebox.machine.models.Binary | None :canonical: archivebox.machine.models.BinaryManager.get_valid_binary ```{autodoc2-docstring} archivebox.machine.models.BinaryManager.get_valid_binary @@ -923,9 +1030,20 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} machine_id +:canonical: archivebox.machine.models.Binary.machine_id +:type: uuid.UUID +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Binary.machine_id +``` + +```` + ````{py:attribute} state_machine_name :canonical: archivebox.machine.models.Binary.state_machine_name -:type: str +:type: str | None :value: > 'archivebox.machine.models.BinaryMachine' @@ -947,7 +1065,6 @@ Bases: {py:obj}`django.db.models.TextChoices` ````{py:attribute} objects :canonical: archivebox.machine.models.Binary.objects -:type: archivebox.machine.models.BinaryManager :value: > 'BinaryManager(...)' @@ -959,8 +1076,7 @@ Bases: {py:obj}`django.db.models.TextChoices` `````{py:class} Meta :canonical: archivebox.machine.models.Binary.Meta -```{autodoc2-docstring} archivebox.machine.models.Binary.Meta -``` +Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta` ````{py:attribute} app_label :canonical: archivebox.machine.models.Binary.Meta.app_label @@ -1043,7 +1159,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} from_json(record: dict, overrides: dict = None) +````{py:method} from_json(record: dict[str, typing.Any], overrides: dict[str, typing.Any] | None = None) :canonical: archivebox.machine.models.Binary.from_json :staticmethod: @@ -1052,14 +1168,6 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} update_and_requeue(**kwargs) -:canonical: archivebox.machine.models.Binary.update_and_requeue - -```{autodoc2-docstring} archivebox.machine.models.Binary.update_and_requeue -``` - -```` - ````{py:method} _allowed_binproviders() -> set[str] | None :canonical: archivebox.machine.models.Binary._allowed_binproviders @@ -1092,6 +1200,14 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:method} symlink_to_lib_bin_after_commit(lib_bin_dir: str | pathlib.Path) -> None +:canonical: archivebox.machine.models.Binary.symlink_to_lib_bin_after_commit + +```{autodoc2-docstring} archivebox.machine.models.Binary.symlink_to_lib_bin_after_commit +``` + +```` + `````` `````{py:class} ProcessManager @@ -1110,7 +1226,7 @@ Bases: {py:obj}`django.db.models.Manager` ```` -````{py:method} get_by_pid(pid: int, machine: archivebox.machine.models.Machine = None) -> Process | None +````{py:method} get_by_pid(pid: int, machine: archivebox.machine.models.Machine | None = None) -> archivebox.machine.models.Process | None :canonical: archivebox.machine.models.ProcessManager.get_by_pid ```{autodoc2-docstring} archivebox.machine.models.ProcessManager.get_by_pid @@ -1131,7 +1247,7 @@ Bases: {py:obj}`django.db.models.Manager` ``````{py:class} Process(*args, **kwargs) :canonical: archivebox.machine.models.Process -Bases: {py:obj}`django.db.models.Model` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`django.db.models.Model` ```{autodoc2-docstring} archivebox.machine.models.Process ``` @@ -1204,6 +1320,46 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} SERVER +:canonical: archivebox.machine.models.Process.TypeChoices.SERVER +:value: > + ('server', 'Server') + +```{autodoc2-docstring} archivebox.machine.models.Process.TypeChoices.SERVER +``` + +```` + +````{py:attribute} UPDATE +:canonical: archivebox.machine.models.Process.TypeChoices.UPDATE +:value: > + ('update', 'Update') + +```{autodoc2-docstring} archivebox.machine.models.Process.TypeChoices.UPDATE +``` + +```` + +````{py:attribute} ADD +:canonical: archivebox.machine.models.Process.TypeChoices.ADD +:value: > + ('add', 'Add') + +```{autodoc2-docstring} archivebox.machine.models.Process.TypeChoices.ADD +``` + +```` + +````{py:attribute} SEARCH +:canonical: archivebox.machine.models.Process.TypeChoices.SEARCH +:value: > + ('search', 'Search') + +```{autodoc2-docstring} archivebox.machine.models.Process.TypeChoices.SEARCH +``` + +```` + ````{py:attribute} WORKER :canonical: archivebox.machine.models.Process.TypeChoices.WORKER :value: > @@ -1466,6 +1622,61 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} machine_id +:canonical: archivebox.machine.models.Process.machine_id +:type: uuid.UUID +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Process.machine_id +``` + +```` + +````{py:attribute} parent_id +:canonical: archivebox.machine.models.Process.parent_id +:type: uuid.UUID | None +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Process.parent_id +``` + +```` + +````{py:attribute} binary_id +:canonical: archivebox.machine.models.Process.binary_id +:type: uuid.UUID | None +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Process.binary_id +``` + +```` + +````{py:attribute} children +:canonical: archivebox.machine.models.Process.children +:type: django.db.models.Manager[archivebox.machine.models.Process] +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Process.children +``` + +```` + +````{py:attribute} archiveresult +:canonical: archivebox.machine.models.Process.archiveresult +:type: archivebox.core.models.ArchiveResult +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Process.archiveresult +``` + +```` + ````{py:attribute} state_machine_name :canonical: archivebox.machine.models.Process.state_machine_name :type: str @@ -1477,9 +1688,18 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} delete_after_final_statuses +:canonical: archivebox.machine.models.Process.delete_after_final_statuses +:value: > + () + +```{autodoc2-docstring} archivebox.machine.models.Process.delete_after_final_statuses +``` + +```` + ````{py:attribute} objects :canonical: archivebox.machine.models.Process.objects -:type: archivebox.machine.models.ProcessManager :value: > 'ProcessManager(...)' @@ -1491,8 +1711,7 @@ Bases: {py:obj}`django.db.models.TextChoices` `````{py:class} Meta :canonical: archivebox.machine.models.Process.Meta -```{autodoc2-docstring} archivebox.machine.models.Process.Meta -``` +Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta` ````{py:attribute} app_label :canonical: archivebox.machine.models.Process.Meta.app_label @@ -1534,6 +1753,16 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} constraints +:canonical: archivebox.machine.models.Process.Meta.constraints +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.Process.Meta.constraints +``` + +```` + ````` ````{py:method} __str__() -> str @@ -1541,6 +1770,23 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:method} get_delete_after_config_value() +:canonical: archivebox.machine.models.Process.get_delete_after_config_value + +```{autodoc2-docstring} archivebox.machine.models.Process.get_delete_after_config_value +``` + +```` + +````{py:method} missing_delete_at_candidates() +:canonical: archivebox.machine.models.Process.missing_delete_at_candidates +:classmethod: + +```{autodoc2-docstring} archivebox.machine.models.Process.missing_delete_at_candidates +``` + +```` + ````{py:property} cmd_version :canonical: archivebox.machine.models.Process.cmd_version :type: str @@ -1585,6 +1831,14 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:method} hydrate_binary_from_context(*, plugin_name: str = '', hook_path: str = '') -> archivebox.machine.models.Binary | None +:canonical: archivebox.machine.models.Process.hydrate_binary_from_context + +```{autodoc2-docstring} archivebox.machine.models.Process.hydrate_binary_from_context +``` + +```` + ````{py:method} parse_records_from_text(text: str) -> list[dict] :canonical: archivebox.machine.models.Process.parse_records_from_text :classmethod: @@ -1602,7 +1856,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} from_json(record: dict, overrides: dict = None) +````{py:method} from_json(record: dict[str, typing.Any], overrides: dict[str, typing.Any] | None = None) :canonical: archivebox.machine.models.Process.from_json :staticmethod: @@ -1611,7 +1865,15 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} update_and_requeue(**kwargs) +````{py:method} safe_update(update_fields: dict[str, typing.Any], *, refresh: bool = True, extra_filter: dict[str, typing.Any] | None = None) -> bool +:canonical: archivebox.machine.models.Process.safe_update + +```{autodoc2-docstring} archivebox.machine.models.Process.safe_update +``` + +```` + +````{py:method} update_and_requeue(**kwargs) -> bool :canonical: archivebox.machine.models.Process.update_and_requeue ```{autodoc2-docstring} archivebox.machine.models.Process.update_and_requeue @@ -1619,6 +1881,30 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:method} mark_running(*, process_type: str | None = None, pwd: str | pathlib.Path | None = None, url: str | None = None, worker_type: str = '', timeout: int | None = None) -> None +:canonical: archivebox.machine.models.Process.mark_running + +```{autodoc2-docstring} archivebox.machine.models.Process.mark_running +``` + +```` + +````{py:method} heartbeat() -> None +:canonical: archivebox.machine.models.Process.heartbeat + +```{autodoc2-docstring} archivebox.machine.models.Process.heartbeat +``` + +```` + +````{py:method} mark_exited(*, exit_code: int = 0) -> None +:canonical: archivebox.machine.models.Process.mark_exited + +```{autodoc2-docstring} archivebox.machine.models.Process.mark_exited +``` + +```` + ````{py:method} current() -> archivebox.machine.models.Process :canonical: archivebox.machine.models.Process.current :classmethod: @@ -1628,7 +1914,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} _find_parent_process(machine: archivebox.machine.models.Machine = None) -> Process | None +````{py:method} _find_parent_process(machine: archivebox.machine.models.Machine | None = None) -> archivebox.machine.models.Process | None :canonical: archivebox.machine.models.Process._find_parent_process :classmethod: @@ -1646,7 +1932,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} cleanup_stale_running(machine: archivebox.machine.models.Machine = None) -> int +````{py:method} cleanup_stale_running(machine: archivebox.machine.models.Machine | None = None) -> int :canonical: archivebox.machine.models.Process.cleanup_stale_running :classmethod: @@ -1740,27 +2026,9 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:property} pid_file -:canonical: archivebox.machine.models.Process.pid_file -:type: pathlib.Path - -```{autodoc2-docstring} archivebox.machine.models.Process.pid_file -``` - -```` - -````{py:property} cmd_file -:canonical: archivebox.machine.models.Process.cmd_file -:type: pathlib.Path - -```{autodoc2-docstring} archivebox.machine.models.Process.cmd_file -``` - -```` - ````{py:property} stdout_file :canonical: archivebox.machine.models.Process.stdout_file -:type: pathlib.Path +:type: pathlib.Path | None ```{autodoc2-docstring} archivebox.machine.models.Process.stdout_file ``` @@ -1769,7 +2037,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ````{py:property} stderr_file :canonical: archivebox.machine.models.Process.stderr_file -:type: pathlib.Path +:type: pathlib.Path | None ```{autodoc2-docstring} archivebox.machine.models.Process.stderr_file ``` @@ -1826,22 +2094,6 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} _write_pid_file() -> None -:canonical: archivebox.machine.models.Process._write_pid_file - -```{autodoc2-docstring} archivebox.machine.models.Process._write_pid_file -``` - -```` - -````{py:method} _write_cmd_file() -> None -:canonical: archivebox.machine.models.Process._write_cmd_file - -```{autodoc2-docstring} archivebox.machine.models.Process._write_cmd_file -``` - -```` - ````{py:method} ensure_log_files() -> None :canonical: archivebox.machine.models.Process.ensure_log_files @@ -1914,7 +2166,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} get_running(process_type: str = None, machine: archivebox.machine.models.Machine = None) -> QuerySet[Process] +````{py:method} get_running(process_type: str | None = None, machine: archivebox.machine.models.Machine | None = None) -> django.db.models.QuerySet[archivebox.machine.models.Process] :canonical: archivebox.machine.models.Process.get_running :classmethod: @@ -1923,7 +2175,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} get_running_count(process_type: str = None, machine: archivebox.machine.models.Machine = None) -> int +````{py:method} get_running_count(process_type: str | None = None, machine: archivebox.machine.models.Machine | None = None) -> int :canonical: archivebox.machine.models.Process.get_running_count :classmethod: @@ -1932,7 +2184,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} stop_all(process_type: str = None, machine: archivebox.machine.models.Machine = None, graceful: bool = True) -> int +````{py:method} stop_all(process_type: str | None = None, machine: archivebox.machine.models.Machine | None = None, graceful: bool = True) -> int :canonical: archivebox.machine.models.Process.stop_all :classmethod: @@ -1941,7 +2193,7 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` -````{py:method} get_next_worker_id(process_type: str = 'worker', machine: archivebox.machine.models.Machine = None) -> int +````{py:method} get_next_worker_id(process_type: str = 'worker', machine: archivebox.machine.models.Machine | None = None) -> int :canonical: archivebox.machine.models.Process.get_next_worker_id :classmethod: @@ -1994,6 +2246,17 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:attribute} binary +:canonical: archivebox.machine.models.BinaryMachine.binary +:type: archivebox.machine.models.Binary +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.binary +``` + +```` + ````{py:attribute} queued :canonical: archivebox.machine.models.BinaryMachine.queued :value: > @@ -2082,6 +2345,17 @@ Bases: {py:obj}`archivebox.workers.models.BaseStateMachine` ```` +````{py:attribute} process +:canonical: archivebox.machine.models.ProcessMachine.process +:type: archivebox.machine.models.Process +:value: > + None + +```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.process +``` + +```` + ````{py:attribute} queued :canonical: archivebox.machine.models.ProcessMachine.queued :value: > diff --git a/docs/apidocs/archivebox/archivebox.mcp.server.md b/docs/apidocs/archivebox/archivebox.mcp.server.md index 5fa6b64a..b599c2e9 100644 --- a/docs/apidocs/archivebox/archivebox.mcp.server.md +++ b/docs/apidocs/archivebox/archivebox.mcp.server.md @@ -65,21 +65,21 @@ Bases: {py:obj}`json.JSONEncoder` ```{autodoc2-docstring} archivebox.mcp.server.MCPJSONEncoder.__init__ ``` -````{py:method} default(obj) +````{py:method} default(o) :canonical: archivebox.mcp.server.MCPJSONEncoder.default ```` ````` -````{py:function} click_type_to_json_schema_type(click_type) -> dict +````{py:function} click_type_to_json_schema_type(click_type: click.ParamType) -> dict[str, typing.Any] :canonical: archivebox.mcp.server.click_type_to_json_schema_type ```{autodoc2-docstring} archivebox.mcp.server.click_type_to_json_schema_type ``` ```` -````{py:function} click_command_to_mcp_tool(cmd_name: str, click_command: click.Command) -> dict +````{py:function} click_command_to_mcp_tool(cmd_name: str, click_command: click.Command) -> dict[str, typing.Any] :canonical: archivebox.mcp.server.click_command_to_mcp_tool ```{autodoc2-docstring} archivebox.mcp.server.click_command_to_mcp_tool @@ -105,7 +105,7 @@ Bases: {py:obj}`json.JSONEncoder` ```{autodoc2-docstring} archivebox.mcp.server.MCPServer.__init__ ``` -````{py:method} get_click_command(cmd_name: str) -> typing.Optional[click.Command] +````{py:method} get_click_command(cmd_name: str) -> click.Command | None :canonical: archivebox.mcp.server.MCPServer.get_click_command ```{autodoc2-docstring} archivebox.mcp.server.MCPServer.get_click_command diff --git a/docs/apidocs/archivebox/archivebox.md b/docs/apidocs/archivebox/archivebox.md index aceb34c1..3fd38cbf 100644 --- a/docs/apidocs/archivebox/archivebox.md +++ b/docs/apidocs/archivebox/archivebox.md @@ -25,8 +25,8 @@ archivebox.personas archivebox.machine archivebox.api archivebox.workers -archivebox.ideas archivebox.base_models +archivebox.services ``` ## Submodules @@ -43,6 +43,28 @@ archivebox.__main__ ## Package Contents +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_ReconfigurableStream ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`__getattr__ ` + - ```{autodoc2-docstring} archivebox.__getattr__ + :summary: + ``` +```` + ### Data ````{list-table} @@ -57,22 +79,6 @@ archivebox.__main__ - ```{autodoc2-docstring} archivebox.PACKAGE_DIR :summary: ``` -* - {py:obj}`BUILTIN_PLUGINS_DIR ` - - ```{autodoc2-docstring} archivebox.BUILTIN_PLUGINS_DIR - :summary: - ``` -* - {py:obj}`USER_PLUGINS_DIR ` - - ```{autodoc2-docstring} archivebox.USER_PLUGINS_DIR - :summary: - ``` -* - {py:obj}`ALL_PLUGINS ` - - ```{autodoc2-docstring} archivebox.ALL_PLUGINS - :summary: - ``` -* - {py:obj}`LOADED_PLUGINS ` - - ```{autodoc2-docstring} archivebox.LOADED_PLUGINS - :summary: - ``` * - {py:obj}`__version__ ` - ```{autodoc2-docstring} archivebox.__version__ :summary: @@ -85,6 +91,10 @@ archivebox.__main__ - ```{autodoc2-docstring} archivebox.__license__ :summary: ``` +* - {py:obj}`__all__ ` + - ```{autodoc2-docstring} archivebox.__all__ + :summary: + ``` * - {py:obj}`ASCII_ICON ` - ```{autodoc2-docstring} archivebox.ASCII_ICON :summary: @@ -93,6 +103,21 @@ archivebox.__main__ ### API +`````{py:class} _ReconfigurableStream +:canonical: archivebox._ReconfigurableStream + +Bases: {py:obj}`typing.Protocol` + +````{py:method} reconfigure(*, line_buffering: bool) -> object +:canonical: archivebox._ReconfigurableStream.reconfigure + +```{autodoc2-docstring} archivebox._ReconfigurableStream.reconfigure +``` + +```` + +````` + ````{py:data} ASCII_LOGO :canonical: archivebox.ASCII_LOGO :value: @@ -112,46 +137,6 @@ archivebox.__main__ ```` -````{py:data} BUILTIN_PLUGINS_DIR -:canonical: archivebox.BUILTIN_PLUGINS_DIR -:value: > - 'resolve(...)' - -```{autodoc2-docstring} archivebox.BUILTIN_PLUGINS_DIR -``` - -```` - -````{py:data} USER_PLUGINS_DIR -:canonical: archivebox.USER_PLUGINS_DIR -:value: > - None - -```{autodoc2-docstring} archivebox.USER_PLUGINS_DIR -``` - -```` - -````{py:data} ALL_PLUGINS -:canonical: archivebox.ALL_PLUGINS -:value: > - None - -```{autodoc2-docstring} archivebox.ALL_PLUGINS -``` - -```` - -````{py:data} LOADED_PLUGINS -:canonical: archivebox.LOADED_PLUGINS -:value: > - None - -```{autodoc2-docstring} archivebox.LOADED_PLUGINS -``` - -```` - ````{py:data} __version__ :canonical: archivebox.__version__ :value: > @@ -182,6 +167,23 @@ archivebox.__main__ ```` +````{py:function} __getattr__(name: str) +:canonical: archivebox.__getattr__ + +```{autodoc2-docstring} archivebox.__getattr__ +``` +```` + +````{py:data} __all__ +:canonical: archivebox.__all__ +:value: > + ('ASCII_LOGO', 'ASCII_ICON', 'PACKAGE_DIR', 'DATA_DIR', 'CONSTANTS', 'VERSION', 'BUILTIN_PLUGINS_DIR... + +```{autodoc2-docstring} archivebox.__all__ +``` + +```` + ````{py:data} ASCII_ICON :canonical: archivebox.ASCII_ICON :value: diff --git a/docs/apidocs/archivebox/archivebox.misc.checks.md b/docs/apidocs/archivebox/archivebox.misc.checks.md index 3a1fdcc2..2430aaf1 100644 --- a/docs/apidocs/archivebox/archivebox.misc.checks.md +++ b/docs/apidocs/archivebox/archivebox.misc.checks.md @@ -51,14 +51,14 @@ ### API -````{py:function} check_data_folder() -> None +````{py:function} check_data_folder(config=None, **config_kwargs) -> None :canonical: archivebox.misc.checks.check_data_folder ```{autodoc2-docstring} archivebox.misc.checks.check_data_folder ``` ```` -````{py:function} check_migrations() +````{py:function} check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_delay: int = 3) -> list[str] :canonical: archivebox.misc.checks.check_migrations ```{autodoc2-docstring} archivebox.misc.checks.check_migrations @@ -86,21 +86,21 @@ ``` ```` -````{py:function} check_data_dir_permissions() +````{py:function} check_data_dir_permissions(config=None, **config_kwargs) :canonical: archivebox.misc.checks.check_data_dir_permissions ```{autodoc2-docstring} archivebox.misc.checks.check_data_dir_permissions ``` ```` -````{py:function} check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True) +````{py:function} check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs) :canonical: archivebox.misc.checks.check_tmp_dir ```{autodoc2-docstring} archivebox.misc.checks.check_tmp_dir ``` ```` -````{py:function} check_lib_dir(lib_dir: pathlib.Path | None = None, throw=False, quiet=False, must_exist=True) +````{py:function} check_lib_dir(lib_dir: pathlib.Path | None = None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs) :canonical: archivebox.misc.checks.check_lib_dir ```{autodoc2-docstring} archivebox.misc.checks.check_lib_dir diff --git a/docs/apidocs/archivebox/archivebox.misc.db.md b/docs/apidocs/archivebox/archivebox.misc.db.md index fe55e33b..d95a48d1 100644 --- a/docs/apidocs/archivebox/archivebox.misc.db.md +++ b/docs/apidocs/archivebox/archivebox.misc.db.md @@ -15,39 +15,128 @@ :class: autosummary longtable :align: left -* - {py:obj}`list_migrations ` - - ```{autodoc2-docstring} archivebox.misc.db.list_migrations +* - {py:obj}`run_db_analyze_batch ` + - ```{autodoc2-docstring} archivebox.misc.db.run_db_analyze_batch + :summary: + ``` +* - {py:obj}`compact_command ` + - ```{autodoc2-docstring} archivebox.misc.db.compact_command + :summary: + ``` +* - {py:obj}`sqlite_lock_holders ` + - ```{autodoc2-docstring} archivebox.misc.db.sqlite_lock_holders + :summary: + ``` +* - {py:obj}`sqlite_lock_error ` + - ```{autodoc2-docstring} archivebox.misc.db.sqlite_lock_error + :summary: + ``` +* - {py:obj}`retry_sqlite_locks ` + - ```{autodoc2-docstring} archivebox.misc.db.retry_sqlite_locks + :summary: + ``` +* - {py:obj}`migration_lock ` + - ```{autodoc2-docstring} archivebox.misc.db.migration_lock + :summary: + ``` +* - {py:obj}`migration_state ` + - ```{autodoc2-docstring} archivebox.misc.db.migration_state + :summary: + ``` +* - {py:obj}`pending_migrations ` + - ```{autodoc2-docstring} archivebox.misc.db.pending_migrations :summary: ``` * - {py:obj}`apply_migrations ` - ```{autodoc2-docstring} archivebox.misc.db.apply_migrations :summary: ``` -* - {py:obj}`get_admins ` - - ```{autodoc2-docstring} archivebox.misc.db.get_admins +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`HISTORICAL_GHOST_MIGRATIONS ` + - ```{autodoc2-docstring} archivebox.misc.db.HISTORICAL_GHOST_MIGRATIONS :summary: ``` ```` ### API -````{py:function} list_migrations(out_dir: pathlib.Path = DATA_DIR) -> typing.List[typing.Tuple[bool, str]] -:canonical: archivebox.misc.db.list_migrations +````{py:function} run_db_analyze_batch(remaining: list[str] | None, *, max_seconds_per_table: float = 120.0) -> list[str] +:canonical: archivebox.misc.db.run_db_analyze_batch -```{autodoc2-docstring} archivebox.misc.db.list_migrations +```{autodoc2-docstring} archivebox.misc.db.run_db_analyze_batch ``` ```` -````{py:function} apply_migrations(out_dir: pathlib.Path = DATA_DIR) -> typing.List[str] +````{py:function} compact_command(cmdline: list[str] | None, fallback: str = '') -> str +:canonical: archivebox.misc.db.compact_command + +```{autodoc2-docstring} archivebox.misc.db.compact_command +``` +```` + +````{py:function} sqlite_lock_holders(db_path: pathlib.Path = DATA_DIR / 'index.sqlite3') -> list[str] +:canonical: archivebox.misc.db.sqlite_lock_holders + +```{autodoc2-docstring} archivebox.misc.db.sqlite_lock_holders +``` +```` + +````{py:function} sqlite_lock_error(error: BaseException) -> bool +:canonical: archivebox.misc.db.sqlite_lock_error + +```{autodoc2-docstring} archivebox.misc.db.sqlite_lock_error +``` +```` + +````{py:function} retry_sqlite_locks(action: collections.abc.Callable[[], typing.Any], *, label: str, stderr: typing.TextIO | None = None) -> typing.Any +:canonical: archivebox.misc.db.retry_sqlite_locks + +```{autodoc2-docstring} archivebox.misc.db.retry_sqlite_locks +``` +```` + +````{py:function} migration_lock(stdout: typing.TextIO | None = None) +:canonical: archivebox.misc.db.migration_lock + +```{autodoc2-docstring} archivebox.misc.db.migration_lock +``` +```` + +````{py:data} HISTORICAL_GHOST_MIGRATIONS +:canonical: archivebox.misc.db.HISTORICAL_GHOST_MIGRATIONS +:type: frozenset[tuple[str, str]] +:value: > + 'frozenset(...)' + +```{autodoc2-docstring} archivebox.misc.db.HISTORICAL_GHOST_MIGRATIONS +``` + +```` + +````{py:function} migration_state(out_dir: pathlib.Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]] +:canonical: archivebox.misc.db.migration_state + +```{autodoc2-docstring} archivebox.misc.db.migration_state +``` +```` + +````{py:function} pending_migrations(out_dir: pathlib.Path = DATA_DIR) -> list[str] +:canonical: archivebox.misc.db.pending_migrations + +```{autodoc2-docstring} archivebox.misc.db.pending_migrations +``` +```` + +````{py:function} apply_migrations(out_dir: pathlib.Path = DATA_DIR, stdout: typing.TextIO | None = None, stderr: typing.TextIO | None = None, verbosity: int = 1) -> list[str] :canonical: archivebox.misc.db.apply_migrations ```{autodoc2-docstring} archivebox.misc.db.apply_migrations ``` ```` - -````{py:function} get_admins(out_dir: pathlib.Path = DATA_DIR) -> typing.List -:canonical: archivebox.misc.db.get_admins - -```{autodoc2-docstring} archivebox.misc.db.get_admins -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.misc.debugging.md b/docs/apidocs/archivebox/archivebox.misc.debugging.md deleted file mode 100644 index eb351416..00000000 --- a/docs/apidocs/archivebox/archivebox.misc.debugging.md +++ /dev/null @@ -1,31 +0,0 @@ -# {py:mod}`archivebox.misc.debugging` - -```{py:module} archivebox.misc.debugging -``` - -```{autodoc2-docstring} archivebox.misc.debugging -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`timed_function ` - - ```{autodoc2-docstring} archivebox.misc.debugging.timed_function - :summary: - ``` -```` - -### API - -````{py:function} timed_function(func) -:canonical: archivebox.misc.debugging.timed_function - -```{autodoc2-docstring} archivebox.misc.debugging.timed_function -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.misc.folders.md b/docs/apidocs/archivebox/archivebox.misc.folders.md deleted file mode 100644 index 9d05e896..00000000 --- a/docs/apidocs/archivebox/archivebox.misc.folders.md +++ /dev/null @@ -1,31 +0,0 @@ -# {py:mod}`archivebox.misc.folders` - -```{py:module} archivebox.misc.folders -``` - -```{autodoc2-docstring} archivebox.misc.folders -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`fix_invalid_folder_locations ` - - ```{autodoc2-docstring} archivebox.misc.folders.fix_invalid_folder_locations - :summary: - ``` -```` - -### API - -````{py:function} fix_invalid_folder_locations(out_dir: pathlib.Path = DATA_DIR) -> typing.Tuple[typing.List[str], typing.List[str]] -:canonical: archivebox.misc.folders.fix_invalid_folder_locations - -```{autodoc2-docstring} archivebox.misc.folders.fix_invalid_folder_locations -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.misc.hashing.md b/docs/apidocs/archivebox/archivebox.misc.hashing.md index fb95421d..2f6a1d54 100644 --- a/docs/apidocs/archivebox/archivebox.misc.hashing.md +++ b/docs/apidocs/archivebox/archivebox.misc.hashing.md @@ -57,14 +57,14 @@ ``` ```` -````{py:function} get_dir_hashes(dir_path: pathlib.Path, pwd: pathlib.Path | None = None, filter_func: typing.Callable | None = None, max_depth: int = -1) -> dict[str, str] +````{py:function} get_dir_hashes(dir_path: pathlib.Path, pwd: pathlib.Path | None = None, filter_func: collections.abc.Callable | None = None, max_depth: int = -1) -> dict[str, str] :canonical: archivebox.misc.hashing.get_dir_hashes ```{autodoc2-docstring} archivebox.misc.hashing.get_dir_hashes ``` ```` -````{py:function} get_dir_entries(dir_path: pathlib.Path, pwd: pathlib.Path | None = None, recursive: bool = True, include_files: bool = True, include_dirs: bool = True, include_hidden: bool = False, filter_func: typing.Callable | None = None, max_depth: int = -1) -> tuple[str, ...] +````{py:function} get_dir_entries(dir_path: pathlib.Path, pwd: pathlib.Path | None = None, recursive: bool = True, include_files: bool = True, include_dirs: bool = True, include_hidden: bool = False, filter_func: collections.abc.Callable | None = None, max_depth: int = -1) -> tuple[str, ...] :canonical: archivebox.misc.hashing.get_dir_entries ```{autodoc2-docstring} archivebox.misc.hashing.get_dir_entries @@ -78,7 +78,7 @@ ``` ```` -````{py:function} get_dir_info(dir_path: pathlib.Path, pwd: pathlib.Path | None = None, filter_func: typing.Callable | None = None, max_depth: int = -1) -> dict +````{py:function} get_dir_info(dir_path: pathlib.Path, pwd: pathlib.Path | None = None, filter_func: collections.abc.Callable | None = None, max_depth: int = -1) -> dict :canonical: archivebox.misc.hashing.get_dir_info ```{autodoc2-docstring} archivebox.misc.hashing.get_dir_info diff --git a/docs/apidocs/archivebox/archivebox.misc.jsonl.md b/docs/apidocs/archivebox/archivebox.misc.jsonl.md index 3d62d30b..99018b0d 100644 --- a/docs/apidocs/archivebox/archivebox.misc.jsonl.md +++ b/docs/apidocs/archivebox/archivebox.misc.jsonl.md @@ -35,10 +35,6 @@ - ```{autodoc2-docstring} archivebox.misc.jsonl.write_record :summary: ``` -* - {py:obj}`write_records ` - - ```{autodoc2-docstring} archivebox.misc.jsonl.write_records - :summary: - ``` ```` ### Data @@ -63,6 +59,10 @@ - ```{autodoc2-docstring} archivebox.misc.jsonl.TYPE_CRAWL :summary: ``` +* - {py:obj}`TYPE_BINARYREQUEST ` + - ```{autodoc2-docstring} archivebox.misc.jsonl.TYPE_BINARYREQUEST + :summary: + ``` * - {py:obj}`TYPE_BINARY ` - ```{autodoc2-docstring} archivebox.misc.jsonl.TYPE_BINARY :summary: @@ -123,6 +123,16 @@ ```` +````{py:data} TYPE_BINARYREQUEST +:canonical: archivebox.misc.jsonl.TYPE_BINARYREQUEST +:value: > + 'BinaryRequest' + +```{autodoc2-docstring} archivebox.misc.jsonl.TYPE_BINARYREQUEST +``` + +```` + ````{py:data} TYPE_BINARY :canonical: archivebox.misc.jsonl.TYPE_BINARY :value: > @@ -163,44 +173,37 @@ ```` -````{py:function} parse_line(line: str) -> typing.Optional[typing.Dict[str, typing.Any]] +````{py:function} parse_line(line: str) -> dict[str, typing.Any] | None :canonical: archivebox.misc.jsonl.parse_line ```{autodoc2-docstring} archivebox.misc.jsonl.parse_line ``` ```` -````{py:function} read_stdin(stream: typing.Optional[typing.TextIO] = None) -> typing.Iterator[typing.Dict[str, typing.Any]] +````{py:function} read_stdin(stream: typing.TextIO | None = None) -> collections.abc.Iterator[dict[str, typing.Any]] :canonical: archivebox.misc.jsonl.read_stdin ```{autodoc2-docstring} archivebox.misc.jsonl.read_stdin ``` ```` -````{py:function} read_file(path: pathlib.Path) -> typing.Iterator[typing.Dict[str, typing.Any]] +````{py:function} read_file(path: pathlib.Path) -> collections.abc.Iterator[dict[str, typing.Any]] :canonical: archivebox.misc.jsonl.read_file ```{autodoc2-docstring} archivebox.misc.jsonl.read_file ``` ```` -````{py:function} read_args_or_stdin(args: tuple, stream: typing.Optional[typing.TextIO] = None) -> typing.Iterator[typing.Dict[str, typing.Any]] +````{py:function} read_args_or_stdin(args: collections.abc.Iterable[str], stream: typing.TextIO | None = None) -> collections.abc.Iterator[dict[str, typing.Any]] :canonical: archivebox.misc.jsonl.read_args_or_stdin ```{autodoc2-docstring} archivebox.misc.jsonl.read_args_or_stdin ``` ```` -````{py:function} write_record(record: typing.Dict[str, typing.Any], stream: typing.Optional[typing.TextIO] = None) -> None +````{py:function} write_record(record: dict[str, typing.Any], stream: typing.TextIO | None = None) -> None :canonical: archivebox.misc.jsonl.write_record ```{autodoc2-docstring} archivebox.misc.jsonl.write_record ``` ```` - -````{py:function} write_records(records: typing.Iterator[typing.Dict[str, typing.Any]], stream: typing.Optional[typing.TextIO] = None) -> int -:canonical: archivebox.misc.jsonl.write_records - -```{autodoc2-docstring} archivebox.misc.jsonl.write_records -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.misc.legacy.md b/docs/apidocs/archivebox/archivebox.misc.legacy.md deleted file mode 100644 index e2f821af..00000000 --- a/docs/apidocs/archivebox/archivebox.misc.legacy.md +++ /dev/null @@ -1,125 +0,0 @@ -# {py:mod}`archivebox.misc.legacy` - -```{py:module} archivebox.misc.legacy -``` - -```{autodoc2-docstring} archivebox.misc.legacy -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`SnapshotDict ` - - ```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict - :summary: - ``` -```` - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`parse_json_main_index ` - - ```{autodoc2-docstring} archivebox.misc.legacy.parse_json_main_index - :summary: - ``` -* - {py:obj}`parse_json_links_details ` - - ```{autodoc2-docstring} archivebox.misc.legacy.parse_json_links_details - :summary: - ``` -```` - -### API - -`````{py:class} SnapshotDict() -:canonical: archivebox.misc.legacy.SnapshotDict - -Bases: {py:obj}`typing.TypedDict` - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict.__init__ -``` - -````{py:attribute} url -:canonical: archivebox.misc.legacy.SnapshotDict.url -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict.url -``` - -```` - -````{py:attribute} timestamp -:canonical: archivebox.misc.legacy.SnapshotDict.timestamp -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict.timestamp -``` - -```` - -````{py:attribute} title -:canonical: archivebox.misc.legacy.SnapshotDict.title -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict.title -``` - -```` - -````{py:attribute} tags -:canonical: archivebox.misc.legacy.SnapshotDict.tags -:type: str -:value: > - None - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict.tags -``` - -```` - -````{py:attribute} sources -:canonical: archivebox.misc.legacy.SnapshotDict.sources -:type: typing.List[str] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.legacy.SnapshotDict.sources -``` - -```` - -````` - -````{py:function} parse_json_main_index(out_dir: pathlib.Path) -> typing.Iterator[archivebox.misc.legacy.SnapshotDict] -:canonical: archivebox.misc.legacy.parse_json_main_index - -```{autodoc2-docstring} archivebox.misc.legacy.parse_json_main_index -``` -```` - -````{py:function} parse_json_links_details(out_dir: pathlib.Path) -> typing.Iterator[archivebox.misc.legacy.SnapshotDict] -:canonical: archivebox.misc.legacy.parse_json_links_details - -```{autodoc2-docstring} archivebox.misc.legacy.parse_json_links_details -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.misc.logging.md b/docs/apidocs/archivebox/archivebox.misc.logging.md index 6fa29c2d..9071b4f4 100644 --- a/docs/apidocs/archivebox/archivebox.misc.logging.md +++ b/docs/apidocs/archivebox/archivebox.misc.logging.md @@ -15,6 +15,8 @@ :class: autosummary longtable :align: left +* - {py:obj}`AttrDict ` + - * - {py:obj}`RainbowHighlighter ` - ```` @@ -107,6 +109,40 @@ ```` +`````{py:class} AttrDict(*args, **kwargs) +:canonical: archivebox.misc.logging.AttrDict + +Bases: {py:obj}`dict` + +````{py:method} _wrap(value) +:canonical: archivebox.misc.logging.AttrDict._wrap +:classmethod: + +```{autodoc2-docstring} archivebox.misc.logging.AttrDict._wrap +``` + +```` + +````{py:method} __setitem__(key, value) +:canonical: archivebox.misc.logging.AttrDict.__setitem__ + +```` + +````{py:method} update(*args, **kwargs) +:canonical: archivebox.misc.logging.AttrDict.update + +```` + +````{py:method} __getattr__(key) +:canonical: archivebox.misc.logging.AttrDict.__getattr__ + +```{autodoc2-docstring} archivebox.misc.logging.AttrDict.__getattr__ +``` + +```` + +````` + `````{py:class} RainbowHighlighter :canonical: archivebox.misc.logging.RainbowHighlighter @@ -132,7 +168,7 @@ Bases: {py:obj}`rich.highlighter.Highlighter` ````{py:data} DEFAULT_CLI_COLORS :canonical: archivebox.misc.logging.DEFAULT_CLI_COLORS :value: > - 'benedict(...)' + 'AttrDict(...)' ```{autodoc2-docstring} archivebox.misc.logging.DEFAULT_CLI_COLORS ``` @@ -142,7 +178,7 @@ Bases: {py:obj}`rich.highlighter.Highlighter` ````{py:data} ANSI :canonical: archivebox.misc.logging.ANSI :value: > - 'benedict(...)' + 'AttrDict(...)' ```{autodoc2-docstring} archivebox.misc.logging.ANSI ``` @@ -159,21 +195,21 @@ Bases: {py:obj}`rich.highlighter.Highlighter` ```` -````{py:function} stdout(*args, color: typing.Optional[str] = None, prefix: str = '', config: typing.Optional[benedict.benedict] = None) -> None +````{py:function} stdout(*args, color: str | None = None, prefix: str = '', config: dict | None = None) -> None :canonical: archivebox.misc.logging.stdout ```{autodoc2-docstring} archivebox.misc.logging.stdout ``` ```` -````{py:function} stderr(*args, color: typing.Optional[str] = None, prefix: str = '', config: typing.Optional[benedict.benedict] = None) -> None +````{py:function} stderr(*args, color: str | None = None, prefix: str = '', config: dict | None = None) -> None :canonical: archivebox.misc.logging.stderr ```{autodoc2-docstring} archivebox.misc.logging.stderr ``` ```` -````{py:function} hint(text: typing.Union[typing.Tuple[str, ...], typing.List[str], str], prefix=' ', config: typing.Optional[benedict.benedict] = None) -> None +````{py:function} hint(text: tuple[str, ...] | list[str] | str, prefix=' ', config: dict | None = None) -> None :canonical: archivebox.misc.logging.hint ```{autodoc2-docstring} archivebox.misc.logging.hint diff --git a/docs/apidocs/archivebox/archivebox.misc.logging_util.md b/docs/apidocs/archivebox/archivebox.misc.logging_util.md index 473210d8..35426997 100644 --- a/docs/apidocs/archivebox/archivebox.misc.logging_util.md +++ b/docs/apidocs/archivebox/archivebox.misc.logging_util.md @@ -15,10 +15,6 @@ :class: autosummary longtable :align: left -* - {py:obj}`RuntimeStats ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats - :summary: - ``` * - {py:obj}`TimedProgress ` - ```{autodoc2-docstring} archivebox.misc.logging_util.TimedProgress :summary: @@ -39,70 +35,6 @@ - ```{autodoc2-docstring} archivebox.misc.logging_util.log_cli_command :summary: ``` -* - {py:obj}`log_importing_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_importing_started - :summary: - ``` -* - {py:obj}`log_source_saved ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_source_saved - :summary: - ``` -* - {py:obj}`log_parsing_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_parsing_finished - :summary: - ``` -* - {py:obj}`log_deduping_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_deduping_finished - :summary: - ``` -* - {py:obj}`log_crawl_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_crawl_started - :summary: - ``` -* - {py:obj}`log_indexing_process_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_process_started - :summary: - ``` -* - {py:obj}`log_indexing_process_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_process_finished - :summary: - ``` -* - {py:obj}`log_indexing_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_started - :summary: - ``` -* - {py:obj}`log_indexing_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_finished - :summary: - ``` -* - {py:obj}`log_archiving_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_archiving_started - :summary: - ``` -* - {py:obj}`log_archiving_paused ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_archiving_paused - :summary: - ``` -* - {py:obj}`log_archiving_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_archiving_finished - :summary: - ``` -* - {py:obj}`log_snapshot_archiving_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_snapshot_archiving_started - :summary: - ``` -* - {py:obj}`log_snapshot_archiving_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_snapshot_archiving_finished - :summary: - ``` -* - {py:obj}`log_archive_method_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_archive_method_started - :summary: - ``` -* - {py:obj}`log_archive_method_finished ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_archive_method_finished - :summary: - ``` * - {py:obj}`log_list_started ` - ```{autodoc2-docstring} archivebox.misc.logging_util.log_list_started :summary: @@ -119,10 +51,6 @@ - ```{autodoc2-docstring} archivebox.misc.logging_util.log_removal_finished :summary: ``` -* - {py:obj}`log_index_started ` - - ```{autodoc2-docstring} archivebox.misc.logging_util.log_index_started - :summary: - ``` * - {py:obj}`pretty_path ` - ```{autodoc2-docstring} archivebox.misc.logging_util.pretty_path :summary: @@ -161,138 +89,9 @@ ``` ```` -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_LAST_RUN_STATS ` - - ```{autodoc2-docstring} archivebox.misc.logging_util._LAST_RUN_STATS - :summary: - ``` -```` - ### API -`````{py:class} RuntimeStats -:canonical: archivebox.misc.logging_util.RuntimeStats - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats -``` - -````{py:attribute} skipped -:canonical: archivebox.misc.logging_util.RuntimeStats.skipped -:type: int -:value: > - 0 - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.skipped -``` - -```` - -````{py:attribute} succeeded -:canonical: archivebox.misc.logging_util.RuntimeStats.succeeded -:type: int -:value: > - 0 - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.succeeded -``` - -```` - -````{py:attribute} failed -:canonical: archivebox.misc.logging_util.RuntimeStats.failed -:type: int -:value: > - 0 - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.failed -``` - -```` - -````{py:attribute} parse_start_ts -:canonical: archivebox.misc.logging_util.RuntimeStats.parse_start_ts -:type: typing.Optional[datetime.datetime] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.parse_start_ts -``` - -```` - -````{py:attribute} parse_end_ts -:canonical: archivebox.misc.logging_util.RuntimeStats.parse_end_ts -:type: typing.Optional[datetime.datetime] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.parse_end_ts -``` - -```` - -````{py:attribute} index_start_ts -:canonical: archivebox.misc.logging_util.RuntimeStats.index_start_ts -:type: typing.Optional[datetime.datetime] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.index_start_ts -``` - -```` - -````{py:attribute} index_end_ts -:canonical: archivebox.misc.logging_util.RuntimeStats.index_end_ts -:type: typing.Optional[datetime.datetime] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.index_end_ts -``` - -```` - -````{py:attribute} archiving_start_ts -:canonical: archivebox.misc.logging_util.RuntimeStats.archiving_start_ts -:type: typing.Optional[datetime.datetime] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.archiving_start_ts -``` - -```` - -````{py:attribute} archiving_end_ts -:canonical: archivebox.misc.logging_util.RuntimeStats.archiving_end_ts -:type: typing.Optional[datetime.datetime] -:value: > - None - -```{autodoc2-docstring} archivebox.misc.logging_util.RuntimeStats.archiving_end_ts -``` - -```` - -````` - -````{py:data} _LAST_RUN_STATS -:canonical: archivebox.misc.logging_util._LAST_RUN_STATS -:value: > - 'RuntimeStats(...)' - -```{autodoc2-docstring} archivebox.misc.logging_util._LAST_RUN_STATS -``` - -```` - -`````{py:class} TimedProgress(seconds, prefix='') +`````{py:class} TimedProgress(seconds, prefix='', config=None, **config_kwargs) :canonical: archivebox.misc.logging_util.TimedProgress ```{autodoc2-docstring} archivebox.misc.logging_util.TimedProgress @@ -314,133 +113,21 @@ ````` -````{py:function} progress_bar(seconds: int, prefix: str = '', ANSI: typing.Dict[str, str] = ANSI) -> None +````{py:function} progress_bar(seconds: int, prefix: str = '', ANSI: dict[str, str] = ANSI, config=None, **config_kwargs) -> None :canonical: archivebox.misc.logging_util.progress_bar ```{autodoc2-docstring} archivebox.misc.logging_util.progress_bar ``` ```` -````{py:function} log_cli_command(subcommand: str, subcommand_args: typing.Iterable[str] = (), stdin: str | typing.IO | None = None, pwd: str = '.') +````{py:function} log_cli_command(subcommand: str, subcommand_args: collections.abc.Iterable[str] = (), stdin: str | typing.IO | None = None, pwd: str = '.') :canonical: archivebox.misc.logging_util.log_cli_command ```{autodoc2-docstring} archivebox.misc.logging_util.log_cli_command ``` ```` -````{py:function} log_importing_started(urls: typing.Union[str, typing.List[str]], depth: int, index_only: bool) -:canonical: archivebox.misc.logging_util.log_importing_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_importing_started -``` -```` - -````{py:function} log_source_saved(source_file: str) -:canonical: archivebox.misc.logging_util.log_source_saved - -```{autodoc2-docstring} archivebox.misc.logging_util.log_source_saved -``` -```` - -````{py:function} log_parsing_finished(num_parsed: int, parser_name: str) -:canonical: archivebox.misc.logging_util.log_parsing_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_parsing_finished -``` -```` - -````{py:function} log_deduping_finished(num_new_links: int) -:canonical: archivebox.misc.logging_util.log_deduping_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_deduping_finished -``` -```` - -````{py:function} log_crawl_started(new_links) -:canonical: archivebox.misc.logging_util.log_crawl_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_crawl_started -``` -```` - -````{py:function} log_indexing_process_started(num_links: int) -:canonical: archivebox.misc.logging_util.log_indexing_process_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_process_started -``` -```` - -````{py:function} log_indexing_process_finished() -:canonical: archivebox.misc.logging_util.log_indexing_process_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_process_finished -``` -```` - -````{py:function} log_indexing_started(out_path: str) -:canonical: archivebox.misc.logging_util.log_indexing_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_started -``` -```` - -````{py:function} log_indexing_finished(out_path: str) -:canonical: archivebox.misc.logging_util.log_indexing_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_indexing_finished -``` -```` - -````{py:function} log_archiving_started(num_links: int, resume: typing.Optional[float] = None) -:canonical: archivebox.misc.logging_util.log_archiving_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_archiving_started -``` -```` - -````{py:function} log_archiving_paused(num_links: int, idx: int, timestamp: str) -:canonical: archivebox.misc.logging_util.log_archiving_paused - -```{autodoc2-docstring} archivebox.misc.logging_util.log_archiving_paused -``` -```` - -````{py:function} log_archiving_finished(num_links: int) -:canonical: archivebox.misc.logging_util.log_archiving_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_archiving_finished -``` -```` - -````{py:function} log_snapshot_archiving_started(snapshot: archivebox.core.models.Snapshot, out_dir: str, is_new: bool) -:canonical: archivebox.misc.logging_util.log_snapshot_archiving_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_snapshot_archiving_started -``` -```` - -````{py:function} log_snapshot_archiving_finished(snapshot: archivebox.core.models.Snapshot, out_dir: str, is_new: bool, stats: dict, start_ts: datetime.datetime) -:canonical: archivebox.misc.logging_util.log_snapshot_archiving_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_snapshot_archiving_finished -``` -```` - -````{py:function} log_archive_method_started(method: str) -:canonical: archivebox.misc.logging_util.log_archive_method_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_archive_method_started -``` -```` - -````{py:function} log_archive_method_finished(result: dict) -:canonical: archivebox.misc.logging_util.log_archive_method_finished - -```{autodoc2-docstring} archivebox.misc.logging_util.log_archive_method_finished -``` -```` - -````{py:function} log_list_started(filter_patterns: typing.Optional[typing.List[str]], filter_type: str) +````{py:function} log_list_started(filter_patterns: list[str] | None, filter_type: str) :canonical: archivebox.misc.logging_util.log_list_started ```{autodoc2-docstring} archivebox.misc.logging_util.log_list_started @@ -454,7 +141,7 @@ ``` ```` -````{py:function} log_removal_started(snapshots, yes: bool, delete: bool) +````{py:function} log_removal_started(snapshots, yes: bool) :canonical: archivebox.misc.logging_util.log_removal_started ```{autodoc2-docstring} archivebox.misc.logging_util.log_removal_started @@ -468,21 +155,14 @@ ``` ```` -````{py:function} log_index_started(url: str) -:canonical: archivebox.misc.logging_util.log_index_started - -```{autodoc2-docstring} archivebox.misc.logging_util.log_index_started -``` -```` - -````{py:function} pretty_path(path: typing.Union[pathlib.Path, str], pwd: typing.Union[pathlib.Path, str] = DATA_DIR, color: bool = True) -> str +````{py:function} pretty_path(path: pathlib.Path | str, pwd: pathlib.Path | str = DATA_DIR, color: bool = True) -> str :canonical: archivebox.misc.logging_util.pretty_path ```{autodoc2-docstring} archivebox.misc.logging_util.pretty_path ``` ```` -````{py:function} printable_filesize(num_bytes: typing.Union[int, float]) -> str +````{py:function} printable_filesize(num_bytes: int | float) -> str :canonical: archivebox.misc.logging_util.printable_filesize ```{autodoc2-docstring} archivebox.misc.logging_util.printable_filesize @@ -503,14 +183,14 @@ ``` ```` -````{py:function} log_worker_event(worker_type: str, event: str, indent_level: int = 0, pid: typing.Optional[int] = None, worker_id: typing.Optional[str] = None, url: typing.Optional[str] = None, plugin: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, typing.Any]] = None, error: typing.Optional[Exception] = None) -> None +````{py:function} log_worker_event(worker_type: str, event: str, indent_level: int = 0, pid: int | None = None, worker_id: str | None = None, url: str | None = None, plugin: str | None = None, metadata: dict[str, typing.Any] | None = None, error: Exception | None = None) -> None :canonical: archivebox.misc.logging_util.log_worker_event ```{autodoc2-docstring} archivebox.misc.logging_util.log_worker_event ``` ```` -````{py:function} printable_folders(folders: typing.Dict[str, typing.Optional[archivebox.core.models.Snapshot]], with_headers: bool = False) -> str +````{py:function} printable_folders(folders: dict[str, typing.Optional[archivebox.core.models.Snapshot]], with_headers: bool = False) -> str :canonical: archivebox.misc.logging_util.printable_folders ```{autodoc2-docstring} archivebox.misc.logging_util.printable_folders @@ -524,14 +204,14 @@ ``` ```` -````{py:function} printable_folder_status(name: str, folder: typing.Dict) -> str +````{py:function} printable_folder_status(name: str, folder: dict) -> str :canonical: archivebox.misc.logging_util.printable_folder_status ```{autodoc2-docstring} archivebox.misc.logging_util.printable_folder_status ``` ```` -````{py:function} printable_dependency_version(name: str, dependency: typing.Dict) -> str +````{py:function} printable_dependency_version(name: str, dependency: dict) -> str :canonical: archivebox.misc.logging_util.printable_dependency_version ```{autodoc2-docstring} archivebox.misc.logging_util.printable_dependency_version diff --git a/docs/apidocs/archivebox/archivebox.misc.md b/docs/apidocs/archivebox/archivebox.misc.md index 1107d94e..f199edb6 100644 --- a/docs/apidocs/archivebox/archivebox.misc.md +++ b/docs/apidocs/archivebox/archivebox.misc.md @@ -16,18 +16,14 @@ archivebox.misc.logging archivebox.misc.db archivebox.misc.system -archivebox.misc.legacy archivebox.misc.shell_welcome_message archivebox.misc.util archivebox.misc.checks archivebox.misc.paginators -archivebox.misc.debugging archivebox.misc.monkey_patches archivebox.misc.logging_util archivebox.misc.jsonl -archivebox.misc.folders archivebox.misc.hashing archivebox.misc.toml_util archivebox.misc.serve_static -archivebox.misc.progress_layout ``` diff --git a/docs/apidocs/archivebox/archivebox.misc.monkey_patches.md b/docs/apidocs/archivebox/archivebox.misc.monkey_patches.md index fe0d985d..f57ce7cb 100644 --- a/docs/apidocs/archivebox/archivebox.misc.monkey_patches.md +++ b/docs/apidocs/archivebox/archivebox.misc.monkey_patches.md @@ -21,8 +21,30 @@ ``` ```` +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`SENSITIVE_QUERY_PARAM_RE ` + - ```{autodoc2-docstring} archivebox.misc.monkey_patches.SENSITIVE_QUERY_PARAM_RE + :summary: + ``` +```` + ### API +````{py:data} SENSITIVE_QUERY_PARAM_RE +:canonical: archivebox.misc.monkey_patches.SENSITIVE_QUERY_PARAM_RE +:value: > + 'compile(...)' + +```{autodoc2-docstring} archivebox.misc.monkey_patches.SENSITIVE_QUERY_PARAM_RE +``` + +```` + `````{py:class} ModifiedAccessLogGenerator(stream) :canonical: archivebox.misc.monkey_patches.ModifiedAccessLogGenerator @@ -37,7 +59,12 @@ Bases: {py:obj}`daphne.access.AccessLogGenerator` ```{autodoc2-docstring} archivebox.misc.monkey_patches.ModifiedAccessLogGenerator.__init__ ``` -````{py:method} write_entry(host, date, request, status=None, length=None, ident=None, user=None) +````{py:method} __call__(protocol, action, details) +:canonical: archivebox.misc.monkey_patches.ModifiedAccessLogGenerator.__call__ + +```` + +````{py:method} write_entry(host, date, request, status=None, length=None, ident=None, user=None, time_taken=None) :canonical: archivebox.misc.monkey_patches.ModifiedAccessLogGenerator.write_entry ```` diff --git a/docs/apidocs/archivebox/archivebox.misc.paginators.md b/docs/apidocs/archivebox/archivebox.misc.paginators.md index dc0855c5..5c44b9b8 100644 --- a/docs/apidocs/archivebox/archivebox.misc.paginators.md +++ b/docs/apidocs/archivebox/archivebox.misc.paginators.md @@ -15,30 +15,100 @@ :class: autosummary longtable :align: left -* - {py:obj}`AccelleratedPaginator ` - - ```{autodoc2-docstring} archivebox.misc.paginators.AccelleratedPaginator +* - {py:obj}`CountlessPage ` + - +* - {py:obj}`CountlessPaginator ` + - ```{autodoc2-docstring} archivebox.misc.paginators.CountlessPaginator + :summary: + ``` +* - {py:obj}`AcceleratedPaginator ` + - ```{autodoc2-docstring} archivebox.misc.paginators.AcceleratedPaginator :summary: ``` ```` ### API -`````{py:class} AccelleratedPaginator(object_list, per_page, orphans=0, allow_empty_first_page=True, error_messages=None) -:canonical: archivebox.misc.paginators.AccelleratedPaginator +`````{py:class} CountlessPage(object_list, number, paginator, has_next_page=False) +:canonical: archivebox.misc.paginators.CountlessPage + +Bases: {py:obj}`django.core.paginator.Page` + +````{py:method} has_next() +:canonical: archivebox.misc.paginators.CountlessPage.has_next + +```{autodoc2-docstring} archivebox.misc.paginators.CountlessPage.has_next +``` + +```` + +````` + +`````{py:class} CountlessPaginator(object_list, per_page, orphans=0, allow_empty_first_page=True, error_messages=None) +:canonical: archivebox.misc.paginators.CountlessPaginator Bases: {py:obj}`django.core.paginator.Paginator` -```{autodoc2-docstring} archivebox.misc.paginators.AccelleratedPaginator +```{autodoc2-docstring} archivebox.misc.paginators.CountlessPaginator ``` ```{rubric} Initialization ``` -```{autodoc2-docstring} archivebox.misc.paginators.AccelleratedPaginator.__init__ +```{autodoc2-docstring} archivebox.misc.paginators.CountlessPaginator.__init__ ``` +````{py:attribute} has_exact_count +:canonical: archivebox.misc.paginators.CountlessPaginator.has_exact_count +:value: > + False + +```{autodoc2-docstring} archivebox.misc.paginators.CountlessPaginator.has_exact_count +``` + +```` + ````{py:method} count() -:canonical: archivebox.misc.paginators.AccelleratedPaginator.count +:canonical: archivebox.misc.paginators.CountlessPaginator.count + +```` + +````{py:method} num_pages() +:canonical: archivebox.misc.paginators.CountlessPaginator.num_pages + +```` + +````{py:method} validate_number(number) +:canonical: archivebox.misc.paginators.CountlessPaginator.validate_number + +```{autodoc2-docstring} archivebox.misc.paginators.CountlessPaginator.validate_number +``` + +```` + +````{py:method} page(number) +:canonical: archivebox.misc.paginators.CountlessPaginator.page + +```` + +````` + +`````{py:class} AcceleratedPaginator(object_list, per_page, orphans=0, allow_empty_first_page=True, error_messages=None) +:canonical: archivebox.misc.paginators.AcceleratedPaginator + +Bases: {py:obj}`django.core.paginator.Paginator` + +```{autodoc2-docstring} archivebox.misc.paginators.AcceleratedPaginator +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.misc.paginators.AcceleratedPaginator.__init__ +``` + +````{py:method} count() +:canonical: archivebox.misc.paginators.AcceleratedPaginator.count ```` diff --git a/docs/apidocs/archivebox/archivebox.misc.progress_layout.md b/docs/apidocs/archivebox/archivebox.misc.progress_layout.md deleted file mode 100644 index 6df20cbd..00000000 --- a/docs/apidocs/archivebox/archivebox.misc.progress_layout.md +++ /dev/null @@ -1,455 +0,0 @@ -# {py:mod}`archivebox.misc.progress_layout` - -```{py:module} archivebox.misc.progress_layout -``` - -```{autodoc2-docstring} archivebox.misc.progress_layout -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`CrawlQueuePanel ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueuePanel - :summary: - ``` -* - {py:obj}`ProcessLogPanel ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel - :summary: - ``` -* - {py:obj}`WorkerLogPanel ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout.WorkerLogPanel - :summary: - ``` -* - {py:obj}`CrawlQueueTreePanel ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel - :summary: - ``` -* - {py:obj}`ArchiveBoxProgressLayout ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout - :summary: - ``` -```` - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_strip_rich ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout._strip_rich - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_RICH_TAG_RE ` - - ```{autodoc2-docstring} archivebox.misc.progress_layout._RICH_TAG_RE - :summary: - ``` -```` - -### API - -````{py:data} _RICH_TAG_RE -:canonical: archivebox.misc.progress_layout._RICH_TAG_RE -:value: > - 'compile(...)' - -```{autodoc2-docstring} archivebox.misc.progress_layout._RICH_TAG_RE -``` - -```` - -````{py:function} _strip_rich(text: str) -> str -:canonical: archivebox.misc.progress_layout._strip_rich - -```{autodoc2-docstring} archivebox.misc.progress_layout._strip_rich -``` -```` - -`````{py:class} CrawlQueuePanel() -:canonical: archivebox.misc.progress_layout.CrawlQueuePanel - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueuePanel -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueuePanel.__init__ -``` - -````{py:method} __rich__() -> rich.panel.Panel -:canonical: archivebox.misc.progress_layout.CrawlQueuePanel.__rich__ - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueuePanel.__rich__ -``` - -```` - -````` - -`````{py:class} ProcessLogPanel(process: typing.Any, max_lines: int = 8, compact: bool | None = None, bg_terminating: bool = False) -:canonical: archivebox.misc.progress_layout.ProcessLogPanel - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel.__init__ -``` - -````{py:method} __rich__() -> rich.panel.Panel -:canonical: archivebox.misc.progress_layout.ProcessLogPanel.__rich__ - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel.__rich__ -``` - -```` - -````{py:method} plain_lines() -> list[str] -:canonical: archivebox.misc.progress_layout.ProcessLogPanel.plain_lines - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel.plain_lines -``` - -```` - -````{py:method} _title() -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._title - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._title -``` - -```` - -````{py:method} _is_background_hook() -> bool -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._is_background_hook - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._is_background_hook -``` - -```` - -````{py:method} _is_pending() -> bool -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._is_pending - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._is_pending -``` - -```` - -````{py:method} _completed_ok() -> bool -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._completed_ok - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._completed_ok -``` - -```` - -````{py:method} _completed_output_line() -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._completed_output_line - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._completed_output_line -``` - -```` - -````{py:method} _has_output_files() -> bool -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._has_output_files - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._has_output_files -``` - -```` - -````{py:method} _border_style(is_pending: bool) -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._border_style - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._border_style -``` - -```` - -````{py:method} _worker_label(worker_type: str) -> tuple[str, str] -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._worker_label - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._worker_label -``` - -```` - -````{py:method} _extract_arg(cmd: list[str], key: str) -> str | None -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._extract_arg -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._extract_arg -``` - -```` - -````{py:method} _abbrev_urls(urls: list[str], max_len: int = 48) -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._abbrev_urls - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._abbrev_urls -``` - -```` - -````{py:method} _extract_url() -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._extract_url - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._extract_url -``` - -```` - -````{py:method} _abbrev_url(url: str, max_len: int = 48) -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._abbrev_url - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._abbrev_url -``` - -```` - -````{py:method} _chrome_launch_line(stderr_lines: list[str], stdout_lines: list[str]) -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._chrome_launch_line - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._chrome_launch_line -``` - -```` - -````{py:method} _elapsed_suffix() -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._elapsed_suffix - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._elapsed_suffix -``` - -```` - -````{py:method} _output_line() -> str -:canonical: archivebox.misc.progress_layout.ProcessLogPanel._output_line - -```{autodoc2-docstring} archivebox.misc.progress_layout.ProcessLogPanel._output_line -``` - -```` - -````` - -`````{py:class} WorkerLogPanel(title: str, empty_message: str, running_message: str, max_lines: int = 8) -:canonical: archivebox.misc.progress_layout.WorkerLogPanel - -```{autodoc2-docstring} archivebox.misc.progress_layout.WorkerLogPanel -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.progress_layout.WorkerLogPanel.__init__ -``` - -````{py:method} update_from_process(process: typing.Any) -:canonical: archivebox.misc.progress_layout.WorkerLogPanel.update_from_process - -```{autodoc2-docstring} archivebox.misc.progress_layout.WorkerLogPanel.update_from_process -``` - -```` - -````{py:method} __rich__() -> rich.panel.Panel -:canonical: archivebox.misc.progress_layout.WorkerLogPanel.__rich__ - -```{autodoc2-docstring} archivebox.misc.progress_layout.WorkerLogPanel.__rich__ -``` - -```` - -````` - -`````{py:class} CrawlQueueTreePanel(max_crawls: int = 8, max_snapshots: int = 16) -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel.__init__ -``` - -````{py:method} update_crawls(crawls: list[dict[str, typing.Any]]) -> None -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel.update_crawls - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel.update_crawls -``` - -```` - -````{py:method} __rich__() -> rich.panel.Panel -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel.__rich__ - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel.__rich__ -``` - -```` - -````{py:method} _status_icon(status: str) -> str -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._status_icon -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._status_icon -``` - -```` - -````{py:method} _hook_style(status: str, is_bg: bool = False, is_running: bool = False, is_pending: bool = False) -> tuple[str, str] -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._hook_style -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._hook_style -``` - -```` - -````{py:method} _hook_stats(size: str = '', elapsed: str = '', timeout: str = '', status: str = '') -> str -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._hook_stats -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._hook_stats -``` - -```` - -````{py:method} _terminal_width() -> int -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._terminal_width -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._terminal_width -``` - -```` - -````{py:method} _truncate_to_width(text: str, max_width: int) -> str -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._truncate_to_width -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._truncate_to_width -``` - -```` - -````{py:method} _truncate_tail(text: str, max_width: int) -> str -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._truncate_tail -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._truncate_tail -``` - -```` - -````{py:method} _available_width(left_text: str, indent: int = 0) -> int -:canonical: archivebox.misc.progress_layout.CrawlQueueTreePanel._available_width - -```{autodoc2-docstring} archivebox.misc.progress_layout.CrawlQueueTreePanel._available_width -``` - -```` - -````` - -`````{py:class} ArchiveBoxProgressLayout(crawl_id: typing.Optional[str] = None) -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.__init__ -``` - -````{py:method} _make_layout() -> rich.layout.Layout -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout._make_layout - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout._make_layout -``` - -```` - -````{py:method} update_orchestrator_status(status: str, crawl_queue_count: int = 0, crawl_workers_count: int = 0, binary_queue_count: int = 0, binary_workers_count: int = 0, max_crawl_workers: int = 8) -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout.update_orchestrator_status - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.update_orchestrator_status -``` - -```` - -````{py:method} update_process_panels(processes: typing.List[typing.Any], pending: typing.Optional[typing.List[typing.Any]] = None) -> None -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout.update_process_panels - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.update_process_panels -``` - -```` - -````{py:method} update_crawl_tree(crawls: list[dict[str, typing.Any]]) -> None -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout.update_crawl_tree - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.update_crawl_tree -``` - -```` - -````{py:method} log_event(message: str, style: str = 'white') -> None -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout.log_event - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.log_event -``` - -```` - -````{py:method} get_layout() -> rich.layout.Layout -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout.get_layout - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.get_layout -``` - -```` - -````{py:method} plain_lines() -> list[tuple[str, str]] -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout.plain_lines - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout.plain_lines -``` - -```` - -````{py:method} _has_log_lines(process: typing.Any) -> bool -:canonical: archivebox.misc.progress_layout.ArchiveBoxProgressLayout._has_log_lines -:staticmethod: - -```{autodoc2-docstring} archivebox.misc.progress_layout.ArchiveBoxProgressLayout._has_log_lines -``` - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.misc.serve_static.md b/docs/apidocs/archivebox/archivebox.misc.serve_static.md index cc1a82d8..e82e8267 100644 --- a/docs/apidocs/archivebox/archivebox.misc.serve_static.md +++ b/docs/apidocs/archivebox/archivebox.misc.serve_static.md @@ -15,6 +15,10 @@ :class: autosummary longtable :align: left +* - {py:obj}`_StreamingQueueWriter ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter + :summary: + ``` * - {py:obj}`RangedFileReader ` - ```{autodoc2-docstring} archivebox.misc.serve_static.RangedFileReader :summary: @@ -35,10 +39,42 @@ - ```{autodoc2-docstring} archivebox.misc.serve_static._hash_for_path :summary: ``` +* - {py:obj}`_resolve_archive_path ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._resolve_archive_path + :summary: + ``` * - {py:obj}`_cache_policy ` - ```{autodoc2-docstring} archivebox.misc.serve_static._cache_policy :summary: ``` +* - {py:obj}`_render_mhtml_preview_document ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._render_mhtml_preview_document + :summary: + ``` +* - {py:obj}`_format_direntry_timestamp ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._format_direntry_timestamp + :summary: + ``` +* - {py:obj}`_safe_zip_stem ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._safe_zip_stem + :summary: + ``` +* - {py:obj}`_iter_visible_files ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._iter_visible_files + :summary: + ``` +* - {py:obj}`_build_directory_zip_response ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._build_directory_zip_response + :summary: + ``` +* - {py:obj}`_stream_ranged_file_async ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._stream_ranged_file_async + :summary: + ``` +* - {py:obj}`_render_directory_index ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._render_directory_index + :summary: + ``` * - {py:obj}`_extract_markdown_candidate ` - ```{autodoc2-docstring} archivebox.misc.serve_static._extract_markdown_candidate :summary: @@ -47,6 +83,14 @@ - ```{autodoc2-docstring} archivebox.misc.serve_static._looks_like_markdown :summary: ``` +* - {py:obj}`_render_text_preview_document ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._render_text_preview_document + :summary: + ``` +* - {py:obj}`_render_image_preview_document ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._render_image_preview_document + :summary: + ``` * - {py:obj}`_render_markdown_fallback ` - ```{autodoc2-docstring} archivebox.misc.serve_static._render_markdown_fallback :summary: @@ -55,6 +99,18 @@ - ```{autodoc2-docstring} archivebox.misc.serve_static._render_markdown_document :summary: ``` +* - {py:obj}`_content_type_base ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._content_type_base + :summary: + ``` +* - {py:obj}`_is_risky_replay_document ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._is_risky_replay_document + :summary: + ``` +* - {py:obj}`_apply_archive_replay_headers ` + - ```{autodoc2-docstring} archivebox.misc.serve_static._apply_archive_replay_headers + :summary: + ``` * - {py:obj}`serve_static_with_byterange_support ` - ```{autodoc2-docstring} archivebox.misc.serve_static.serve_static_with_byterange_support :summary: @@ -103,6 +159,18 @@ - ```{autodoc2-docstring} archivebox.misc.serve_static.HTML_BODY_RE :summary: ``` +* - {py:obj}`RISKY_REPLAY_MIMETYPES ` + - ```{autodoc2-docstring} archivebox.misc.serve_static.RISKY_REPLAY_MIMETYPES + :summary: + ``` +* - {py:obj}`RISKY_REPLAY_EXTENSIONS ` + - ```{autodoc2-docstring} archivebox.misc.serve_static.RISKY_REPLAY_EXTENSIONS + :summary: + ``` +* - {py:obj}`RISKY_REPLAY_MARKERS ` + - ```{autodoc2-docstring} archivebox.misc.serve_static.RISKY_REPLAY_MARKERS + :summary: + ``` ```` ### API @@ -132,13 +200,132 @@ ``` ```` -````{py:function} _cache_policy() -> str +````{py:function} _resolve_archive_path(document_root: str | pathlib.Path, rel_path: str) -> tuple[pathlib.Path, str] +:canonical: archivebox.misc.serve_static._resolve_archive_path + +```{autodoc2-docstring} archivebox.misc.serve_static._resolve_archive_path +``` +```` + +````{py:function} _cache_policy(config=None, **config_kwargs) -> str :canonical: archivebox.misc.serve_static._cache_policy ```{autodoc2-docstring} archivebox.misc.serve_static._cache_policy ``` ```` +````{py:function} _render_mhtml_preview_document(filename: str, output_path: str) -> str +:canonical: archivebox.misc.serve_static._render_mhtml_preview_document + +```{autodoc2-docstring} archivebox.misc.serve_static._render_mhtml_preview_document +``` +```` + +````{py:function} _format_direntry_timestamp(stat_result: os.stat_result) -> str +:canonical: archivebox.misc.serve_static._format_direntry_timestamp + +```{autodoc2-docstring} archivebox.misc.serve_static._format_direntry_timestamp +``` +```` + +````{py:function} _safe_zip_stem(name: str) -> str +:canonical: archivebox.misc.serve_static._safe_zip_stem + +```{autodoc2-docstring} archivebox.misc.serve_static._safe_zip_stem +``` +```` + +`````{py:class} _StreamingQueueWriter(output_queue: queue.Queue[bytes | BaseException | object]) +:canonical: archivebox.misc.serve_static._StreamingQueueWriter + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.__init__ +``` + +````{py:method} write(data: bytes) -> int +:canonical: archivebox.misc.serve_static._StreamingQueueWriter.write + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.write +``` + +```` + +````{py:method} tell() -> int +:canonical: archivebox.misc.serve_static._StreamingQueueWriter.tell + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.tell +``` + +```` + +````{py:method} flush() -> None +:canonical: archivebox.misc.serve_static._StreamingQueueWriter.flush + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.flush +``` + +```` + +````{py:method} close() -> None +:canonical: archivebox.misc.serve_static._StreamingQueueWriter.close + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.close +``` + +```` + +````{py:method} writable() -> bool +:canonical: archivebox.misc.serve_static._StreamingQueueWriter.writable + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.writable +``` + +```` + +````{py:method} seekable() -> bool +:canonical: archivebox.misc.serve_static._StreamingQueueWriter.seekable + +```{autodoc2-docstring} archivebox.misc.serve_static._StreamingQueueWriter.seekable +``` + +```` + +````` + +````{py:function} _iter_visible_files(root: pathlib.Path) +:canonical: archivebox.misc.serve_static._iter_visible_files + +```{autodoc2-docstring} archivebox.misc.serve_static._iter_visible_files +``` +```` + +````{py:function} _build_directory_zip_response(fullpath: pathlib.Path, path: str, *, is_archive_replay: bool, use_async_stream: bool, config=None) -> django.http.StreamingHttpResponse +:canonical: archivebox.misc.serve_static._build_directory_zip_response + +```{autodoc2-docstring} archivebox.misc.serve_static._build_directory_zip_response +``` +```` + +````{py:function} _stream_ranged_file_async(ranged_file: RangedFileReader) +:canonical: archivebox.misc.serve_static._stream_ranged_file_async +:async: + +```{autodoc2-docstring} archivebox.misc.serve_static._stream_ranged_file_async +``` +```` + +````{py:function} _render_directory_index(request, path: str, fullpath: pathlib.Path) -> django.http.HttpResponse +:canonical: archivebox.misc.serve_static._render_directory_index + +```{autodoc2-docstring} archivebox.misc.serve_static._render_directory_index +``` +```` + ````{py:data} MARKDOWN_INLINE_LINK_RE :canonical: archivebox.misc.serve_static.MARKDOWN_INLINE_LINK_RE :value: > @@ -199,6 +386,36 @@ ```` +````{py:data} RISKY_REPLAY_MIMETYPES +:canonical: archivebox.misc.serve_static.RISKY_REPLAY_MIMETYPES +:value: > + None + +```{autodoc2-docstring} archivebox.misc.serve_static.RISKY_REPLAY_MIMETYPES +``` + +```` + +````{py:data} RISKY_REPLAY_EXTENSIONS +:canonical: archivebox.misc.serve_static.RISKY_REPLAY_EXTENSIONS +:value: > + None + +```{autodoc2-docstring} archivebox.misc.serve_static.RISKY_REPLAY_EXTENSIONS +``` + +```` + +````{py:data} RISKY_REPLAY_MARKERS +:canonical: archivebox.misc.serve_static.RISKY_REPLAY_MARKERS +:value: > + (' str :canonical: archivebox.misc.serve_static._extract_markdown_candidate @@ -213,6 +430,20 @@ ``` ```` +````{py:function} _render_text_preview_document(text: str, title: str) -> str +:canonical: archivebox.misc.serve_static._render_text_preview_document + +```{autodoc2-docstring} archivebox.misc.serve_static._render_text_preview_document +``` +```` + +````{py:function} _render_image_preview_document(image_url: str, title: str) -> str +:canonical: archivebox.misc.serve_static._render_image_preview_document + +```{autodoc2-docstring} archivebox.misc.serve_static._render_image_preview_document +``` +```` + ````{py:function} _render_markdown_fallback(text: str) -> str :canonical: archivebox.misc.serve_static._render_markdown_fallback @@ -227,7 +458,28 @@ ``` ```` -````{py:function} serve_static_with_byterange_support(request, path, document_root=None, show_indexes=False) +````{py:function} _content_type_base(content_type: str) -> str +:canonical: archivebox.misc.serve_static._content_type_base + +```{autodoc2-docstring} archivebox.misc.serve_static._content_type_base +``` +```` + +````{py:function} _is_risky_replay_document(fullpath: pathlib.Path, content_type: str) -> bool +:canonical: archivebox.misc.serve_static._is_risky_replay_document + +```{autodoc2-docstring} archivebox.misc.serve_static._is_risky_replay_document +``` +```` + +````{py:function} _apply_archive_replay_headers(response: django.http.HttpResponse, *, fullpath: pathlib.Path, content_type: str, is_archive_replay: bool, config=None, **config_kwargs) -> django.http.HttpResponse +:canonical: archivebox.misc.serve_static._apply_archive_replay_headers + +```{autodoc2-docstring} archivebox.misc.serve_static._apply_archive_replay_headers +``` +```` + +````{py:function} serve_static_with_byterange_support(request, path, document_root=None, show_indexes=False, is_archive_replay: bool = False) :canonical: archivebox.misc.serve_static.serve_static_with_byterange_support ```{autodoc2-docstring} archivebox.misc.serve_static.serve_static_with_byterange_support @@ -278,4 +530,12 @@ ```` +````{py:method} close() +:canonical: archivebox.misc.serve_static.RangedFileReader.close + +```{autodoc2-docstring} archivebox.misc.serve_static.RangedFileReader.close +``` + +```` + ````` diff --git a/docs/apidocs/archivebox/archivebox.misc.shell_welcome_message.md b/docs/apidocs/archivebox/archivebox.misc.shell_welcome_message.md index 5e16f46c..e17b8f70 100644 --- a/docs/apidocs/archivebox/archivebox.misc.shell_welcome_message.md +++ b/docs/apidocs/archivebox/archivebox.misc.shell_welcome_message.md @@ -6,29 +6,3 @@ ```{autodoc2-docstring} archivebox.misc.shell_welcome_message :allowtitles: ``` - -## Module Contents - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`CONFIG ` - - ```{autodoc2-docstring} archivebox.misc.shell_welcome_message.CONFIG - :summary: - ``` -```` - -### API - -````{py:data} CONFIG -:canonical: archivebox.misc.shell_welcome_message.CONFIG -:value: > - 'get_config(...)' - -```{autodoc2-docstring} archivebox.misc.shell_welcome_message.CONFIG -``` - -```` diff --git a/docs/apidocs/archivebox/archivebox.misc.system.md b/docs/apidocs/archivebox/archivebox.misc.system.md index 77a9f2fc..b8da53ad 100644 --- a/docs/apidocs/archivebox/archivebox.misc.system.md +++ b/docs/apidocs/archivebox/archivebox.misc.system.md @@ -9,40 +9,16 @@ ## Module Contents -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`suppress_output ` - - ```{autodoc2-docstring} archivebox.misc.system.suppress_output - :summary: - ``` -```` - ### Functions ````{list-table} :class: autosummary longtable :align: left -* - {py:obj}`run ` - - ```{autodoc2-docstring} archivebox.misc.system.run - :summary: - ``` * - {py:obj}`atomic_write ` - ```{autodoc2-docstring} archivebox.misc.system.atomic_write :summary: ``` -* - {py:obj}`chmod_file ` - - ```{autodoc2-docstring} archivebox.misc.system.chmod_file - :summary: - ``` -* - {py:obj}`copy_and_overwrite ` - - ```{autodoc2-docstring} archivebox.misc.system.copy_and_overwrite - :summary: - ``` * - {py:obj}`get_dir_size ` - ```{autodoc2-docstring} archivebox.misc.system.get_dir_size :summary: @@ -51,69 +27,16 @@ ### API -````{py:function} run(cmd, *args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs) -:canonical: archivebox.misc.system.run - -```{autodoc2-docstring} archivebox.misc.system.run -``` -```` - -````{py:function} atomic_write(path: typing.Union[pathlib.Path, str], contents: typing.Union[dict, str, bytes], overwrite: bool = True) -> None +````{py:function} atomic_write(path: pathlib.Path | str, contents: dict | str | bytes, overwrite: bool = True, config=None, **config_kwargs) -> None :canonical: archivebox.misc.system.atomic_write ```{autodoc2-docstring} archivebox.misc.system.atomic_write ``` ```` -````{py:function} chmod_file(path: str, cwd: str = '') -> None -:canonical: archivebox.misc.system.chmod_file - -```{autodoc2-docstring} archivebox.misc.system.chmod_file -``` -```` - -````{py:function} copy_and_overwrite(from_path: typing.Union[str, pathlib.Path], to_path: typing.Union[str, pathlib.Path]) -:canonical: archivebox.misc.system.copy_and_overwrite - -```{autodoc2-docstring} archivebox.misc.system.copy_and_overwrite -``` -```` - -````{py:function} get_dir_size(path: typing.Union[str, pathlib.Path], recursive: bool = True, pattern: typing.Optional[str] = None) -> typing.Tuple[int, int, int] +````{py:function} get_dir_size(path: str | pathlib.Path, recursive: bool = True, pattern: str | None = None) -> tuple[int, int, int] :canonical: archivebox.misc.system.get_dir_size ```{autodoc2-docstring} archivebox.misc.system.get_dir_size ``` ```` - -`````{py:class} suppress_output(stdout=True, stderr=True) -:canonical: archivebox.misc.system.suppress_output - -Bases: {py:obj}`object` - -```{autodoc2-docstring} archivebox.misc.system.suppress_output -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.misc.system.suppress_output.__init__ -``` - -````{py:method} __enter__() -:canonical: archivebox.misc.system.suppress_output.__enter__ - -```{autodoc2-docstring} archivebox.misc.system.suppress_output.__enter__ -``` - -```` - -````{py:method} __exit__(*_) -:canonical: archivebox.misc.system.suppress_output.__exit__ - -```{autodoc2-docstring} archivebox.misc.system.suppress_output.__exit__ -``` - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.misc.toml_util.md b/docs/apidocs/archivebox/archivebox.misc.toml_util.md index 484c788f..1575c139 100644 --- a/docs/apidocs/archivebox/archivebox.misc.toml_util.md +++ b/docs/apidocs/archivebox/archivebox.misc.toml_util.md @@ -110,7 +110,7 @@ Bases: {py:obj}`pydantic.json_schema.GenerateJsonSchema` ```{autodoc2-docstring} archivebox.misc.toml_util.JSONSchemaWithLambdas.__init__ ``` -````{py:method} encode_default(default: typing.Any) -> typing.Any +````{py:method} encode_default(dft: typing.Any) -> typing.Any :canonical: archivebox.misc.toml_util.JSONSchemaWithLambdas.encode_default ```` diff --git a/docs/apidocs/archivebox/archivebox.misc.util.md b/docs/apidocs/archivebox/archivebox.misc.util.md index 0039441c..9bd8ff62 100644 --- a/docs/apidocs/archivebox/archivebox.misc.util.md +++ b/docs/apidocs/archivebox/archivebox.misc.util.md @@ -27,6 +27,26 @@ :class: autosummary longtable :align: left +* - {py:obj}`filter_queryset_by_uuid_substring ` + - ```{autodoc2-docstring} archivebox.misc.util.filter_queryset_by_uuid_substring + :summary: + ``` +* - {py:obj}`ts_to_date_str ` + - ```{autodoc2-docstring} archivebox.misc.util.ts_to_date_str + :summary: + ``` +* - {py:obj}`sanitize_extracted_url ` + - ```{autodoc2-docstring} archivebox.misc.util.sanitize_extracted_url + :summary: + ``` +* - {py:obj}`validate_url_length ` + - ```{autodoc2-docstring} archivebox.misc.util.validate_url_length + :summary: + ``` +* - {py:obj}`validate_url ` + - ```{autodoc2-docstring} archivebox.misc.util.validate_url + :summary: + ``` * - {py:obj}`parens_are_matched ` - ```{autodoc2-docstring} archivebox.misc.util.parens_are_matched :summary: @@ -35,12 +55,16 @@ - ```{autodoc2-docstring} archivebox.misc.util.fix_url_from_markdown :summary: ``` +* - {py:obj}`split_comma_separated_urls ` + - ```{autodoc2-docstring} archivebox.misc.util.split_comma_separated_urls + :summary: + ``` * - {py:obj}`find_all_urls ` - ```{autodoc2-docstring} archivebox.misc.util.find_all_urls :summary: ``` -* - {py:obj}`is_static_file ` - - ```{autodoc2-docstring} archivebox.misc.util.is_static_file +* - {py:obj}`parse_filesize_to_bytes ` + - ```{autodoc2-docstring} archivebox.misc.util.parse_filesize_to_bytes :summary: ``` * - {py:obj}`enforce_types ` @@ -51,10 +75,6 @@ - ```{autodoc2-docstring} archivebox.misc.util.docstring :summary: ``` -* - {py:obj}`str_between ` - - ```{autodoc2-docstring} archivebox.misc.util.str_between - :summary: - ``` * - {py:obj}`parse_date ` - ```{autodoc2-docstring} archivebox.misc.util.parse_date :summary: @@ -63,26 +83,14 @@ - ```{autodoc2-docstring} archivebox.misc.util.download_url :summary: ``` -* - {py:obj}`get_headers ` - - ```{autodoc2-docstring} archivebox.misc.util.get_headers - :summary: - ``` * - {py:obj}`ansi_to_html ` - ```{autodoc2-docstring} archivebox.misc.util.ansi_to_html :summary: ``` -* - {py:obj}`dedupe ` - - ```{autodoc2-docstring} archivebox.misc.util.dedupe - :summary: - ``` * - {py:obj}`to_json ` - ```{autodoc2-docstring} archivebox.misc.util.to_json :summary: ``` -* - {py:obj}`chrome_cleanup ` - - ```{autodoc2-docstring} archivebox.misc.util.chrome_cleanup - :summary: - ``` ```` ### Data @@ -139,18 +147,6 @@ - ```{autodoc2-docstring} archivebox.misc.util.base_url :summary: ``` -* - {py:obj}`without_www ` - - ```{autodoc2-docstring} archivebox.misc.util.without_www - :summary: - ``` -* - {py:obj}`without_trailing_slash ` - - ```{autodoc2-docstring} archivebox.misc.util.without_trailing_slash - :summary: - ``` -* - {py:obj}`hashurl ` - - ```{autodoc2-docstring} archivebox.misc.util.hashurl - :summary: - ``` * - {py:obj}`urlencode ` - ```{autodoc2-docstring} archivebox.misc.util.urlencode :summary: @@ -167,18 +163,6 @@ - ```{autodoc2-docstring} archivebox.misc.util.htmldecode :summary: ``` -* - {py:obj}`short_ts ` - - ```{autodoc2-docstring} archivebox.misc.util.short_ts - :summary: - ``` -* - {py:obj}`ts_to_date_str ` - - ```{autodoc2-docstring} archivebox.misc.util.ts_to_date_str - :summary: - ``` -* - {py:obj}`ts_to_iso ` - - ```{autodoc2-docstring} archivebox.misc.util.ts_to_iso - :summary: - ``` * - {py:obj}`COLOR_REGEX ` - ```{autodoc2-docstring} archivebox.misc.util.COLOR_REGEX :summary: @@ -187,18 +171,37 @@ - ```{autodoc2-docstring} archivebox.misc.util.URL_REGEX :summary: ``` -* - {py:obj}`URL_REGEX_TESTS ` - - ```{autodoc2-docstring} archivebox.misc.util.URL_REGEX_TESTS +* - {py:obj}`MAX_URL_LENGTH ` + - ```{autodoc2-docstring} archivebox.misc.util.MAX_URL_LENGTH :summary: ``` -* - {py:obj}`_test_url_strs ` - - ```{autodoc2-docstring} archivebox.misc.util._test_url_strs +* - {py:obj}`QUOTE_DELIMITERS ` + - ```{autodoc2-docstring} archivebox.misc.util.QUOTE_DELIMITERS + :summary: + ``` +* - {py:obj}`QUOTE_ENTITY_DELIMITERS ` + - ```{autodoc2-docstring} archivebox.misc.util.QUOTE_ENTITY_DELIMITERS + :summary: + ``` +* - {py:obj}`URL_ENTITY_REPLACEMENTS ` + - ```{autodoc2-docstring} archivebox.misc.util.URL_ENTITY_REPLACEMENTS + :summary: + ``` +* - {py:obj}`FILESIZE_UNITS ` + - ```{autodoc2-docstring} archivebox.misc.util.FILESIZE_UNITS :summary: ``` ```` ### API +````{py:function} filter_queryset_by_uuid_substring(queryset, slug: str, field: str = 'id') +:canonical: archivebox.misc.util.filter_queryset_by_uuid_substring + +```{autodoc2-docstring} archivebox.misc.util.filter_queryset_by_uuid_substring +``` +```` + ````{py:data} scheme :canonical: archivebox.misc.util.scheme :value: > @@ -319,36 +322,6 @@ ```` -````{py:data} without_www -:canonical: archivebox.misc.util.without_www -:value: > - None - -```{autodoc2-docstring} archivebox.misc.util.without_www -``` - -```` - -````{py:data} without_trailing_slash -:canonical: archivebox.misc.util.without_trailing_slash -:value: > - None - -```{autodoc2-docstring} archivebox.misc.util.without_trailing_slash -``` - -```` - -````{py:data} hashurl -:canonical: archivebox.misc.util.hashurl -:value: > - None - -```{autodoc2-docstring} archivebox.misc.util.hashurl -``` - -```` - ````{py:data} urlencode :canonical: archivebox.misc.util.urlencode :value: > @@ -389,34 +362,11 @@ ```` -````{py:data} short_ts -:canonical: archivebox.misc.util.short_ts -:value: > - None - -```{autodoc2-docstring} archivebox.misc.util.short_ts -``` - -```` - -````{py:data} ts_to_date_str +````{py:function} ts_to_date_str(ts: typing.Any) -> str | None :canonical: archivebox.misc.util.ts_to_date_str -:value: > - None ```{autodoc2-docstring} archivebox.misc.util.ts_to_date_str ``` - -```` - -````{py:data} ts_to_iso -:canonical: archivebox.misc.util.ts_to_iso -:value: > - None - -```{autodoc2-docstring} archivebox.misc.util.ts_to_iso -``` - ```` ````{py:data} COLOR_REGEX @@ -439,6 +389,78 @@ ```` +````{py:data} MAX_URL_LENGTH +:canonical: archivebox.misc.util.MAX_URL_LENGTH +:value: > + 65535 + +```{autodoc2-docstring} archivebox.misc.util.MAX_URL_LENGTH +``` + +```` + +````{py:data} QUOTE_DELIMITERS +:canonical: archivebox.misc.util.QUOTE_DELIMITERS +:value: > + ('"', "'", '`', '“', '”', '‘', '’') + +```{autodoc2-docstring} archivebox.misc.util.QUOTE_DELIMITERS +``` + +```` + +````{py:data} QUOTE_ENTITY_DELIMITERS +:canonical: archivebox.misc.util.QUOTE_ENTITY_DELIMITERS +:value: > + ('"', '"', '"', ''', ''', ''') + +```{autodoc2-docstring} archivebox.misc.util.QUOTE_ENTITY_DELIMITERS +``` + +```` + +````{py:data} URL_ENTITY_REPLACEMENTS +:canonical: archivebox.misc.util.URL_ENTITY_REPLACEMENTS +:value: > + (('&', '&'), ('&', '&'), ('&', '&')) + +```{autodoc2-docstring} archivebox.misc.util.URL_ENTITY_REPLACEMENTS +``` + +```` + +````{py:data} FILESIZE_UNITS +:canonical: archivebox.misc.util.FILESIZE_UNITS +:type: dict[str, int] +:value: > + None + +```{autodoc2-docstring} archivebox.misc.util.FILESIZE_UNITS +``` + +```` + +````{py:function} sanitize_extracted_url(url: str) -> str +:canonical: archivebox.misc.util.sanitize_extracted_url + +```{autodoc2-docstring} archivebox.misc.util.sanitize_extracted_url +``` +```` + +````{py:function} validate_url_length(url: str) -> str +:canonical: archivebox.misc.util.validate_url_length + +```{autodoc2-docstring} archivebox.misc.util.validate_url_length +``` +```` + +````{py:function} validate_url(url: str) -> str +:canonical: archivebox.misc.util.validate_url + +```{autodoc2-docstring} archivebox.misc.util.validate_url +``` +```` + ````{py:function} parens_are_matched(string: str, open_char='(', close_char=')') :canonical: archivebox.misc.util.parens_are_matched @@ -453,6 +475,13 @@ ``` ```` +````{py:function} split_comma_separated_urls(url: str) +:canonical: archivebox.misc.util.split_comma_separated_urls + +```{autodoc2-docstring} archivebox.misc.util.split_comma_separated_urls +``` +```` + ````{py:function} find_all_urls(urls_str: str) :canonical: archivebox.misc.util.find_all_urls @@ -460,10 +489,10 @@ ``` ```` -````{py:function} is_static_file(url: str) -:canonical: archivebox.misc.util.is_static_file +````{py:function} parse_filesize_to_bytes(value: str | int | float | None) -> int +:canonical: archivebox.misc.util.parse_filesize_to_bytes -```{autodoc2-docstring} archivebox.misc.util.is_static_file +```{autodoc2-docstring} archivebox.misc.util.parse_filesize_to_bytes ``` ```` @@ -474,41 +503,27 @@ ``` ```` -````{py:function} docstring(text: typing.Optional[str]) +````{py:function} docstring(text: str | None) :canonical: archivebox.misc.util.docstring ```{autodoc2-docstring} archivebox.misc.util.docstring ``` ```` -````{py:function} str_between(string: str, start: str, end: str = None) -> str -:canonical: archivebox.misc.util.str_between - -```{autodoc2-docstring} archivebox.misc.util.str_between -``` -```` - -````{py:function} parse_date(date: typing.Any) -> datetime.datetime +````{py:function} parse_date(date: typing.Any) -> datetime.datetime | None :canonical: archivebox.misc.util.parse_date ```{autodoc2-docstring} archivebox.misc.util.parse_date ``` ```` -````{py:function} download_url(url: str, timeout: int = None) -> str +````{py:function} download_url(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str :canonical: archivebox.misc.util.download_url ```{autodoc2-docstring} archivebox.misc.util.download_url ``` ```` -````{py:function} get_headers(url: str, timeout: int | None = None) -> str -:canonical: archivebox.misc.util.get_headers - -```{autodoc2-docstring} archivebox.misc.util.get_headers -``` -```` - ````{py:function} ansi_to_html(text: str) -> str :canonical: archivebox.misc.util.ansi_to_html @@ -516,13 +531,6 @@ ``` ```` -````{py:function} dedupe(options: typing.List[str]) -> typing.List[str] -:canonical: archivebox.misc.util.dedupe - -```{autodoc2-docstring} archivebox.misc.util.dedupe -``` -```` - `````{py:class} ExtendedEncoder(*, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None) :canonical: archivebox.misc.util.ExtendedEncoder @@ -537,43 +545,16 @@ Bases: {py:obj}`json.JSONEncoder` ```{autodoc2-docstring} archivebox.misc.util.ExtendedEncoder.__init__ ``` -````{py:method} default(obj) +````{py:method} default(o) :canonical: archivebox.misc.util.ExtendedEncoder.default ```` ````` -````{py:function} to_json(obj: typing.Any, indent: typing.Optional[int] = 4, sort_keys: bool = True) -> str +````{py:function} to_json(obj: typing.Any, indent: int | None = 4, sort_keys: bool = True) -> str :canonical: archivebox.misc.util.to_json ```{autodoc2-docstring} archivebox.misc.util.to_json ``` ```` - -````{py:data} URL_REGEX_TESTS -:canonical: archivebox.misc.util.URL_REGEX_TESTS -:value: > - [('https://example.com', ['https://example.com']), ('http://abc-file234example.com/abc?def=abc&23423... - -```{autodoc2-docstring} archivebox.misc.util.URL_REGEX_TESTS -``` - -```` - -````{py:data} _test_url_strs -:canonical: archivebox.misc.util._test_url_strs -:value: > - None - -```{autodoc2-docstring} archivebox.misc.util._test_url_strs -``` - -```` - -````{py:function} chrome_cleanup() -:canonical: archivebox.misc.util.chrome_cleanup - -```{autodoc2-docstring} archivebox.misc.util.chrome_cleanup -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.personas.admin.md b/docs/apidocs/archivebox/archivebox.personas.admin.md index 2592636a..159e4f86 100644 --- a/docs/apidocs/archivebox/archivebox.personas.admin.md +++ b/docs/apidocs/archivebox/archivebox.personas.admin.md @@ -6,3 +6,206 @@ ```{autodoc2-docstring} archivebox.personas.admin :allowtitles: ``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`PersonaAdmin ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`register_admin ` + - ```{autodoc2-docstring} archivebox.personas.admin.register_admin + :summary: + ``` +```` + +### API + +`````{py:class} PersonaAdmin(model, admin_site) +:canonical: archivebox.personas.admin.PersonaAdmin + +Bases: {py:obj}`archivebox.base_models.admin.ConfigEditorMixin`, {py:obj}`archivebox.base_models.admin.BaseModelAdmin` + +````{py:attribute} form +:canonical: archivebox.personas.admin.PersonaAdmin.form +:value: > + None + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.form +``` + +```` + +````{py:attribute} change_form_template +:canonical: archivebox.personas.admin.PersonaAdmin.change_form_template +:value: > + 'admin/personas/persona/change_form.html' + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.change_form_template +``` + +```` + +````{py:attribute} list_display +:canonical: archivebox.personas.admin.PersonaAdmin.list_display +:value: > + ('name', 'created_by', 'created_at', 'chrome_profile_state', 'cookies_state', 'auth_state') + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.list_display +``` + +```` + +````{py:attribute} search_fields +:canonical: archivebox.personas.admin.PersonaAdmin.search_fields +:value: > + ('name', 'created_by__username') + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.search_fields +``` + +```` + +````{py:attribute} list_filter +:canonical: archivebox.personas.admin.PersonaAdmin.list_filter +:value: > + ('created_by',) + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.list_filter +``` + +```` + +````{py:attribute} ordering +:canonical: archivebox.personas.admin.PersonaAdmin.ordering +:value: > + ['name'] + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.ordering +``` + +```` + +````{py:attribute} list_per_page +:canonical: archivebox.personas.admin.PersonaAdmin.list_per_page +:value: > + 100 + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.list_per_page +``` + +```` + +````{py:attribute} readonly_fields +:canonical: archivebox.personas.admin.PersonaAdmin.readonly_fields +:value: > + ('id', 'created_at', 'persona_paths', 'import_artifact_status') + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.readonly_fields +``` + +```` + +````{py:attribute} add_fieldsets +:canonical: archivebox.personas.admin.PersonaAdmin.add_fieldsets +:value: > + (('Persona',), ('Browser Import',), ('Advanced',)) + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.add_fieldsets +``` + +```` + +````{py:attribute} change_fieldsets +:canonical: archivebox.personas.admin.PersonaAdmin.change_fieldsets +:value: > + () + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.change_fieldsets +``` + +```` + +````{py:method} chrome_profile_state(obj: archivebox.personas.models.Persona) -> str +:canonical: archivebox.personas.admin.PersonaAdmin.chrome_profile_state + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.chrome_profile_state +``` + +```` + +````{py:method} cookies_state(obj: archivebox.personas.models.Persona) -> str +:canonical: archivebox.personas.admin.PersonaAdmin.cookies_state + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.cookies_state +``` + +```` + +````{py:method} auth_state(obj: archivebox.personas.models.Persona) -> str +:canonical: archivebox.personas.admin.PersonaAdmin.auth_state + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.auth_state +``` + +```` + +````{py:method} persona_paths(obj: archivebox.personas.models.Persona) -> str +:canonical: archivebox.personas.admin.PersonaAdmin.persona_paths + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.persona_paths +``` + +```` + +````{py:method} import_artifact_status(obj: archivebox.personas.models.Persona) -> str +:canonical: archivebox.personas.admin.PersonaAdmin.import_artifact_status + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.import_artifact_status +``` + +```` + +````{py:method} get_fieldsets(request, obj=None) +:canonical: archivebox.personas.admin.PersonaAdmin.get_fieldsets + +```` + +````{py:method} get_form(request, obj=None, change=False, **kwargs) +:canonical: archivebox.personas.admin.PersonaAdmin.get_form + +```` + +````{py:method} render_change_form(request, context, add=False, change=False, form_url='', obj=None) +:canonical: archivebox.personas.admin.PersonaAdmin.render_change_form + +```{autodoc2-docstring} archivebox.personas.admin.PersonaAdmin.render_change_form +``` + +```` + +````{py:method} save_model(request, obj, form, change) +:canonical: archivebox.personas.admin.PersonaAdmin.save_model + +```` + +````` + +````{py:function} register_admin(admin_site: django.contrib.admin.AdminSite) -> None +:canonical: archivebox.personas.admin.register_admin + +```{autodoc2-docstring} archivebox.personas.admin.register_admin +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.personas.forms.md b/docs/apidocs/archivebox/archivebox.personas.forms.md new file mode 100644 index 00000000..bdd343f6 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.personas.forms.md @@ -0,0 +1,177 @@ +# {py:mod}`archivebox.personas.forms` + +```{py:module} archivebox.personas.forms +``` + +```{autodoc2-docstring} archivebox.personas.forms +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`PersonaAdminForm ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_mode_label ` + - ```{autodoc2-docstring} archivebox.personas.forms._mode_label + :summary: + ``` +```` + +### API + +````{py:function} _mode_label(title: str, description: str) -> str +:canonical: archivebox.personas.forms._mode_label + +```{autodoc2-docstring} archivebox.personas.forms._mode_label +``` +```` + +``````{py:class} PersonaAdminForm(*args: typing.Any, **kwargs: typing.Any) +:canonical: archivebox.personas.forms.PersonaAdminForm + +Bases: {py:obj}`archivebox.core.forms.PluginConfigFormMixin`, {py:obj}`django.forms.ModelForm` + +````{py:attribute} permissions +:canonical: archivebox.personas.forms.PersonaAdminForm.permissions +:value: > + 'ChoiceField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.permissions +``` + +```` + +````{py:attribute} import_mode +:canonical: archivebox.personas.forms.PersonaAdminForm.import_mode +:value: > + 'ChoiceField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_mode +``` + +```` + +````{py:attribute} import_discovered_profile +:canonical: archivebox.personas.forms.PersonaAdminForm.import_discovered_profile +:value: > + 'ChoiceField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_discovered_profile +``` + +```` + +````{py:attribute} import_source +:canonical: archivebox.personas.forms.PersonaAdminForm.import_source +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_source +``` + +```` + +````{py:attribute} import_profile_name +:canonical: archivebox.personas.forms.PersonaAdminForm.import_profile_name +:value: > + 'CharField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_profile_name +``` + +```` + +````{py:attribute} import_copy_profile +:canonical: archivebox.personas.forms.PersonaAdminForm.import_copy_profile +:value: > + 'BooleanField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_copy_profile +``` + +```` + +````{py:attribute} import_extract_cookies +:canonical: archivebox.personas.forms.PersonaAdminForm.import_extract_cookies +:value: > + 'BooleanField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_extract_cookies +``` + +```` + +````{py:attribute} import_capture_storage +:canonical: archivebox.personas.forms.PersonaAdminForm.import_capture_storage +:value: > + 'BooleanField(...)' + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.import_capture_storage +``` + +```` + +`````{py:class} Meta +:canonical: archivebox.personas.forms.PersonaAdminForm.Meta + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.Meta +``` + +````{py:attribute} model +:canonical: archivebox.personas.forms.PersonaAdminForm.Meta.model +:value: > + None + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.Meta.model +``` + +```` + +````{py:attribute} fields +:canonical: archivebox.personas.forms.PersonaAdminForm.Meta.fields +:value: > + ('name', 'created_by', 'config') + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.Meta.fields +``` + +```` + +````` + +````{py:method} clean_name() -> str +:canonical: archivebox.personas.forms.PersonaAdminForm.clean_name + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.clean_name +``` + +```` + +````{py:method} clean() -> dict[str, typing.Any] +:canonical: archivebox.personas.forms.PersonaAdminForm.clean + +```` + +````{py:method} apply_import(persona: archivebox.personas.models.Persona) -> archivebox.personas.importers.PersonaImportResult | None +:canonical: archivebox.personas.forms.PersonaAdminForm.apply_import + +```{autodoc2-docstring} archivebox.personas.forms.PersonaAdminForm.apply_import +``` + +```` + +`````` diff --git a/docs/apidocs/archivebox/archivebox.personas.importers.md b/docs/apidocs/archivebox/archivebox.personas.importers.md new file mode 100644 index 00000000..deb19874 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.personas.importers.md @@ -0,0 +1,627 @@ +# {py:mod}`archivebox.personas.importers` + +```{py:module} archivebox.personas.importers +``` + +```{autodoc2-docstring} archivebox.personas.importers +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`PersonaImportSource ` + - ```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource + :summary: + ``` +* - {py:obj}`PersonaImportResult ` + - ```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult + :summary: + ``` +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`get_chrome_user_data_dir ` + - ```{autodoc2-docstring} archivebox.personas.importers.get_chrome_user_data_dir + :summary: + ``` +* - {py:obj}`get_brave_user_data_dir ` + - ```{autodoc2-docstring} archivebox.personas.importers.get_brave_user_data_dir + :summary: + ``` +* - {py:obj}`get_edge_user_data_dir ` + - ```{autodoc2-docstring} archivebox.personas.importers.get_edge_user_data_dir + :summary: + ``` +* - {py:obj}`get_browser_binary ` + - ```{autodoc2-docstring} archivebox.personas.importers.get_browser_binary + :summary: + ``` +* - {py:obj}`validate_persona_name ` + - ```{autodoc2-docstring} archivebox.personas.importers.validate_persona_name + :summary: + ``` +* - {py:obj}`discover_local_browser_profiles ` + - ```{autodoc2-docstring} archivebox.personas.importers.discover_local_browser_profiles + :summary: + ``` +* - {py:obj}`discover_persona_template_profiles ` + - ```{autodoc2-docstring} archivebox.personas.importers.discover_persona_template_profiles + :summary: + ``` +* - {py:obj}`resolve_browser_import_source ` + - ```{autodoc2-docstring} archivebox.personas.importers.resolve_browser_import_source + :summary: + ``` +* - {py:obj}`resolve_browser_profile_source ` + - ```{autodoc2-docstring} archivebox.personas.importers.resolve_browser_profile_source + :summary: + ``` +* - {py:obj}`resolve_custom_import_source ` + - ```{autodoc2-docstring} archivebox.personas.importers.resolve_custom_import_source + :summary: + ``` +* - {py:obj}`pick_default_profile_dir ` + - ```{autodoc2-docstring} archivebox.personas.importers.pick_default_profile_dir + :summary: + ``` +* - {py:obj}`import_persona_from_source ` + - ```{autodoc2-docstring} archivebox.personas.importers.import_persona_from_source + :summary: + ``` +* - {py:obj}`copy_browser_user_data_dir ` + - ```{autodoc2-docstring} archivebox.personas.importers.copy_browser_user_data_dir + :summary: + ``` +* - {py:obj}`export_browser_state ` + - ```{autodoc2-docstring} archivebox.personas.importers.export_browser_state + :summary: + ``` +* - {py:obj}`_list_profile_names ` + - ```{autodoc2-docstring} archivebox.personas.importers._list_profile_names + :summary: + ``` +* - {py:obj}`_looks_like_profile_dir ` + - ```{autodoc2-docstring} archivebox.personas.importers._looks_like_profile_dir + :summary: + ``` +* - {py:obj}`_looks_like_cdp_url ` + - ```{autodoc2-docstring} archivebox.personas.importers._looks_like_cdp_url + :summary: + ``` +* - {py:obj}`_parse_netscape_cookies ` + - ```{autodoc2-docstring} archivebox.personas.importers._parse_netscape_cookies + :summary: + ``` +* - {py:obj}`_write_netscape_cookies ` + - ```{autodoc2-docstring} archivebox.personas.importers._write_netscape_cookies + :summary: + ``` +* - {py:obj}`_merge_netscape_cookies ` + - ```{autodoc2-docstring} archivebox.personas.importers._merge_netscape_cookies + :summary: + ``` +* - {py:obj}`_merge_auth_storage ` + - ```{autodoc2-docstring} archivebox.personas.importers._merge_auth_storage + :summary: + ``` +* - {py:obj}`_load_auth_storage ` + - ```{autodoc2-docstring} archivebox.personas.importers._load_auth_storage + :summary: + ``` +* - {py:obj}`_merge_cookie_dicts ` + - ```{autodoc2-docstring} archivebox.personas.importers._merge_cookie_dicts + :summary: + ``` +* - {py:obj}`_apply_imported_user_agent ` + - ```{autodoc2-docstring} archivebox.personas.importers._apply_imported_user_agent + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`BROWSER_LABELS ` + - ```{autodoc2-docstring} archivebox.personas.importers.BROWSER_LABELS + :summary: + ``` +* - {py:obj}`BROWSER_PROFILE_DIR_NAMES ` + - ```{autodoc2-docstring} archivebox.personas.importers.BROWSER_PROFILE_DIR_NAMES + :summary: + ``` +* - {py:obj}`VOLATILE_PROFILE_COPY_PATTERNS ` + - ```{autodoc2-docstring} archivebox.personas.importers.VOLATILE_PROFILE_COPY_PATTERNS + :summary: + ``` +* - {py:obj}`PERSONA_PROFILE_DIR_CANDIDATES ` + - ```{autodoc2-docstring} archivebox.personas.importers.PERSONA_PROFILE_DIR_CANDIDATES + :summary: + ``` +* - {py:obj}`BROWSER_PROFILE_FINDERS ` + - ```{autodoc2-docstring} archivebox.personas.importers.BROWSER_PROFILE_FINDERS + :summary: + ``` +* - {py:obj}`CHROMIUM_BROWSERS ` + - ```{autodoc2-docstring} archivebox.personas.importers.CHROMIUM_BROWSERS + :summary: + ``` +* - {py:obj}`NETSCAPE_COOKIE_HEADER ` + - ```{autodoc2-docstring} archivebox.personas.importers.NETSCAPE_COOKIE_HEADER + :summary: + ``` +```` + +### API + +````{py:data} BROWSER_LABELS +:canonical: archivebox.personas.importers.BROWSER_LABELS +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.BROWSER_LABELS +``` + +```` + +````{py:data} BROWSER_PROFILE_DIR_NAMES +:canonical: archivebox.personas.importers.BROWSER_PROFILE_DIR_NAMES +:value: > + ('Default', 'Profile ', 'Guest Profile') + +```{autodoc2-docstring} archivebox.personas.importers.BROWSER_PROFILE_DIR_NAMES +``` + +```` + +````{py:data} VOLATILE_PROFILE_COPY_PATTERNS +:canonical: archivebox.personas.importers.VOLATILE_PROFILE_COPY_PATTERNS +:value: > + ('Cache', 'Code Cache', 'GPUCache', 'ShaderCache', 'Service Worker', 'GCM Store', '*.log', 'Crashpad... + +```{autodoc2-docstring} archivebox.personas.importers.VOLATILE_PROFILE_COPY_PATTERNS +``` + +```` + +````{py:data} PERSONA_PROFILE_DIR_CANDIDATES +:canonical: archivebox.personas.importers.PERSONA_PROFILE_DIR_CANDIDATES +:value: > + ('chrome_profile', 'chrome_user_data') + +```{autodoc2-docstring} archivebox.personas.importers.PERSONA_PROFILE_DIR_CANDIDATES +``` + +```` + +`````{py:class} PersonaImportSource +:canonical: archivebox.personas.importers.PersonaImportSource + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource +``` + +````{py:attribute} kind +:canonical: archivebox.personas.importers.PersonaImportSource.kind +:type: str +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.kind +``` + +```` + +````{py:attribute} browser +:canonical: archivebox.personas.importers.PersonaImportSource.browser +:type: str +:value: > + 'custom' + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.browser +``` + +```` + +````{py:attribute} source_name +:canonical: archivebox.personas.importers.PersonaImportSource.source_name +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.source_name +``` + +```` + +````{py:attribute} user_data_dir +:canonical: archivebox.personas.importers.PersonaImportSource.user_data_dir +:type: pathlib.Path | None +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.user_data_dir +``` + +```` + +````{py:attribute} profile_dir +:canonical: archivebox.personas.importers.PersonaImportSource.profile_dir +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.profile_dir +``` + +```` + +````{py:attribute} browser_binary +:canonical: archivebox.personas.importers.PersonaImportSource.browser_binary +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.browser_binary +``` + +```` + +````{py:attribute} cdp_url +:canonical: archivebox.personas.importers.PersonaImportSource.cdp_url +:type: str | None +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.cdp_url +``` + +```` + +````{py:property} browser_label +:canonical: archivebox.personas.importers.PersonaImportSource.browser_label +:type: str + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.browser_label +``` + +```` + +````{py:property} profile_path +:canonical: archivebox.personas.importers.PersonaImportSource.profile_path +:type: pathlib.Path | None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.profile_path +``` + +```` + +````{py:property} display_label +:canonical: archivebox.personas.importers.PersonaImportSource.display_label +:type: str + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.display_label +``` + +```` + +````{py:property} choice_value +:canonical: archivebox.personas.importers.PersonaImportSource.choice_value +:type: str + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.choice_value +``` + +```` + +````{py:method} as_choice_label() -> django.utils.safestring.SafeString +:canonical: archivebox.personas.importers.PersonaImportSource.as_choice_label + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.as_choice_label +``` + +```` + +````{py:method} from_choice_value(value: str) -> archivebox.personas.importers.PersonaImportSource +:canonical: archivebox.personas.importers.PersonaImportSource.from_choice_value +:classmethod: + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportSource.from_choice_value +``` + +```` + +````` + +`````{py:class} PersonaImportResult +:canonical: archivebox.personas.importers.PersonaImportResult + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult +``` + +````{py:attribute} source +:canonical: archivebox.personas.importers.PersonaImportResult.source +:type: archivebox.personas.importers.PersonaImportSource +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.source +``` + +```` + +````{py:attribute} profile_copied +:canonical: archivebox.personas.importers.PersonaImportResult.profile_copied +:type: bool +:value: > + False + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.profile_copied +``` + +```` + +````{py:attribute} cookies_imported +:canonical: archivebox.personas.importers.PersonaImportResult.cookies_imported +:type: bool +:value: > + False + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.cookies_imported +``` + +```` + +````{py:attribute} storage_captured +:canonical: archivebox.personas.importers.PersonaImportResult.storage_captured +:type: bool +:value: > + False + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.storage_captured +``` + +```` + +````{py:attribute} user_agent_imported +:canonical: archivebox.personas.importers.PersonaImportResult.user_agent_imported +:type: bool +:value: > + False + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.user_agent_imported +``` + +```` + +````{py:attribute} warnings +:canonical: archivebox.personas.importers.PersonaImportResult.warnings +:type: list[str] +:value: > + 'field(...)' + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.warnings +``` + +```` + +````{py:property} did_work +:canonical: archivebox.personas.importers.PersonaImportResult.did_work +:type: bool + +```{autodoc2-docstring} archivebox.personas.importers.PersonaImportResult.did_work +``` + +```` + +````` + +````{py:function} get_chrome_user_data_dir() -> pathlib.Path | None +:canonical: archivebox.personas.importers.get_chrome_user_data_dir + +```{autodoc2-docstring} archivebox.personas.importers.get_chrome_user_data_dir +``` +```` + +````{py:function} get_brave_user_data_dir() -> pathlib.Path | None +:canonical: archivebox.personas.importers.get_brave_user_data_dir + +```{autodoc2-docstring} archivebox.personas.importers.get_brave_user_data_dir +``` +```` + +````{py:function} get_edge_user_data_dir() -> pathlib.Path | None +:canonical: archivebox.personas.importers.get_edge_user_data_dir + +```{autodoc2-docstring} archivebox.personas.importers.get_edge_user_data_dir +``` +```` + +````{py:function} get_browser_binary(browser: str) -> str | None +:canonical: archivebox.personas.importers.get_browser_binary + +```{autodoc2-docstring} archivebox.personas.importers.get_browser_binary +``` +```` + +````{py:data} BROWSER_PROFILE_FINDERS +:canonical: archivebox.personas.importers.BROWSER_PROFILE_FINDERS +:value: > + None + +```{autodoc2-docstring} archivebox.personas.importers.BROWSER_PROFILE_FINDERS +``` + +```` + +````{py:data} CHROMIUM_BROWSERS +:canonical: archivebox.personas.importers.CHROMIUM_BROWSERS +:value: > + 'tuple(...)' + +```{autodoc2-docstring} archivebox.personas.importers.CHROMIUM_BROWSERS +``` + +```` + +````{py:data} NETSCAPE_COOKIE_HEADER +:canonical: archivebox.personas.importers.NETSCAPE_COOKIE_HEADER +:value: > + ['# Netscape HTTP Cookie File', '# https://curl.se/docs/http-cookies.html', '# This file was generat... + +```{autodoc2-docstring} archivebox.personas.importers.NETSCAPE_COOKIE_HEADER +``` + +```` + +````{py:function} validate_persona_name(name: str) -> tuple[bool, str] +:canonical: archivebox.personas.importers.validate_persona_name + +```{autodoc2-docstring} archivebox.personas.importers.validate_persona_name +``` +```` + +````{py:function} discover_local_browser_profiles() -> list[archivebox.personas.importers.PersonaImportSource] +:canonical: archivebox.personas.importers.discover_local_browser_profiles + +```{autodoc2-docstring} archivebox.personas.importers.discover_local_browser_profiles +``` +```` + +````{py:function} discover_persona_template_profiles(personas_dir: pathlib.Path | None = None) -> list[archivebox.personas.importers.PersonaImportSource] +:canonical: archivebox.personas.importers.discover_persona_template_profiles + +```{autodoc2-docstring} archivebox.personas.importers.discover_persona_template_profiles +``` +```` + +````{py:function} resolve_browser_import_source(browser: str, profile_dir: str | None = None) -> archivebox.personas.importers.PersonaImportSource +:canonical: archivebox.personas.importers.resolve_browser_import_source + +```{autodoc2-docstring} archivebox.personas.importers.resolve_browser_import_source +``` +```` + +````{py:function} resolve_browser_profile_source(browser: str, user_data_dir: pathlib.Path, profile_dir: str, source_name: str | None = None, browser_binary: str | None = None) -> archivebox.personas.importers.PersonaImportSource +:canonical: archivebox.personas.importers.resolve_browser_profile_source + +```{autodoc2-docstring} archivebox.personas.importers.resolve_browser_profile_source +``` +```` + +````{py:function} resolve_custom_import_source(raw_value: str, profile_dir: str | None = None) -> archivebox.personas.importers.PersonaImportSource +:canonical: archivebox.personas.importers.resolve_custom_import_source + +```{autodoc2-docstring} archivebox.personas.importers.resolve_custom_import_source +``` +```` + +````{py:function} pick_default_profile_dir(user_data_dir: pathlib.Path) -> str | None +:canonical: archivebox.personas.importers.pick_default_profile_dir + +```{autodoc2-docstring} archivebox.personas.importers.pick_default_profile_dir +``` +```` + +````{py:function} import_persona_from_source(persona: archivebox.personas.models.Persona, source: archivebox.personas.importers.PersonaImportSource, *, copy_profile: bool = True, import_cookies: bool = True, capture_storage: bool = False) -> archivebox.personas.importers.PersonaImportResult +:canonical: archivebox.personas.importers.import_persona_from_source + +```{autodoc2-docstring} archivebox.personas.importers.import_persona_from_source +``` +```` + +````{py:function} copy_browser_user_data_dir(source_dir: pathlib.Path, destination_dir: pathlib.Path) -> None +:canonical: archivebox.personas.importers.copy_browser_user_data_dir + +```{autodoc2-docstring} archivebox.personas.importers.copy_browser_user_data_dir +``` +```` + +````{py:function} export_browser_state(*, user_data_dir: pathlib.Path | None = None, cdp_url: str | None = None, profile_dir: str | None = None, chrome_binary: str | None = None, cookies_output_file: pathlib.Path | None = None, auth_output_file: pathlib.Path | None = None) -> tuple[bool, dict | None, str] +:canonical: archivebox.personas.importers.export_browser_state + +```{autodoc2-docstring} archivebox.personas.importers.export_browser_state +``` +```` + +````{py:function} _list_profile_names(user_data_dir: pathlib.Path) -> list[str] +:canonical: archivebox.personas.importers._list_profile_names + +```{autodoc2-docstring} archivebox.personas.importers._list_profile_names +``` +```` + +````{py:function} _looks_like_profile_dir(path: pathlib.Path) -> bool +:canonical: archivebox.personas.importers._looks_like_profile_dir + +```{autodoc2-docstring} archivebox.personas.importers._looks_like_profile_dir +``` +```` + +````{py:function} _looks_like_cdp_url(value: str) -> bool +:canonical: archivebox.personas.importers._looks_like_cdp_url + +```{autodoc2-docstring} archivebox.personas.importers._looks_like_cdp_url +``` +```` + +````{py:function} _parse_netscape_cookies(path: pathlib.Path) -> dict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]] +:canonical: archivebox.personas.importers._parse_netscape_cookies + +```{autodoc2-docstring} archivebox.personas.importers._parse_netscape_cookies +``` +```` + +````{py:function} _write_netscape_cookies(path: pathlib.Path, cookies: dict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]) -> None +:canonical: archivebox.personas.importers._write_netscape_cookies + +```{autodoc2-docstring} archivebox.personas.importers._write_netscape_cookies +``` +```` + +````{py:function} _merge_netscape_cookies(existing_file: pathlib.Path, new_file: pathlib.Path) -> None +:canonical: archivebox.personas.importers._merge_netscape_cookies + +```{autodoc2-docstring} archivebox.personas.importers._merge_netscape_cookies +``` +```` + +````{py:function} _merge_auth_storage(existing_file: pathlib.Path, new_file: pathlib.Path) -> None +:canonical: archivebox.personas.importers._merge_auth_storage + +```{autodoc2-docstring} archivebox.personas.importers._merge_auth_storage +``` +```` + +````{py:function} _load_auth_storage(path: pathlib.Path) -> dict +:canonical: archivebox.personas.importers._load_auth_storage + +```{autodoc2-docstring} archivebox.personas.importers._load_auth_storage +``` +```` + +````{py:function} _merge_cookie_dicts(existing: list[dict], new: list[dict]) -> list[dict] +:canonical: archivebox.personas.importers._merge_cookie_dicts + +```{autodoc2-docstring} archivebox.personas.importers._merge_cookie_dicts +``` +```` + +````{py:function} _apply_imported_user_agent(persona: archivebox.personas.models.Persona, auth_payload: dict | None) -> bool +:canonical: archivebox.personas.importers._apply_imported_user_agent + +```{autodoc2-docstring} archivebox.personas.importers._apply_imported_user_agent +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.personas.md b/docs/apidocs/archivebox/archivebox.personas.md index 244f9baa..fd6e6b1b 100644 --- a/docs/apidocs/archivebox/archivebox.personas.md +++ b/docs/apidocs/archivebox/archivebox.personas.md @@ -14,8 +14,8 @@ :maxdepth: 1 archivebox.personas.models +archivebox.personas.importers archivebox.personas.apps -archivebox.personas.runtime +archivebox.personas.forms archivebox.personas.admin -archivebox.personas.views ``` diff --git a/docs/apidocs/archivebox/archivebox.personas.models.md b/docs/apidocs/archivebox/archivebox.personas.models.md index e30f2c33..178a54fe 100644 --- a/docs/apidocs/archivebox/archivebox.personas.models.md +++ b/docs/apidocs/archivebox/archivebox.personas.models.md @@ -21,8 +21,59 @@ ``` ```` +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_fcntl ` + - ```{autodoc2-docstring} archivebox.personas.models._fcntl + :summary: + ``` +* - {py:obj}`VOLATILE_PROFILE_DIR_NAMES ` + - ```{autodoc2-docstring} archivebox.personas.models.VOLATILE_PROFILE_DIR_NAMES + :summary: + ``` +* - {py:obj}`VOLATILE_PROFILE_FILE_NAMES ` + - ```{autodoc2-docstring} archivebox.personas.models.VOLATILE_PROFILE_FILE_NAMES + :summary: + ``` +```` + ### API +````{py:data} _fcntl +:canonical: archivebox.personas.models._fcntl +:type: typing.Any | None +:value: > + None + +```{autodoc2-docstring} archivebox.personas.models._fcntl +``` + +```` + +````{py:data} VOLATILE_PROFILE_DIR_NAMES +:canonical: archivebox.personas.models.VOLATILE_PROFILE_DIR_NAMES +:value: > + None + +```{autodoc2-docstring} archivebox.personas.models.VOLATILE_PROFILE_DIR_NAMES +``` + +```` + +````{py:data} VOLATILE_PROFILE_FILE_NAMES +:canonical: archivebox.personas.models.VOLATILE_PROFILE_FILE_NAMES +:value: > + None + +```{autodoc2-docstring} archivebox.personas.models.VOLATILE_PROFILE_FILE_NAMES +``` + +```` + ``````{py:class} Persona(*args, **kwargs) :canonical: archivebox.personas.models.Persona @@ -77,11 +128,20 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithConfig` ```` +````{py:attribute} permissions +:canonical: archivebox.personas.models.Persona.permissions +:value: > + 'GeneratedField(...)' + +```{autodoc2-docstring} archivebox.personas.models.Persona.permissions +``` + +```` + `````{py:class} Meta :canonical: archivebox.personas.models.Persona.Meta -```{autodoc2-docstring} archivebox.personas.models.Persona.Meta -``` +Bases: {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta` ````{py:attribute} app_label :canonical: archivebox.personas.models.Persona.Meta.app_label @@ -95,6 +155,11 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithConfig` ````` +````{py:method} save(*args, **kwargs) +:canonical: archivebox.personas.models.Persona.save + +```` + ````{py:method} __str__() -> str :canonical: archivebox.personas.models.Persona.__str__ @@ -145,6 +210,15 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithConfig` ```` +````{py:property} AUTH_STORAGE_FILE +:canonical: archivebox.personas.models.Persona.AUTH_STORAGE_FILE +:type: str + +```{autodoc2-docstring} archivebox.personas.models.Persona.AUTH_STORAGE_FILE +``` + +```` + ````{py:method} get_derived_config() -> dict :canonical: archivebox.personas.models.Persona.get_derived_config @@ -161,6 +235,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithConfig` ```` +````{py:method} cleanup_chrome_profile(profile_dir: pathlib.Path) -> bool +:canonical: archivebox.personas.models.Persona.cleanup_chrome_profile + +```{autodoc2-docstring} archivebox.personas.models.Persona.cleanup_chrome_profile +``` + +```` + ````{py:method} cleanup_chrome() -> bool :canonical: archivebox.personas.models.Persona.cleanup_chrome @@ -169,6 +251,62 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithConfig` ```` +````{py:method} lock_runtime_for_crawl() +:canonical: archivebox.personas.models.Persona.lock_runtime_for_crawl + +```{autodoc2-docstring} archivebox.personas.models.Persona.lock_runtime_for_crawl +``` + +```` + +````{py:method} runtime_root_for_crawl(crawl) -> pathlib.Path +:canonical: archivebox.personas.models.Persona.runtime_root_for_crawl + +```{autodoc2-docstring} archivebox.personas.models.Persona.runtime_root_for_crawl +``` + +```` + +````{py:method} runtime_profile_dir_for_crawl(crawl) -> pathlib.Path +:canonical: archivebox.personas.models.Persona.runtime_profile_dir_for_crawl + +```{autodoc2-docstring} archivebox.personas.models.Persona.runtime_profile_dir_for_crawl +``` + +```` + +````{py:method} runtime_downloads_dir_for_crawl(crawl) -> pathlib.Path +:canonical: archivebox.personas.models.Persona.runtime_downloads_dir_for_crawl + +```{autodoc2-docstring} archivebox.personas.models.Persona.runtime_downloads_dir_for_crawl +``` + +```` + +````{py:method} copy_chrome_profile(source_dir: pathlib.Path, destination_dir: pathlib.Path) -> None +:canonical: archivebox.personas.models.Persona.copy_chrome_profile + +```{autodoc2-docstring} archivebox.personas.models.Persona.copy_chrome_profile +``` + +```` + +````{py:method} prepare_runtime_for_crawl(crawl, chrome_binary: str = '') -> dict[str, str] +:canonical: archivebox.personas.models.Persona.prepare_runtime_for_crawl + +```{autodoc2-docstring} archivebox.personas.models.Persona.prepare_runtime_for_crawl +``` + +```` + +````{py:method} cleanup_runtime_for_crawl(crawl) -> None +:canonical: archivebox.personas.models.Persona.cleanup_runtime_for_crawl + +```{autodoc2-docstring} archivebox.personas.models.Persona.cleanup_runtime_for_crawl +``` + +```` + ````{py:method} get_or_create_default() -> archivebox.personas.models.Persona :canonical: archivebox.personas.models.Persona.get_or_create_default :classmethod: diff --git a/docs/apidocs/archivebox/archivebox.personas.runtime.md b/docs/apidocs/archivebox/archivebox.personas.runtime.md deleted file mode 100644 index 63d6df93..00000000 --- a/docs/apidocs/archivebox/archivebox.personas.runtime.md +++ /dev/null @@ -1,111 +0,0 @@ -# {py:mod}`archivebox.personas.runtime` - -```{py:module} archivebox.personas.runtime -``` - -```{autodoc2-docstring} archivebox.personas.runtime -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`crawl_persona_lock ` - - ```{autodoc2-docstring} archivebox.personas.runtime.crawl_persona_lock - :summary: - ``` -* - {py:obj}`_remove_volatile_profile_state ` - - ```{autodoc2-docstring} archivebox.personas.runtime._remove_volatile_profile_state - :summary: - ``` -* - {py:obj}`_copy_profile_template ` - - ```{autodoc2-docstring} archivebox.personas.runtime._copy_profile_template - :summary: - ``` -* - {py:obj}`prepare_crawl_persona_runtime ` - - ```{autodoc2-docstring} archivebox.personas.runtime.prepare_crawl_persona_runtime - :summary: - ``` -* - {py:obj}`cleanup_crawl_persona_runtime ` - - ```{autodoc2-docstring} archivebox.personas.runtime.cleanup_crawl_persona_runtime - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`VOLATILE_PROFILE_DIR_NAMES ` - - ```{autodoc2-docstring} archivebox.personas.runtime.VOLATILE_PROFILE_DIR_NAMES - :summary: - ``` -* - {py:obj}`VOLATILE_PROFILE_FILE_NAMES ` - - ```{autodoc2-docstring} archivebox.personas.runtime.VOLATILE_PROFILE_FILE_NAMES - :summary: - ``` -```` - -### API - -````{py:data} VOLATILE_PROFILE_DIR_NAMES -:canonical: archivebox.personas.runtime.VOLATILE_PROFILE_DIR_NAMES -:value: > - None - -```{autodoc2-docstring} archivebox.personas.runtime.VOLATILE_PROFILE_DIR_NAMES -``` - -```` - -````{py:data} VOLATILE_PROFILE_FILE_NAMES -:canonical: archivebox.personas.runtime.VOLATILE_PROFILE_FILE_NAMES -:value: > - None - -```{autodoc2-docstring} archivebox.personas.runtime.VOLATILE_PROFILE_FILE_NAMES -``` - -```` - -````{py:function} crawl_persona_lock(persona_path: pathlib.Path) -:canonical: archivebox.personas.runtime.crawl_persona_lock - -```{autodoc2-docstring} archivebox.personas.runtime.crawl_persona_lock -``` -```` - -````{py:function} _remove_volatile_profile_state(profile_dir: pathlib.Path) -> None -:canonical: archivebox.personas.runtime._remove_volatile_profile_state - -```{autodoc2-docstring} archivebox.personas.runtime._remove_volatile_profile_state -``` -```` - -````{py:function} _copy_profile_template(source_dir: pathlib.Path, destination_dir: pathlib.Path) -> None -:canonical: archivebox.personas.runtime._copy_profile_template - -```{autodoc2-docstring} archivebox.personas.runtime._copy_profile_template -``` -```` - -````{py:function} prepare_crawl_persona_runtime(crawl, persona, chrome_binary: str = '') -> dict[str, str] -:canonical: archivebox.personas.runtime.prepare_crawl_persona_runtime - -```{autodoc2-docstring} archivebox.personas.runtime.prepare_crawl_persona_runtime -``` -```` - -````{py:function} cleanup_crawl_persona_runtime(crawl) -> None -:canonical: archivebox.personas.runtime.cleanup_crawl_persona_runtime - -```{autodoc2-docstring} archivebox.personas.runtime.cleanup_crawl_persona_runtime -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.personas.views.md b/docs/apidocs/archivebox/archivebox.personas.views.md deleted file mode 100644 index 70b872bc..00000000 --- a/docs/apidocs/archivebox/archivebox.personas.views.md +++ /dev/null @@ -1,8 +0,0 @@ -# {py:mod}`archivebox.personas.views` - -```{py:module} archivebox.personas.views -``` - -```{autodoc2-docstring} archivebox.personas.views -:allowtitles: -``` diff --git a/docs/apidocs/archivebox/archivebox.search.admin.md b/docs/apidocs/archivebox/archivebox.search.admin.md index b2858cb2..850795db 100644 --- a/docs/apidocs/archivebox/archivebox.search.admin.md +++ b/docs/apidocs/archivebox/archivebox.search.admin.md @@ -15,17 +15,133 @@ :class: autosummary longtable :align: left +* - {py:obj}`SearchResultsChangeList ` + - ```{autodoc2-docstring} archivebox.search.admin.SearchResultsChangeList + :summary: + ``` * - {py:obj}`SearchResultsAdminMixin ` - ```` +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`get_admin_search_cache_key ` + - ```{autodoc2-docstring} archivebox.search.admin.get_admin_search_cache_key + :summary: + ``` +* - {py:obj}`get_cached_admin_search_ids ` + - ```{autodoc2-docstring} archivebox.search.admin.get_cached_admin_search_ids + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`SEARCH_RESULT_CACHE_TTL ` + - ```{autodoc2-docstring} archivebox.search.admin.SEARCH_RESULT_CACHE_TTL + :summary: + ``` +```` + ### API +````{py:data} SEARCH_RESULT_CACHE_TTL +:canonical: archivebox.search.admin.SEARCH_RESULT_CACHE_TTL +:value: > + 60 + +```{autodoc2-docstring} archivebox.search.admin.SEARCH_RESULT_CACHE_TTL +``` + +```` + +````{py:function} get_admin_search_cache_key(request, url: str | None = None) -> str +:canonical: archivebox.search.admin.get_admin_search_cache_key + +```{autodoc2-docstring} archivebox.search.admin.get_admin_search_cache_key +``` +```` + +````{py:function} get_cached_admin_search_ids(request) -> list[str] | None +:canonical: archivebox.search.admin.get_cached_admin_search_ids + +```{autodoc2-docstring} archivebox.search.admin.get_cached_admin_search_ids +``` +```` + +`````{py:class} SearchResultsChangeList(request, *args, **kwargs) +:canonical: archivebox.search.admin.SearchResultsChangeList + +Bases: {py:obj}`django.contrib.admin.views.main.ChangeList` + +```{autodoc2-docstring} archivebox.search.admin.SearchResultsChangeList +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.search.admin.SearchResultsChangeList.__init__ +``` + +````{py:method} get_results(request) +:canonical: archivebox.search.admin.SearchResultsChangeList.get_results + +```{autodoc2-docstring} archivebox.search.admin.SearchResultsChangeList.get_results +``` + +```` + +````{py:method} get_filters_params(params=None) +:canonical: archivebox.search.admin.SearchResultsChangeList.get_filters_params + +```` + +````` + `````{py:class} SearchResultsAdminMixin(model, admin_site) :canonical: archivebox.search.admin.SearchResultsAdminMixin Bases: {py:obj}`django.contrib.admin.ModelAdmin` +````{py:attribute} show_search_mode_selector +:canonical: archivebox.search.admin.SearchResultsAdminMixin.show_search_mode_selector +:value: > + True + +```{autodoc2-docstring} archivebox.search.admin.SearchResultsAdminMixin.show_search_mode_selector +``` + +```` + +````{py:method} get_changelist(request, **kwargs) +:canonical: archivebox.search.admin.SearchResultsAdminMixin.get_changelist + +```` + +````{py:method} get_default_search_mode() +:canonical: archivebox.search.admin.SearchResultsAdminMixin.get_default_search_mode + +```{autodoc2-docstring} archivebox.search.admin.SearchResultsAdminMixin.get_default_search_mode +``` + +```` + +````{py:method} get_search_mode_options() +:canonical: archivebox.search.admin.SearchResultsAdminMixin.get_search_mode_options + +```{autodoc2-docstring} archivebox.search.admin.SearchResultsAdminMixin.get_search_mode_options +``` + +```` + ````{py:method} get_search_results(request, queryset, search_term: str) :canonical: archivebox.search.admin.SearchResultsAdminMixin.get_search_results diff --git a/docs/apidocs/archivebox/archivebox.search.md b/docs/apidocs/archivebox/archivebox.search.md index 993b77c7..0d80b205 100644 --- a/docs/apidocs/archivebox/archivebox.search.md +++ b/docs/apidocs/archivebox/archivebox.search.md @@ -13,6 +13,7 @@ :titlesonly: :maxdepth: 1 +archivebox.search.sonic_daemon archivebox.search.admin ``` @@ -24,6 +25,42 @@ archivebox.search.admin :class: autosummary longtable :align: left +* - {py:obj}`search_backend_env ` + - ```{autodoc2-docstring} archivebox.search.search_backend_env + :summary: + ``` +* - {py:obj}`normalize_search_backend_name ` + - ```{autodoc2-docstring} archivebox.search.normalize_search_backend_name + :summary: + ``` +* - {py:obj}`get_search_backend_display_name ` + - ```{autodoc2-docstring} archivebox.search.get_search_backend_display_name + :summary: + ``` +* - {py:obj}`get_default_search_mode ` + - ```{autodoc2-docstring} archivebox.search.get_default_search_mode + :summary: + ``` +* - {py:obj}`get_search_mode ` + - ```{autodoc2-docstring} archivebox.search.get_search_mode + :summary: + ``` +* - {py:obj}`get_search_mode_base ` + - ```{autodoc2-docstring} archivebox.search.get_search_mode_base + :summary: + ``` +* - {py:obj}`get_search_mode_backend ` + - ```{autodoc2-docstring} archivebox.search.get_search_mode_backend + :summary: + ``` +* - {py:obj}`get_search_mode_options ` + - ```{autodoc2-docstring} archivebox.search.get_search_mode_options + :summary: + ``` +* - {py:obj}`prioritize_metadata_matches ` + - ```{autodoc2-docstring} archivebox.search.prioritize_metadata_matches + :summary: + ``` * - {py:obj}`get_available_backends ` - ```{autodoc2-docstring} archivebox.search.get_available_backends :summary: @@ -36,6 +73,10 @@ archivebox.search.admin - ```{autodoc2-docstring} archivebox.search.query_search_index :summary: ``` +* - {py:obj}`iter_query_search_ids ` + - ```{autodoc2-docstring} archivebox.search.iter_query_search_ids + :summary: + ``` * - {py:obj}`flush_search_index ` - ```{autodoc2-docstring} archivebox.search.flush_search_index :summary: @@ -52,13 +93,25 @@ archivebox.search.admin - ```{autodoc2-docstring} archivebox.search._search_backends_cache :summary: ``` +* - {py:obj}`SEARCH_MODES ` + - ```{autodoc2-docstring} archivebox.search.SEARCH_MODES + :summary: + ``` +* - {py:obj}`SEARCH_BACKEND_UI_NAMES ` + - ```{autodoc2-docstring} archivebox.search.SEARCH_BACKEND_UI_NAMES + :summary: + ``` +* - {py:obj}`MAX_SEARCH_RANK_IDS ` + - ```{autodoc2-docstring} archivebox.search.MAX_SEARCH_RANK_IDS + :summary: + ``` ```` ### API ````{py:data} _search_backends_cache :canonical: archivebox.search._search_backends_cache -:type: typing.Optional[dict] +:type: dict | None :value: > None @@ -67,6 +120,99 @@ archivebox.search.admin ```` +````{py:data} SEARCH_MODES +:canonical: archivebox.search.SEARCH_MODES +:value: > + ('meta', 'contents', 'deep') + +```{autodoc2-docstring} archivebox.search.SEARCH_MODES +``` + +```` + +````{py:data} SEARCH_BACKEND_UI_NAMES +:canonical: archivebox.search.SEARCH_BACKEND_UI_NAMES +:value: > + None + +```{autodoc2-docstring} archivebox.search.SEARCH_BACKEND_UI_NAMES +``` + +```` + +````{py:data} MAX_SEARCH_RANK_IDS +:canonical: archivebox.search.MAX_SEARCH_RANK_IDS +:value: > + 500 + +```{autodoc2-docstring} archivebox.search.MAX_SEARCH_RANK_IDS +``` + +```` + +````{py:function} search_backend_env(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) +:canonical: archivebox.search.search_backend_env + +```{autodoc2-docstring} archivebox.search.search_backend_env +``` +```` + +````{py:function} normalize_search_backend_name(backend_name: str | None) -> str +:canonical: archivebox.search.normalize_search_backend_name + +```{autodoc2-docstring} archivebox.search.normalize_search_backend_name +``` +```` + +````{py:function} get_search_backend_display_name(backend_name: str) -> str +:canonical: archivebox.search.get_search_backend_display_name + +```{autodoc2-docstring} archivebox.search.get_search_backend_display_name +``` +```` + +````{py:function} get_default_search_mode(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.search.get_default_search_mode + +```{autodoc2-docstring} archivebox.search.get_default_search_mode +``` +```` + +````{py:function} get_search_mode(search_mode: str | None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.search.get_search_mode + +```{autodoc2-docstring} archivebox.search.get_search_mode +``` +```` + +````{py:function} get_search_mode_base(search_mode: str | None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str +:canonical: archivebox.search.get_search_mode_base + +```{autodoc2-docstring} archivebox.search.get_search_mode_base +``` +```` + +````{py:function} get_search_mode_backend(search_mode: str | None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str | None +:canonical: archivebox.search.get_search_mode_backend + +```{autodoc2-docstring} archivebox.search.get_search_mode_backend +``` +```` + +````{py:function} get_search_mode_options(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> list[dict[str, str]] +:canonical: archivebox.search.get_search_mode_options + +```{autodoc2-docstring} archivebox.search.get_search_mode_options +``` +```` + +````{py:function} prioritize_metadata_matches(base_queryset: django.db.models.QuerySet, metadata_queryset: django.db.models.QuerySet, fulltext_queryset: django.db.models.QuerySet, *, deep_queryset: django.db.models.QuerySet | None = None, ordering: list[str] | tuple[str, ...] | None = None) -> django.db.models.QuerySet +:canonical: archivebox.search.prioritize_metadata_matches + +```{autodoc2-docstring} archivebox.search.prioritize_metadata_matches +``` +```` + ````{py:function} get_available_backends() -> dict :canonical: archivebox.search.get_available_backends @@ -74,21 +220,28 @@ archivebox.search.admin ``` ```` -````{py:function} get_backend() -> typing.Any +````{py:function} get_backend(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> typing.Any :canonical: archivebox.search.get_backend ```{autodoc2-docstring} archivebox.search.get_backend ``` ```` -````{py:function} query_search_index(query: str) -> django.db.models.QuerySet +````{py:function} query_search_index(query: str, search_mode: str | None = None, config: dict[str, typing.Any] | None = None, max_results: int | None = None, **config_kwargs: typing.Any) -> django.db.models.QuerySet :canonical: archivebox.search.query_search_index ```{autodoc2-docstring} archivebox.search.query_search_index ``` ```` -````{py:function} flush_search_index(snapshots: django.db.models.QuerySet) -> None +````{py:function} iter_query_search_ids(query: str, search_mode: str | None = None, config: dict[str, typing.Any] | None = None, max_results: int | None = None, **config_kwargs: typing.Any) +:canonical: archivebox.search.iter_query_search_ids + +```{autodoc2-docstring} archivebox.search.iter_query_search_ids +``` +```` + +````{py:function} flush_search_index(snapshots: django.db.models.QuerySet, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> None :canonical: archivebox.search.flush_search_index ```{autodoc2-docstring} archivebox.search.flush_search_index diff --git a/docs/apidocs/archivebox/archivebox.search.sonic_daemon.md b/docs/apidocs/archivebox/archivebox.search.sonic_daemon.md new file mode 100644 index 00000000..fbd6a55a --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.search.sonic_daemon.md @@ -0,0 +1,31 @@ +# {py:mod}`archivebox.search.sonic_daemon` + +```{py:module} archivebox.search.sonic_daemon +``` + +```{autodoc2-docstring} archivebox.search.sonic_daemon +:allowtitles: +``` + +## Module Contents + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`register_sonic_daemon_event_handler ` + - ```{autodoc2-docstring} archivebox.search.sonic_daemon.register_sonic_daemon_event_handler + :summary: + ``` +```` + +### API + +````{py:function} register_sonic_daemon_event_handler(bus) -> None +:canonical: archivebox.search.sonic_daemon.register_sonic_daemon_event_handler + +```{autodoc2-docstring} archivebox.search.sonic_daemon.register_sonic_daemon_event_handler +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.services.archive_result_service.md b/docs/apidocs/archivebox/archivebox.services.archive_result_service.md new file mode 100644 index 00000000..178722c2 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.archive_result_service.md @@ -0,0 +1,207 @@ +# {py:mod}`archivebox.services.archive_result_service` + +```{py:module} archivebox.services.archive_result_service +``` + +```{autodoc2-docstring} archivebox.services.archive_result_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`ArchiveResultService ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_collect_output_metadata ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._collect_output_metadata + :summary: + ``` +* - {py:obj}`_coerce_output_file_size ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._coerce_output_file_size + :summary: + ``` +* - {py:obj}`_normalize_output_files ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._normalize_output_files + :summary: + ``` +* - {py:obj}`_has_structured_output_metadata ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._has_structured_output_metadata + :summary: + ``` +* - {py:obj}`_summarize_output_files ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._summarize_output_files + :summary: + ``` +* - {py:obj}`_resolve_output_metadata ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._resolve_output_metadata + :summary: + ``` +* - {py:obj}`_normalize_status ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._normalize_status + :summary: + ``` +* - {py:obj}`_normalize_snapshot_title ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._normalize_snapshot_title + :summary: + ``` +* - {py:obj}`_extract_snapshot_title ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._extract_snapshot_title + :summary: + ``` +* - {py:obj}`_should_update_snapshot_title ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._should_update_snapshot_title + :summary: + ``` +* - {py:obj}`_has_content_files ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._has_content_files + :summary: + ``` +* - {py:obj}`_iter_archiveresult_records ` + - ```{autodoc2-docstring} archivebox.services.archive_result_service._iter_archiveresult_records + :summary: + ``` +```` + +### API + +````{py:function} _collect_output_metadata(plugin_dir: pathlib.Path) -> tuple[dict[str, dict], int, str] +:canonical: archivebox.services.archive_result_service._collect_output_metadata + +```{autodoc2-docstring} archivebox.services.archive_result_service._collect_output_metadata +``` +```` + +````{py:function} _coerce_output_file_size(value: typing.Any) -> int +:canonical: archivebox.services.archive_result_service._coerce_output_file_size + +```{autodoc2-docstring} archivebox.services.archive_result_service._coerce_output_file_size +``` +```` + +````{py:function} _normalize_output_files(raw_output_files: typing.Any) -> dict[str, dict] +:canonical: archivebox.services.archive_result_service._normalize_output_files + +```{autodoc2-docstring} archivebox.services.archive_result_service._normalize_output_files +``` +```` + +````{py:function} _has_structured_output_metadata(output_files: dict[str, dict]) -> bool +:canonical: archivebox.services.archive_result_service._has_structured_output_metadata + +```{autodoc2-docstring} archivebox.services.archive_result_service._has_structured_output_metadata +``` +```` + +````{py:function} _summarize_output_files(output_files: dict[str, dict]) -> tuple[int, str] +:canonical: archivebox.services.archive_result_service._summarize_output_files + +```{autodoc2-docstring} archivebox.services.archive_result_service._summarize_output_files +``` +```` + +````{py:function} _resolve_output_metadata(raw_output_files: typing.Any, plugin_dir: pathlib.Path) -> tuple[dict[str, dict], int, str] +:canonical: archivebox.services.archive_result_service._resolve_output_metadata + +```{autodoc2-docstring} archivebox.services.archive_result_service._resolve_output_metadata +``` +```` + +````{py:function} _normalize_status(status: str) -> str +:canonical: archivebox.services.archive_result_service._normalize_status + +```{autodoc2-docstring} archivebox.services.archive_result_service._normalize_status +``` +```` + +````{py:function} _normalize_snapshot_title(candidate: str, *, snapshot_url: str) -> str +:canonical: archivebox.services.archive_result_service._normalize_snapshot_title + +```{autodoc2-docstring} archivebox.services.archive_result_service._normalize_snapshot_title +``` +```` + +````{py:function} _extract_snapshot_title(snapshot_output_dir: str, plugin: str, output_str: str, *, snapshot_url: str) -> str +:canonical: archivebox.services.archive_result_service._extract_snapshot_title + +```{autodoc2-docstring} archivebox.services.archive_result_service._extract_snapshot_title +``` +```` + +````{py:function} _should_update_snapshot_title(current_title: str, next_title: str, *, snapshot_url: str) -> bool +:canonical: archivebox.services.archive_result_service._should_update_snapshot_title + +```{autodoc2-docstring} archivebox.services.archive_result_service._should_update_snapshot_title +``` +```` + +````{py:function} _has_content_files(output_files: typing.Any) -> bool +:canonical: archivebox.services.archive_result_service._has_content_files + +```{autodoc2-docstring} archivebox.services.archive_result_service._has_content_files +``` +```` + +````{py:function} _iter_archiveresult_records(stdout: str) -> list[dict] +:canonical: archivebox.services.archive_result_service._iter_archiveresult_records + +```{autodoc2-docstring} archivebox.services.archive_result_service._iter_archiveresult_records +``` +```` + +`````{py:class} ArchiveResultService(bus) +:canonical: archivebox.services.archive_result_service.ArchiveResultService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.archive_result_service.ArchiveResultService.LISTENS_TO +:value: > + None + +```{autodoc2-docstring} archivebox.services.archive_result_service.ArchiveResultService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.archive_result_service.ArchiveResultService.EMITS +:value: > + [] + +```{autodoc2-docstring} archivebox.services.archive_result_service.ArchiveResultService.EMITS +``` + +```` + +````{py:method} on_ArchiveResultEvent__save_to_db(event: abx_dl.events.ArchiveResultEvent) -> None +:canonical: archivebox.services.archive_result_service.ArchiveResultService.on_ArchiveResultEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.archive_result_service.ArchiveResultService.on_ArchiveResultEvent__save_to_db +``` + +```` + +````{py:method} on_ProcessCompletedEvent__save_to_db(event: abx_dl.events.ProcessCompletedEvent) -> None +:canonical: archivebox.services.archive_result_service.ArchiveResultService.on_ProcessCompletedEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.archive_result_service.ArchiveResultService.on_ProcessCompletedEvent__save_to_db +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.services.binary_service.md b/docs/apidocs/archivebox/archivebox.services.binary_service.md new file mode 100644 index 00000000..22a6aac3 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.binary_service.md @@ -0,0 +1,67 @@ +# {py:mod}`archivebox.services.binary_service` + +```{py:module} archivebox.services.binary_service +``` + +```{autodoc2-docstring} archivebox.services.binary_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`BinaryService ` + - +```` + +### API + +`````{py:class} BinaryService(bus) +:canonical: archivebox.services.binary_service.BinaryService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.binary_service.BinaryService.LISTENS_TO +:value: > + None + +```{autodoc2-docstring} archivebox.services.binary_service.BinaryService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.binary_service.BinaryService.EMITS +:value: > + [] + +```{autodoc2-docstring} archivebox.services.binary_service.BinaryService.EMITS +``` + +```` + +````{py:method} on_BinaryRequestEvent(event: abx_dl.events.BinaryRequestEvent) -> str | None +:canonical: archivebox.services.binary_service.BinaryService.on_BinaryRequestEvent +:async: + +```{autodoc2-docstring} archivebox.services.binary_service.BinaryService.on_BinaryRequestEvent +``` + +```` + +````{py:method} on_BinaryEvent(event: abx_dl.events.BinaryEvent) -> None +:canonical: archivebox.services.binary_service.BinaryService.on_BinaryEvent +:async: + +```{autodoc2-docstring} archivebox.services.binary_service.BinaryService.on_BinaryEvent +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.services.crawl_service.md b/docs/apidocs/archivebox/archivebox.services.crawl_service.md new file mode 100644 index 00000000..9287d06b --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.crawl_service.md @@ -0,0 +1,85 @@ +# {py:mod}`archivebox.services.crawl_service` + +```{py:module} archivebox.services.crawl_service +``` + +```{autodoc2-docstring} archivebox.services.crawl_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`CrawlService ` + - +```` + +### API + +`````{py:class} CrawlService(bus, *, crawl_id: str) +:canonical: archivebox.services.crawl_service.CrawlService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.crawl_service.CrawlService.LISTENS_TO +:value: > + None + +```{autodoc2-docstring} archivebox.services.crawl_service.CrawlService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.crawl_service.CrawlService.EMITS +:value: > + [] + +```{autodoc2-docstring} archivebox.services.crawl_service.CrawlService.EMITS +``` + +```` + +````{py:method} on_CrawlSetupEvent__save_to_db(event: abx_dl.events.CrawlSetupEvent) -> None +:canonical: archivebox.services.crawl_service.CrawlService.on_CrawlSetupEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.crawl_service.CrawlService.on_CrawlSetupEvent__save_to_db +``` + +```` + +````{py:method} on_CrawlStartEvent__save_to_db(event: abx_dl.events.CrawlStartEvent) -> None +:canonical: archivebox.services.crawl_service.CrawlService.on_CrawlStartEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.crawl_service.CrawlService.on_CrawlStartEvent__save_to_db +``` + +```` + +````{py:method} on_CrawlCleanupEvent__save_to_db(event: abx_dl.events.CrawlCleanupEvent) -> None +:canonical: archivebox.services.crawl_service.CrawlService.on_CrawlCleanupEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.crawl_service.CrawlService.on_CrawlCleanupEvent__save_to_db +``` + +```` + +````{py:method} on_CrawlCompletedEvent__save_to_db(event: abx_dl.events.CrawlCompletedEvent) -> None +:canonical: archivebox.services.crawl_service.CrawlService.on_CrawlCompletedEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.crawl_service.CrawlService.on_CrawlCompletedEvent__save_to_db +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.services.machine_service.md b/docs/apidocs/archivebox/archivebox.services.machine_service.md new file mode 100644 index 00000000..c62dba08 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.machine_service.md @@ -0,0 +1,110 @@ +# {py:mod}`archivebox.services.machine_service` + +```{py:module} archivebox.services.machine_service +``` + +```{autodoc2-docstring} archivebox.services.machine_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`MachineService ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_is_binary_event_key ` + - ```{autodoc2-docstring} archivebox.services.machine_service._is_binary_event_key + :summary: + ``` +* - {py:obj}`_strip_to_binary_keys ` + - ```{autodoc2-docstring} archivebox.services.machine_service._strip_to_binary_keys + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_BINARY_EVENT_ALLOWED_KEYS ` + - ```{autodoc2-docstring} archivebox.services.machine_service._BINARY_EVENT_ALLOWED_KEYS + :summary: + ``` +```` + +### API + +````{py:data} _BINARY_EVENT_ALLOWED_KEYS +:canonical: archivebox.services.machine_service._BINARY_EVENT_ALLOWED_KEYS +:value: > + 'frozenset(...)' + +```{autodoc2-docstring} archivebox.services.machine_service._BINARY_EVENT_ALLOWED_KEYS +``` + +```` + +````{py:function} _is_binary_event_key(key: str) -> bool +:canonical: archivebox.services.machine_service._is_binary_event_key + +```{autodoc2-docstring} archivebox.services.machine_service._is_binary_event_key +``` +```` + +````{py:function} _strip_to_binary_keys(config: dict[str, typing.Any] | None) -> dict[str, typing.Any] +:canonical: archivebox.services.machine_service._strip_to_binary_keys + +```{autodoc2-docstring} archivebox.services.machine_service._strip_to_binary_keys +``` +```` + +`````{py:class} MachineService(bus) +:canonical: archivebox.services.machine_service.MachineService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.machine_service.MachineService.LISTENS_TO +:value: > + None + +```{autodoc2-docstring} archivebox.services.machine_service.MachineService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.machine_service.MachineService.EMITS +:value: > + [] + +```{autodoc2-docstring} archivebox.services.machine_service.MachineService.EMITS +``` + +```` + +````{py:method} on_MachineEvent__save_to_db(event: abx_dl.events.MachineEvent) -> None +:canonical: archivebox.services.machine_service.MachineService.on_MachineEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.machine_service.MachineService.on_MachineEvent__save_to_db +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.services.md b/docs/apidocs/archivebox/archivebox.services.md new file mode 100644 index 00000000..501f63c3 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.md @@ -0,0 +1,50 @@ +# {py:mod}`archivebox.services` + +```{py:module} archivebox.services +``` + +```{autodoc2-docstring} archivebox.services +:allowtitles: +``` + +## Submodules + +```{toctree} +:titlesonly: +:maxdepth: 1 + +archivebox.services.binary_service +archivebox.services.runner +archivebox.services.process_service +archivebox.services.crawl_service +archivebox.services.tag_service +archivebox.services.snapshot_service +archivebox.services.archive_result_service +archivebox.services.machine_service +``` + +## Package Contents + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`__all__ ` + - ```{autodoc2-docstring} archivebox.services.__all__ + :summary: + ``` +```` + +### API + +````{py:data} __all__ +:canonical: archivebox.services.__all__ +:value: > + ['ArchiveResultService', 'BinaryService', 'CrawlService', 'MachineService', 'ProcessService', 'Snaps... + +```{autodoc2-docstring} archivebox.services.__all__ +``` + +```` diff --git a/docs/apidocs/archivebox/archivebox.services.process_service.md b/docs/apidocs/archivebox/archivebox.services.process_service.md new file mode 100644 index 00000000..e29058c3 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.process_service.md @@ -0,0 +1,152 @@ +# {py:mod}`archivebox.services.process_service` + +```{py:module} archivebox.services.process_service +``` + +```{autodoc2-docstring} archivebox.services.process_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`ProcessService ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`parse_event_datetime ` + - ```{autodoc2-docstring} archivebox.services.process_service.parse_event_datetime + :summary: + ``` +* - {py:obj}`current_network_interface_with_machine ` + - ```{autodoc2-docstring} archivebox.services.process_service.current_network_interface_with_machine + :summary: + ``` +```` + +### API + +````{py:function} parse_event_datetime(value: str | None) +:canonical: archivebox.services.process_service.parse_event_datetime + +```{autodoc2-docstring} archivebox.services.process_service.parse_event_datetime +``` +```` + +````{py:function} current_network_interface_with_machine() +:canonical: archivebox.services.process_service.current_network_interface_with_machine + +```{autodoc2-docstring} archivebox.services.process_service.current_network_interface_with_machine +``` +```` + +`````{py:class} ProcessService(bus) +:canonical: archivebox.services.process_service.ProcessService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.process_service.ProcessService.LISTENS_TO +:type: typing.ClassVar[list[type[abxbus.BaseEvent]]] +:value: > + None + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.process_service.ProcessService.EMITS +:type: typing.ClassVar[list[type[abxbus.BaseEvent]]] +:value: > + [] + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.EMITS +``` + +```` + +````{py:method} current_iface() +:canonical: archivebox.services.process_service.ProcessService.current_iface +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.current_iface +``` + +```` + +````{py:method} on_ProcessStartedEvent__save_to_db(event: abx_dl.events.ProcessStartedEvent) -> None +:canonical: archivebox.services.process_service.ProcessService.on_ProcessStartedEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.on_ProcessStartedEvent__save_to_db +``` + +```` + +````{py:method} _completed_worker_loop() -> None +:canonical: archivebox.services.process_service.ProcessService._completed_worker_loop +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService._completed_worker_loop +``` + +```` + +````{py:method} _ensure_completed_worker() -> None +:canonical: archivebox.services.process_service.ProcessService._ensure_completed_worker + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService._ensure_completed_worker +``` + +```` + +````{py:method} on_ProcessCompletedEvent__save_to_db(event: abx_dl.events.ProcessCompletedEvent) -> None +:canonical: archivebox.services.process_service.ProcessService.on_ProcessCompletedEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.on_ProcessCompletedEvent__save_to_db +``` + +```` + +````{py:method} on_CrawlCleanupEvent__flush_completed(event: abx_dl.events.CrawlCleanupEvent) -> None +:canonical: archivebox.services.process_service.ProcessService.on_CrawlCleanupEvent__flush_completed +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.on_CrawlCleanupEvent__flush_completed +``` + +```` + +````{py:method} on_CrawlCompletedEvent__flush_completed(event: abx_dl.events.CrawlCompletedEvent) -> None +:canonical: archivebox.services.process_service.ProcessService.on_CrawlCompletedEvent__flush_completed +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService.on_CrawlCompletedEvent__flush_completed +``` + +```` + +````{py:method} _save_completed_process_to_db(event: abx_dl.events.ProcessCompletedEvent) -> None +:canonical: archivebox.services.process_service.ProcessService._save_completed_process_to_db +:async: + +```{autodoc2-docstring} archivebox.services.process_service.ProcessService._save_completed_process_to_db +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.services.runner.md b/docs/apidocs/archivebox/archivebox.services.runner.md new file mode 100644 index 00000000..1dca66a9 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.runner.md @@ -0,0 +1,551 @@ +# {py:mod}`archivebox.services.runner` + +```{py:module} archivebox.services.runner +``` + +```{autodoc2-docstring} archivebox.services.runner +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`CrawlRunner ` + - ```{autodoc2-docstring} archivebox.services.runner.CrawlRunner + :summary: + ``` +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`_bus_name ` + - ```{autodoc2-docstring} archivebox.services.runner._bus_name + :summary: + ``` +* - {py:obj}`_runner_short_id ` + - ```{autodoc2-docstring} archivebox.services.runner._runner_short_id + :summary: + ``` +* - {py:obj}`_runner_label ` + - ```{autodoc2-docstring} archivebox.services.runner._runner_label + :summary: + ``` +* - {py:obj}`_runner_console_line ` + - ```{autodoc2-docstring} archivebox.services.runner._runner_console_line + :summary: + ``` +* - {py:obj}`_count_selected_hooks ` + - ```{autodoc2-docstring} archivebox.services.runner._count_selected_hooks + :summary: + ``` +* - {py:obj}`_normalize_runtime_config ` + - ```{autodoc2-docstring} archivebox.services.runner._normalize_runtime_config + :summary: + ``` +* - {py:obj}`_runner_task_context ` + - ```{autodoc2-docstring} archivebox.services.runner._runner_task_context + :summary: + ``` +* - {py:obj}`_is_external_task_cancelled ` + - ```{autodoc2-docstring} archivebox.services.runner._is_external_task_cancelled + :summary: + ``` +* - {py:obj}`_emit_machine_config ` + - ```{autodoc2-docstring} archivebox.services.runner._emit_machine_config + :summary: + ``` +* - {py:obj}`_run_event_now ` + - ```{autodoc2-docstring} archivebox.services.runner._run_event_now + :summary: + ``` +* - {py:obj}`ensure_background_runner ` + - ```{autodoc2-docstring} archivebox.services.runner.ensure_background_runner + :summary: + ``` +* - {py:obj}`run_crawl ` + - ```{autodoc2-docstring} archivebox.services.runner.run_crawl + :summary: + ``` +* - {py:obj}`_run_binary ` + - ```{autodoc2-docstring} archivebox.services.runner._run_binary + :summary: + ``` +* - {py:obj}`run_binary ` + - ```{autodoc2-docstring} archivebox.services.runner.run_binary + :summary: + ``` +* - {py:obj}`_snapshot_hook_names_by_plugin ` + - ```{autodoc2-docstring} archivebox.services.runner._snapshot_hook_names_by_plugin + :summary: + ``` +* - {py:obj}`queued_plugins_for_snapshot ` + - ```{autodoc2-docstring} archivebox.services.runner.queued_plugins_for_snapshot + :summary: + ``` +* - {py:obj}`run_snapshot_maintenance ` + - ```{autodoc2-docstring} archivebox.services.runner.run_snapshot_maintenance + :summary: + ``` +* - {py:obj}`run_due_crawl ` + - ```{autodoc2-docstring} archivebox.services.runner.run_due_crawl + :summary: + ``` +* - {py:obj}`run_due_snapshot ` + - ```{autodoc2-docstring} archivebox.services.runner.run_due_snapshot + :summary: + ``` +* - {py:obj}`run_due_binary ` + - ```{autodoc2-docstring} archivebox.services.runner.run_due_binary + :summary: + ``` +* - {py:obj}`_run_install ` + - ```{autodoc2-docstring} archivebox.services.runner._run_install + :summary: + ``` +* - {py:obj}`run_install ` + - ```{autodoc2-docstring} archivebox.services.runner.run_install + :summary: + ``` +* - {py:obj}`_first_due_id ` + - ```{autodoc2-docstring} archivebox.services.runner._first_due_id + :summary: + ``` +* - {py:obj}`_run_due_crawl_status ` + - ```{autodoc2-docstring} archivebox.services.runner._run_due_crawl_status + :summary: + ``` +* - {py:obj}`_run_due_snapshot_query ` + - ```{autodoc2-docstring} archivebox.services.runner._run_due_snapshot_query + :summary: + ``` +* - {py:obj}`_run_due_snapshot_id ` + - ```{autodoc2-docstring} archivebox.services.runner._run_due_snapshot_id + :summary: + ``` +* - {py:obj}`_run_due_queued_download_result ` + - ```{autodoc2-docstring} archivebox.services.runner._run_due_queued_download_result + :summary: + ``` +* - {py:obj}`_run_due_binary ` + - ```{autodoc2-docstring} archivebox.services.runner._run_due_binary + :summary: + ``` +* - {py:obj}`run_pending_crawls ` + - ```{autodoc2-docstring} archivebox.services.runner.run_pending_crawls + :summary: + ``` +```` + +### API + +````{py:function} _bus_name(prefix: str, identifier: str) -> str +:canonical: archivebox.services.runner._bus_name + +```{autodoc2-docstring} archivebox.services.runner._bus_name +``` +```` + +````{py:function} _runner_short_id(identifier) -> str +:canonical: archivebox.services.runner._runner_short_id + +```{autodoc2-docstring} archivebox.services.runner._runner_short_id +``` +```` + +````{py:function} _runner_label(value: str, *, reserve: int) -> str +:canonical: archivebox.services.runner._runner_label + +```{autodoc2-docstring} archivebox.services.runner._runner_label +``` +```` + +````{py:function} _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: str = 'STARTED') -> None +:canonical: archivebox.services.runner._runner_console_line + +```{autodoc2-docstring} archivebox.services.runner._runner_console_line +``` +```` + +````{py:function} _count_selected_hooks(plugins: dict[str, abx_dl.models.Plugin], selected_plugins: list[str] | None) -> int +:canonical: archivebox.services.runner._count_selected_hooks + +```{autodoc2-docstring} archivebox.services.runner._count_selected_hooks +``` +```` + +````{py:function} _normalize_runtime_config(config: archivebox.config.configset.BaseConfigSet | collections.abc.Mapping[str, typing.Any] | str | None) -> dict[str, typing.Any] +:canonical: archivebox.services.runner._normalize_runtime_config + +```{autodoc2-docstring} archivebox.services.runner._normalize_runtime_config +``` +```` + +````{py:function} _runner_task_context() -> contextvars.Context +:canonical: archivebox.services.runner._runner_task_context + +```{autodoc2-docstring} archivebox.services.runner._runner_task_context +``` +```` + +````{py:function} _is_external_task_cancelled(error: asyncio.CancelledError) -> bool +:canonical: archivebox.services.runner._is_external_task_cancelled + +```{autodoc2-docstring} archivebox.services.runner._is_external_task_cancelled +``` +```` + +````{py:function} _emit_machine_config(bus, *, config: dict[str, typing.Any], derived_config: dict[str, typing.Any], parent_event=None) -> None +:canonical: archivebox.services.runner._emit_machine_config +:async: + +```{autodoc2-docstring} archivebox.services.runner._emit_machine_config +``` +```` + +````{py:function} _run_event_now(event, timeout: float | None = None) +:canonical: archivebox.services.runner._run_event_now +:async: + +```{autodoc2-docstring} archivebox.services.runner._run_event_now +``` +```` + +````{py:function} ensure_background_runner(*, allow_under_pytest: bool = False) -> bool +:canonical: archivebox.services.runner.ensure_background_runner + +```{autodoc2-docstring} archivebox.services.runner.ensure_background_runner +``` +```` + +`````{py:class} CrawlRunner(crawl, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, show_progress: bool = True, interactive_interrupts: bool = False) +:canonical: archivebox.services.runner.CrawlRunner + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.__init__ +``` + +````{py:method} _request_abort_from_signal(_sig: signal.Signals) -> None +:canonical: archivebox.services.runner.CrawlRunner._request_abort_from_signal + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner._request_abort_from_signal +``` + +```` + +````{py:method} crawl_is_cancelled() -> bool +:canonical: archivebox.services.runner.CrawlRunner.crawl_is_cancelled +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.crawl_is_cancelled +``` + +```` + +````{py:method} crawl_is_paused() -> bool +:canonical: archivebox.services.runner.CrawlRunner.crawl_is_paused +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.crawl_is_paused +``` + +```` + +````{py:method} watch_for_cancelled_crawl(parent_event: abxbus.BaseEvent, *, poll_interval: float = 1.0) -> None +:canonical: archivebox.services.runner.CrawlRunner.watch_for_cancelled_crawl +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.watch_for_cancelled_crawl +``` + +```` + +````{py:method} runtime_plugins() -> dict[str, abx_dl.models.Plugin] +:canonical: archivebox.services.runner.CrawlRunner.runtime_plugins + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.runtime_plugins +``` + +```` + +````{py:property} allow_maintenance_on_inactive_crawl +:canonical: archivebox.services.runner.CrawlRunner.allow_maintenance_on_inactive_crawl +:type: bool + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.allow_maintenance_on_inactive_crawl +``` + +```` + +````{py:method} run() -> None +:canonical: archivebox.services.runner.CrawlRunner.run +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.run +``` + +```` + +````{py:method} enqueue_snapshot(snapshot_id: str, crawl_start_event: abx_dl.events.CrawlStartEvent | None = None) -> None +:canonical: archivebox.services.runner.CrawlRunner.enqueue_snapshot +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.enqueue_snapshot +``` + +```` + +````{py:method} stop_snapshot_tasks() -> None +:canonical: archivebox.services.runner.CrawlRunner.stop_snapshot_tasks +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.stop_snapshot_tasks +``` + +```` + +````{py:method} wait_for_snapshot_tasks() -> None +:canonical: archivebox.services.runner.CrawlRunner.wait_for_snapshot_tasks +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.wait_for_snapshot_tasks +``` + +```` + +````{py:method} heartbeat_active_leases() -> None +:canonical: archivebox.services.runner.CrawlRunner.heartbeat_active_leases +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.heartbeat_active_leases +``` + +```` + +````{py:method} drain_snapshot_tasks() -> None +:canonical: archivebox.services.runner.CrawlRunner.drain_snapshot_tasks +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.drain_snapshot_tasks +``` + +```` + +````{py:method} enqueue_pending_snapshots_from_projection() -> None +:canonical: archivebox.services.runner.CrawlRunner.enqueue_pending_snapshots_from_projection +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.enqueue_pending_snapshots_from_projection +``` + +```` + +````{py:method} load_run_state() -> list[str] +:canonical: archivebox.services.runner.CrawlRunner.load_run_state + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.load_run_state +``` + +```` + +````{py:method} finalize_run_state() -> None +:canonical: archivebox.services.runner.CrawlRunner.finalize_run_state + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.finalize_run_state +``` + +```` + +````{py:method} _create_live_ui() -> abx_dl.cli.LiveBusUI | None +:canonical: archivebox.services.runner.CrawlRunner._create_live_ui + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner._create_live_ui +``` + +```` + +````{py:method} load_snapshot_payload(snapshot_id: str) -> dict[str, typing.Any] +:canonical: archivebox.services.runner.CrawlRunner.load_snapshot_payload + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.load_snapshot_payload +``` + +```` + +````{py:method} enqueue_discovered_snapshots_from_outputs(snapshot_payload: dict[str, typing.Any]) -> None +:canonical: archivebox.services.runner.CrawlRunner.enqueue_discovered_snapshots_from_outputs +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.enqueue_discovered_snapshots_from_outputs +``` + +```` + +````{py:method} run_crawl(root_snapshot_id: str, snapshot_ids: list[str]) -> None +:canonical: archivebox.services.runner.CrawlRunner.run_crawl +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.run_crawl +``` + +```` + +````{py:method} run_snapshot(snapshot_id: str, crawl_start_event: abx_dl.events.CrawlStartEvent | None = None) -> None +:canonical: archivebox.services.runner.CrawlRunner.run_snapshot +:async: + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.run_snapshot +``` + +```` + +````{py:method} seal_snapshot_due_to_limit(snapshot_id: str) -> None +:canonical: archivebox.services.runner.CrawlRunner.seal_snapshot_due_to_limit + +```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.seal_snapshot_due_to_limit +``` + +```` + +````` + +````{py:function} run_crawl(crawl_id: str, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, show_progress: bool = True, interactive_interrupts: bool = False) -> None +:canonical: archivebox.services.runner.run_crawl + +```{autodoc2-docstring} archivebox.services.runner.run_crawl +``` +```` + +````{py:function} _run_binary(binary_id: str) -> None +:canonical: archivebox.services.runner._run_binary +:async: + +```{autodoc2-docstring} archivebox.services.runner._run_binary +``` +```` + +````{py:function} run_binary(binary_id: str) -> None +:canonical: archivebox.services.runner.run_binary + +```{autodoc2-docstring} archivebox.services.runner.run_binary +``` +```` + +````{py:function} _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]] +:canonical: archivebox.services.runner._snapshot_hook_names_by_plugin + +```{autodoc2-docstring} archivebox.services.runner._snapshot_hook_names_by_plugin +``` +```` + +````{py:function} queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None +:canonical: archivebox.services.runner.queued_plugins_for_snapshot + +```{autodoc2-docstring} archivebox.services.runner.queued_plugins_for_snapshot +``` +```` + +````{py:function} run_snapshot_maintenance(snapshot_id: str) -> bool +:canonical: archivebox.services.runner.run_snapshot_maintenance + +```{autodoc2-docstring} archivebox.services.runner.run_snapshot_maintenance +``` +```` + +````{py:function} run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) -> bool +:canonical: archivebox.services.runner.run_due_crawl + +```{autodoc2-docstring} archivebox.services.runner.run_due_crawl +``` +```` + +````{py:function} run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) -> bool +:canonical: archivebox.services.runner.run_due_snapshot + +```{autodoc2-docstring} archivebox.services.runner.run_due_snapshot +``` +```` + +````{py:function} run_due_binary(binary, *, lock_seconds: int) -> bool +:canonical: archivebox.services.runner.run_due_binary + +```{autodoc2-docstring} archivebox.services.runner.run_due_binary +``` +```` + +````{py:function} _run_install(plugin_names: list[str] | None = None) -> None +:canonical: archivebox.services.runner._run_install +:async: + +```{autodoc2-docstring} archivebox.services.runner._run_install +``` +```` + +````{py:function} run_install(*, plugin_names: list[str] | None = None) -> None +:canonical: archivebox.services.runner.run_install + +```{autodoc2-docstring} archivebox.services.runner.run_install +``` +```` + +````{py:function} _first_due_id(queryset) +:canonical: archivebox.services.runner._first_due_id + +```{autodoc2-docstring} archivebox.services.runner._first_due_id +``` +```` + +````{py:function} _run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: int, interactive_interrupts: bool) -> bool +:canonical: archivebox.services.runner._run_due_crawl_status + +```{autodoc2-docstring} archivebox.services.runner._run_due_crawl_status +``` +```` + +````{py:function} _run_due_snapshot_query(queryset, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool +:canonical: archivebox.services.runner._run_due_snapshot_query + +```{autodoc2-docstring} archivebox.services.runner._run_due_snapshot_query +``` +```` + +````{py:function} _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool +:canonical: archivebox.services.runner._run_due_snapshot_id + +```{autodoc2-docstring} archivebox.services.runner._run_due_snapshot_id +``` +```` + +````{py:function} _run_due_queued_download_result(download_plugin_names: frozenset[str], *, crawl_id: str | None, lock_seconds: int, interactive_interrupts: bool, runtime_config) -> bool +:canonical: archivebox.services.runner._run_due_queued_download_result + +```{autodoc2-docstring} archivebox.services.runner._run_due_queued_download_result +``` +```` + +````{py:function} _run_due_binary() -> bool +:canonical: archivebox.services.runner._run_due_binary + +```{autodoc2-docstring} archivebox.services.runner._run_due_binary +``` +```` + +````{py:function} run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False, interactive_interrupts: bool = False) -> int +:canonical: archivebox.services.runner.run_pending_crawls + +```{autodoc2-docstring} archivebox.services.runner.run_pending_crawls +``` +```` diff --git a/docs/apidocs/archivebox/archivebox.services.snapshot_service.md b/docs/apidocs/archivebox/archivebox.services.snapshot_service.md new file mode 100644 index 00000000..71629010 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.snapshot_service.md @@ -0,0 +1,97 @@ +# {py:mod}`archivebox.services.snapshot_service` + +```{py:module} archivebox.services.snapshot_service +``` + +```{autodoc2-docstring} archivebox.services.snapshot_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`SnapshotService ` + - +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`finalize_completed_snapshot ` + - ```{autodoc2-docstring} archivebox.services.snapshot_service.finalize_completed_snapshot + :summary: + ``` +* - {py:obj}`_crawl_limit_stop_reason ` + - ```{autodoc2-docstring} archivebox.services.snapshot_service._crawl_limit_stop_reason + :summary: + ``` +```` + +### API + +````{py:function} finalize_completed_snapshot(snapshot_id: str) -> None +:canonical: archivebox.services.snapshot_service.finalize_completed_snapshot + +```{autodoc2-docstring} archivebox.services.snapshot_service.finalize_completed_snapshot +``` +```` + +````{py:function} _crawl_limit_stop_reason(crawl) -> str +:canonical: archivebox.services.snapshot_service._crawl_limit_stop_reason + +```{autodoc2-docstring} archivebox.services.snapshot_service._crawl_limit_stop_reason +``` +```` + +`````{py:class} SnapshotService(bus, *, crawl_id: str, schedule_snapshot) +:canonical: archivebox.services.snapshot_service.SnapshotService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.snapshot_service.SnapshotService.LISTENS_TO +:value: > + None + +```{autodoc2-docstring} archivebox.services.snapshot_service.SnapshotService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.snapshot_service.SnapshotService.EMITS +:value: > + [] + +```{autodoc2-docstring} archivebox.services.snapshot_service.SnapshotService.EMITS +``` + +```` + +````{py:method} on_SnapshotEvent(event: abx_dl.events.SnapshotEvent) -> None +:canonical: archivebox.services.snapshot_service.SnapshotService.on_SnapshotEvent +:async: + +```{autodoc2-docstring} archivebox.services.snapshot_service.SnapshotService.on_SnapshotEvent +``` + +```` + +````{py:method} on_SnapshotCompletedEvent(event: abx_dl.events.SnapshotCompletedEvent) -> None +:canonical: archivebox.services.snapshot_service.SnapshotService.on_SnapshotCompletedEvent +:async: + +```{autodoc2-docstring} archivebox.services.snapshot_service.SnapshotService.on_SnapshotCompletedEvent +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.services.tag_service.md b/docs/apidocs/archivebox/archivebox.services.tag_service.md new file mode 100644 index 00000000..fbf6985e --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.services.tag_service.md @@ -0,0 +1,58 @@ +# {py:mod}`archivebox.services.tag_service` + +```{py:module} archivebox.services.tag_service +``` + +```{autodoc2-docstring} archivebox.services.tag_service +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`TagService ` + - +```` + +### API + +`````{py:class} TagService(bus) +:canonical: archivebox.services.tag_service.TagService + +Bases: {py:obj}`abx_dl.services.base.BaseService` + +````{py:attribute} LISTENS_TO +:canonical: archivebox.services.tag_service.TagService.LISTENS_TO +:value: > + None + +```{autodoc2-docstring} archivebox.services.tag_service.TagService.LISTENS_TO +``` + +```` + +````{py:attribute} EMITS +:canonical: archivebox.services.tag_service.TagService.EMITS +:value: > + [] + +```{autodoc2-docstring} archivebox.services.tag_service.TagService.EMITS +``` + +```` + +````{py:method} on_TagEvent__save_to_db(event: abx_dl.events.TagEvent) -> None +:canonical: archivebox.services.tag_service.TagService.on_TagEvent__save_to_db +:async: + +```{autodoc2-docstring} archivebox.services.tag_service.TagService.on_TagEvent__save_to_db +``` + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.uuid_compat.md b/docs/apidocs/archivebox/archivebox.uuid_compat.md index f52d92af..5476106a 100644 --- a/docs/apidocs/archivebox/archivebox.uuid_compat.md +++ b/docs/apidocs/archivebox/archivebox.uuid_compat.md @@ -35,7 +35,7 @@ ### API -````{py:function} uuid7() +````{py:function} uuid7() -> uuid.UUID :canonical: archivebox.uuid_compat.uuid7 ```{autodoc2-docstring} archivebox.uuid_compat.uuid7 diff --git a/docs/apidocs/archivebox/archivebox.workers.management.commands.md b/docs/apidocs/archivebox/archivebox.workers.management.commands.md index a135897a..2952b99d 100644 --- a/docs/apidocs/archivebox/archivebox.workers.management.commands.md +++ b/docs/apidocs/archivebox/archivebox.workers.management.commands.md @@ -13,6 +13,6 @@ :titlesonly: :maxdepth: 1 -archivebox.workers.management.commands.orchestrator_watch -archivebox.workers.management.commands.orchestrator +archivebox.workers.management.commands.runner_watch +archivebox.workers.management.commands.supervisord_watchdog ``` diff --git a/docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator.md b/docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator.md deleted file mode 100644 index 6496c248..00000000 --- a/docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator.md +++ /dev/null @@ -1,49 +0,0 @@ -# {py:mod}`archivebox.workers.management.commands.orchestrator` - -```{py:module} archivebox.workers.management.commands.orchestrator -``` - -```{autodoc2-docstring} archivebox.workers.management.commands.orchestrator -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`Command ` - - -```` - -### API - -`````{py:class} Command(stdout=None, stderr=None, no_color=False, force_color=False) -:canonical: archivebox.workers.management.commands.orchestrator.Command - -Bases: {py:obj}`django.core.management.base.BaseCommand` - -````{py:attribute} help -:canonical: archivebox.workers.management.commands.orchestrator.Command.help -:value: > - 'Run the archivebox orchestrator' - -```{autodoc2-docstring} archivebox.workers.management.commands.orchestrator.Command.help -``` - -```` - -````{py:method} add_arguments(parser) -:canonical: archivebox.workers.management.commands.orchestrator.Command.add_arguments - -```` - -````{py:method} handle(*args, **kwargs) -:canonical: archivebox.workers.management.commands.orchestrator.Command.handle - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator_watch.md b/docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator_watch.md deleted file mode 100644 index 6aad1c0c..00000000 --- a/docs/apidocs/archivebox/archivebox.workers.management.commands.orchestrator_watch.md +++ /dev/null @@ -1,49 +0,0 @@ -# {py:mod}`archivebox.workers.management.commands.orchestrator_watch` - -```{py:module} archivebox.workers.management.commands.orchestrator_watch -``` - -```{autodoc2-docstring} archivebox.workers.management.commands.orchestrator_watch -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`Command ` - - -```` - -### API - -`````{py:class} Command(stdout=None, stderr=None, no_color=False, force_color=False) -:canonical: archivebox.workers.management.commands.orchestrator_watch.Command - -Bases: {py:obj}`django.core.management.base.BaseCommand` - -````{py:attribute} help -:canonical: archivebox.workers.management.commands.orchestrator_watch.Command.help -:value: > - 'Watch the runserver autoreload PID file and restart orchestrator on reloads.' - -```{autodoc2-docstring} archivebox.workers.management.commands.orchestrator_watch.Command.help -``` - -```` - -````{py:method} add_arguments(parser) -:canonical: archivebox.workers.management.commands.orchestrator_watch.Command.add_arguments - -```` - -````{py:method} handle(*args, **kwargs) -:canonical: archivebox.workers.management.commands.orchestrator_watch.Command.handle - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.workers.management.commands.runner_watch.md b/docs/apidocs/archivebox/archivebox.workers.management.commands.runner_watch.md new file mode 100644 index 00000000..e3810d8c --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.workers.management.commands.runner_watch.md @@ -0,0 +1,49 @@ +# {py:mod}`archivebox.workers.management.commands.runner_watch` + +```{py:module} archivebox.workers.management.commands.runner_watch +``` + +```{autodoc2-docstring} archivebox.workers.management.commands.runner_watch +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`Command ` + - +```` + +### API + +`````{py:class} Command(stdout=None, stderr=None, no_color=False, force_color=False) +:canonical: archivebox.workers.management.commands.runner_watch.Command + +Bases: {py:obj}`django.core.management.base.BaseCommand` + +````{py:attribute} help +:canonical: archivebox.workers.management.commands.runner_watch.Command.help +:value: > + 'Watch the debug runserver Process row and restart the background runner on autoreloads.' + +```{autodoc2-docstring} archivebox.workers.management.commands.runner_watch.Command.help +``` + +```` + +````{py:method} add_arguments(parser) +:canonical: archivebox.workers.management.commands.runner_watch.Command.add_arguments + +```` + +````{py:method} handle(*args, **kwargs) +:canonical: archivebox.workers.management.commands.runner_watch.Command.handle + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.workers.management.commands.supervisord_watchdog.md b/docs/apidocs/archivebox/archivebox.workers.management.commands.supervisord_watchdog.md new file mode 100644 index 00000000..86eb4374 --- /dev/null +++ b/docs/apidocs/archivebox/archivebox.workers.management.commands.supervisord_watchdog.md @@ -0,0 +1,49 @@ +# {py:mod}`archivebox.workers.management.commands.supervisord_watchdog` + +```{py:module} archivebox.workers.management.commands.supervisord_watchdog +``` + +```{autodoc2-docstring} archivebox.workers.management.commands.supervisord_watchdog +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`Command ` + - +```` + +### API + +`````{py:class} Command(stdout=None, stderr=None, no_color=False, force_color=False) +:canonical: archivebox.workers.management.commands.supervisord_watchdog.Command + +Bases: {py:obj}`django.core.management.base.BaseCommand` + +````{py:attribute} help +:canonical: archivebox.workers.management.commands.supervisord_watchdog.Command.help +:value: > + 'Stop a foreground-owned supervisord if its exact ArchiveBox owner Process exits.' + +```{autodoc2-docstring} archivebox.workers.management.commands.supervisord_watchdog.Command.help +``` + +```` + +````{py:method} add_arguments(parser) +:canonical: archivebox.workers.management.commands.supervisord_watchdog.Command.add_arguments + +```` + +````{py:method} handle(*args, **kwargs) +:canonical: archivebox.workers.management.commands.supervisord_watchdog.Command.handle + +```` + +````` diff --git a/docs/apidocs/archivebox/archivebox.workers.md b/docs/apidocs/archivebox/archivebox.workers.md index a26ca538..c5360c4d 100644 --- a/docs/apidocs/archivebox/archivebox.workers.md +++ b/docs/apidocs/archivebox/archivebox.workers.md @@ -22,12 +22,9 @@ archivebox.workers.management :titlesonly: :maxdepth: 1 -archivebox.workers.worker -archivebox.workers.tasks archivebox.workers.models archivebox.workers.apps archivebox.workers.admin -archivebox.workers.orchestrator archivebox.workers.supervisord_util ``` diff --git a/docs/apidocs/archivebox/archivebox.workers.models.md b/docs/apidocs/archivebox/archivebox.workers.models.md index 561141d1..67b1b644 100644 --- a/docs/apidocs/archivebox/archivebox.workers.models.md +++ b/docs/apidocs/archivebox/archivebox.workers.models.md @@ -41,6 +41,30 @@ - ```{autodoc2-docstring} archivebox.workers.models.default_retry_at_field :summary: ``` +* - {py:obj}`RETRY_AT_MAX ` + - ```{autodoc2-docstring} archivebox.workers.models.RETRY_AT_MAX + :summary: + ``` +* - {py:obj}`ACTIVE_STATE_LEASE_SECONDS ` + - ```{autodoc2-docstring} archivebox.workers.models.ACTIVE_STATE_LEASE_SECONDS + :summary: + ``` +* - {py:obj}`logger ` + - ```{autodoc2-docstring} archivebox.workers.models.logger + :summary: + ``` +* - {py:obj}`MODULE_PATH ` + - ```{autodoc2-docstring} archivebox.workers.models.MODULE_PATH + :summary: + ``` +* - {py:obj}`REPO_ROOT ` + - ```{autodoc2-docstring} archivebox.workers.models.REPO_ROOT + :summary: + ``` +* - {py:obj}`PACKAGE_ROOT ` + - ```{autodoc2-docstring} archivebox.workers.models.PACKAGE_ROOT + :summary: + ``` * - {py:obj}`ObjectState ` - ```{autodoc2-docstring} archivebox.workers.models.ObjectState :summary: @@ -78,6 +102,16 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:attribute} PAUSED +:canonical: archivebox.workers.models.DefaultStatusChoices.PAUSED +:value: > + ('paused', 'Paused') + +```{autodoc2-docstring} archivebox.workers.models.DefaultStatusChoices.PAUSED +``` + +```` + ````{py:attribute} SEALED :canonical: archivebox.workers.models.DefaultStatusChoices.SEALED :value: > @@ -112,6 +146,66 @@ Bases: {py:obj}`django.db.models.TextChoices` ```` +````{py:data} RETRY_AT_MAX +:canonical: archivebox.workers.models.RETRY_AT_MAX +:value: > + 'replace(...)' + +```{autodoc2-docstring} archivebox.workers.models.RETRY_AT_MAX +``` + +```` + +````{py:data} ACTIVE_STATE_LEASE_SECONDS +:canonical: archivebox.workers.models.ACTIVE_STATE_LEASE_SECONDS +:value: > + 60 + +```{autodoc2-docstring} archivebox.workers.models.ACTIVE_STATE_LEASE_SECONDS +``` + +```` + +````{py:data} logger +:canonical: archivebox.workers.models.logger +:value: > + 'getLogger(...)' + +```{autodoc2-docstring} archivebox.workers.models.logger +``` + +```` + +````{py:data} MODULE_PATH +:canonical: archivebox.workers.models.MODULE_PATH +:value: > + 'resolve(...)' + +```{autodoc2-docstring} archivebox.workers.models.MODULE_PATH +``` + +```` + +````{py:data} REPO_ROOT +:canonical: archivebox.workers.models.REPO_ROOT +:value: > + None + +```{autodoc2-docstring} archivebox.workers.models.REPO_ROOT +``` + +```` + +````{py:data} PACKAGE_ROOT +:canonical: archivebox.workers.models.PACKAGE_ROOT +:value: > + None + +```{autodoc2-docstring} archivebox.workers.models.PACKAGE_ROOT +``` + +```` + ````{py:data} ObjectState :canonical: archivebox.workers.models.ObjectState :value: > @@ -137,20 +231,9 @@ Bases: {py:obj}`django.db.models.TextChoices` Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMixin` -````{py:attribute} id -:canonical: archivebox.workers.models.BaseModelWithStateMachine.id -:type: django.db.models.UUIDField -:value: > - None - -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.id -``` - -```` - ````{py:attribute} StatusChoices :canonical: archivebox.workers.models.BaseModelWithStateMachine.StatusChoices -:type: typing.ClassVar[typing.Type[django.db.models.TextChoices]] +:type: typing.ClassVar[type[archivebox.workers.models.DefaultStatusChoices]] :value: > None @@ -161,7 +244,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ````{py:attribute} state_machine_name :canonical: archivebox.workers.models.BaseModelWithStateMachine.state_machine_name -:type: typing.ClassVar[str] +:type: str | None :value: > None @@ -172,7 +255,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ````{py:attribute} state_field_name :canonical: archivebox.workers.models.BaseModelWithStateMachine.state_field_name -:type: typing.ClassVar[str] +:type: str :value: > None @@ -183,7 +266,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ````{py:attribute} state_machine_attr :canonical: archivebox.workers.models.BaseModelWithStateMachine.state_machine_attr -:type: typing.ClassVar[str] +:type: str :value: > 'sm' @@ -194,7 +277,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ````{py:attribute} bind_events_as_methods :canonical: archivebox.workers.models.BaseModelWithStateMachine.bind_events_as_methods -:type: typing.ClassVar[bool] +:type: bool :value: > True @@ -205,7 +288,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ````{py:attribute} active_state :canonical: archivebox.workers.models.BaseModelWithStateMachine.active_state -:type: typing.ClassVar[archivebox.workers.models.ObjectState] +:type: archivebox.workers.models.ObjectState :value: > None @@ -216,7 +299,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ````{py:attribute} retry_at_field_name :canonical: archivebox.workers.models.BaseModelWithStateMachine.retry_at_field_name -:type: typing.ClassVar[str] +:type: str :value: > None @@ -228,8 +311,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix `````{py:class} Meta :canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta -```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.Meta -``` +Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta` ````{py:attribute} app_label :canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta.app_label @@ -297,7 +379,45 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ```` -````{py:method} update_and_requeue(**kwargs) -> bool +````{py:property} is_paused +:canonical: archivebox.workers.models.BaseModelWithStateMachine.is_paused +:type: bool + +```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.is_paused +``` + +```` + +````{py:method} safe_update(update_fields: dict[str, typing.Any], *, refresh: bool = True, extra_filter: dict[str, typing.Any] | None = None) -> bool +:canonical: archivebox.workers.models.BaseModelWithStateMachine.safe_update + +```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.safe_update +``` + +```` + +````{py:method} save(*args, **kwargs) +:canonical: archivebox.workers.models.BaseModelWithStateMachine.save + +```` + +````{py:method} pause(*, save: bool = True) -> bool +:canonical: archivebox.workers.models.BaseModelWithStateMachine.pause + +```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.pause +``` + +```` + +````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool +:canonical: archivebox.workers.models.BaseModelWithStateMachine.resume + +```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.resume +``` + +```` + +````{py:method} update_and_requeue(*, refresh: bool = True, **kwargs) -> bool :canonical: archivebox.workers.models.BaseModelWithStateMachine.update_and_requeue ```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.update_and_requeue @@ -323,6 +443,22 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ```` +````{py:method} claim_processing_lock(lock_seconds: int = 60) -> bool +:canonical: archivebox.workers.models.BaseModelWithStateMachine.claim_processing_lock + +```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.claim_processing_lock +``` + +```` + +````{py:method} tick_claimed(lock_seconds: int = 60) -> bool +:canonical: archivebox.workers.models.BaseModelWithStateMachine.tick_claimed + +```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.tick_claimed +``` + +```` + ````{py:method} ACTIVE_STATE() -> str :canonical: archivebox.workers.models.BaseModelWithStateMachine.ACTIVE_STATE @@ -355,7 +491,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ```` -````{py:method} extend_choices(base_choices: typing.Type[django.db.models.TextChoices]) +````{py:method} extend_choices(base_choices: type[django.db.models.TextChoices]) :canonical: archivebox.workers.models.BaseModelWithStateMachine.extend_choices :classmethod: @@ -382,7 +518,7 @@ Bases: {py:obj}`django.db.models.Model`, {py:obj}`statemachine.mixins.MachineMix ```` -````{py:method} StateMachineClass() -> typing.Type[statemachine.StateMachine] +````{py:method} StateMachineClass() -> type[statemachine.StateMachine] :canonical: archivebox.workers.models.BaseModelWithStateMachine.StateMachineClass ```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StateMachineClass @@ -399,7 +535,6 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} StatusChoices :canonical: archivebox.workers.models.ModelWithStateMachine.StatusChoices -:type: typing.ClassVar[typing.Type[archivebox.workers.models.DefaultStatusChoices]] :value: > None @@ -432,7 +567,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} state_machine_name :canonical: archivebox.workers.models.ModelWithStateMachine.state_machine_name -:type: typing.ClassVar[str] +:type: str | None :value: > None @@ -443,7 +578,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} state_field_name :canonical: archivebox.workers.models.ModelWithStateMachine.state_field_name -:type: typing.ClassVar[str] +:type: str :value: > 'status' @@ -454,7 +589,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} state_machine_attr :canonical: archivebox.workers.models.ModelWithStateMachine.state_machine_attr -:type: typing.ClassVar[str] +:type: str :value: > 'sm' @@ -465,7 +600,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} bind_events_as_methods :canonical: archivebox.workers.models.ModelWithStateMachine.bind_events_as_methods -:type: typing.ClassVar[bool] +:type: bool :value: > True @@ -476,7 +611,6 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} active_state :canonical: archivebox.workers.models.ModelWithStateMachine.active_state -:type: typing.ClassVar[str] :value: > None @@ -487,7 +621,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} retry_at_field_name :canonical: archivebox.workers.models.ModelWithStateMachine.retry_at_field_name -:type: typing.ClassVar[str] +:type: str :value: > 'retry_at' @@ -499,18 +633,7 @@ Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` `````{py:class} Meta :canonical: archivebox.workers.models.ModelWithStateMachine.Meta -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.Meta -``` - -````{py:attribute} app_label -:canonical: archivebox.workers.models.ModelWithStateMachine.Meta.app_label -:value: > - 'workers' - -```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.Meta.app_label -``` - -```` +Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine` ````{py:attribute} abstract :canonical: archivebox.workers.models.ModelWithStateMachine.Meta.abstract diff --git a/docs/apidocs/archivebox/archivebox.workers.orchestrator.md b/docs/apidocs/archivebox/archivebox.workers.orchestrator.md deleted file mode 100644 index 74c2873e..00000000 --- a/docs/apidocs/archivebox/archivebox.workers.orchestrator.md +++ /dev/null @@ -1,311 +0,0 @@ -# {py:mod}`archivebox.workers.orchestrator` - -```{py:module} archivebox.workers.orchestrator -``` - -```{autodoc2-docstring} archivebox.workers.orchestrator -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`Orchestrator ` - - ```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator - :summary: - ``` -```` - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_run_orchestrator_process ` - - ```{autodoc2-docstring} archivebox.workers.orchestrator._run_orchestrator_process - :summary: - ``` -```` - -### API - -````{py:function} _run_orchestrator_process(exit_on_idle: bool) -> None -:canonical: archivebox.workers.orchestrator._run_orchestrator_process - -```{autodoc2-docstring} archivebox.workers.orchestrator._run_orchestrator_process -``` -```` - -`````{py:class} Orchestrator(exit_on_idle: bool = True, crawl_id: str | None = None) -:canonical: archivebox.workers.orchestrator.Orchestrator - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.__init__ -``` - -````{py:attribute} WORKER_TYPES -:canonical: archivebox.workers.orchestrator.Orchestrator.WORKER_TYPES -:type: list[typing.Type[archivebox.workers.worker.Worker]] -:value: > - None - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.WORKER_TYPES -``` - -```` - -````{py:attribute} POLL_INTERVAL -:canonical: archivebox.workers.orchestrator.Orchestrator.POLL_INTERVAL -:type: float -:value: > - 2.0 - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.POLL_INTERVAL -``` - -```` - -````{py:attribute} IDLE_TIMEOUT -:canonical: archivebox.workers.orchestrator.Orchestrator.IDLE_TIMEOUT -:type: int -:value: > - 3 - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.IDLE_TIMEOUT -``` - -```` - -````{py:attribute} MAX_CRAWL_WORKERS -:canonical: archivebox.workers.orchestrator.Orchestrator.MAX_CRAWL_WORKERS -:type: int -:value: > - 8 - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.MAX_CRAWL_WORKERS -``` - -```` - -````{py:attribute} MAX_BINARY_WORKERS -:canonical: archivebox.workers.orchestrator.Orchestrator.MAX_BINARY_WORKERS -:type: int -:value: > - 1 - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.MAX_BINARY_WORKERS -``` - -```` - -````{py:method} __repr__() -> str -:canonical: archivebox.workers.orchestrator.Orchestrator.__repr__ - -```` - -````{py:method} is_running() -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator.is_running -:classmethod: - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.is_running -``` - -```` - -````{py:method} on_startup() -> None -:canonical: archivebox.workers.orchestrator.Orchestrator.on_startup - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.on_startup -``` - -```` - -````{py:method} terminate_all_workers() -> None -:canonical: archivebox.workers.orchestrator.Orchestrator.terminate_all_workers - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.terminate_all_workers -``` - -```` - -````{py:method} on_shutdown(error: BaseException | None = None) -> None -:canonical: archivebox.workers.orchestrator.Orchestrator.on_shutdown - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.on_shutdown -``` - -```` - -````{py:method} get_total_worker_count() -> int -:canonical: archivebox.workers.orchestrator.Orchestrator.get_total_worker_count - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.get_total_worker_count -``` - -```` - -````{py:method} get_running_workers_for_type(WorkerClass: typing.Type[archivebox.workers.worker.Worker]) -> int -:canonical: archivebox.workers.orchestrator.Orchestrator.get_running_workers_for_type - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.get_running_workers_for_type -``` - -```` - -````{py:method} _get_scoped_running_workers() -:canonical: archivebox.workers.orchestrator.Orchestrator._get_scoped_running_workers - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator._get_scoped_running_workers -``` - -```` - -````{py:method} should_spawn_worker(WorkerClass: typing.Type[archivebox.workers.worker.Worker], queue_count: int) -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator.should_spawn_worker - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.should_spawn_worker -``` - -```` - -````{py:method} spawn_worker(WorkerClass: typing.Type[archivebox.workers.worker.Worker]) -> int | None -:canonical: archivebox.workers.orchestrator.Orchestrator.spawn_worker - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.spawn_worker -``` - -```` - -````{py:method} check_queues_and_spawn_workers() -> dict[str, int] -:canonical: archivebox.workers.orchestrator.Orchestrator.check_queues_and_spawn_workers - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.check_queues_and_spawn_workers -``` - -```` - -````{py:method} _should_process_schedules() -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator._should_process_schedules - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator._should_process_schedules -``` - -```` - -````{py:method} _materialize_due_schedules() -> None -:canonical: archivebox.workers.orchestrator.Orchestrator._materialize_due_schedules - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator._materialize_due_schedules -``` - -```` - -````{py:method} _enforce_hard_timeouts() -> None -:canonical: archivebox.workers.orchestrator.Orchestrator._enforce_hard_timeouts - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator._enforce_hard_timeouts -``` - -```` - -````{py:method} _claim_crawl(crawl) -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator._claim_crawl - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator._claim_crawl -``` - -```` - -````{py:method} has_pending_work(queue_sizes: dict[str, int]) -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator.has_pending_work - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.has_pending_work -``` - -```` - -````{py:method} has_running_workers() -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator.has_running_workers - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.has_running_workers -``` - -```` - -````{py:method} has_future_work() -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator.has_future_work - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.has_future_work -``` - -```` - -````{py:method} on_tick(queue_sizes: dict[str, int]) -> None -:canonical: archivebox.workers.orchestrator.Orchestrator.on_tick - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.on_tick -``` - -```` - -````{py:method} on_idle() -> None -:canonical: archivebox.workers.orchestrator.Orchestrator.on_idle - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.on_idle -``` - -```` - -````{py:method} should_exit(queue_sizes: dict[str, int]) -> bool -:canonical: archivebox.workers.orchestrator.Orchestrator.should_exit - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.should_exit -``` - -```` - -````{py:method} runloop() -> None -:canonical: archivebox.workers.orchestrator.Orchestrator.runloop - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.runloop -``` - -```` - -````{py:method} _run_orchestrator_loop(progress_layout, plain_output: bool = False) -:canonical: archivebox.workers.orchestrator.Orchestrator._run_orchestrator_loop - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator._run_orchestrator_loop -``` - -```` - -````{py:method} start() -> int -:canonical: archivebox.workers.orchestrator.Orchestrator.start - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.start -``` - -```` - -````{py:method} get_or_start(exit_on_idle: bool = True) -> archivebox.workers.orchestrator.Orchestrator -:canonical: archivebox.workers.orchestrator.Orchestrator.get_or_start -:classmethod: - -```{autodoc2-docstring} archivebox.workers.orchestrator.Orchestrator.get_or_start -``` - -```` - -````` diff --git a/docs/apidocs/archivebox/archivebox.workers.supervisord_util.md b/docs/apidocs/archivebox/archivebox.workers.supervisord_util.md index ff85f84e..3ea6a2a7 100644 --- a/docs/apidocs/archivebox/archivebox.workers.supervisord_util.md +++ b/docs/apidocs/archivebox/archivebox.workers.supervisord_util.md @@ -9,22 +9,58 @@ ## Module Contents +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`SupervisordConnectionCache ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.SupervisordConnectionCache + :summary: + ``` +```` + ### Functions ````{list-table} :class: autosummary longtable :align: left +* - {py:obj}`_shell_join ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._shell_join + :summary: + ``` +* - {py:obj}`_record_supervisord_process ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._record_supervisord_process + :summary: + ``` +* - {py:obj}`_fallback_supervisord_process_from_db ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._fallback_supervisord_process_from_db + :summary: + ``` +* - {py:obj}`_live_supervisord_processes_from_db ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._live_supervisord_processes_from_db + :summary: + ``` +* - {py:obj}`_stop_older_supervisord_processes ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._stop_older_supervisord_processes + :summary: + ``` +* - {py:obj}`RUNSERVER_WORKER ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNSERVER_WORKER + :summary: + ``` * - {py:obj}`is_port_in_use ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.is_port_in_use :summary: ``` -* - {py:obj}`get_sock_file ` - - ```{autodoc2-docstring} archivebox.workers.supervisord_util.get_sock_file +* - {py:obj}`_sonic_worker_bind_target ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._sonic_worker_bind_target :summary: ``` -* - {py:obj}`follow ` - - ```{autodoc2-docstring} archivebox.workers.supervisord_util.follow +* - {py:obj}`get_sock_file ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.get_sock_file :summary: ``` * - {py:obj}`create_supervisord_config ` @@ -35,6 +71,14 @@ - ```{autodoc2-docstring} archivebox.workers.supervisord_util.create_worker_config :summary: ``` +* - {py:obj}`_current_foreground_supervisord_process_id ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._current_foreground_supervisord_process_id + :summary: + ``` +* - {py:obj}`sync_supervisord_workers ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.sync_supervisord_workers + :summary: + ``` * - {py:obj}`get_existing_supervisord_process ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.get_existing_supervisord_process :summary: @@ -43,6 +87,14 @@ - ```{autodoc2-docstring} archivebox.workers.supervisord_util.stop_existing_supervisord_process :summary: ``` +* - {py:obj}`stop_own_supervisord_process ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.stop_own_supervisord_process + :summary: + ``` +* - {py:obj}`reap_foreground_supervisord_process ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.reap_foreground_supervisord_process + :summary: + ``` * - {py:obj}`start_new_supervisord_process ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.start_new_supervisord_process :summary: @@ -59,34 +111,78 @@ - ```{autodoc2-docstring} archivebox.workers.supervisord_util.start_worker :summary: ``` +* - {py:obj}`run_runner_worker ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.run_runner_worker + :summary: + ``` * - {py:obj}`get_worker ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.get_worker :summary: ``` +* - {py:obj}`format_runtime_components ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.format_runtime_components + :summary: + ``` +* - {py:obj}`worker_runtime_component ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.worker_runtime_component + :summary: + ``` +* - {py:obj}`runtime_components_for_worker_names ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.runtime_components_for_worker_names + :summary: + ``` +* - {py:obj}`active_supervisord_runtime_components ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.active_supervisord_runtime_components + :summary: + ``` +* - {py:obj}`build_server_worker_plan ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.build_server_worker_plan + :summary: + ``` * - {py:obj}`stop_worker ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.stop_worker :summary: ``` -* - {py:obj}`tail_worker_logs ` - - ```{autodoc2-docstring} archivebox.workers.supervisord_util.tail_worker_logs - :summary: - ``` * - {py:obj}`tail_multiple_worker_logs ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.tail_multiple_worker_logs :summary: ``` -* - {py:obj}`watch_worker ` - - ```{autodoc2-docstring} archivebox.workers.supervisord_util.watch_worker +* - {py:obj}`get_sonic_supervisord_worker_from_plugin ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.get_sonic_supervisord_worker_from_plugin + :summary: + ``` +* - {py:obj}`_proc_cmdline ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._proc_cmdline + :summary: + ``` +* - {py:obj}`_is_sonic_process ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._is_sonic_process + :summary: + ``` +* - {py:obj}`_is_supervisord_process ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._is_supervisord_process + :summary: + ``` +* - {py:obj}`_has_live_archivebox_parent ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._has_live_archivebox_parent + :summary: + ``` +* - {py:obj}`_terminate_process_tree ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._terminate_process_tree + :summary: + ``` +* - {py:obj}`_sonic_listeners ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._sonic_listeners + :summary: + ``` +* - {py:obj}`stop_stale_sonic_processes ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.stop_stale_sonic_processes :summary: ``` * - {py:obj}`start_server_workers ` - ```{autodoc2-docstring} archivebox.workers.supervisord_util.start_server_workers :summary: ``` -* - {py:obj}`start_cli_workers ` - - ```{autodoc2-docstring} archivebox.workers.supervisord_util.start_cli_workers - :summary: - ``` ```` ### Data @@ -115,8 +211,32 @@ - ```{autodoc2-docstring} archivebox.workers.supervisord_util._supervisord_proc :summary: ``` -* - {py:obj}`ORCHESTRATOR_WORKER ` - - ```{autodoc2-docstring} archivebox.workers.supervisord_util.ORCHESTRATOR_WORKER +* - {py:obj}`_desired_supervisord_workers ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._desired_supervisord_workers + :summary: + ``` +* - {py:obj}`_ACTIVE_WORKER_STATES ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._ACTIVE_WORKER_STATES + :summary: + ``` +* - {py:obj}`_RUNTIME_COMPONENT_ORDER ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util._RUNTIME_COMPONENT_ORDER + :summary: + ``` +* - {py:obj}`RUNNER_WORKER ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNNER_WORKER + :summary: + ``` +* - {py:obj}`RUNNER_ONCE_WORKER ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNNER_ONCE_WORKER + :summary: + ``` +* - {py:obj}`RUNNER_WATCH_WORKER ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNNER_WATCH_WORKER + :summary: + ``` +* - {py:obj}`SUPERVISORD_PARENT_WATCHDOG_WORKER ` + - ```{autodoc2-docstring} archivebox.workers.supervisord_util.SUPERVISORD_PARENT_WATCHDOG_WORKER :summary: ``` * - {py:obj}`SERVER_WORKER ` @@ -177,12 +297,108 @@ ```` -````{py:data} ORCHESTRATOR_WORKER -:canonical: archivebox.workers.supervisord_util.ORCHESTRATOR_WORKER +````{py:data} _desired_supervisord_workers +:canonical: archivebox.workers.supervisord_util._desired_supervisord_workers +:type: dict[str, dict[str, str]] :value: > None -```{autodoc2-docstring} archivebox.workers.supervisord_util.ORCHESTRATOR_WORKER +```{autodoc2-docstring} archivebox.workers.supervisord_util._desired_supervisord_workers +``` + +```` + +````{py:data} _ACTIVE_WORKER_STATES +:canonical: archivebox.workers.supervisord_util._ACTIVE_WORKER_STATES +:value: > + None + +```{autodoc2-docstring} archivebox.workers.supervisord_util._ACTIVE_WORKER_STATES +``` + +```` + +````{py:data} _RUNTIME_COMPONENT_ORDER +:canonical: archivebox.workers.supervisord_util._RUNTIME_COMPONENT_ORDER +:value: > + ('orchestrator', 'server', 'sonic') + +```{autodoc2-docstring} archivebox.workers.supervisord_util._RUNTIME_COMPONENT_ORDER +``` + +```` + +````{py:function} _shell_join(args: list[str]) -> str +:canonical: archivebox.workers.supervisord_util._shell_join + +```{autodoc2-docstring} archivebox.workers.supervisord_util._shell_join +``` +```` + +````{py:function} _record_supervisord_process(proc: subprocess.Popen, config_file: pathlib.Path) -> None +:canonical: archivebox.workers.supervisord_util._record_supervisord_process + +```{autodoc2-docstring} archivebox.workers.supervisord_util._record_supervisord_process +``` +```` + +````{py:function} _fallback_supervisord_process_from_db() +:canonical: archivebox.workers.supervisord_util._fallback_supervisord_process_from_db + +```{autodoc2-docstring} archivebox.workers.supervisord_util._fallback_supervisord_process_from_db +``` +```` + +````{py:function} _live_supervisord_processes_from_db() +:canonical: archivebox.workers.supervisord_util._live_supervisord_processes_from_db + +```{autodoc2-docstring} archivebox.workers.supervisord_util._live_supervisord_processes_from_db +``` +```` + +````{py:function} _stop_older_supervisord_processes(*, current_pid: int, current_started_at: float, timeout: float) -> None +:canonical: archivebox.workers.supervisord_util._stop_older_supervisord_processes + +```{autodoc2-docstring} archivebox.workers.supervisord_util._stop_older_supervisord_processes +``` +```` + +````{py:data} RUNNER_WORKER +:canonical: archivebox.workers.supervisord_util.RUNNER_WORKER +:value: > + None + +```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNNER_WORKER +``` + +```` + +````{py:data} RUNNER_ONCE_WORKER +:canonical: archivebox.workers.supervisord_util.RUNNER_ONCE_WORKER +:value: > + None + +```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNNER_ONCE_WORKER +``` + +```` + +````{py:data} RUNNER_WATCH_WORKER +:canonical: archivebox.workers.supervisord_util.RUNNER_WATCH_WORKER +:value: > + None + +```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNNER_WATCH_WORKER +``` + +```` + +````{py:data} SUPERVISORD_PARENT_WATCHDOG_WORKER +:canonical: archivebox.workers.supervisord_util.SUPERVISORD_PARENT_WATCHDOG_WORKER +:value: > + None + +```{autodoc2-docstring} archivebox.workers.supervisord_util.SUPERVISORD_PARENT_WATCHDOG_WORKER ``` ```` @@ -197,6 +413,13 @@ ```` +````{py:function} RUNSERVER_WORKER(host: str, port: str, *, reload: bool, nothreading: bool = False) +:canonical: archivebox.workers.supervisord_util.RUNSERVER_WORKER + +```{autodoc2-docstring} archivebox.workers.supervisord_util.RUNSERVER_WORKER +``` +```` + ````{py:function} is_port_in_use(host: str, port: int) -> bool :canonical: archivebox.workers.supervisord_util.is_port_in_use @@ -204,6 +427,13 @@ ``` ```` +````{py:function} _sonic_worker_bind_target(worker: dict[str, str]) -> tuple[str, int] | None +:canonical: archivebox.workers.supervisord_util._sonic_worker_bind_target + +```{autodoc2-docstring} archivebox.workers.supervisord_util._sonic_worker_bind_target +``` +```` + ````{py:function} get_sock_file() :canonical: archivebox.workers.supervisord_util.get_sock_file @@ -211,13 +441,6 @@ ``` ```` -````{py:function} follow(file, sleep_sec=0.1) -> typing.Iterator[str] -:canonical: archivebox.workers.supervisord_util.follow - -```{autodoc2-docstring} archivebox.workers.supervisord_util.follow -``` -```` - ````{py:function} create_supervisord_config() :canonical: archivebox.workers.supervisord_util.create_supervisord_config @@ -232,13 +455,57 @@ ``` ```` -````{py:function} get_existing_supervisord_process() +````{py:function} _current_foreground_supervisord_process_id() +:canonical: archivebox.workers.supervisord_util._current_foreground_supervisord_process_id + +```{autodoc2-docstring} archivebox.workers.supervisord_util._current_foreground_supervisord_process_id +``` +```` + +````{py:function} sync_supervisord_workers(supervisor, workers: list[tuple[dict[str, str], bool]], *, prune: bool = True) +:canonical: archivebox.workers.supervisord_util.sync_supervisord_workers + +```{autodoc2-docstring} archivebox.workers.supervisord_util.sync_supervisord_workers +``` +```` + +````{py:function} get_existing_supervisord_process(*, quiet: bool = False) :canonical: archivebox.workers.supervisord_util.get_existing_supervisord_process ```{autodoc2-docstring} archivebox.workers.supervisord_util.get_existing_supervisord_process ``` ```` +`````{py:class} SupervisordConnectionCache(*, quiet: bool = False) +:canonical: archivebox.workers.supervisord_util.SupervisordConnectionCache + +```{autodoc2-docstring} archivebox.workers.supervisord_util.SupervisordConnectionCache +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} archivebox.workers.supervisord_util.SupervisordConnectionCache.__init__ +``` + +````{py:method} clear() -> None +:canonical: archivebox.workers.supervisord_util.SupervisordConnectionCache.clear + +```{autodoc2-docstring} archivebox.workers.supervisord_util.SupervisordConnectionCache.clear +``` + +```` + +````{py:method} get() +:canonical: archivebox.workers.supervisord_util.SupervisordConnectionCache.get + +```{autodoc2-docstring} archivebox.workers.supervisord_util.SupervisordConnectionCache.get +``` + +```` + +````` + ````{py:function} stop_existing_supervisord_process() :canonical: archivebox.workers.supervisord_util.stop_existing_supervisord_process @@ -246,6 +513,20 @@ ``` ```` +````{py:function} stop_own_supervisord_process() +:canonical: archivebox.workers.supervisord_util.stop_own_supervisord_process + +```{autodoc2-docstring} archivebox.workers.supervisord_util.stop_own_supervisord_process +``` +```` + +````{py:function} reap_foreground_supervisord_process() -> None +:canonical: archivebox.workers.supervisord_util.reap_foreground_supervisord_process + +```{autodoc2-docstring} archivebox.workers.supervisord_util.reap_foreground_supervisord_process +``` +```` + ````{py:function} start_new_supervisord_process(daemonize=False) :canonical: archivebox.workers.supervisord_util.start_new_supervisord_process @@ -253,7 +534,7 @@ ``` ```` -````{py:function} wait_for_supervisord_ready(max_wait_sec: float = 5.0, interval_sec: float = 0.1) +````{py:function} wait_for_supervisord_ready(max_wait_sec: float = 5.0, interval_sec: float = 0.1, *, quiet: bool = False) :canonical: archivebox.workers.supervisord_util.wait_for_supervisord_ready ```{autodoc2-docstring} archivebox.workers.supervisord_util.wait_for_supervisord_ready @@ -274,6 +555,13 @@ ``` ```` +````{py:function} run_runner_worker(args: list[str], *, name: str = 'worker_runner_once', interactive_interrupts: bool = False) -> int +:canonical: archivebox.workers.supervisord_util.run_runner_worker + +```{autodoc2-docstring} archivebox.workers.supervisord_util.run_runner_worker +``` +```` + ````{py:function} get_worker(supervisor, daemon_name) :canonical: archivebox.workers.supervisord_util.get_worker @@ -281,6 +569,41 @@ ``` ```` +````{py:function} format_runtime_components(components: list[str] | tuple[str, ...]) -> str +:canonical: archivebox.workers.supervisord_util.format_runtime_components + +```{autodoc2-docstring} archivebox.workers.supervisord_util.format_runtime_components +``` +```` + +````{py:function} worker_runtime_component(worker_name: str, *, config=None) -> str | None +:canonical: archivebox.workers.supervisord_util.worker_runtime_component + +```{autodoc2-docstring} archivebox.workers.supervisord_util.worker_runtime_component +``` +```` + +````{py:function} runtime_components_for_worker_names(worker_names: set[str] | list[str] | tuple[str, ...], *, config=None) -> list[str] +:canonical: archivebox.workers.supervisord_util.runtime_components_for_worker_names + +```{autodoc2-docstring} archivebox.workers.supervisord_util.runtime_components_for_worker_names +``` +```` + +````{py:function} active_supervisord_runtime_components(*, config=None, supervisor=None) -> list[str] +:canonical: archivebox.workers.supervisord_util.active_supervisord_runtime_components + +```{autodoc2-docstring} archivebox.workers.supervisord_util.active_supervisord_runtime_components +``` +```` + +````{py:function} build_server_worker_plan(*, config, host: str, port: str, debug: bool, reload: bool, nothreading: bool, supervisor=None) +:canonical: archivebox.workers.supervisord_util.build_server_worker_plan + +```{autodoc2-docstring} archivebox.workers.supervisord_util.build_server_worker_plan +``` +```` + ````{py:function} stop_worker(supervisor, daemon_name) :canonical: archivebox.workers.supervisord_util.stop_worker @@ -288,37 +611,72 @@ ``` ```` -````{py:function} tail_worker_logs(log_path: str) -:canonical: archivebox.workers.supervisord_util.tail_worker_logs - -```{autodoc2-docstring} archivebox.workers.supervisord_util.tail_worker_logs -``` -```` - -````{py:function} tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None) +````{py:function} tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep_running=None) :canonical: archivebox.workers.supervisord_util.tail_multiple_worker_logs ```{autodoc2-docstring} archivebox.workers.supervisord_util.tail_multiple_worker_logs ``` ```` -````{py:function} watch_worker(supervisor, daemon_name, interval=5) -:canonical: archivebox.workers.supervisord_util.watch_worker +````{py:function} get_sonic_supervisord_worker_from_plugin(config) -> dict[str, str] | None +:canonical: archivebox.workers.supervisord_util.get_sonic_supervisord_worker_from_plugin -```{autodoc2-docstring} archivebox.workers.supervisord_util.watch_worker +```{autodoc2-docstring} archivebox.workers.supervisord_util.get_sonic_supervisord_worker_from_plugin ``` ```` -````{py:function} start_server_workers(host='0.0.0.0', port='8000', daemonize=False) +````{py:function} _proc_cmdline(proc: psutil.Process) -> list[str] +:canonical: archivebox.workers.supervisord_util._proc_cmdline + +```{autodoc2-docstring} archivebox.workers.supervisord_util._proc_cmdline +``` +```` + +````{py:function} _is_sonic_process(proc: psutil.Process) -> bool +:canonical: archivebox.workers.supervisord_util._is_sonic_process + +```{autodoc2-docstring} archivebox.workers.supervisord_util._is_sonic_process +``` +```` + +````{py:function} _is_supervisord_process(proc: psutil.Process | None) -> bool +:canonical: archivebox.workers.supervisord_util._is_supervisord_process + +```{autodoc2-docstring} archivebox.workers.supervisord_util._is_supervisord_process +``` +```` + +````{py:function} _has_live_archivebox_parent(proc: psutil.Process | None) -> bool +:canonical: archivebox.workers.supervisord_util._has_live_archivebox_parent + +```{autodoc2-docstring} archivebox.workers.supervisord_util._has_live_archivebox_parent +``` +```` + +````{py:function} _terminate_process_tree(root: psutil.Process, *, timeout: float = 2.0) -> None +:canonical: archivebox.workers.supervisord_util._terminate_process_tree + +```{autodoc2-docstring} archivebox.workers.supervisord_util._terminate_process_tree +``` +```` + +````{py:function} _sonic_listeners(host: str, port: int) -> list[psutil.Process] +:canonical: archivebox.workers.supervisord_util._sonic_listeners + +```{autodoc2-docstring} archivebox.workers.supervisord_util._sonic_listeners +``` +```` + +````{py:function} stop_stale_sonic_processes(sonic_worker: dict[str, str], *, supervisor_pid: int | None, host: str | None = None, port: int | None = None) -> None +:canonical: archivebox.workers.supervisord_util.stop_stale_sonic_processes + +```{autodoc2-docstring} archivebox.workers.supervisord_util.stop_stale_sonic_processes +``` +```` + +````{py:function} start_server_workers(host='0.0.0.0', port='8000', daemonize=False, debug=False, reload=False, nothreading=False, keep_running=None, should_stop_supervisord=None, resumed_from_pid=None) :canonical: archivebox.workers.supervisord_util.start_server_workers ```{autodoc2-docstring} archivebox.workers.supervisord_util.start_server_workers ``` ```` - -````{py:function} start_cli_workers(watch=False) -:canonical: archivebox.workers.supervisord_util.start_cli_workers - -```{autodoc2-docstring} archivebox.workers.supervisord_util.start_cli_workers -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.workers.tasks.md b/docs/apidocs/archivebox/archivebox.workers.tasks.md deleted file mode 100644 index 3601c078..00000000 --- a/docs/apidocs/archivebox/archivebox.workers.tasks.md +++ /dev/null @@ -1,53 +0,0 @@ -# {py:mod}`archivebox.workers.tasks` - -```{py:module} archivebox.workers.tasks -``` - -```{autodoc2-docstring} archivebox.workers.tasks -:allowtitles: -``` - -## Module Contents - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`bg_add ` - - ```{autodoc2-docstring} archivebox.workers.tasks.bg_add - :summary: - ``` -* - {py:obj}`bg_archive_snapshots ` - - ```{autodoc2-docstring} archivebox.workers.tasks.bg_archive_snapshots - :summary: - ``` -* - {py:obj}`bg_archive_snapshot ` - - ```{autodoc2-docstring} archivebox.workers.tasks.bg_archive_snapshot - :summary: - ``` -```` - -### API - -````{py:function} bg_add(add_kwargs: dict) -> int -:canonical: archivebox.workers.tasks.bg_add - -```{autodoc2-docstring} archivebox.workers.tasks.bg_add -``` -```` - -````{py:function} bg_archive_snapshots(snapshots, kwargs: dict | None = None) -> int -:canonical: archivebox.workers.tasks.bg_archive_snapshots - -```{autodoc2-docstring} archivebox.workers.tasks.bg_archive_snapshots -``` -```` - -````{py:function} bg_archive_snapshot(snapshot, overwrite: bool = False, methods: list | None = None) -> int -:canonical: archivebox.workers.tasks.bg_archive_snapshot - -```{autodoc2-docstring} archivebox.workers.tasks.bg_archive_snapshot -``` -```` diff --git a/docs/apidocs/archivebox/archivebox.workers.worker.md b/docs/apidocs/archivebox/archivebox.workers.worker.md deleted file mode 100644 index a6348124..00000000 --- a/docs/apidocs/archivebox/archivebox.workers.worker.md +++ /dev/null @@ -1,549 +0,0 @@ -# {py:mod}`archivebox.workers.worker` - -```{py:module} archivebox.workers.worker -``` - -```{autodoc2-docstring} archivebox.workers.worker -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`Worker ` - - ```{autodoc2-docstring} archivebox.workers.worker.Worker - :summary: - ``` -* - {py:obj}`CrawlWorker ` - - ```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker - :summary: - ``` -* - {py:obj}`SnapshotWorker ` - - ```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker - :summary: - ``` -* - {py:obj}`BinaryWorker ` - - ```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker - :summary: - ``` -```` - -### Functions - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`_run_worker ` - - ```{autodoc2-docstring} archivebox.workers.worker._run_worker - :summary: - ``` -* - {py:obj}`_run_snapshot_worker ` - - ```{autodoc2-docstring} archivebox.workers.worker._run_snapshot_worker - :summary: - ``` -* - {py:obj}`get_worker_class ` - - ```{autodoc2-docstring} archivebox.workers.worker.get_worker_class - :summary: - ``` -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`CPU_COUNT ` - - ```{autodoc2-docstring} archivebox.workers.worker.CPU_COUNT - :summary: - ``` -* - {py:obj}`WORKER_TYPES ` - - ```{autodoc2-docstring} archivebox.workers.worker.WORKER_TYPES - :summary: - ``` -```` - -### API - -````{py:data} CPU_COUNT -:canonical: archivebox.workers.worker.CPU_COUNT -:value: > - 'cpu_count(...)' - -```{autodoc2-docstring} archivebox.workers.worker.CPU_COUNT -``` - -```` - -````{py:data} WORKER_TYPES -:canonical: archivebox.workers.worker.WORKER_TYPES -:type: dict[str, type[archivebox.workers.worker.Worker]] -:value: > - None - -```{autodoc2-docstring} archivebox.workers.worker.WORKER_TYPES -``` - -```` - -````{py:function} _run_worker(worker_class_name: str, worker_id: int, **kwargs) -:canonical: archivebox.workers.worker._run_worker - -```{autodoc2-docstring} archivebox.workers.worker._run_worker -``` -```` - -````{py:function} _run_snapshot_worker(snapshot_id: str, worker_id: int, **kwargs) -:canonical: archivebox.workers.worker._run_snapshot_worker - -```{autodoc2-docstring} archivebox.workers.worker._run_snapshot_worker -``` -```` - -`````{py:class} Worker(worker_id: int = 0, **kwargs: typing.Any) -:canonical: archivebox.workers.worker.Worker - -```{autodoc2-docstring} archivebox.workers.worker.Worker -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.workers.worker.Worker.__init__ -``` - -````{py:attribute} name -:canonical: archivebox.workers.worker.Worker.name -:type: typing.ClassVar[str] -:value: > - 'worker' - -```{autodoc2-docstring} archivebox.workers.worker.Worker.name -``` - -```` - -````{py:attribute} MAX_TICK_TIME -:canonical: archivebox.workers.worker.Worker.MAX_TICK_TIME -:type: typing.ClassVar[int] -:value: > - 60 - -```{autodoc2-docstring} archivebox.workers.worker.Worker.MAX_TICK_TIME -``` - -```` - -````{py:attribute} MAX_CONCURRENT_TASKS -:canonical: archivebox.workers.worker.Worker.MAX_CONCURRENT_TASKS -:type: typing.ClassVar[int] -:value: > - 1 - -```{autodoc2-docstring} archivebox.workers.worker.Worker.MAX_CONCURRENT_TASKS -``` - -```` - -````{py:method} __repr__() -> str -:canonical: archivebox.workers.worker.Worker.__repr__ - -```` - -````{py:method} get_model() -:canonical: archivebox.workers.worker.Worker.get_model -:abstractmethod: - -```{autodoc2-docstring} archivebox.workers.worker.Worker.get_model -``` - -```` - -````{py:method} on_startup() -> None -:canonical: archivebox.workers.worker.Worker.on_startup - -```{autodoc2-docstring} archivebox.workers.worker.Worker.on_startup -``` - -```` - -````{py:method} on_shutdown(error: BaseException | None = None) -> None -:canonical: archivebox.workers.worker.Worker.on_shutdown - -```{autodoc2-docstring} archivebox.workers.worker.Worker.on_shutdown -``` - -```` - -````{py:method} _terminate_background_hooks(background_processes: dict[str, archivebox.machine.models.Process], worker_type: str, indent_level: int) -> None -:canonical: archivebox.workers.worker.Worker._terminate_background_hooks - -```{autodoc2-docstring} archivebox.workers.worker.Worker._terminate_background_hooks -``` - -```` - -````{py:method} start(parent: typing.Any = None, **kwargs: typing.Any) -> int -:canonical: archivebox.workers.worker.Worker.start -:classmethod: - -```{autodoc2-docstring} archivebox.workers.worker.Worker.start -``` - -```` - -````{py:method} get_running_workers() -> list -:canonical: archivebox.workers.worker.Worker.get_running_workers -:classmethod: - -```{autodoc2-docstring} archivebox.workers.worker.Worker.get_running_workers -``` - -```` - -````{py:method} get_worker_count() -> int -:canonical: archivebox.workers.worker.Worker.get_worker_count -:classmethod: - -```{autodoc2-docstring} archivebox.workers.worker.Worker.get_worker_count -``` - -```` - -````` - -`````{py:class} CrawlWorker(crawl_id: str, **kwargs: typing.Any) -:canonical: archivebox.workers.worker.CrawlWorker - -Bases: {py:obj}`archivebox.workers.worker.Worker` - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.__init__ -``` - -````{py:attribute} name -:canonical: archivebox.workers.worker.CrawlWorker.name -:type: typing.ClassVar[str] -:value: > - 'crawl' - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.name -``` - -```` - -````{py:attribute} MAX_TICK_TIME -:canonical: archivebox.workers.worker.CrawlWorker.MAX_TICK_TIME -:type: typing.ClassVar[int] -:value: > - 60 - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.MAX_TICK_TIME -``` - -```` - -````{py:attribute} MAX_SNAPSHOT_WORKERS -:canonical: archivebox.workers.worker.CrawlWorker.MAX_SNAPSHOT_WORKERS -:type: typing.ClassVar[int] -:value: > - 8 - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.MAX_SNAPSHOT_WORKERS -``` - -```` - -````{py:method} get_model() -:canonical: archivebox.workers.worker.CrawlWorker.get_model - -```` - -````{py:method} on_startup() -> None -:canonical: archivebox.workers.worker.CrawlWorker.on_startup - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.on_startup -``` - -```` - -````{py:method} runloop() -> None -:canonical: archivebox.workers.worker.CrawlWorker.runloop - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.runloop -``` - -```` - -````{py:method} _spawn_snapshot_workers() -> None -:canonical: archivebox.workers.worker.CrawlWorker._spawn_snapshot_workers - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker._spawn_snapshot_workers -``` - -```` - -````{py:method} _terminate_running_snapshot_workers() -> None -:canonical: archivebox.workers.worker.CrawlWorker._terminate_running_snapshot_workers - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker._terminate_running_snapshot_workers -``` - -```` - -````{py:method} _is_crawl_finished() -> bool -:canonical: archivebox.workers.worker.CrawlWorker._is_crawl_finished - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker._is_crawl_finished -``` - -```` - -````{py:method} on_shutdown(error: BaseException | None = None) -> None -:canonical: archivebox.workers.worker.CrawlWorker.on_shutdown - -```{autodoc2-docstring} archivebox.workers.worker.CrawlWorker.on_shutdown -``` - -```` - -````` - -`````{py:class} SnapshotWorker(snapshot_id: str, **kwargs: typing.Any) -:canonical: archivebox.workers.worker.SnapshotWorker - -Bases: {py:obj}`archivebox.workers.worker.Worker` - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker.__init__ -``` - -````{py:attribute} name -:canonical: archivebox.workers.worker.SnapshotWorker.name -:type: typing.ClassVar[str] -:value: > - 'snapshot' - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker.name -``` - -```` - -````{py:method} get_model() -:canonical: archivebox.workers.worker.SnapshotWorker.get_model - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker.get_model -``` - -```` - -````{py:method} on_startup() -> None -:canonical: archivebox.workers.worker.SnapshotWorker.on_startup - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker.on_startup -``` - -```` - -````{py:method} runloop() -> None -:canonical: archivebox.workers.worker.SnapshotWorker.runloop - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker.runloop -``` - -```` - -````{py:method} _run_hook(hook_path: pathlib.Path, ar: typing.Any, config: dict) -> typing.Any -:canonical: archivebox.workers.worker.SnapshotWorker._run_hook - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._run_hook -``` - -```` - -````{py:method} _wait_for_hook(process: typing.Any, ar: typing.Any) -> None -:canonical: archivebox.workers.worker.SnapshotWorker._wait_for_hook - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._wait_for_hook -``` - -```` - -````{py:method} _retry_failed_empty_foreground_hooks(hooks: list[tuple[pathlib.Path, typing.Any]], config: dict) -> None -:canonical: archivebox.workers.worker.SnapshotWorker._retry_failed_empty_foreground_hooks - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._retry_failed_empty_foreground_hooks -``` - -```` - -````{py:method} _finalize_background_hooks() -> None -:canonical: archivebox.workers.worker.SnapshotWorker._finalize_background_hooks - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._finalize_background_hooks -``` - -```` - -````{py:method} _reap_background_hooks() -> None -:canonical: archivebox.workers.worker.SnapshotWorker._reap_background_hooks - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._reap_background_hooks -``` - -```` - -````{py:method} _snapshot_exceeded_hard_timeout() -> bool -:canonical: archivebox.workers.worker.SnapshotWorker._snapshot_exceeded_hard_timeout - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._snapshot_exceeded_hard_timeout -``` - -```` - -````{py:method} _seal_snapshot_due_to_timeout() -> None -:canonical: archivebox.workers.worker.SnapshotWorker._seal_snapshot_due_to_timeout - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._seal_snapshot_due_to_timeout -``` - -```` - -````{py:method} on_shutdown(error: BaseException | None = None) -> None -:canonical: archivebox.workers.worker.SnapshotWorker.on_shutdown - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker.on_shutdown -``` - -```` - -````{py:method} _extract_plugin_name(hook_path: pathlib.Path, hook_name: str) -> str -:canonical: archivebox.workers.worker.SnapshotWorker._extract_plugin_name -:staticmethod: - -```{autodoc2-docstring} archivebox.workers.worker.SnapshotWorker._extract_plugin_name -``` - -```` - -````` - -`````{py:class} BinaryWorker(binary_id: str = None, worker_id: int = 0) -:canonical: archivebox.workers.worker.BinaryWorker - -Bases: {py:obj}`archivebox.workers.worker.Worker` - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker -``` - -```{rubric} Initialization -``` - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.__init__ -``` - -````{py:attribute} name -:canonical: archivebox.workers.worker.BinaryWorker.name -:type: typing.ClassVar[str] -:value: > - 'binary' - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.name -``` - -```` - -````{py:attribute} MAX_TICK_TIME -:canonical: archivebox.workers.worker.BinaryWorker.MAX_TICK_TIME -:type: typing.ClassVar[int] -:value: > - 600 - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.MAX_TICK_TIME -``` - -```` - -````{py:attribute} MAX_CONCURRENT_TASKS -:canonical: archivebox.workers.worker.BinaryWorker.MAX_CONCURRENT_TASKS -:type: typing.ClassVar[int] -:value: > - 1 - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.MAX_CONCURRENT_TASKS -``` - -```` - -````{py:attribute} POLL_INTERVAL -:canonical: archivebox.workers.worker.BinaryWorker.POLL_INTERVAL -:type: typing.ClassVar[float] -:value: > - 0.5 - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.POLL_INTERVAL -``` - -```` - -````{py:method} get_model() -:canonical: archivebox.workers.worker.BinaryWorker.get_model - -```` - -````{py:method} get_next_item() -:canonical: archivebox.workers.worker.BinaryWorker.get_next_item - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.get_next_item -``` - -```` - -````{py:method} runloop() -> None -:canonical: archivebox.workers.worker.BinaryWorker.runloop - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker.runloop -``` - -```` - -````{py:method} _process_single_binary() -:canonical: archivebox.workers.worker.BinaryWorker._process_single_binary - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker._process_single_binary -``` - -```` - -````{py:method} _daemon_loop() -:canonical: archivebox.workers.worker.BinaryWorker._daemon_loop - -```{autodoc2-docstring} archivebox.workers.worker.BinaryWorker._daemon_loop -``` - -```` - -````` - -````{py:function} get_worker_class(name: str) -> type[archivebox.workers.worker.Worker] -:canonical: archivebox.workers.worker.get_worker_class - -```{autodoc2-docstring} archivebox.workers.worker.get_worker_class -``` -````