diff --git a/archivebox/core/models.py b/archivebox/core/models.py index e37dbda7..be9aecc7 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -273,7 +273,7 @@ class SnapshotQuerySet(models.QuerySet): last_values = None value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field])) - for _attempt in range(8): + while True: batch_qs = self.order_by(*ordering) if last_values is not None: page_filter = models.Q() @@ -731,7 +731,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if not plugin_names: return False retry_at = when or timezone.now() - while True: + for _attempt in range(8): current = type(self).objects.select_related("crawl").get(pk=self.pk) pending_plugins = {str(name).strip() for name in (current.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()} config = {**(current.config or {}), "RETRY_PLUGINS": sorted(pending_plugins | set(plugin_names))} diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index c5cd9262..cc84a980 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -401,14 +401,19 @@ def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO except (OperationalError, SQLiteOperationalError) as err: if not sqlite_lock_error(err): raise - if retry_timeout and time.monotonic() - started_at >= retry_timeout: - raise + if retry_timeout: + remaining = retry_timeout - (time.monotonic() - started_at) + if remaining <= 0: + raise + sleep_for = min(retry_interval, remaining) + else: + sleep_for = retry_interval connections.close_all() - console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {retry_interval:g}s...[/yellow]") + console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {sleep_for:g}s...[/yellow]") log_sqlite_lock_holders(console) with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"): - time.sleep(retry_interval) + time.sleep(sleep_for) @contextmanager diff --git a/archivebox/tests/test_snapshot_service.py b/archivebox/tests/test_snapshot_service.py index f5c3e041..d59b1ff3 100644 --- a/archivebox/tests/test_snapshot_service.py +++ b/archivebox/tests/test_snapshot_service.py @@ -102,6 +102,17 @@ def test_concurrent_plugin_scheduling_durably_merges_every_request(admin_user): assert snapshot.config["RETRY_PLUGINS"] == sorted(plugins) +def test_snapshot_keyset_iterator_reads_more_than_eight_pages(admin_user): + from archivebox.crawls.models import Crawl + + crawl = Crawl.objects.create(urls="https://example.com/pages", created_by=admin_user) + snapshots = [Snapshot.objects.create(url=f"https://example.com/pages/{idx}", crawl=crawl) for idx in range(10)] + + yielded_ids = [snapshot.id for snapshot in crawl.snapshot_set.order_by("created_at").paged_iterator(chunk_size=1)] + + assert yielded_ids == [snapshot.id for snapshot in snapshots] + + def test_snapshot_merge_consolidates_only_exact_hook_identity(admin_user): from archivebox.crawls.models import Crawl