fix: preserve unbounded maintenance pagination

archivebox/core/models.py

@@ -273,7 +273,7 @@ class SnapshotQuerySet(models.QuerySet):
 
         last_values = None
         value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        for _attempt in range(8):
+        while True:
             batch_qs = self.order_by(*ordering)
             if last_values is not None:
                 page_filter = models.Q()
@@ -731,7 +731,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if not plugin_names:
             return False
         retry_at = when or timezone.now()
-        while True:
+        for _attempt in range(8):
             current = type(self).objects.select_related("crawl").get(pk=self.pk)
             pending_plugins = {str(name).strip() for name in (current.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()}
             config = {**(current.config or {}), "RETRY_PLUGINS": sorted(pending_plugins | set(plugin_names))}

archivebox/misc/db.py

@@ -401,14 +401,19 @@ def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO
         except (OperationalError, SQLiteOperationalError) as err:
             if not sqlite_lock_error(err):
                 raise
-            if retry_timeout and time.monotonic() - started_at >= retry_timeout:
-                raise
+            if retry_timeout:
+                remaining = retry_timeout - (time.monotonic() - started_at)
+                if remaining <= 0:
+                    raise
+                sleep_for = min(retry_interval, remaining)
+            else:
+                sleep_for = retry_interval
 
         connections.close_all()
-        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {retry_interval:g}s...[/yellow]")
+        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in {sleep_for:g}s...[/yellow]")
         log_sqlite_lock_holders(console)
         with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"):
-            time.sleep(retry_interval)
+            time.sleep(sleep_for)
 
 
 @contextmanager

archivebox/tests/test_snapshot_service.py

@@ -102,6 +102,17 @@ def test_concurrent_plugin_scheduling_durably_merges_every_request(admin_user):
     assert snapshot.config["RETRY_PLUGINS"] == sorted(plugins)
 
 
+def test_snapshot_keyset_iterator_reads_more_than_eight_pages(admin_user):
+    from archivebox.crawls.models import Crawl
+
+    crawl = Crawl.objects.create(urls="https://example.com/pages", created_by=admin_user)
+    snapshots = [Snapshot.objects.create(url=f"https://example.com/pages/{idx}", crawl=crawl) for idx in range(10)]
+
+    yielded_ids = [snapshot.id for snapshot in crawl.snapshot_set.order_by("created_at").paged_iterator(chunk_size=1)]
+
+    assert yielded_ids == [snapshot.id for snapshot in snapshots]
+
+
 def test_snapshot_merge_consolidates_only_exact_hook_identity(admin_user):
     from archivebox.crawls.models import Crawl