release: v0.9.33rc51

archivebox/cli/archivebox_add.py

@@ -216,7 +216,7 @@ def add(
     #    - Repeat until max_depth reached
 
     if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
+        # Background mode: just queue work and return (background runner via server will pick it up).
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )

archivebox/core/models.py

@@ -754,11 +754,28 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             self.ensure_legacy_archive_symlink()
             self.ensure_crawl_symlink()
             crawl = self.crawl
-            existing_urls = {url for _raw_line, url in crawl._iter_url_lines() if url}
-            if crawl.url_passes_filters(self.url, snapshot=self) and self.url not in existing_urls:
+            if not crawl.url_passes_filters(self.url, snapshot=self):
+                return
+            # Sibling snapshots in the same crawl race to append into the
+            # same ``crawl.urls`` text field; reload + retry on CAS miss
+            # so concurrent appends don't clobber each other.
+            for _ in range(16):
+                crawl.refresh_from_db()
+                existing_urls = {url for _raw_line, url in crawl._iter_url_lines() if url}
+                if self.url in existing_urls:
+                    return
                 urls = f"{crawl.urls}\n{self.url}"
-                crawl.safe_update({"urls": urls, "modified_at": timezone.now()}, refresh=False)
-                crawl.urls = urls
+                if crawl.safe_update({"urls": urls, "modified_at": timezone.now()}, refresh=False):
+                    crawl.urls = urls
+                    return
+            import logging as _logging
+
+            _logging.getLogger("archivebox.crawls").error(
+                "Snapshot %s could not append its URL to crawl %s.urls: %s",
+                self.id,
+                crawl.id,
+                self.url,
+            )
 
         # get_or_create/update_or_create wrap save() in atomic(); defer filesystem
         # work and crawl maintenance so SQLite commits before touching the disk.

archivebox/machine/models.py

@@ -1608,6 +1608,18 @@ class Process(ModelWithDeleteAfter, models.Model):
     def _detect_process_type(cls) -> str:
         """
         Detect the type of the current process from sys.argv.
+
+        ``archivebox add --bg`` is a fire-and-forget queue write — it does not
+        run the runner or own the runtime stack — so it's classified as CLI
+        instead of ADD. The ADD process_type is reserved for the foreground
+        ``archivebox add`` flow that actually takes over the runtime stack via
+        ``current_command(TypeChoices.ADD, ...)``. Misclassifying ``--bg`` as
+        ADD makes ``runtime_stack_owner`` treat it as a newer stack owner for
+        the few seconds it's alive, knocks the running ``archivebox server``
+        out of leadership, and triggers a supervisord tear-down + respawn
+        cycle (~5s of dead time per add). Detecting bg here at insert time
+        avoids any race window where the row briefly exists as ADD before a
+        higher-level demotion.
         """
         argv_str = " ".join(sys.argv).lower()
 
@@ -1620,6 +1632,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         elif "archivebox update" in argv_str:
             return cls.TypeChoices.UPDATE
         elif "archivebox add" in argv_str:
+            if "--bg" in sys.argv:
+                return cls.TypeChoices.CLI
             return cls.TypeChoices.ADD
         elif "archivebox search" in argv_str or "archivebox list" in argv_str:
             return cls.TypeChoices.SEARCH