Fix bug where all crawls are added to workflow as successful even if failed

Author: tw4l

backend/btrixcloud/basecrawls.py

@@ -372,14 +372,21 @@ async def delete_crawls(
             size += crawl_size
 
             cid = crawl.cid
+            successful = crawl.state in SUCCESSFUL_STATES
             if cid:
                 if cids_to_update.get(cid):
                     cids_to_update[cid]["inc"] += 1
                     cids_to_update[cid]["size"] += crawl_size
+                    if successful:
+                        cids_to_update[cid]["successful"] += 1
                 else:
                     cids_to_update[cid] = {}
                     cids_to_update[cid]["inc"] = 1
                     cids_to_update[cid]["size"] = crawl_size
+                    if successful:
+                        cids_to_update[cid]["successful"] = 1
+                    else:
+                        cids_to_update[cid]["successful"] = 0
 
             if type_ == "crawl":
                 asyncio.create_task(
@@ -866,7 +873,10 @@ async def delete_crawls_all_types(
             for cid, cid_dict in cids_to_update.items():
                 cid_size = cid_dict["size"]
                 cid_inc = cid_dict["inc"]
-                await self.crawl_configs.stats_recompute_last(cid, -cid_size, -cid_inc)
+                cid_successful = cid_dict["successful"]
+                await self.crawl_configs.stats_recompute_last(
+                    cid, -cid_size, -cid_inc, -cid_successful
+                )
 
         if uploads_length:
             upload_delete_list = DeleteCrawlList(crawl_ids=uploads)

backend/btrixcloud/crawlconfigs.py

@@ -920,7 +920,9 @@ async def get_last_successful_crawl_out(
 
         return None
 
-    async def stats_recompute_last(self, cid: UUID, size: int, inc_crawls: int = 1):
+    async def stats_recompute_last(
+        self, cid: UUID, size: int, inc_crawls: int = 1, inc_successful: int = 1
+    ):
         """recompute stats by incrementing size counter and number of crawls"""
         update_query: dict[str, object] = {}
 
@@ -977,7 +979,7 @@ async def stats_recompute_last(self, cid: UUID, size: int, inc_crawls: int = 1):
                 "$inc": {
                     "totalSize": size,
                     "crawlCount": inc_crawls,
-                    "crawlSuccessfulCount": inc_crawls,
+                    "crawlSuccessfulCount": inc_successful,
                 },
             },
         )

backend/btrixcloud/crawls.py

@@ -412,7 +412,10 @@ async def delete_crawls(
         for cid, cid_dict in cids_to_update.items():
             cid_size = cid_dict["size"]
             cid_inc = cid_dict["inc"]
-            await self.crawl_configs.stats_recompute_last(cid, -cid_size, -cid_inc)
+            cid_successful = cid_dict["successful"]
+            await self.crawl_configs.stats_recompute_last(
+                cid, -cid_size, -cid_inc, -cid_successful
+            )
 
         return count, cids_to_update, quota_reached
 
@@ -903,7 +906,9 @@ async def shutdown_crawl(
             if not graceful:
                 await self.update_crawl_state(crawl_id, "canceled")
                 crawl = await self.get_crawl(crawl_id, org)
-                if not await self.crawl_configs.stats_recompute_last(crawl.cid, 0, -1):
+                if not await self.crawl_configs.stats_recompute_last(
+                    crawl.cid, 0, -1, 0
+                ):
                     raise HTTPException(
                         status_code=404,
                         detail=f"crawl_config_not_found: {crawl.cid}",

backend/btrixcloud/operator/crawls.py

@@ -1746,11 +1746,10 @@ async def do_crawl_finished_tasks(
         stats: Optional[CrawlStats],
     ) -> None:
         """Run tasks after crawl completes in asyncio.task coroutine."""
-        await self.crawl_config_ops.stats_recompute_last(
-            crawl.cid, status.filesAddedSize, 1
-        )
-
         if state in SUCCESSFUL_STATES and crawl.oid:
+            await self.crawl_config_ops.stats_recompute_last(
+                crawl.cid, status.filesAddedSize, 1, 1
+            )
             await self.page_ops.set_archived_item_page_counts(crawl.id)
             await self.org_ops.set_last_crawl_finished(crawl.oid)
             await self.coll_ops.add_successful_crawl_to_collections(
@@ -1767,6 +1766,7 @@ async def do_crawl_finished_tasks(
                 )
 
         if state in FAILED_STATES:
+            await self.crawl_config_ops.stats_recompute_last(crawl.cid, 0, 1, 0)
             await self.crawl_ops.delete_failed_crawl_files(crawl.id, crawl.oid)
             await self.page_ops.delete_crawl_pages(crawl.id, crawl.oid)