Fix pixel recursive

Author: dshabin

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2392,11 +2392,14 @@ def upload_annotations_from_folder_to_project(
         annotation_paths=annotation_paths,  # noqa: E203
         client_s3_bucket=from_s3_bucket,
     )
-    with tqdm(
-        total=len(annotation_paths), desc="Uploading annotations"
-    ) as progress_bar:
-        for _ in use_case.execute():
-            progress_bar.update(1)
+    if use_case.is_valid():
+        with tqdm(
+            total=len(use_case.annotations_to_upload), desc="Uploading annotations"
+        ) as progress_bar:
+            for _ in use_case.execute():
+                progress_bar.update(1)
+    else:
+        raise AppException(use_case.response.errors)
     return use_case.data
 
 

src/superannotate/lib/core/usecases/images.py

@@ -2662,6 +2662,8 @@ def execute(self):
                 else:
                     from_s3 = None
 
+                for _ in range(len(annotations_to_upload) - len(response.data.images)):
+                    yield
                 with concurrent.futures.ThreadPoolExecutor(
                     max_workers=self.MAX_WORKERS
                 ) as executor:
@@ -2693,7 +2695,6 @@ def execute(self):
                 failed_annotations = [
                     annotation.path for annotation in failed_annotations
                 ]
-
             self._response.data = (
                 uploaded_annotations,
                 failed_annotations,
@@ -2705,42 +2706,41 @@ def execute(self):
     def upload_to_s3(
         self, image_id: int, image_info, bucket, from_s3, image_id_name_map
     ):
-        if from_s3:
-            file = io.BytesIO()
-            s3_object = from_s3.Object(
-                self._client_s3_bucket, image_id_name_map[image_id].path
-            )
-            s3_object.download_fileobj(file)
-            file.seek(0)
-            annotation_json = json.load(file)
-        else:
-            annotation_json = json.load(open(image_id_name_map[image_id].path))
-
-        self.fill_classes_data(annotation_json)
-
-        if not self._is_valid_json(annotation_json):
-            logger.warning(f"Invalid json {image_id_name_map[image_id].path}")
-            return image_id_name_map[image_id], False
-        bucket.put_object(
-            Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json),
-        )
-        if self._project.project_type == constances.ProjectType.PIXEL.value:
-            mask_filename = (
-                image_id_name_map[image_id].name + constances.ANNOTATION_MASK_POSTFIX
-            )
+        try:
             if from_s3:
                 file = io.BytesIO()
                 s3_object = from_s3.Object(
-                    self._client_s3_bucket, f"{self._folder_path}/{mask_filename}"
+                    self._client_s3_bucket, image_id_name_map[image_id].path
                 )
                 s3_object.download_fileobj(file)
                 file.seek(0)
+                annotation_json = json.load(file)
             else:
-                with open(f"{self._folder_path}/{mask_filename}", "rb") as mask_file:
-                    file = io.BytesIO(mask_file.read())
-
-            bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
-        return image_id_name_map[image_id], True
+                annotation_json = json.load(open(image_id_name_map[image_id].path))
+            self.fill_classes_data(annotation_json)
+            if not self._is_valid_json(annotation_json):
+                logger.warning(f"Invalid json {image_id_name_map[image_id].path}")
+                return image_id_name_map[image_id], False
+            bucket.put_object(
+                Key=image_info["annotation_json_path"], Body=json.dumps(annotation_json),
+            )
+            if self._project.project_type == constances.ProjectType.PIXEL.value:
+                mask_path = image_id_name_map[image_id].path.replace("___pixel.json", constances.ANNOTATION_MASK_POSTFIX)
+                if from_s3:
+                    file = io.BytesIO()
+                    s3_object = from_s3.Object(
+                        self._client_s3_bucket,
+                        mask_path
+                    )
+                    s3_object.download_fileobj(file)
+                    file.seek(0)
+                else:
+                    with open(mask_path, "rb") as mask_file:
+                        file = io.BytesIO(mask_file.read())
+                bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
+            return image_id_name_map[image_id], True
+        except Exception as _:
+            return image_id_name_map[image_id], False
 
     def report_missing_data(self):
         if self.missing_classes:

tests/integration/test_recursive_folder_pixel.py

@@ -0,0 +1,32 @@
+import os
+from os.path import dirname
+
+import src.superannotate as sa
+from tests.integration.base import BaseTestCase
+
+
+class TestRecursiveFolderPixel(BaseTestCase):
+    PROJECT_NAME = "pixel_recursive_test"
+    PROJECT_DESCRIPTION = "Desc"
+    PROJECT_TYPE = "Pixel"
+    S3_FOLDER_PATH = "pixel_all_fuse"
+    JSON_POSTFIX = "*.json"
+
+
+    def test_recursive_upload_pixel(self):
+        uploaded, _, duplicated = sa.upload_images_from_folder_to_project(self.PROJECT_NAME,
+                                                                          self.S3_FOLDER_PATH,
+                                                                          from_s3_bucket="test-openseadragon-1212",
+                                                                          recursive_subfolders=True
+                                                                          )
+
+        uploaded, failed, missing = sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME,
+                                                                          self.S3_FOLDER_PATH,
+                                                                          from_s3_bucket="test-openseadragon-1212",
+                                                                          recursive_subfolders=True
+                                                                          )
+        self.assertEqual(115, len(uploaded))
+        self.assertEqual(0, len(failed))
+        self.assertEqual(0, len(missing))
+
+