Merge pull request #195 from superannotateai/s3_recursive

Fix s3_recursive annotations

Author: dshabin

src/superannotate/lib/core/usecases.py

@@ -2231,9 +2231,10 @@ def execute(self):
                     url=annotation_blue_map_creds["url"],
                     headers=annotation_blue_map_creds["headers"],
                 )
+                data["annotation_mask_filename"] = f"{self._image_name}___save.png"
+
                 if response.ok:
                     data["annotation_mask"] = io.BytesIO(response.content)
-                    data["annotation_mask_filename"] = f"{self._image_name}___save.png"
                     mask_path = (
                         Path(self._destination) / data["annotation_mask_filename"]
                     )
@@ -4480,58 +4481,63 @@ def paths(self):
                         ):
                             paths.append(key)
                             break
-
-        data = []
-        for path in paths:
-            if all(
-                [
-                    True if exclude_pattern not in str(path) else False
-                    for exclude_pattern in self.exclude_file_patterns
-                ]
-            ):
-                data.append(str(path))
-        return data
+        return [str(path) for path in paths]
 
     @property
     def images_to_upload(self):
         if not self._images_to_upload:
             paths = self.paths
             filtered_paths = []
             duplicated_paths = []
+            for path in paths:
+                if path.split("/")[-1] not in [
+                    path_name.split("/")[-1] for path_name in filtered_paths
+                ]:
+                    filtered_paths.append(path)
+                else:
+                    duplicated_paths.append(path)
+            filtered_paths = [
+                path
+                for path in filtered_paths
+                if not any(
+                    [
+                        path.endswith(extension)
+                        for extension in self.exclude_file_patterns
+                    ]
+                )
+            ]
+            duplicated_paths = [
+                path
+                for path in duplicated_paths
+                if not any(
+                    [
+                        path.endswith(extension)
+                        for extension in self.exclude_file_patterns
+                    ]
+                )
+            ]
+
             image_entities = (
                 GetBulkImages(
                     service=self._backend_client,
                     project_id=self._project.uuid,
                     team_id=self._project.team_id,
                     folder_id=self._folder.uuid,
-                    images=[Path(image).name for image in paths],
+                    images=[image.split("/")[-1] for image in filtered_paths],
                 )
                 .execute()
                 .data
             )
+            images_to_upload = []
+            image_list = [image.name for image in image_entities]
 
-            for path in paths:
-                not_in_exclude_list = [
-                    x not in Path(path).name for x in self.exclude_file_patterns
-                ]
-                non_in_service_list = [
-                    x.name not in Path(path).name for x in image_entities
-                ]
-                if (
-                    all(not_in_exclude_list)
-                    and all(non_in_service_list)
-                    and not any(
-                        Path(path).name in filtered_path
-                        for filtered_path in filtered_paths
-                    )
-                ):
-                    filtered_paths.append(path)
-                elif not all(non_in_service_list) or any(
-                    Path(path).name in filtered_path for filtered_path in filtered_paths
-                ):
+            for path in filtered_paths:
+                if path not in image_list:
+                    images_to_upload.append(path)
+                else:
                     duplicated_paths.append(path)
 
-            self._images_to_upload = list(set(filtered_paths)), duplicated_paths
+            self._images_to_upload = list(set(images_to_upload)), duplicated_paths
         return self._images_to_upload
 
     def execute(self):

tests/integration/test_recursive_folder.py

@@ -269,6 +269,23 @@ def test_images_non_recursive_s3(self):
 
         self.assertEqual(len(sa.search_images(self.PROJECT_NAME)), 1)
 
+
+
+
+    def test_images_recursive_s3_122(self):
+        sa.upload_images_from_folder_to_project(self.PROJECT_NAME, '8sep', from_s3_bucket="superannotate-python-sdk-test",recursive_subfolders=True)
+        self.assertEqual(len(sa.search_images(self.PROJECT_NAME)), 122)
+
+
+    def test_annotations_recursive_s3_122(self):
+        sa.upload_images_from_folder_to_project(self.PROJECT_NAME, '8sep', from_s3_bucket="superannotate-python-sdk-test",recursive_subfolders=True)
+        sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME, '8sep', from_s3_bucket="superannotate-python-sdk-test",recursive_subfolders=True)
+        annotations = sa.get_image_annotations(self.PROJECT_NAME,"e13cb60a2bf31c22d2524518b7444f92e37fe5d404b0144390f8c078a0eabd_640.jpg")[
+            "annotation_json"
+        ]
+        self.assertEqual(len(annotations['instances']),1)
+
+
     def test_images_non_recursive(self):
         sa.upload_images_from_folder_to_project(
             self.PROJECT_NAME, self.folder_path, recursive_subfolders=False