merge

Author: dshabin

src/superannotate/lib/app/interface/sdk_interface.py

@@ -30,6 +30,7 @@
 from lib.app.helpers import reformat_metrics_json
 from lib.app.interface.types import AnnotationStatuses
 from lib.app.interface.types import AnnotationType
+from lib.app.interface.types import ImageQualityChoices
 from lib.app.interface.types import NotEmptyStr
 from lib.app.interface.types import Status
 from lib.app.interface.types import validate_arguments
@@ -3471,7 +3472,7 @@ def upload_images_to_project(
     img_paths: List[NotEmptyStr],
     annotation_status: Optional[Status] = "NotStarted",
     from_s3_bucket=None,
-    image_quality_in_editor: Optional[NotEmptyStr] = None,
+    image_quality_in_editor: Optional[ImageQualityChoices] = None,
 ):
     """Uploads all images given in list of path objects in img_paths to the project.
     Sets status of all the uploaded images to set_status if it is not None.

src/superannotate/lib/app/interface/types.py

@@ -34,6 +34,19 @@ def validate(cls, value: Union[str]) -> Union[str]:
         return value
 
 
+class ImageQualityChoices(StrictStr):
+    VALID_CHOICES = ["compressed", "original"]
+
+    @classmethod
+    def validate(cls, value: Union[str]) -> Union[str]:
+        super().validate(value)
+        if value.lower() not in cls.VALID_CHOICES:
+            raise TypeError(
+                f"Image quality should be on of {', '.join(cls.VALID_CHOICES)}."
+            )
+        return value.lower()
+
+
 class AnnotationStatuses(StrictStr):
     @classmethod
     def validate(cls, value: Union[str]) -> Union[str]:

src/superannotate/lib/core/usecases.py

@@ -17,12 +17,12 @@
 from typing import Optional
 
 import boto3
-import botocore.exceptions
 import cv2
 import lib.core as constances
 import numpy as np
 import pandas as pd
 import requests
+from botocore.exceptions import ClientError
 from lib.app.analytics.common import aggregate_annotations_as_df
 from lib.app.analytics.common import consensus_plot
 from lib.app.analytics.common import image_consensus
@@ -805,7 +805,7 @@ def __init__(
     ):
         super().__init__(),
         self._project = project
-        self._folder_names = folder_names
+        self._folder_names = list(folder_names)
         self._backend_service = backend_service_provider
         self._annotation_statuses = annotation_statuses
         self._include_fuse = include_fuse
@@ -855,12 +855,12 @@ def execute(self):
             if "error" in response:
                 raise AppException(response["error"])
 
-            report_message = self._project.name
+            report_message = ""
             if self._folder_names:
-                report_message = f"[{', '.join(self._folder_names)}]"
+                report_message = f"[{', '.join(self._folder_names)}] "
             logger.info(
-                f"Prepared export {response['name']} for project "
-                f"{report_message} (project ID {self._project.uuid})."
+                f"Prepared export {response['name']} for project {self._project.name} "
+                f"{report_message}(project ID {self._project.uuid})."
             )
             self._response.data = response
 
@@ -2156,14 +2156,19 @@ def __init__(
         self._image_path = image_path
 
     def execute(self):
-        image = io.BytesIO()
-        session = boto3.Session()
-        resource = session.resource("s3")
-        image_object = resource.Object(self._s3_bucket, self._image_path)
-        if image_object.content_length > constances.MAX_IMAGE_SIZE:
-            raise AppValidationException(f"File size is {image_object.content_length}")
-        image_object.download_fileobj(image)
-        self._response.data = image
+        try:
+            image = io.BytesIO()
+            session = boto3.Session()
+            resource = session.resource("s3")
+            image_object = resource.Object(self._s3_bucket, self._image_path)
+            if image_object.content_length > constances.MAX_IMAGE_SIZE:
+                raise AppValidationException(
+                    f"File size is {image_object.content_length}"
+                )
+            image_object.download_fileobj(image)
+            self._response.data = image
+        except ClientError as e:
+            self._response.errors = str(e)
         return self._response
 
 
@@ -3431,19 +3436,21 @@ def execute(self):
                 )
                 if self._project.project_type == constances.ProjectType.PIXEL.value:
                     mask_path = None
-                    if os.path.exists(self._annotation_path) and not self._mask:
-                        mask_path = self._annotation_path
+                    png_path = self._annotation_path.replace(
+                        "___pixel.json", "___save.png"
+                    )
+                    if os.path.exists(png_path) and not self._mask:
+                        mask_path = png_path
                     elif self._mask:
                         mask_path = self._mask
 
                     if mask_path:
                         with open(mask_path, "rb") as descriptor:
-                            file = io.BytesIO(descriptor.read())
                             bucket.put_object(
                                 Key=response.data.images[image_data["id"]][
                                     "annotation_bluemap_path"
                                 ],
-                                Body=file,
+                                Body=descriptor.read(),
                             )
                 if self._verbose:
                     logger.info(
@@ -4162,7 +4169,7 @@ def execute(self):
                     mapper_path,
                     os.path.join(self._download_path, "classes_mapper.json"),
                 )
-            except botocore.exceptions.ClientError:
+            except ClientError:
                 logger.info(
                     "The specified model does not contain a classes_mapper and/or a metrics file."
                 )
@@ -4853,11 +4860,20 @@ def _upload_image(self, image_path: str):
             "ProcessedImage", ["uploaded", "path", "entity", "name"]
         )
         if self._from_s3_bucket:
-            image_bytes = (
-                GetS3ImageUseCase(s3_bucket=self._from_s3_bucket, image_path=image_path)
-                .execute()
-                .data
-            )
+            response = GetS3ImageUseCase(
+                s3_bucket=self._from_s3_bucket, image_path=image_path
+            ).execute()
+            if response.errors:
+                logger.warning(
+                    f"Unable to upload image {image_path} \n{response.errors}"
+                )
+                return ProcessedImage(
+                    uploaded=False,
+                    path=image_path,
+                    entity=None,
+                    name=Path(image_path).name,
+                )
+            image_bytes = response.data
         else:
             try:
                 image_bytes = io.BytesIO(open(image_path, "rb").read())
@@ -4934,7 +4950,6 @@ def images_to_upload(self):
                     images_to_upload.append(path)
                 else:
                     duplicated_paths.append(path)
-
             self._images_to_upload = list(set(images_to_upload)), duplicated_paths
         return self._images_to_upload
 
@@ -4979,7 +4994,8 @@ def execute(self):
                 duplications.extend(attach_duplications)
             uploaded = [image["name"] for image in uploaded]
             failed_images = [image.split("/")[-1] for image in failed_images]
-
+            if duplications:
+                logger.info(f"Duplicated images {', '.join(duplications)}")
             self._response.data = uploaded, failed_images, duplications
         return self._response
 

tests/integration/test_interface.py

@@ -1,7 +1,7 @@
 import os
 from os.path import dirname
 import tempfile
-import unittest
+import boto3
 
 import src.superannotate as sa
 from src.superannotate.lib.app.exceptions import AppException
@@ -10,17 +10,21 @@
 
 class TestInterface(BaseTestCase):
     PROJECT_NAME = "Interface test"
-    TEST_FOLDER_PATH = "data_set/sample_project_vector"
+    TEST_FOLDER_PATH = "sample_project_vector"
     TEST_FOLDER_PATH_WITH_MULTIPLE_IMAGERS = "data_set/sample_project_vector"
     PROJECT_DESCRIPTION = "desc"
     PROJECT_TYPE = "Vector"
     TEST_FOLDER_NAME = "folder"
     EXAMPLE_IMAGE_1 = "example_image_1.jpg"
     EXAMPLE_IMAGE_2 = "example_image_2.jpg"
 
+    @property
+    def data_set_path(self):
+        return os.path.join(dirname(dirname(__file__)), "data_set")
+
     @property
     def folder_path(self):
-        return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH)
+        return os.path.join(self.data_set_path, self.TEST_FOLDER_PATH)
 
     @property
     def folder_path_with_multiple_images(self):
@@ -133,3 +137,46 @@ def test_overlay_fuse(self):
                 include_overlay=True,
             )
             self.assertIsNotNone(paths)
+
+    def test_upload_images_to_project_image_quality_in_editor(self):
+        self.assertRaises(
+            AppException,
+            sa.upload_images_to_project,
+            self.PROJECT_NAME,
+            [self.EXAMPLE_IMAGE_1],
+            image_quality_in_editor='random_string'
+        )
+
+
+class TestPixelInterface(BaseTestCase):
+    PROJECT_NAME = "Interface test"
+    TEST_FOLDER_PATH = "sample_project_pixel"
+    PROJECT_DESCRIPTION = "desc"
+    PROJECT_TYPE = "Pixel"
+    TEST_FOLDER_NAME = "folder"
+    EXAMPLE_IMAGE_1 = "example_image_1.jpg"
+
+    @property
+    def data_set_path(self):
+        return os.path.join(dirname(dirname(__file__)), "data_set")
+
+    @property
+    def folder_path(self):
+        return os.path.join(self.data_set_path, self.TEST_FOLDER_PATH)
+
+    def test_export_annotation(self):
+        sa.upload_image_to_project(self.PROJECT_NAME, f"{self.folder_path}/{self.EXAMPLE_IMAGE_1}")
+        sa.create_annotation_classes_from_classes_json(self.PROJECT_NAME, f"{self.folder_path}/classes/classes.json")
+        sa.upload_image_annotations(
+            self.PROJECT_NAME, self.EXAMPLE_IMAGE_1, f"{self.folder_path}/{self.EXAMPLE_IMAGE_1}___pixel.json"
+        )
+        with tempfile.TemporaryDirectory() as export_dir:
+            result = sa.prepare_export(
+                self.PROJECT_NAME,
+            )
+            sa.download_export(self.PROJECT_NAME, result, export_dir, True)
+            with tempfile.TemporaryDirectory() as convert_path:
+                sa.export_annotation(
+                    export_dir, convert_path, "COCO", "data_set_name", "Pixel", "panoptic_segmentation"
+                )
+                pass