merge

Author: dshabin

src/superannotate/lib/app/interface/sdk_interface.py

@@ -5,7 +5,6 @@
 import os
 import tempfile
 import time
-from collections import Counter
 from collections import namedtuple
 from pathlib import Path
 from typing import Iterable
@@ -43,7 +42,6 @@
 from lib.core import LIMITED_FUNCTIONS
 from lib.core.enums import ImageQuality
 from lib.core.exceptions import AppException
-from lib.core.exceptions import AppValidationException
 from lib.core.types import AttributeGroup
 from lib.core.types import ClassesJson
 from lib.core.types import MLModel
@@ -1070,8 +1068,10 @@ def delete_image(project: Union[NotEmptyStr, dict], image_name: str):
     :param image_name: image name
     :type image_name: str
     """
-    project_name, _ = extract_project_folder(project)
-    response = controller.delete_image(image_name=image_name, project_name=project_name)
+    project_name, folder_name = extract_project_folder(project)
+    response = controller.delete_image(
+        image_name=image_name, folder_name=folder_name, project_name=project_name
+    )
     if response.errors:
         raise AppException("Couldn't delete image ")
     logger.info(f"Successfully deleted image {image_name}.")
@@ -2122,7 +2122,7 @@ def move_image(
             is_pinned=1,
         )
 
-    controller.delete_image(image_name, source_project_name)
+    controller.delete_image(image_name, source_project_name, source_folder_name)
 
 
 @Trackable
@@ -3490,104 +3490,29 @@ def upload_images_to_project(
     :return: uploaded, could-not-upload, existing-images filepaths
     :rtype: tuple (3 members) of list of strs
     """
-    uploaded_image_entities = []
-    failed_images = []
     project_name, folder_name = extract_project_folder(project)
-    project = controller.get_project_metadata(project_name).data
-    if project["project"].project_type in [
-        constances.ProjectType.VIDEO.value,
-        constances.ProjectType.DOCUMENT.value,
-    ]:
-        raise AppException(LIMITED_FUNCTIONS[project["project"].project_type])
-
-    ProcessedImage = namedtuple("ProcessedImage", ["uploaded", "path", "entity"])
 
-    def _upload_local_image(image_path: str):
-        try:
-            with open(image_path, "rb") as image:
-                image_bytes = io.BytesIO(image.read())
-                upload_response = controller.upload_image_to_s3(
-                    project_name=project_name,
-                    image_path=image_path,
-                    image_bytes=image_bytes,
-                    folder_name=folder_name,
-                    image_quality_in_editor=image_quality_in_editor,
-                )
-
-                if not upload_response.errors and upload_response.data:
-                    entity = upload_response.data
-                    return ProcessedImage(
-                        uploaded=True, path=entity.path, entity=entity
-                    )
-                else:
-                    return ProcessedImage(uploaded=False, path=image_path, entity=None)
-        except FileNotFoundError:
-            return ProcessedImage(uploaded=False, path=image_path, entity=None)
-
-    def _upload_s3_image(image_path: str):
-        try:
-            image_bytes = controller.get_image_from_s3(
-                s3_bucket=from_s3_bucket, image_path=image_path
-            ).data
-        except AppValidationException as e:
-            logger.warning(e)
-            return image_path
-        upload_response = controller.upload_image_to_s3(
-            project_name=project_name,
-            image_path=image_path,
-            image_bytes=image_bytes,
-            folder_name=folder_name,
-            image_quality_in_editor=image_quality_in_editor,
-        )
-        if not upload_response.errors and upload_response.data:
-            entity = upload_response.data
-            return ProcessedImage(uploaded=True, path=entity.path, entity=entity)
-        else:
-            return ProcessedImage(uploaded=False, path=image_path, entity=None)
-
-    filtered_paths = img_paths
-    duplication_counter = Counter(filtered_paths)
-    images_to_upload, duplicated_images = (
-        set(filtered_paths),
-        [item for item in duplication_counter if duplication_counter[item] > 1],
+    use_case = controller.upload_images_to_project(
+        project_name=project_name,
+        folder_name=folder_name,
+        paths=img_paths,
+        annotation_status=annotation_status,
+        image_quality_in_editor=image_quality_in_editor,
+        from_s3_bucket=from_s3_bucket,
     )
-    upload_method = _upload_s3_image if from_s3_bucket else _upload_local_image
-
-    logger.info(f"Uploading {len(images_to_upload)} images to project.")
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-        results = [
-            executor.submit(upload_method, image_path)
-            for image_path in images_to_upload
-        ]
-        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
-            for future in concurrent.futures.as_completed(results):
-                processed_image = future.result()
-                if processed_image.uploaded and processed_image.entity:
-                    uploaded_image_entities.append(processed_image.entity)
-                else:
-                    failed_images.append(processed_image.path)
-                progress_bar.update(1)
-    uploaded = []
-    duplicates = []
-
-    for i in range(0, len(uploaded_image_entities), 500):
-        response = controller.upload_images(
-            project_name=project_name,
-            folder_name=folder_name,
-            images=uploaded_image_entities[i : i + 500],  # noqa: E203
-            annotation_status=annotation_status,
-        )
-        attachments, duplications = response.data
-        uploaded.extend([attachment["name"] for attachment in attachments])
-        duplicates.extend(duplications)
 
+    images_to_upload, duplicates = use_case.images_to_upload
     if len(duplicates):
         logger.warning(
             "%s already existing images found that won't be uploaded.", len(duplicates)
         )
-
-    return uploaded, failed_images, duplicates
+    logger.info(f"Uploading {len(images_to_upload)} images to project {project}.")
+    if use_case.is_valid():
+        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
+            for _ in use_case.execute():
+                progress_bar.update(1)
+        return use_case.data
+    raise AppException(use_case.response.errors)
 
 
 @Trackable

src/superannotate/lib/core/usecases.py

@@ -341,6 +341,36 @@ def validate_project_type(self):
                 constances.LIMITED_FUNCTIONS[self._project.project_type]
             )
 
+    def validate_project_name(self):
+        if self._project_to_create.name:
+            if (
+                len(
+                    set(self._project_to_create.name).intersection(
+                        constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES
+                    )
+                )
+                > 0
+            ):
+                self._project_to_create.name = "".join(
+                    "_"
+                    if char in constances.SPECIAL_CHARACTERS_IN_PROJECT_FOLDER_NAMES
+                    else char
+                    for char in self._project_to_create.name
+                )
+                logger.warning(
+                    "New folder name has special characters. Special characters will be replaced by underscores."
+                )
+            condition = Condition("name", self._project_to_create.name, EQ) & Condition(
+                "team_id", self._project.team_id, EQ
+            )
+            for project in self._projects.get_all(condition):
+                if project.name == self._project_to_create.name:
+                    logger.error("There are duplicated names.")
+                    raise AppValidationException(
+                        f"Project name {self._project_to_create.name} is not unique. "
+                        f"To use SDK please make project names unique."
+                    )
+
     def execute(self):
         if self.is_valid():
             project = self._projects.insert(self._project_to_create)
@@ -3263,6 +3293,9 @@ def __init__(
         self._mask = mask
         self._verbose = verbose
         self._annotation_path = annotation_path
+        self.unknown_classes = []
+        self.unknown_attribute_groups = []
+        self.unknown_attributes = []
 
     def validate_project_type(self):
         if self._project.project_type in constances.LIMITED_FUNCTIONS:
@@ -3303,18 +3336,21 @@ def fill_classes_data(self, annotations: dict):
         annotation_classes = self.annotation_classes_name_map
         if "instances" not in annotations:
             return
+
         unknown_classes = {}
         for annotation in [i for i in annotations["instances"] if "className" in i]:
             if "className" not in annotation:
                 return
             annotation_class_name = annotation["className"]
-            if annotation_class_name not in annotation_classes:
+            if annotation_class_name not in annotation_classes.keys():
                 if annotation_class_name not in unknown_classes:
+                    self.unknown_classes.append(annotation_class_name)
                     unknown_classes[annotation_class_name] = {
                         "id": -(len(unknown_classes) + 1),
                         "attribute_groups": {},
                     }
-        annotation_classes.update(unknown_classes)
+        if unknown_classes:
+            annotation_classes.update(unknown_classes)
         templates = self.get_templates_mapping()
         for annotation in (
             i for i in annotations["instances"] if i.get("type", None) == "template"
@@ -3325,14 +3361,15 @@ def fill_classes_data(self, annotations: dict):
 
         for annotation in [i for i in annotations["instances"] if "className" in i]:
             annotation_class_name = annotation["className"]
-            if annotation_class_name not in annotation_classes:
+            if annotation_class_name not in annotation_classes.keys():
                 continue
             annotation["classId"] = annotation_classes[annotation_class_name]["id"]
             for attribute in annotation["attributes"]:
                 if (
                     attribute["groupName"]
                     not in annotation_classes[annotation_class_name]["attribute_groups"]
                 ):
+                    self.unknown_attributes.append(attribute["groupName"])
                     continue
                 attribute["groupId"] = annotation_classes[annotation_class_name][
                     "attribute_groups"
@@ -3344,11 +3381,22 @@ def fill_classes_data(self, annotations: dict):
                     ][attribute["groupName"]]["attributes"]
                 ):
                     del attribute["groupId"]
+                    self.unknown_attributes.append(attribute["name"])
                     continue
                 attribute["id"] = annotation_classes[annotation_class_name][
                     "attribute_groups"
                 ][attribute["groupName"]]["attributes"][attribute["name"]]
 
+    def report_unknown_data(self):
+        if self.unknown_classes:
+            logger.warning(f"Unknown classes [{', '.join(self.unknown_classes)}]")
+        if self.unknown_attribute_groups:
+            logger.warning(
+                f"Unknown attribute_groups [{', '.join(self.unknown_attribute_groups)}]"
+            )
+        if self.unknown_attributes:
+            logger.warning(f"Unknown attributes [{', '.join(self.unknown_attributes)}]")
+
     def execute(self):
         if self.is_valid():
             image_data = self._backend_service.get_bulk_images(
@@ -3376,6 +3424,7 @@ def execute(self):
                 resource = session.resource("s3")
                 bucket = resource.Bucket(response.data.bucket)
                 self.fill_classes_data(self._annotations)
+                self.report_unknown_data()
                 bucket.put_object(
                     Key=response.data.images[image_data["id"]]["annotation_json_path"],
                     Body=json.dumps(self._annotations),
@@ -4802,7 +4851,15 @@ def _upload_image(self, image_path: str):
                 .data
             )
         else:
-            image_bytes = io.BytesIO(open(image_path, "rb").read())
+            try:
+                image_bytes = io.BytesIO(open(image_path, "rb").read())
+            except FileNotFoundError:
+                return ProcessedImage(
+                    uploaded=False,
+                    path=image_path,
+                    entity=None,
+                    name=Path(image_path).name,
+                )
         upload_response = UploadImageS3UseCase(
             project=self._project,
             project_settings=self._settings,

src/superannotate/lib/infrastructure/controller.py

@@ -636,9 +636,8 @@ def search_images(
         return use_case.execute()
 
     def _get_image(
-        self, project: ProjectEntity, image_name: str, folder_path: str = None,
+        self, project: ProjectEntity, image_name: str, folder: FolderEntity = None,
     ) -> ImageEntity:
-        folder = self._get_folder(project, folder_path)
         use_case = usecases.GetImageUseCase(
             service=self._backend_client,
             project=project,
@@ -651,7 +650,9 @@ def _get_image(
     def get_image(
         self, project_name: str, image_name: str, folder_path: str = None
     ) -> ImageEntity:
-        return self._get_image(self._get_project(project_name), image_name, folder_path)
+        project = self._get_project(project_name)
+        folder = self._get_folder(project, folder_path)
+        return self._get_image(project, image_name, folder)
 
     def update_folder(self, project_name: str, folder_name: str, folder_data: dict):
         project = self._get_project(project_name)
@@ -669,7 +670,8 @@ def get_image_bytes(
         image_variant: str = None,
     ):
         project = self._get_project(project_name)
-        image = self._get_image(project, image_name, folder_name)
+        folder = self._get_folder(project, folder_name)
+        image = self._get_image(project, image_name, folder)
         use_case = usecases.GetImageBytesUseCase(
             image=image,
             backend_service_provider=self._backend_client,
@@ -686,12 +688,12 @@ def copy_image_annotation_classes(
         image_name: str,
     ):
         from_project = self._get_project(from_project_name)
-        image = self._get_image(
-            from_project, folder_path=from_folder_name, image_name=image_name
-        )
+        from_folder = self._get_folder(from_project, from_folder_name)
+        image = self._get_image(from_project, folder=from_folder, image_name=image_name)
         to_project = self._get_project(to_project_name)
+        to_folder = self._get_folder(to_project, to_folder_name)
         uploaded_image = self._get_image(
-            to_project, folder_path=to_folder_name, image_name=image_name
+            to_project, folder=to_folder, image_name=image_name
         )
 
         use_case = usecases.CopyImageAnnotationClasses(
@@ -857,14 +859,16 @@ def set_project_settings(self, project_name: str, new_settings: List[dict]):
         )
         return use_case.execute()
 
-    def delete_image(self, image_name, project_name):
-        image = self.get_image(project_name=project_name, image_name=image_name)
-        project_entity = self._get_project(project_name)
+    def delete_image(self, project_name: str, image_name: str, folder_name: str):
+        project = self._get_project(project_name)
+        folder = self._get_folder(project, folder_name)
+        image = self._get_image(project=project, image_name=image_name, folder=folder)
+
         use_case = usecases.DeleteImageUseCase(
             images=ImageRepository(service=self._backend_client),
             image=image,
-            team_id=project_entity.team_id,
-            project_id=project_entity.uuid,
+            team_id=project.team_id,
+            project_id=project.uuid,
         )
         return use_case.execute()
 
@@ -1231,7 +1235,7 @@ def download_image(
     ):
         project = self._get_project(project_name)
         folder = self._get_folder(project, folder_name)
-        image = self._get_image(project, image_name, folder_name)
+        image = self._get_image(project, image_name, folder)
 
         use_case = usecases.DownloadImageUseCase(
             project=project,

src/superannotate/lib/infrastructure/repositories.py

@@ -361,8 +361,8 @@ def insert(self, entity: ImageEntity) -> ImageEntity:
         raise NotImplementedError
 
     def delete(self, uuid: int, team_id: int, project_id: int):
-        self._service.delete_image(
-            image_id=uuid, team_id=team_id, project_id=project_id
+        self._service.delete_images(
+            image_ids=[uuid], team_id=team_id, project_id=project_id
         )
 
     def update(self, entity: ImageEntity):