Tests moved to annotations folder

Author: Vaghinak Basentsyan

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2387,9 +2387,6 @@ def upload_annotations_from_folder_to_project(
         )
     logger.info("The JSON files should follow a specific naming convention, matching file names already present "
                 "on the platform. Existing annotations will be overwritten")
-    logger.info(
-        "Uploading all annotations from %s to project %s.", folder_path, project_name
-    )
 
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders

src/superannotate/lib/core/helpers.py

@@ -60,7 +60,8 @@ def fill_annotation_ids(
         annotation_class_name = annotation["className"]
         if annotation_class_name not in annotation_classes_name_maps.keys():
             if annotation_class_name not in unknown_classes:
-                reporter.store_message("Couldn't find classes ", annotation_class_name)
+                reporter.log_warning(f"Couldn't find class {annotation_class_name}")
+                reporter.store_message("missing_classes", annotation_class_name)
                 unknown_classes[annotation_class_name] = {
                     "id": -(len(unknown_classes) + 1),
                     "attribute_groups": {},
@@ -94,9 +95,7 @@ def fill_annotation_ids(
                 reporter.log_warning(
                     f"Couldn't find annotation group {attribute['groupName']}."
                 )
-                reporter.store_message(
-                    "Couldn't find annotation groups", attribute["groupName"]
-                )
+                reporter.store_message("missing_attribute_groups", f"{annotation['className']}.{attribute['groupName']}")
                 continue
             attribute["groupId"] = annotation_classes_name_maps[annotation_class_name][
                 "attribute_groups"
@@ -112,7 +111,7 @@ def fill_annotation_ids(
                     f"Couldn't find annotation name {attribute['name']} in"
                     f" annotation group {attribute['groupName']}",
                 )
-                reporter.store_message("Couldn't find attributes", attribute["name"])
+                reporter.store_message("missing_attributes", attribute["name"])
                 continue
             attribute["id"] = annotation_classes_name_maps[annotation_class_name][
                 "attribute_groups"

src/superannotate/lib/core/types.py

@@ -1,7 +1,7 @@
+from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Union
-from typing import Dict
 
 from pydantic import BaseModel
 from pydantic import constr
@@ -48,15 +48,15 @@ class Metadata(BaseModel):
 
 
 class PointLabels(BaseModel):
-    __root__: Dict[constr(regex=r"^[0-9]*$"), str]
+    __root__: Dict[constr(regex=r'^[0-9]*$'), str]
 
 
 class BaseInstance(BaseModel):
     type: AnnotationType
-    classId: int
+    classId: Optional[int]
     groupId: Optional[int]
     attributes: List[Attribute]
-    point_labels: Optional[PointLabels]
+    # point_labels: Optional[PointLabels]
 
     class Config:
         error_msg_templates = {
@@ -96,20 +96,20 @@ class Ellipse(BaseInstance):
 
 
 class TemplatePoint(BaseModel):
-    id: int
+    id: Optional[int]
     x: float
     y: float
 
 
 class TemplateConnection(BaseModel):
-    id: int
+    id: Optional[int]
     to: int
 
 
 class Template(BaseInstance):
     points: List[TemplatePoint]
     connections: List[Optional[TemplateConnection]]
-    templateId: int
+    templateId: Optional[int]
 
 
 class EmptyPoint(BaseModel):
@@ -133,7 +133,7 @@ class PixelAnnotationPart(BaseModel):
 
 
 class PixelAnnotationInstance(BaseModel):
-    classId: int
+    classId: Optional[int]
     groupId: Optional[int]
     parts: List[PixelAnnotationPart]
     attributes: List[Attribute]
@@ -189,7 +189,7 @@ class Config:
 
 class MLModel(BaseModel):
     name: NotEmptyStr
-    id: int
+    id: Optional[int]
     path: NotEmptyStr
     config_path: NotEmptyStr
     team_id: Optional[int]
@@ -206,7 +206,7 @@ class VideoMetaData(BaseModel):
 
 class VideoInstanceMeta(BaseModel):
     type: NotEmptyStr
-    classId: int
+    classId: Optional[int]
 
 
 class VideoTimeStamp(BaseModel):

src/superannotate/lib/core/usecases/annotations.py

@@ -13,7 +13,6 @@
 from lib.core.entities import ImageEntity
 from lib.core.entities import ProjectEntity
 from lib.core.exceptions import AppException
-from lib.core.exceptions import AppValidationException
 from lib.core.helpers import convert_to_video_editor_json
 from lib.core.helpers import fill_annotation_ids
 from lib.core.helpers import map_annotation_classes_name
@@ -177,64 +176,84 @@ def _upload_annotation(
             # raise e
             return path, False
 
+    def get_bucket_to_upload(self, ids: List[int]):
+        upload_data = self.get_annotation_upload_data(ids)
+        if upload_data:
+            session = boto3.Session(
+                aws_access_key_id=upload_data.access_key,
+                aws_secret_access_key=upload_data.secret_key,
+                aws_session_token=upload_data.session_token,
+                region_name=upload_data.region,
+            )
+            resource = session.resource("s3")
+            return resource.Bucket(upload_data.bucket)
+
+    def _log_report(self):
+        for key, values in self.reporter.custom_messages.items():
+            template = key + ": {}"
+            if key == "missing_classes":
+                template = "Could not find annotation classes matching existing classes on the platform: [{}]"
+            elif key == "missing_attribute_groups":
+                template = "Could not find attribute groups matching existing attribute groups on the platform: [{}]"
+            elif key == "missing_attributes":
+                template = "Could not find attributes matching existing attributes on the platform: [{}]"
+            logger.warning(
+                template.format(", ".join(values))
+            )
+
     def execute(self):
         uploaded_annotations = []
         failed_annotations = []
-        iterations_range = range(
-            0, len(self.annotations_to_upload), self.AUTH_DATA_CHUNK_SIZE
-        )
-        self.reporter.start_progress(iterations_range,description="Uploading Annotations")
-        for _ in iterations_range:
-            annotations_to_upload = self.annotations_to_upload[
-                _ : _ + self.AUTH_DATA_CHUNK_SIZE  # noqa: E203
-            ]
-            upload_data = self.get_annotation_upload_data(
-                [int(image.id) for image in annotations_to_upload]
+        if self.annotations_to_upload:
+            iterations_range = range(
+                0, len(self.annotations_to_upload), self.AUTH_DATA_CHUNK_SIZE
             )
-            if upload_data:
-                session = boto3.Session(
-                    aws_access_key_id=upload_data.access_key,
-                    aws_secret_access_key=upload_data.secret_key,
-                    aws_session_token=upload_data.session_token,
-                    region_name=upload_data.region,
+            self.reporter.start_progress(iterations_range, description="Uploading Annotations")
+            for _ in iterations_range:
+                annotations_to_upload = self.annotations_to_upload[
+                    _ : _ + self.AUTH_DATA_CHUNK_SIZE  # noqa: E203
+                ]
+                upload_data = self.get_annotation_upload_data(
+                    [int(image.id) for image in annotations_to_upload]
                 )
-                resource = session.resource("s3")
-                bucket = resource.Bucket(upload_data.bucket)
-                image_id_name_map = {
-                    image.id: image for image in self.annotations_to_upload
-                }
-                # dummy progress
-                for _ in range(len(annotations_to_upload) - len(upload_data.images)):
-                    self.reporter.update_progress()
-                with concurrent.futures.ThreadPoolExecutor(
-                    max_workers=self.MAX_WORKERS
-                ) as executor:
-                    results = [
-                        executor.submit(
-                            self._upload_annotation,
-                            image_id,
-                            image_id_name_map[image_id].name,
-                            upload_data,
-                            image_id_name_map[image_id].path,
-                            bucket,
-                        )
-                        for image_id, image_data in upload_data.images.items()
-                    ]
-                    for future in concurrent.futures.as_completed(results):
-                        annotation, uploaded = future.result()
-                        if uploaded:
-                            uploaded_annotations.append(annotation)
-                        else:
-                            failed_annotations.append(annotation)
+                bucket = self.get_bucket_to_upload([int(image.id) for image in annotations_to_upload])
+                if bucket:
+                    image_id_name_map = {
+                        image.id: image for image in self.annotations_to_upload
+                    }
+                    # dummy progress
+                    for _ in range(len(annotations_to_upload) - len(upload_data.images)):
                         self.reporter.update_progress()
+                    with concurrent.futures.ThreadPoolExecutor(
+                        max_workers=self.MAX_WORKERS
+                    ) as executor:
+                        results = [
+                            executor.submit(
+                                self._upload_annotation,
+                                image_id,
+                                image_id_name_map[image_id].name,
+                                upload_data,
+                                image_id_name_map[image_id].path,
+                                bucket,
+                            )
+                            for image_id, image_data in upload_data.images.items()
+                        ]
+                        for future in concurrent.futures.as_completed(results):
+                            annotation, uploaded = future.result()
+                            if uploaded:
+                                uploaded_annotations.append(annotation)
+                            else:
+                                failed_annotations.append(annotation)
+                            self.reporter.update_progress()
 
-        self._response.data = (
-            uploaded_annotations,
-            failed_annotations,
-            [annotation.path for annotation in self._missing_annotations],
-        )
-        for message in self.reporter.messages:
-            logger.warning(message)
+            self._response.data = (
+                uploaded_annotations,
+                failed_annotations,
+                [annotation.path for annotation in self._missing_annotations],
+            )
+            self._log_report()
+        else:
+            self._response.errors = "Could not find annotations matching existing items on the platform."
         return self._response
 
 
@@ -401,5 +420,5 @@ def execute(self):
                         self._project.name,
                     )
             else:
-                self._response.errors = f"Invalid json"
+                self._response.errors = "Invalid json"
         return self._response