Merge pull request #262 from superannotateai/friday

Friday

Author: VaghinakDev

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2447,22 +2447,23 @@ def upload_annotations_from_folder_to_project(
     project_name, folder_name = extract_project_folder(project)
     project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
 
-
     if recursive_subfolders:
         logger.info(
             "When using recursive subfolder parsing same name annotations in different "
             "subfolders will overwrite each other.",
         )
-    logger.info("The JSON files should follow a specific naming convention, matching file names already present "
-                "on the platform. Existing annotations will be overwritten")
+    logger.info(
+        "The JSON files should follow a specific naming convention, matching file names already present "
+        "on the platform. Existing annotations will be overwritten"
+    )
 
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
-    if not annotation_paths:
-        raise AppException("Could not find annotations matching existing items on the platform.")
 
-    logger.info(f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}.")
+    logger.info(
+        f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}."
+    )
     response = controller.upload_annotations_from_folder(
         project_name=project_name,
         folder_name=folder_name,
@@ -2518,16 +2519,17 @@ def upload_preannotations_from_folder_to_project(
             "When using recursive subfolder parsing same name annotations in different "
             "subfolders will overwrite each other.",
         )
-    logger.info("The JSON files should follow a specific naming convention, matching file names already present "
-                "on the platform. Existing annotations will be overwritten")
+    logger.info(
+        "The JSON files should follow a specific naming convention, matching file names already present "
+        "on the platform. Existing annotations will be overwritten"
+    )
     logger.info("Existing annotations will be overwritten.",)
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
-    if not annotation_paths:
-        raise AppException("Could not find annotations matching existing items on the platform.")
     logger.info(
-        f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}.")
+        f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}."
+    )
     response = controller.upload_annotations_from_folder(
         project_name=project_name,
         folder_name=folder_name,

src/superannotate/lib/core/helpers.py

@@ -8,7 +8,7 @@
 def map_annotation_classes_name(annotation_classes, reporter: Reporter) -> dict:
     classes_data = defaultdict(dict)
     for annotation_class in annotation_classes:
-        class_info = {"id": annotation_class.uuid}
+        class_info = {"id": annotation_class.uuid, "attribute_groups": {}}
         if annotation_class.attribute_groups:
             for attribute_group in annotation_class.attribute_groups:
                 attribute_group_data = defaultdict(dict)
@@ -27,12 +27,11 @@ def map_annotation_classes_name(annotation_classes, reporter: Reporter) -> dict:
                         " Only one of the annotation class attribute groups will be used."
                         " This will result in errors in annotation upload."
                     )
-                class_info["attribute_groups"] = {
-                    attribute_group["name"]: {
-                        "id": attribute_group["id"],
-                        "attributes": attribute_group_data,
-                    }
+                class_info["attribute_groups"][attribute_group["name"]] = {
+                    "id": attribute_group["id"],
+                    "attributes": attribute_group_data,
                 }
+
         if annotation_class.name in classes_data.keys():
             reporter.log_warning(
                 f"Duplicate annotation class name {annotation_class.name}."
@@ -43,6 +42,16 @@ def map_annotation_classes_name(annotation_classes, reporter: Reporter) -> dict:
     return classes_data
 
 
+def fill_document_tags(
+    annotations: dict, annotation_classes: dict,
+):
+    new_tags = []
+    for tag in annotations["tags"]:
+        if annotation_classes.get(tag):
+            new_tags.append(annotation_classes[tag]["id"])
+    annotations["tags"] = new_tags
+
+
 def fill_annotation_ids(
     annotations: dict,
     annotation_classes_name_maps: dict,
@@ -95,7 +104,10 @@ def fill_annotation_ids(
                 reporter.log_warning(
                     f"Couldn't find annotation group {attribute['groupName']}."
                 )
-                reporter.store_message("missing_attribute_groups", f"{annotation['className']}.{attribute['groupName']}")
+                reporter.store_message(
+                    "missing_attribute_groups",
+                    f"{annotation['className']}.{attribute['groupName']}",
+                )
                 continue
             attribute["groupId"] = annotation_classes_name_maps[annotation_class_name][
                 "attribute_groups"
@@ -118,14 +130,16 @@ def fill_annotation_ids(
             ][attribute["groupName"]]["attributes"][attribute["name"]]
 
 
-def convert_to_video_editor_json(data: dict, class_name_mapper: dict, reporter: Reporter):
+def convert_to_video_editor_json(
+    data: dict, class_name_mapper: dict, reporter: Reporter
+):
     id_generator = ClassIdGenerator()
 
     def safe_time(timestamp):
         return "0" if str(timestamp) == "0.0" else timestamp
 
     def convert_timestamp(timestamp):
-        return timestamp / 10 ** 6
+        return timestamp / 10 ** 6 if timestamp else "0"
 
     editor_data = {
         "instances": [],
@@ -134,8 +148,8 @@ def convert_timestamp(timestamp):
         "metadata": {
             "duration": convert_timestamp(data["metadata"]["duration"]),
             "name": data["metadata"]["name"],
-            "width": data["metadata"]["width"],
-            "height": data["metadata"]["height"],
+            "width": data["metadata"].get("width"),
+            "height": data["metadata"].get("height"),
         },
     }
     for instance in data["instances"]:
@@ -145,10 +159,13 @@ def convert_timestamp(timestamp):
             "attributes": [],
             "timeline": {},
             "type": meta["type"],
-            "locked": True,
+            # TODO check
+            "locked": False,
         }
         if class_name:
-            editor_instance["classId"] = class_name_mapper.get(class_name, {}).get("id", id_generator.send(class_name))
+            editor_instance["classId"] = class_name_mapper.get(class_name, {}).get(
+                "id", id_generator.send(class_name)
+            )
         else:
             editor_instance["classId"] = id_generator.send("unknown_class")
         if meta.get("pointLabels", None):
@@ -181,13 +198,33 @@ def convert_timestamp(timestamp):
 
                 existing_attributes_in_current_instance = set()
                 for attribute in timestamp_data["attributes"]:
-                    group_name, attr_name = attribute.get("groupName"), attribute.get("name")
-                    if not class_name_mapper[class_name].get("attribute_groups", {}).get(group_name):
-                        reporter.store_message("missing_attribute_groups", f"{class_name}.{group_name}")
-                    elif not class_name_mapper[class_name]["attribute_groups"][group_name].get("attributes", {}).get(attr_name):
-                        reporter.store_message("missing_attributes", f"{class_name}.{group_name}.{attr_name}")
+                    group_name, attr_name = (
+                        attribute.get("groupName"),
+                        attribute.get("name"),
+                    )
+                    if (
+                        not class_name_mapper[class_name]
+                        .get("attribute_groups", {})
+                        .get(group_name)
+                    ):
+                        reporter.store_message(
+                            "missing_attribute_groups", f"{class_name}.{group_name}"
+                        )
+                    elif (
+                        not class_name_mapper[class_name]["attribute_groups"][
+                            group_name
+                        ]
+                        .get("attributes", {})
+                        .get(attr_name)
+                    ):
+                        reporter.store_message(
+                            "missing_attributes",
+                            f"{class_name}.{group_name}.{attr_name}",
+                        )
                     else:
-                        existing_attributes_in_current_instance.add((group_name, attr_name))
+                        existing_attributes_in_current_instance.add(
+                            (group_name, attr_name)
+                        )
                 attributes_to_add = (
                     existing_attributes_in_current_instance - active_attributes
                 )

src/superannotate/lib/core/types.py

@@ -48,7 +48,7 @@ class Metadata(BaseModel):
 
 
 class PointLabels(BaseModel):
-    __root__: Dict[constr(regex=r'^[0-9]*$'), str]
+    __root__: Dict[constr(regex=r"^[0-9]*$"), str]
 
 
 class BaseInstance(BaseModel):
@@ -227,10 +227,10 @@ class VideoInstance(BaseModel):
 
 class VideoAnnotation(BaseModel):
     metadata: VideoMetaData
-    instances: List[VideoInstance]
-    tags: List[str]
+    instances: Optional[List[VideoInstance]]
+    tags: Optional[List[str]]
 
 
 class DocumentAnnotation(BaseModel):
     instances: list
-    tags: List[str]
+    tags: Optional[List[str]]

src/superannotate/lib/core/usecases/annotations.py

@@ -12,9 +12,9 @@
 from lib.core.entities import FolderEntity
 from lib.core.entities import ImageEntity
 from lib.core.entities import ProjectEntity
-from lib.core.exceptions import AppException
 from lib.core.helpers import convert_to_video_editor_json
 from lib.core.helpers import fill_annotation_ids
+from lib.core.helpers import fill_document_tags
 from lib.core.helpers import map_annotation_classes_name
 from lib.core.reporter import Reporter
 from lib.core.service_types import UploadAnnotationAuthData
@@ -110,6 +110,7 @@ def annotations_to_upload(self):
                 for idx, detail in enumerate(images_detail):
                     if detail.name == image_data.name:
                         images_detail[idx] = detail._replace(id=image_data.uuid)
+                        break
 
             missing_annotations = list(
                 filter(lambda image_detail: image_detail.id is None, images_detail)
@@ -118,12 +119,9 @@ def annotations_to_upload(self):
                 filter(lambda image_detail: image_detail.id is not None, images_detail)
             )
             if missing_annotations:
-                for missing in missing_annotations:
-                    logger.warning(
-                        f"Couldn't find image {missing.path} for annotation upload."
-                    )
-            if not annotations_to_upload:
-                raise AppException("No image to attach annotations.")
+                logger.warning(
+                    f"Couldn't find {len(missing_annotations)}/{len(annotations_to_upload + missing_annotations)} items on the platform that match the annotations you want to upload."
+                )
             self._missing_annotations = missing_annotations
             self._annotations_to_upload = annotations_to_upload
         return self._annotations_to_upload
@@ -196,9 +194,7 @@ def _log_report(self):
                 template = "Could not find attribute groups matching existing attribute groups on the platform: [{}]"
             elif key == "missing_attributes":
                 template = "Could not find attributes matching existing attributes on the platform: [{}]"
-            logger.warning(
-                template.format("', '".join(values))
-            )
+            logger.warning(template.format("', '".join(values)))
 
     def execute(self):
         uploaded_annotations = []
@@ -207,21 +203,27 @@ def execute(self):
             iterations_range = range(
                 0, len(self.annotations_to_upload), self.AUTH_DATA_CHUNK_SIZE
             )
-            self.reporter.start_progress(iterations_range, description="Uploading Annotations")
-            for _ in iterations_range:
+            self.reporter.start_progress(
+                len(self.annotations_to_upload), description="Uploading Annotations"
+            )
+            for step in iterations_range:
                 annotations_to_upload = self.annotations_to_upload[
-                    _ : _ + self.AUTH_DATA_CHUNK_SIZE  # noqa: E203
-                ]
+                    step : step + self.AUTH_DATA_CHUNK_SIZE
+                ]  # noqa: E203
                 upload_data = self.get_annotation_upload_data(
                     [int(image.id) for image in annotations_to_upload]
                 )
-                bucket = self.get_bucket_to_upload([int(image.id) for image in annotations_to_upload])
+                bucket = self.get_bucket_to_upload(
+                    [int(image.id) for image in annotations_to_upload]
+                )
                 if bucket:
                     image_id_name_map = {
                         image.id: image for image in self.annotations_to_upload
                     }
                     # dummy progress
-                    for _ in range(len(annotations_to_upload) - len(upload_data.images)):
+                    for _ in range(
+                        len(annotations_to_upload) - len(upload_data.images)
+                    ):
                         self.reporter.update_progress()
                     with concurrent.futures.ThreadPoolExecutor(
                         max_workers=self.MAX_WORKERS
@@ -251,8 +253,6 @@ def execute(self):
                 [annotation.path for annotation in self._missing_annotations],
             )
             self._log_report()
-        else:
-            self._response.errors = "Could not find annotations matching existing items on the platform."
         return self._response
 
 
@@ -337,13 +337,26 @@ def from_s3(self):
     def set_annotation_json(self):
         if not self._annotation_json:
             if self._client_s3_bucket:
-                self._annotation_json = json.load(self.get_s3_file(self.from_s3, self._annotation_path))
+                self._annotation_json = json.load(
+                    self.get_s3_file(self.from_s3, self._annotation_path)
+                )
                 if self._project.project_type == constances.ProjectType.PIXEL.value:
-                    self._mask = self.get_s3_file(self.from_s3, self._annotation_path.replace(constances.PIXEL_ANNOTATION_POSTFIX, constances.ANNOTATION_MASK_POSTFIX))
+                    self._mask = self.get_s3_file(
+                        self.from_s3,
+                        self._annotation_path.replace(
+                            constances.PIXEL_ANNOTATION_POSTFIX,
+                            constances.ANNOTATION_MASK_POSTFIX,
+                        ),
+                    )
             else:
                 self._annotation_json = json.load(open(self._annotation_path))
                 if self._project.project_type == constances.ProjectType.PIXEL.value:
-                    self._mask = open(self._annotation_path.replace(constances.PIXEL_ANNOTATION_POSTFIX, constances.ANNOTATION_MASK_POSTFIX))
+                    self._mask = open(
+                        self._annotation_path.replace(
+                            constances.PIXEL_ANNOTATION_POSTFIX,
+                            constances.ANNOTATION_MASK_POSTFIX,
+                        )
+                    )
 
     def _is_valid_json(self, json_data: dict):
         use_case = ValidateAnnotationUseCase(
@@ -361,22 +374,28 @@ def prepare_annotations(
         templates: List[dict],
         reporter: Reporter,
     ) -> dict:
+        annotation_classes_name_maps = map_annotation_classes_name(
+            annotation_classes, reporter
+        )
         if project_type in (
             constances.ProjectType.VECTOR.value,
             constances.ProjectType.PIXEL.value,
             constances.ProjectType.DOCUMENT.value,
         ):
             fill_annotation_ids(
                 annotations=annotations,
-                annotation_classes_name_maps=map_annotation_classes_name(
-                    annotation_classes, reporter
-                ),
+                annotation_classes_name_maps=annotation_classes_name_maps,
                 templates=templates,
                 reporter=reporter,
             )
         elif project_type == constances.ProjectType.VIDEO.value:
             annotations = convert_to_video_editor_json(
-                annotations, map_annotation_classes_name(annotation_classes, reporter), reporter
+                annotations, annotation_classes_name_maps, reporter
+            )
+        if project_type == constances.ProjectType.DOCUMENT.value:
+            fill_document_tags(
+                annotations=annotations,
+                annotation_classes=annotation_classes_name_maps,
             )
         return annotations
 
@@ -408,7 +427,10 @@ def execute(self):
                     ],
                     Body=json.dumps(annotation_json),
                 )
-                if self._project.project_type == constances.ProjectType.PIXEL.value and self._mask:
+                if (
+                    self._project.project_type == constances.ProjectType.PIXEL.value
+                    and self._mask
+                ):
                     bucket.put_object(
                         Key=self.annotation_upload_data.images[self._image.uuid][
                             "annotation_bluemap_path"

src/superannotate/lib/infrastructure/services.py

@@ -1,3 +1,4 @@
+import time
 from contextlib import contextmanager
 from datetime import datetime
 from typing import Dict
@@ -653,6 +654,7 @@ def get_bulk_images(
         self, project_id: int, team_id: int, folder_id: int, images: List[str]
     ) -> List[dict]:
         bulk_get_images_url = urljoin(self.api_url, self.URL_BULK_GET_IMAGES)
+        time.sleep(1)
 
         res = self._request(
             bulk_get_images_url,

tests/data_set/document_annotation/text_file_example_1.json

@@ -1 +1 @@
-{"metadata":{"name":"text_file_example_1","status":"Completed","url":"https://sa-public-files.s3.us-west-2.amazonaws.com/Text+project/text_file_example_1.txt","projectId":160158,"annotatorEmail":null,"qaEmail":null,"lastAction":{"email":"shab.prog@gmail.com","timestamp":1634899229953}},"instances":[{"start":253,"end":593,"classId":873208,"createdAt":"2021-10-22T10:40:26.151Z","createdBy":{"email":"shab.prog@gmail.com","role":"Admin"},"updatedAt":"2021-10-22T10:40:29.953Z","updatedBy":{"email":"shab.prog@gmail.com","role":"Admin"},"attributes":[],"creationType":"Manual","className":"vid"}],"tags":[],"freeText":""}
+{"metadata":{"name":"text_file_example_1","status":"Completed","url":"https://sa-public-files.s3.us-west-2.amazonaws.com/Text+project/text_file_example_1.txt","projectId":160158,"annotatorEmail":null,"qaEmail":null,"lastAction":{"email":"shab.prog@gmail.com","timestamp":1634899229953}},"instances":[{"start":253,"end":593,"classId":873208,"createdAt":"2021-10-22T10:40:26.151Z","createdBy":{"email":"shab.prog@gmail.com","role":"Admin"},"updatedAt":"2021-10-22T10:40:29.953Z","updatedBy":{"email":"shab.prog@gmail.com","role":"Admin"},"attributes":[],"creationType":"Manual","className":"vid"}],"tags":["vid"],"freeText":""}