Merge pull request #258 from superannotateai/friday

Friday

Author: VaghinakDev

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2445,6 +2445,8 @@ def upload_annotations_from_folder_to_project(
     """
 
     project_name, folder_name = extract_project_folder(project)
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
+
 
     if recursive_subfolders:
         logger.info(
@@ -2460,9 +2462,7 @@ def upload_annotations_from_folder_to_project(
     if not annotation_paths:
         raise AppException("Could not find annotations matching existing items on the platform.")
 
-    logger.info(
-        "Uploading %s annotations to project %s.", len(annotation_paths), project_name
-    )
+    logger.info(f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}.")
     response = controller.upload_annotations_from_folder(
         project_name=project_name,
         folder_name=folder_name,
@@ -2506,32 +2506,28 @@ def upload_preannotations_from_folder_to_project(
     :rtype: tuple of list of strs
     """
     project_name, folder_name = extract_project_folder(project)
+    project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
     project = controller.get_project_metadata(project_name).data
     if project["project"].project_type in [
         constances.ProjectType.VIDEO.value,
         constances.ProjectType.DOCUMENT.value,
     ]:
         raise AppException(LIMITED_FUNCTIONS[project["project"].project_type])
-
     if recursive_subfolders:
         logger.info(
-            "When using recursive subfolder parsing same name annotations in different subfolders will overwrite each other.",
+            "When using recursive subfolder parsing same name annotations in different "
+            "subfolders will overwrite each other.",
         )
-
-    logger.info(
-        "The JSON files should follow specific naming convention. For Vector projects they should be named '<image_name>___objects.json', for Pixel projects JSON file should be names '<image_name>___pixel.json' and also second mask image file should be present with the name '<image_name>___save.png'. In both cases image with <image_name> should be already present on the platform."
-    )
+    logger.info("The JSON files should follow a specific naming convention, matching file names already present "
+                "on the platform. Existing annotations will be overwritten")
     logger.info("Existing annotations will be overwritten.",)
-    logger.info(
-        "Uploading all annotations from %s to project %s.", folder_path, project_name
-    )
-
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
+    if not annotation_paths:
+        raise AppException("Could not find annotations matching existing items on the platform.")
     logger.info(
-        "Uploading %s annotations to project %s.", len(annotation_paths), project_name
-    )
+        f"Uploading {len(annotation_paths)} annotations from {folder_path} to the project {project_folder_name}.")
     response = controller.upload_annotations_from_folder(
         project_name=project_name,
         folder_name=folder_name,

src/superannotate/lib/core/helpers.py

@@ -119,6 +119,8 @@ def fill_annotation_ids(
 
 
 def convert_to_video_editor_json(data: dict, class_name_mapper: dict, reporter: Reporter):
+    id_generator = ClassIdGenerator()
+
     def safe_time(timestamp):
         return "0" if str(timestamp) == "0.0" else timestamp
 
@@ -138,14 +140,17 @@ def convert_timestamp(timestamp):
     }
     for instance in data["instances"]:
         meta = instance["meta"]
-        class_name = meta["className"]
+        class_name = meta.get("className")
         editor_instance = {
             "attributes": [],
             "timeline": {},
             "type": meta["type"],
-            "classId": class_name_mapper.get(class_name, {}).get("id", -1),
             "locked": True,
         }
+        if class_name:
+            editor_instance["classId"] = class_name_mapper.get(class_name, {}).get("id", id_generator.send(class_name))
+        else:
+            editor_instance["classId"] = id_generator.send("unknown_class")
         if meta.get("pointLabels", None):
             editor_instance["pointLabels"] = meta["pointLabels"]
         active_attributes = set()
@@ -168,8 +173,9 @@ def convert_timestamp(timestamp):
                     editor_instance["timeline"][timestamp]["points"] = timestamp_data[
                         "points"
                     ]
-
-                if not class_name_mapper.get(meta["className"], None):
+                if not class_name:
+                    continue
+                elif not class_name_mapper.get(class_name):
                     reporter.store_message("missing_classes", meta["className"])
                     continue
 
@@ -227,3 +233,15 @@ def default(self, obj):
         if isinstance(obj, set):
             return list(obj)
         return json.JSONEncoder.default(self, obj)
+
+
+class ClassIdGenerator:
+    def __init__(self):
+        self.classes = defaultdict(int)
+        self.idx = -1
+
+    def send(self, class_name: str):
+        if class_name not in self.classes:
+            self.classes[class_name] = self.idx
+            self.idx -= 1
+        return self.classes[class_name]

src/superannotate/lib/core/plugin.py

@@ -259,7 +259,7 @@ def get_extractable_frames(
         total_with_fps = sum(
             1
             for _ in VideoPlugin.frames_generator(
-                video_path,start_time, end_time, target_fps, log=False
+                video_path, start_time, end_time, target_fps, log=False
             )
         )
         zero_fill_count = len(str(total))
@@ -288,7 +288,7 @@ def extract_frames(
         for frame in VideoPlugin.frames_generator(
             video_path, start_time, end_time, target_fps
         ):
-            if len(extracted_frames_paths) > limit:
+            if len(extracted_frames_paths) >= limit:
                 break
             path = str(
                 Path(extract_path)

src/superannotate/lib/core/usecases/annotations.py

@@ -173,7 +173,6 @@ def _upload_annotation(
                 return path, False
             return path, True
         except Exception as e:
-            # raise e
             return path, False
 
     def get_bucket_to_upload(self, ids: List[int]):
@@ -322,25 +321,29 @@ def s3_bucket(self):
                 self._s3_bucket = resource.Bucket(upload_data.bucket)
         return self._s3_bucket
 
-    def get_s3_annotation(self, s3, path: str):
+    def get_s3_file(self, s3, path: str):
         file = io.BytesIO()
         s3_object = s3.Object(self._client_s3_bucket, path)
         s3_object.download_fileobj(file)
         file.seek(0)
-        return json.load(file)
+        return file
 
     @property
     def from_s3(self):
         if self._client_s3_bucket:
             from_session = boto3.Session()
             return from_session.resource("s3")
 
-    def get_annotation_json(self):
+    def set_annotation_json(self):
         if not self._annotation_json:
             if self._client_s3_bucket:
-                return self.get_s3_annotation(self.from_s3, self._annotation_path)
-            return json.load(open(self._annotation_path))
-        return self._annotation_json
+                self._annotation_json = json.load(self.get_s3_file(self.from_s3, self._annotation_path))
+                if self._project.project_type == constances.ProjectType.PIXEL.value:
+                    self._mask = self.get_s3_file(self.from_s3, self._annotation_path.replace(constances.PIXEL_ANNOTATION_POSTFIX, constances.ANNOTATION_MASK_POSTFIX))
+            else:
+                self._annotation_json = json.load(open(self._annotation_path))
+                if self._project.project_type == constances.ProjectType.PIXEL.value:
+                    self._mask = open(self._annotation_path.replace(constances.PIXEL_ANNOTATION_POSTFIX, constances.ANNOTATION_MASK_POSTFIX))
 
     def _is_valid_json(self, json_data: dict):
         use_case = ValidateAnnotationUseCase(
@@ -389,12 +392,12 @@ def is_valid_json(
 
     def execute(self):
         if self.is_valid():
-            annotation_json = self.get_annotation_json()
-            if self.is_valid_json(annotation_json):
+            self.set_annotation_json()
+            if self.is_valid_json(self._annotation_json):
                 bucket = self.s3_bucket
                 annotation_json = self.prepare_annotations(
                     project_type=self._project.project_type,
-                    annotations=annotation_json,
+                    annotations=self._annotation_json,
                     annotation_classes=self._annotation_classes,
                     templates=self._templates,
                     reporter=self.reporter,
@@ -405,18 +408,13 @@ def execute(self):
                     ],
                     Body=json.dumps(annotation_json),
                 )
-                if self._project.project_type == constances.ProjectType.PIXEL.value:
-                    mask_path = self._annotation_path.replace(
-                        "___pixel.json", constances.ANNOTATION_MASK_POSTFIX
+                if self._project.project_type == constances.ProjectType.PIXEL.value and self._mask:
+                    bucket.put_object(
+                        Key=self.annotation_upload_data.images[self._image.uuid][
+                            "annotation_bluemap_path"
+                        ],
+                        Body=self._mask,
                     )
-                    with open(mask_path, "rb") as mask_file:
-                        file = io.BytesIO(mask_file.read())
-                        bucket.put_object(
-                            Key=self.annotation_upload_data.images[self._image.uuid][
-                                "annotation_bluemap_path"
-                            ],
-                            Body=file,
-                        )
                 if self._verbose:
                     logger.info(
                         "Uploading annotations for image %s in project %s.",