merge friday

Author: dshabin

src/superannotate/__init__.py

@@ -304,4 +304,3 @@
 logging.config.fileConfig(
     os.path.join(WORKING_DIR, "logging.conf"), disable_existing_loggers=False
 )
-sys.tracebacklimit = 0

src/superannotate/lib/app/analytics/common.py

@@ -212,6 +212,8 @@ def aggregate_annotations_as_df(
         "updatorRole": [],
         "updatorEmail": [],
         "folderName": [],
+        "imageAnnotator": [],
+        "imageQA": [],
     }
 
     if include_comments:
@@ -255,6 +257,8 @@ def __get_image_metadata(image_name, annotations):
         image_metadata["imageWidth"] = annotations["metadata"].get("width")
         image_metadata["imageStatus"] = annotations["metadata"].get("status")
         image_metadata["imagePinned"] = annotations["metadata"].get("pinned")
+        image_metadata["imageAnnotator"] = annotations["metadata"].get("annotatorEmail")
+        image_metadata["imageQA"] = annotations["metadata"].get("qaEmail")
         return image_metadata
 
     def __get_user_metadata(annotation):

src/superannotate/lib/app/helpers.py

@@ -9,6 +9,7 @@
 import boto3
 import pandas as pd
 from superannotate.lib.app.exceptions import PathError
+from superannotate.lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX
 from superannotate.lib.core import PIXEL_ANNOTATION_POSTFIX
 from superannotate.lib.core import VECTOR_ANNOTATION_POSTFIX
 
@@ -55,7 +56,13 @@ def get_local_annotation_paths(
         [
             str(i)
             for i in all_not_folders
-            if i.name.endswith((VECTOR_ANNOTATION_POSTFIX, PIXEL_ANNOTATION_POSTFIX))
+            if i.name.endswith(
+                (
+                    VECTOR_ANNOTATION_POSTFIX,
+                    PIXEL_ANNOTATION_POSTFIX,
+                    ATTACHED_VIDEO_ANNOTATION_POSTFIX,
+                )
+            )
         ]
     )
     if recursive:
@@ -82,8 +89,10 @@ def get_s3_annotation_paths(folder_path, s3_bucket, annotation_paths, recursive)
     for data in paginator.paginate(Bucket=s3_bucket, Prefix=folder_path):
         for annotation in data["Contents"]:
             key = annotation["Key"]
-            if key.endswith(VECTOR_ANNOTATION_POSTFIX) or key.endswith(
-                PIXEL_ANNOTATION_POSTFIX
+            if (
+                key.endswith(VECTOR_ANNOTATION_POSTFIX)
+                or key.endswith(PIXEL_ANNOTATION_POSTFIX)
+                or key.endswith(ATTACHED_VIDEO_ANNOTATION_POSTFIX)
             ):
                 if not recursive and "/" in key[len(folder_path) + 1 :]:
                     continue

src/superannotate/lib/app/interface/sdk_interface.py

@@ -31,7 +31,6 @@
 from lib.app.interface.types import ImageQualityChoices
 from lib.app.interface.types import NotEmptyStr
 from lib.app.interface.types import ProjectTypes
-from lib.app.interface.types import Status
 from lib.app.interface.types import validate_arguments
 from lib.app.mixp.decorators import Trackable
 from lib.app.serializers import BaseSerializers
@@ -282,7 +281,7 @@ def clone_project(
 def search_images(
     project: Union[NotEmptyStr, dict],
     image_name_prefix: Optional[NotEmptyStr] = None,
-    annotation_status: Optional[Status] = None,
+    annotation_status: Optional[AnnotationStatuses] = None,
     return_metadata: Optional[StrictBool] = False,
 ):
     """Search images by name_prefix (case-insensitive) and annotation status
@@ -335,10 +334,6 @@ def create_folder(project: NotEmptyStr, folder_name: NotEmptyStr):
     res = controller.create_folder(project=project, folder_name=folder_name)
     if res.data:
         folder = res.data
-        if folder and folder.name != folder_name:
-            logger.warning(
-                f"Created folder has name {folder.name}, since folder with name {folder_name} already existed.",
-            )
         logger.info(f"Folder {folder.name} created in project {project}")
         return folder.to_dict()
     if res.errors:
@@ -605,7 +600,7 @@ def upload_images_from_public_urls_to_project(
     project: Union[NotEmptyStr, dict],
     img_urls: List[NotEmptyStr],
     img_names: Optional[List[NotEmptyStr]] = None,
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
     image_quality_in_editor: Optional[NotEmptyStr] = None,
 ):
     """Uploads all images given in the list of URL strings in img_urls to the project.
@@ -1643,7 +1638,7 @@ def upload_videos_from_folder_to_project(
     target_fps: Optional[int] = None,
     start_time: Optional[float] = 0.0,
     end_time: Optional[float] = None,
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
     image_quality_in_editor: Optional[ImageQualityChoices] = None,
 ):
     """Uploads image frames from all videos with given extensions from folder_path to the project.
@@ -1793,7 +1788,7 @@ def upload_video_to_project(
     target_fps: Optional[int] = None,
     start_time: Optional[float] = 0.0,
     end_time: Optional[float] = None,
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
     image_quality_in_editor: Optional[ImageQualityChoices] = None,
 ):
     """Uploads image frames from video to platform. Uploaded images will have
@@ -2291,7 +2286,7 @@ def download_image(
 def attach_image_urls_to_project(
     project: Union[NotEmptyStr, dict],
     attachments: Union[str, Path],
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
 ):
     """Link images on external storage to SuperAnnotate.
 
@@ -2354,7 +2349,7 @@ def attach_image_urls_to_project(
 def attach_video_urls_to_project(
     project: Union[NotEmptyStr, dict],
     attachments: Union[str, Path],
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
 ):
     """Link videos on external storage to SuperAnnotate.
 
@@ -2443,51 +2438,36 @@ def upload_annotations_from_folder_to_project(
     """
 
     project_name, folder_name = extract_project_folder(project)
-    project = controller.get_project_metadata(project_name).data
-    if project["project"].project_type in [
-        constances.ProjectType.VIDEO.value,
-        constances.ProjectType.DOCUMENT.value,
-    ]:
-        raise AppException(LIMITED_FUNCTIONS[project["project"].project_type])
 
     if recursive_subfolders:
         logger.info(
-            "When using recursive subfolder parsing same name annotations in different subfolders will overwrite each other.",
+            "When using recursive subfolder parsing same name annotations in different "
+            "subfolders will overwrite each other.",
         )
-
-    logger.info(
-        "The JSON files should follow specific naming convention. For Vector projects they should be named '<image_name>___objects.json', for Pixel projects JSON file should be names '<image_name>___pixel.json' and also second mask image file should be present with the name '<image_name>___save.png'. In both cases image with <image_name> should be already present on the platform."
-    )
-    logger.info("Existing annotations will be overwritten.",)
+    logger.info("The JSON files should follow a specific naming convention, matching file names already present "
+                "on the platform. Existing annotations will be overwritten")
     logger.info(
         "Uploading all annotations from %s to project %s.", folder_path, project_name
     )
 
     annotation_paths = get_annotation_paths(
         folder_path, from_s3_bucket, recursive_subfolders
     )
+    if not annotation_paths:
+        raise AppException("Could not find annotations matching existing items on the platform.")
+
     logger.info(
         "Uploading %s annotations to project %s.", len(annotation_paths), project_name
     )
-    use_case = controller.upload_annotations_from_folder(
+    response = controller.upload_annotations_from_folder(
         project_name=project_name,
         folder_name=folder_name,
-        folder_path=folder_path,
         annotation_paths=annotation_paths,  # noqa: E203
         client_s3_bucket=from_s3_bucket,
     )
-    if use_case.is_valid():
-        with tqdm(
-            total=len(use_case.annotations_to_upload), desc="Uploading annotations"
-        ) as progress_bar:
-            for _ in use_case.execute():
-                progress_bar.update(1)
-    else:
-        raise AppException(use_case.response.errors)
-    if use_case.response.report:
-        for i in use_case.response.report_messages:
-            logger.info(i)
-    return use_case.data
+    if response.errors:
+        raise AppException(response.errors)
+    return response.data
 
 
 @Trackable
@@ -2548,20 +2528,16 @@ def upload_preannotations_from_folder_to_project(
     logger.info(
         "Uploading %s annotations to project %s.", len(annotation_paths), project_name
     )
-    use_case = controller.upload_annotations_from_folder(
+    response = controller.upload_annotations_from_folder(
         project_name=project_name,
         folder_name=folder_name,
-        folder_path=folder_path,
         annotation_paths=annotation_paths,  # noqa: E203
         client_s3_bucket=from_s3_bucket,
         is_pre_annotations=True,
     )
-    with tqdm(
-        total=len(annotation_paths), desc="Uploading annotations"
-    ) as progress_bar:
-        for _ in use_case.execute():
-            progress_bar.update(1)
-    return use_case.data
+    if response.errors:
+        raise AppException(response.errors)
+    return response.data
 
 
 @Trackable
@@ -2570,7 +2546,7 @@ def upload_image_annotations(
     project: Union[NotEmptyStr, dict],
     image_name: str,
     annotation_json: Union[str, Path, dict],
-    mask: Optional[Union[str, Path, dict]] = None,
+    mask: Optional[Union[str, Path, bytes]] = None,
     verbose: Optional[StrictBool] = True,
 ):
     """Upload annotations from JSON (also mask for pixel annotations)
@@ -2585,15 +2561,17 @@ def upload_image_annotations(
     :param mask: BytesIO object or filepath to mask annotation for pixel projects in SuperAnnotate format
     :type mask: BytesIO or Path-like (str or Path)
     """
-    annotation_path = f"{image_name}___save.png"
-    if isinstance(annotation_json, str) or isinstance(annotation_json, Path):
-        annotation_path = str(annotation_json).replace("___pixel.json", "___save.png")
-    if isinstance(annotation_json, list):
-        raise AppException(
-            "Annotation JSON should be a dict object. You are using list object."
-            " If this is an old annotation format you can convert it to new format with superannotate."
-            "update_json_format SDK function"
-        )
+    if not mask:
+        if not isinstance(annotation_json, dict):
+            mask_path = str(annotation_json).replace("___pixel.json", "___save.png")
+        else:
+            mask_path = f"{image_name}___save.png"
+        if os.path.exists(mask_path):
+            mask = open(mask_path, "rb").read()
+    elif isinstance(mask, str) or isinstance(mask, Path):
+        if os.path.exists(mask):
+            mask = open(mask, "rb").read()
+
     if not isinstance(annotation_json, dict):
         if verbose:
             logger.info("Uploading annotations from %s.", annotation_json)
@@ -2606,7 +2584,6 @@ def upload_image_annotations(
         annotations=annotation_json,
         mask=mask,
         verbose=verbose,
-        annotation_path=annotation_path,
     )
     if response.errors:
         raise AppException(response.errors)
@@ -3370,7 +3347,7 @@ def upload_image_to_project(
     project: NotEmptyStr,
     img,
     image_name: Optional[NotEmptyStr] = None,
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
     from_s3_bucket=None,
     image_quality_in_editor: Optional[NotEmptyStr] = None,
 ):
@@ -3445,7 +3422,7 @@ def search_models(
 def upload_images_to_project(
     project: NotEmptyStr,
     img_paths: List[NotEmptyStr],
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
     from_s3_bucket=None,
     image_quality_in_editor: Optional[ImageQualityChoices] = None,
 ):
@@ -3579,7 +3556,7 @@ def delete_annotations(
 def attach_document_urls_to_project(
     project: Union[NotEmptyStr, dict],
     attachments: Union[Path, NotEmptyStr],
-    annotation_status: Optional[Status] = "NotStarted",
+    annotation_status: Optional[AnnotationStatuses] = "NotStarted",
 ):
     """Link documents on external storage to SuperAnnotate.
 

src/superannotate/lib/app/interface/types.py

@@ -19,7 +19,9 @@ def validate(cls, value: Union[str]) -> Union[str]:
         if cls.curtail_length and len(value) > cls.curtail_length:
             value = value[: cls.curtail_length]
         if value.lower() not in AnnotationStatus.values():
-            raise TypeError(f"Available statuses is {', '.join(AnnotationStatus)}. ")
+            raise TypeError(
+                f"Available statuses is {', '.join(AnnotationStatus.titles())}. "
+            )
         return value
 
 

src/superannotate/lib/core/__init__.py

@@ -45,6 +45,7 @@
 VECTOR_ANNOTATION_POSTFIX = "___objects.json"
 PIXEL_ANNOTATION_POSTFIX = "___pixel.json"
 ANNOTATION_MASK_POSTFIX = "___save.png"
+ATTACHED_VIDEO_ANNOTATION_POSTFIX = ".json"
 
 NON_PLOTABLE_KEYS = ["eta_seconds", "iteration", "data_time", "time", "model"]