Merge pull request #316 from superannotateai/friday

Friday

Author: dshabin

src/superannotate/__init__.py

@@ -22,7 +22,6 @@
 from superannotate.lib.app.input_converters.conversion import export_annotation
 from superannotate.lib.app.input_converters.conversion import import_annotation
 from superannotate.lib.app.input_converters.df_converter import df_to_annotations
-
 from superannotate.lib.app.input_converters.dicom_converter import dicom_to_rgb_sequence
 from superannotate.lib.app.interface.sdk_interface import add_annotation_bbox_to_image
 from superannotate.lib.app.interface.sdk_interface import (

src/superannotate/lib/app/analytics/aggregators.py

@@ -1,14 +1,14 @@
 import copy
 import json
 import logging
+from dataclasses import dataclass
 from pathlib import Path
 from typing import List
 from typing import Optional
 from typing import Union
 
 import lib.core as constances
 import pandas as pd
-from dataclasses import dataclass
 from lib.app.exceptions import AppException
 from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX
 from lib.core import PIXEL_ANNOTATION_POSTFIX
@@ -58,10 +58,10 @@ class VideoRawData:
 
 class DataAggregator:
     def __init__(
-            self,
-            project_type: str,
-            project_root: Union[str, Path],
-            folder_names: Optional[List[Union[Path, str]]] = None
+        self,
+        project_type: str,
+        project_root: Union[str, Path],
+        folder_names: Optional[List[Union[Path, str]]] = None,
     ):
         self.project_type = project_type
         self.project_root = Path(project_root)
@@ -87,10 +87,18 @@ def get_annotation_paths(self):
                 if path.is_file() and path.suffix == self.annotation_suffix:
                     annotations_paths.append(path)
                 elif path.is_dir() and path.name != "classes":
-                    annotations_paths.extend(list(path.rglob(f"*{self.annotation_suffix}")))
+                    annotations_paths.extend(
+                        list(path.rglob(f"*{self.annotation_suffix}"))
+                    )
         else:
             for folder_name in self.folder_names:
-                annotations_paths.extend(list((self.project_root / folder_name).rglob(f"*{self.annotation_suffix:}")))
+                annotations_paths.extend(
+                    list(
+                        (self.project_root / folder_name).rglob(
+                            f"*{self.annotation_suffix:}"
+                        )
+                    )
+                )
 
         if not annotations_paths:
             logger.warning(f"Could not find annotations in {self.project_root}.")
@@ -103,11 +111,16 @@ def check_classes_path(self):
             )
 
     def aggregate_annotations_as_df(self):
-        logger.info(f"Aggregating annotations from {self.project_root} as pandas DataFrame")
+        logger.info(
+            f"Aggregating annotations from {self.project_root} as pandas DataFrame"
+        )
         self.check_classes_path()
         annotation_paths = self.get_annotation_paths()
 
-        if self.project_type in (constances.ProjectType.VECTOR.name, constances.ProjectType.PIXEL.name):
+        if self.project_type in (
+            constances.ProjectType.VECTOR.name,
+            constances.ProjectType.PIXEL.name,
+        ):
             return self.aggregate_image_annotations_as_df(annotation_paths)
         elif self.project_type == constances.ProjectType.VIDEO.name:
             return self.aggregate_video_annotations_as_df(annotation_paths)
@@ -120,7 +133,11 @@ def aggregate_video_annotations_as_df(self, annotation_paths: List[str]):
             raw_data = VideoRawData()
             # metadata
             raw_data.videoName = annotation_data["metadata"]["name"]
-            raw_data.folderName = annotation_path.parent.name if annotation_path.parent != self.project_root else None
+            raw_data.folderName = (
+                annotation_path.parent.name
+                if annotation_path.parent != self.project_root
+                else None
+            )
             raw_data.videoHeight = annotation_data["metadata"].get("height")
             raw_data.videoWidth = annotation_data["metadata"].get("width")
             raw_data.videoStatus = annotation_data["metadata"].get("status")
@@ -146,11 +163,19 @@ def aggregate_video_annotations_as_df(self, annotation_paths: List[str]):
                 instance_raw.type = instance["meta"].get("type")
                 instance_raw.className = instance["meta"].get("className")
                 instance_raw.createdAt = instance["meta"].get("createdAt")
-                instance_raw.createdBy = instance["meta"].get("createdBy", {}).get("email")
-                instance_raw.creatorRole = instance["meta"].get("createdBy", {}).get("role")
+                instance_raw.createdBy = (
+                    instance["meta"].get("createdBy", {}).get("email")
+                )
+                instance_raw.creatorRole = (
+                    instance["meta"].get("createdBy", {}).get("role")
+                )
                 instance_raw.updatedAt = instance["meta"].get("updatedAt")
-                instance_raw.updatedBy = instance["meta"].get("updatedBy", {}).get("email")
-                instance_raw.updatorRole = instance["meta"].get("updatedBy", {}).get("role")
+                instance_raw.updatedBy = (
+                    instance["meta"].get("updatedBy", {}).get("email")
+                )
+                instance_raw.updatorRole = (
+                    instance["meta"].get("updatedBy", {}).get("role")
+                )
                 instance_raw.pointLabels = instance["meta"].get("pointLabels")
                 parameters = instance.get("parameters", [])
                 for parameter_id, parameter in enumerate(parameters):
@@ -167,7 +192,9 @@ def aggregate_video_annotations_as_df(self, annotation_paths: List[str]):
                         for attribute_id, attribute in enumerate(attributes):
                             attribute_raw = copy.copy(timestamp_raw)
                             attribute_raw.attributeId = attribute_id
-                            attribute_raw.attributeGroupName = attribute.get("groupName")
+                            attribute_raw.attributeGroupName = attribute.get(
+                                "groupName"
+                            )
                             attribute_raw.attributeName = attribute.get("name")
                             raws.append(attribute_raw)
                         if not attributes:
@@ -212,7 +239,7 @@ def aggregate_image_annotations_as_df(self, annotations_paths: List[str]):
             "imageAnnotator": [],
             "imageQA": [],
             "commentResolved": [],
-            "tag": []
+            "tag": [],
         }
 
         classes_json = json.load(open(self.classes_path))
@@ -233,7 +260,9 @@ def aggregate_image_annotations_as_df(self, annotations_paths: List[str]):
         def __append_annotation(annotation_dict):
             for annotation_key in annotation_data:
                 if annotation_key in annotation_dict:
-                    annotation_data[annotation_key].append(annotation_dict[annotation_key])
+                    annotation_data[annotation_key].append(
+                        annotation_dict[annotation_key]
+                    )
                 else:
                     annotation_data[annotation_key].append(None)
 
@@ -271,8 +300,8 @@ def __append_annotation(annotation_dict):
                 annotation_type = annotation.get("type", "mask")
                 annotation_class_name = annotation.get("className")
                 if (
-                        annotation_class_name is None
-                        or annotation_class_name not in class_name_to_color
+                    annotation_class_name is None
+                    or annotation_class_name not in class_name_to_color
                 ):
                     logger.warning(
                         "Annotation class %s not found in classes json. Skipping.",
@@ -308,7 +337,7 @@ def __append_annotation(annotation_dict):
                 annotation_probability = annotation.get("probability")
                 annotation_point_labels = annotation.get("pointLabels")
                 attributes = annotation.get("attributes")
-                user_metadata =self.__get_user_metadata(annotation)
+                user_metadata = self.__get_user_metadata(annotation)
                 folder_name = None
                 if annotation_path.parent != Path(self.project_root):
                     folder_name = annotation_path.parent.name
@@ -339,19 +368,19 @@ def __append_annotation(annotation_dict):
                         attribute_group = attribute.get("groupName")
                         attribute_name = attribute.get("name")
                         if (
-                                attribute_group
-                                not in class_group_name_to_values[annotation_class_name]
+                            attribute_group
+                            not in class_group_name_to_values[annotation_class_name]
                         ):
                             logger.warning(
                                 "Annotation class group %s not in classes json. Skipping.",
                                 attribute_group,
                             )
                             continue
                         if (
-                                attribute_name
-                                not in class_group_name_to_values[annotation_class_name][
-                            attribute_group
-                        ]
+                            attribute_name
+                            not in class_group_name_to_values[annotation_class_name][
+                                attribute_group
+                            ]
                         ):
                             logger.warning(
                                 "Annotation class group value %s not in classes json. Skipping.",

src/superannotate/lib/app/analytics/class_analytics.py

@@ -67,4 +67,3 @@ def class_distribution(export_root, project_names, visualize=False):
         fig.show()
 
     return df
-

src/superannotate/lib/app/interface/sdk_interface.py

@@ -3528,8 +3528,14 @@ def aggregate_annotations_as_df(
     :return: DataFrame on annotations
     :rtype: pandas DataFrame
     """
-    if project_type in (constances.ProjectType.VECTOR.name, constances.ProjectType.PIXEL.name):
-        from superannotate.lib.app.analytics.common import aggregate_image_annotations_as_df
+    if project_type in (
+        constances.ProjectType.VECTOR.name,
+        constances.ProjectType.PIXEL.name,
+    ):
+        from superannotate.lib.app.analytics.common import (
+            aggregate_image_annotations_as_df,
+        )
+
         return aggregate_image_annotations_as_df(
             project_root=project_root,
             include_classes_wo_annotations=False,
@@ -3539,10 +3545,11 @@ def aggregate_annotations_as_df(
         )
     elif project_type == constances.ProjectType.VIDEO.name:
         from superannotate.lib.app.analytics.aggregators import DataAggregator
+
         return DataAggregator(
             project_type=project_type,
             project_root=project_root,
-            folder_names=folder_names
+            folder_names=folder_names,
         ).aggregate_annotations_as_df()
     else:
         raise AppException(constances.DEPRECATED_DOCUMENT_PROJECTS_MESSAGE)

src/superannotate/lib/core/entities/document.py

@@ -7,16 +7,18 @@
 from lib.core.entities.utils import MetadataBase
 from lib.core.entities.utils import Tag
 from pydantic import Field
+from pydantic import StrictInt
+from pydantic import StrictStr
 
 
 class DocumentInstance(BaseInstance):
-    start: int
-    end: int
+    start: StrictInt
+    end: StrictInt
     attributes: Optional[List[Attribute]] = Field(list())
 
 
 class DocumentAnnotation(BaseModel):
     metadata: MetadataBase
     instances: Optional[List[DocumentInstance]] = Field(list())
     tags: Optional[List[Tag]] = Field(list())
-    free_text: Optional[str] = Field(None, alias="freeText")
+    free_text: Optional[StrictStr] = Field(None, alias="freeText")

src/superannotate/lib/core/entities/pixel.py

@@ -7,13 +7,14 @@
 from lib.core.entities.utils import Metadata
 from lib.core.entities.utils import Tag
 from pydantic import Field
+from pydantic import StrictBool
 from pydantic import validator
 from pydantic.color import Color
 from pydantic.color import ColorType
 
 
 class PixelMetaData(Metadata):
-    is_segmented: Optional[bool] = Field(None, alias="isSegmented")
+    is_segmented: Optional[StrictBool] = Field(None, alias="isSegmented")
 
 
 class PixelAnnotationPart(BaseModel):

src/superannotate/lib/core/entities/utils.py

@@ -9,8 +9,11 @@
 from pydantic import EmailStr
 from pydantic import Extra
 from pydantic import Field
+from pydantic import StrictStr
 from pydantic import StrRegexError
+from pydantic import ValidationError
 from pydantic import validator
+from pydantic.error_wrappers import ErrorWrapper
 from pydantic.errors import EnumMemberError
 
 
@@ -21,16 +24,20 @@ def enum_error_handling(self) -> str:
 
 EnumMemberError.__str__ = enum_error_handling
 
-
 NotEmptyStr = constr(strict=True, min_length=1)
 
-
 DATE_REGEX = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d(?:\.\d{3})Z"
 
 DATE_TIME_FORMAT_ERROR_MESSAGE = (
     "does not match expected format YYYY-MM-DDTHH:MM:SS.fffZ"
 )
 
+POINT_LABEL_KEY_FORMAT_ERROR_MESSAGE = "does not match expected format ^[0-9]*$"
+
+POINT_LABEL_VALUE_FORMAT_ERROR_MESSAGE = "str type expected"
+
+INVALID_DICT_MESSAGE = "value is not a valid dict"
+
 
 class BaseModel(PyDanticBaseModel):
     class Config:
@@ -144,7 +151,7 @@ class LastUserAction(BaseModel):
 
 class BaseInstance(TrackableModel, TimedBaseModel):
     class_id: Optional[int] = Field(None, alias="classId")
-    class_name: NotEmptyStr = Field(alias="className")
+    class_name: Optional[NotEmptyStr] = Field(None, alias="className")
 
 
 class MetadataBase(BaseModel):
@@ -159,10 +166,6 @@ class MetadataBase(BaseModel):
     status: Optional[AnnotationStatusEnum]
 
 
-class PointLabels(BaseModel):
-    __root__: Dict[constr(regex=r"^[0-9]*$"), NotEmptyStr]  # noqa: F722 E261
-
-
 class Correspondence(BaseModel):
     text: NotEmptyStr
     email: EmailStr
@@ -188,6 +191,56 @@ class Config:
         }
 
 
+class StringA(BaseModel):
+    string: StrictStr
+
+
+class PointLabels(BaseModel):
+    __root__: Dict[constr(regex=r"^[0-9]*$"), str]
+
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate_type
+        yield cls.validate_value
+
+    @validator("__root__", pre=True)
+    def validate_value(cls, values):
+        result = {}
+        errors = []
+        validate_key = None
+        validate_value = None
+        for key, value in values.items():
+            try:
+                validate_key = constr(regex=r"^[0-9]*$", min_length=1).validate(key)
+            except ValueError:
+                errors.append(
+                    ErrorWrapper(
+                        ValueError(POINT_LABEL_KEY_FORMAT_ERROR_MESSAGE), str(key)
+                    )
+                )
+            try:
+                validate_value = StringA(string=value)
+            except ValueError:
+                errors.append(
+                    ErrorWrapper(
+                        ValueError(POINT_LABEL_VALUE_FORMAT_ERROR_MESSAGE), str(key)
+                    )
+                )
+
+        if validate_key and validate_value:
+            result.update({key: value})
+
+        if errors:
+            raise ValidationError(errors, cls)
+        return result
+
+    @classmethod
+    def validate_type(cls, values):
+        if not issubclass(type(values), dict):
+            raise TypeError(INVALID_DICT_MESSAGE)
+        return values
+
+
 class BaseVectorInstance(BaseImageInstance):
     type: VectorAnnotationTypeEnum
     point_labels: Optional[PointLabels] = Field(None, alias="pointLabels")