Merge pull request #493 from superannotateai/fix_validate_annotations

changed some fields validations

Author: VaghinakDev

src/superannotate/lib/core/entities/base.py

@@ -29,6 +29,7 @@
     from pydantic import MappingIntStrAny  # noqa
 except ImportError:
     pass
+_missing = object()
 
 
 class BaseModel(PydanticBaseModel):
@@ -37,6 +38,70 @@ class BaseModel(PydanticBaseModel):
     - use_enum_names: that's for BaseTitledEnum to use names instead of enum objects
     """
 
+    def _iter(
+        self,
+        to_dict: bool = False,
+        by_alias: bool = False,
+        include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None,
+        exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None,
+        exclude_unset: bool = False,
+        exclude_defaults: bool = False,
+        exclude_none: bool = False,
+    ) -> "TupleGenerator":  # noqa
+
+        # Merge field set excludes with explicit exclude parameter with explicit overriding field set options.
+        # The extra "is not None" guards are not logically necessary but optimizes performance for the simple case.
+        if exclude is not None or self.__exclude_fields__ is not None:
+            exclude = ValueItems.merge(self.__exclude_fields__, exclude)
+
+        if include is not None or self.__include_fields__ is not None:
+            include = ValueItems.merge(self.__include_fields__, include, intersect=True)
+
+        allowed_keys = self._calculate_keys(
+            include=include, exclude=exclude, exclude_unset=exclude_unset  # type: ignore
+        )
+        if allowed_keys is None and not (
+            by_alias or exclude_unset or exclude_defaults or exclude_none
+        ):
+            # huge boost for plain _iter()
+            yield from self.__dict__.items()
+            return
+
+        value_exclude = ValueItems(self, exclude) if exclude is not None else None
+        value_include = ValueItems(self, include) if include is not None else None
+
+        for field_key, v in self.__dict__.items():
+            if (allowed_keys is not None and field_key not in allowed_keys) or (
+                exclude_none and v is None
+            ):
+                continue
+
+            if exclude_defaults:
+                model_field = self.__fields__.get(field_key)
+                if (
+                    not getattr(model_field, "required", True)
+                    and getattr(model_field, "default", _missing) == v
+                ):
+                    continue
+
+            if by_alias and field_key in self.__fields__:
+                dict_key = self.__fields__[field_key].alias
+            else:
+                dict_key = field_key
+
+            # if to_dict or value_include or value_exclude:
+            v = self._get_value(
+                v,
+                to_dict=to_dict,
+                by_alias=by_alias,
+                include=value_include and value_include.for_element(field_key),
+                exclude=value_exclude and value_exclude.for_element(field_key),
+                exclude_unset=exclude_unset,
+                exclude_defaults=exclude_defaults,
+                exclude_none=exclude_none,
+            )
+            yield dict_key, v
+
     @classmethod
     @no_type_check
     def _get_value(

src/superannotate/lib/core/entities/classes.py

@@ -102,5 +102,4 @@ class Config:
             BaseTitledEnum: lambda v: v.value,
         }
         validate_assignment = True
-        # exclude_none = True
         use_enum_names = True

src/superannotate/lib/core/usecases/annotations.py

@@ -1145,7 +1145,8 @@ class ValidateAnnotationUseCase(BaseReportableUseCase):
     DEFAULT_VERSION = "V1.00"
     SCHEMAS: Dict[str, Draft7Validator] = {}
     PATTERN_MAP = {
-        "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "YYYY-MM-DDTHH:MM:SS.fffZ"
+        "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d(?:\\.\\d{3})Z": "does not match YYYY-MM-DDTHH:MM:SS.fffZ",
+        "^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$": "invalid email",
     }
 
     def __init__(
@@ -1228,7 +1229,7 @@ def oneOf(validator, oneOf, instance, schema):  # noqa
     def _pattern(validator, patrn, instance, schema):
         if validator.is_type(instance, "string") and not re.search(patrn, instance):
             yield ValidationError(
-                f"{instance} does not match {ValidateAnnotationUseCase.PATTERN_MAP.get(patrn, patrn)}"
+                f"{instance}  {ValidateAnnotationUseCase.PATTERN_MAP.get(patrn, patrn)}"
             )
 
     @staticmethod

src/superannotate/lib/infrastructure/services.py

@@ -46,7 +46,7 @@
 class PydanticEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, BaseModel):
-            return json.loads(obj.json(exclude_unset=True, exclude_none=True))
+            return json.loads(obj.json(exclude_none=True))
         return json.JSONEncoder.default(self, obj)
 
 
@@ -555,7 +555,8 @@ def create_annotation_classes(
             "post",
             params=params,
             data={
-                "classes": [i.dict(exclude_none=True, exclude_unset=True) for i in data]
+                # "classes": [json.loads(i.json(exclude_none=True, exclude_unset=True)) for i in data]
+                "classes": data
             },
             content_type=List[AnnotationClassEntity],
         )

tests/integration/annotations/validations/test_vector_annotation_validation.py

@@ -59,36 +59,4 @@ def test_validate_create_dby(self, mock_print):
             """instances[0]                                    'points' is a required property
 instances[0].createdBy.role                     'dmin' is not one of ['Customer', 'Admin', 'Annotator', 'QA']
 instances[1]                                    'points' is a required property"""
-        )
-
-    def test_should_be_valid(self):
-        is_valid = sa.validate_annotations(
-            "Vector",
-            {
-                "metadata": {"name": "12"},
-                "instances": [
-                    {
-                        "type": "bbox",
-                        "createdAt": '2021-11-18T13.36.53.293Z',
-                        "createdBy": {'email': 'arturn@superannotate.com', 'role': 'dmin'},
-                        "points": {
-                            "x1": 437.16,
-                            "x2": 465.23,
-                            "y1": 341.5,
-                            "y2": 357.09
-                        },
-                        "attributes": [
-                            {
-                                "name": 4,
-                                "groupName": "Num doors1"
-                            },
-                            {
-                                "name": 4,
-                                "groupName": "Num doors1"
-                            }
-                        ]
-
-                    },
-                ]
-            }
-        )
+        )

tests/integration/classes/test_classes_serialization.py

@@ -39,14 +39,14 @@ def test_empty_multiselect_excluded(self):
         annotation_class = AnnotationClassEntity(name="asd", color="blue",
                                                  attribute_groups=[AttributeGroup(name="sad")])
         serializer_data = json.loads(json.dumps(annotation_class, cls=PydanticEncoder))
-        assert {'name': 'asd', 'color': '#0000FF', 'attribute_groups': [{'name': 'sad'}]} == serializer_data
+        assert {'type': 1, 'name': 'asd', 'color': '#0000FF', 'attribute_groups': [{'name': 'sad'}]} == serializer_data
 
     def test_empty_multiselect_bool_serializer(self):
         annotation_class = AnnotationClassEntity(
             name="asd", color="blue", attribute_groups=[AttributeGroup(name="sad", is_multiselect="True")]
         )
         serializer_data = json.loads(json.dumps(annotation_class, cls=PydanticEncoder))
-        assert {'name': 'asd', 'color': '#0000FF',
+        assert {'type': 1,'name': 'asd', 'color': '#0000FF',
                 'attribute_groups': [{'name': 'sad', 'is_multiselect': True}]} == serializer_data
 
     def test_group_type_wrong_arg(self):