make validate - add debug

Author: dshabin

src/superannotate/__init__.py

@@ -233,7 +233,7 @@
             },
             "fileFormatter": {
                 "format": "SA-PYTHON-SDK - %(levelname)s - %(asctime)s - %(message)s"
-            }
+            },
         },
         "root": {  # root logger
             "level": "DEBUG",
@@ -257,7 +257,9 @@ def log_version_info():
                 pip_version = max(pip_version, ver)
         if pip_version.major > local_version.major:
             logging.warning(
-                constances.PACKAGE_VERSION_MAJOR_UPGRADE.format(local_version, pip_version)
+                constances.PACKAGE_VERSION_MAJOR_UPGRADE.format(
+                    local_version, pip_version
+                )
             )
         elif pip_version > local_version:
             logging.warning(

src/superannotate/lib/app/analytics/aggregators.py

@@ -1,14 +1,14 @@
 import copy
 import json
 import logging
+from dataclasses import dataclass
 from pathlib import Path
 from typing import List
 from typing import Optional
 from typing import Union
 
 import lib.core as constances
 import pandas as pd
-from dataclasses import dataclass
 from lib.app.exceptions import AppException
 from lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX
 from lib.core import PIXEL_ANNOTATION_POSTFIX

src/superannotate/lib/app/annotation_helpers.py

@@ -34,7 +34,12 @@ def _postprocess_annotation_json(annotation_json, path):
 
 
 def add_annotation_comment_to_json(
-    annotation_json, comment_text, comment_coords, comment_author, resolved=False, image_name=""
+    annotation_json,
+    comment_text,
+    comment_coords,
+    comment_author,
+    resolved=False,
+    image_name="",
 ):
     """Add a comment to SuperAnnotate format annotation JSON
 
@@ -54,7 +59,9 @@ def add_annotation_comment_to_json(
     if len(comment_coords) != 2:
         raise AppException("Comment should have two values")
 
-    annotation_json, path = _preprocess_annotation_json(annotation_json, image_name=image_name)
+    annotation_json, path = _preprocess_annotation_json(
+        annotation_json, image_name=image_name
+    )
 
     annotation = {
         "type": "comment",

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2475,7 +2475,9 @@ def add_annotation_bbox_to_image(
         image_name,
     )
 
-    controller.upload_image_annotations(*extract_project_folder(project), image_name, annotations)
+    controller.upload_image_annotations(
+        *extract_project_folder(project), image_name, annotations
+    )
 
 
 @Trackable
@@ -2509,7 +2511,9 @@ def add_annotation_point_to_image(
     annotations = add_annotation_point_to_json(
         annotations, point, annotation_class_name, annotation_class_attributes, error
     )
-    controller.upload_image_annotations(*extract_project_folder(project), image_name, annotations)
+    controller.upload_image_annotations(
+        *extract_project_folder(project), image_name, annotations
+    )
 
 
 @Trackable
@@ -2538,9 +2542,16 @@ def add_annotation_comment_to_image(
     """
     annotations = get_image_annotations(project, image_name)["annotation_json"]
     annotations = add_annotation_comment_to_json(
-        annotations, comment_text, comment_coords, comment_author, resolved=resolved, image_name=image_name
+        annotations,
+        comment_text,
+        comment_coords,
+        comment_author,
+        resolved=resolved,
+        image_name=image_name,
+    )
+    controller.upload_image_annotations(
+        *extract_project_folder(project), image_name, annotations
     )
-    controller.upload_image_annotations(*extract_project_folder(project), image_name, annotations)
 
 
 @validate_arguments

src/superannotate/lib/app/mixp/utils/parsers.py

@@ -97,11 +97,12 @@ def clone_project(*args, **kwargs):
     project_metadata = result.data["project"]
     project_type = ProjectType.get_name(project_metadata.project_type)
 
-
     return {
         "event_name": "clone_project",
         "properties": {
-            "External": bool(project_metadata.upload_state == constances.UploadState.EXTERNAL.value),
+            "External": bool(
+                project_metadata.upload_state == constances.UploadState.EXTERNAL.value
+            ),
             "Project Type": project_type,
             "Copy Classes": bool(
                 args[3:4] or kwargs.get("copy_annotation_classes", None)

src/superannotate/lib/core/entities/utils.py

@@ -199,7 +199,7 @@ class StringA(BaseModel):
 
 
 class PointLabels(BaseModel):
-    __root__: Dict[constr(regex=r"^[0-9]+$"), StrictStr] # noqa F722
+    __root__: Dict[constr(regex=r"^[0-9]+$"), StrictStr]  # noqa F722
 
     @classmethod
     def __get_validators__(cls):

src/superannotate/lib/core/entities/video.py

@@ -47,7 +47,9 @@ class BaseVideoInstance(BaseInstance):
 
 
 class BboxInstance(BaseVideoInstance):
-    point_labels: Optional[Dict[constr(regex=r"^[0-9]+$"), NotEmptyStr]] = Field( # noqa F722
+    point_labels: Optional[
+        Dict[constr(regex=r"^[0-9]+$"), NotEmptyStr]
+    ] = Field(  # noqa F722
         None, alias="pointLabels"
     )
     timeline: Dict[float, BboxTimeStamp]

src/superannotate/lib/core/reporter.py

@@ -11,13 +11,16 @@ def __init__(
         log_info: bool = True,
         log_warning: bool = True,
         disable_progress_bar: bool = False,
+        log_debug: bool = True,
     ):
         self.logger = logging.getLogger("root")
         self._log_info = log_info
         self._log_warning = log_warning
+        self._log_debug = log_debug
         self._disable_progress_bar = disable_progress_bar
         self.info_messages = []
         self.warning_messages = []
+        self.debug_messages = []
         self.custom_messages = defaultdict(set)
         self.progress_bar = None
 
@@ -31,6 +34,11 @@ def log_warning(self, value: str):
             self.logger.warning(value)
         self.warning_messages.append(value)
 
+    def log_debug(self, value: str):
+        if self._log_debug:
+            self.logger.debug(value)
+        self.debug_messages.append(value)
+
     def start_progress(
         self, iterations: Union[int, range], description: str = "Processing"
     ):

src/superannotate/lib/core/usecases/annotations.py

@@ -166,7 +166,12 @@ def _upload_annotation(
                 project=self._project,
                 folder=self._folder,
                 team=self._team,
-                image=ImageEntity(uuid=image_id, name=image_name, team_id=self._project.team_id, project_id=self._project.uuid),
+                image=ImageEntity(
+                    uuid=image_id,
+                    name=image_name,
+                    team_id=self._project.team_id,
+                    project_id=self._project.uuid,
+                ),
                 images=self._images,
                 annotation_classes=self._annotation_classes,
                 backend_service_provider=self._backend_service,
@@ -209,8 +214,10 @@ def _log_report(self):
                 if key == "missing_classes":
                     template = "Could not find annotation classes matching existing classes on the platform: [{}]"
                 elif key == "missing_attribute_groups":
-                    template = "Could not find attribute groups matching existing attribute groups" \
-                               " on the platform: [{}]"
+                    template = (
+                        "Could not find attribute groups matching existing attribute groups"
+                        " on the platform: [{}]"
+                    )
                 elif key == "missing_attributes":
                     template = "Could not find attributes matching existing attributes on the platform: [{}]"
                 logger.warning(template.format("', '".join(values)))
@@ -465,7 +472,9 @@ def execute(self):
                         ],
                         Body=self._mask,
                     )
-                self._image.annotation_status_code = constances.AnnotationStatus.IN_PROGRESS.value
+                self._image.annotation_status_code = (
+                    constances.AnnotationStatus.IN_PROGRESS.value
+                )
                 self._images.update(self._image)
                 if self._verbose:
                     logger.info(

src/superannotate/lib/core/usecases/projects.py

@@ -21,6 +21,7 @@
 from lib.core.serviceproviders import SuerannotateServiceProvider
 from lib.core.usecases.base import BaseReportableUseCae
 from lib.core.usecases.base import BaseUseCase
+from requests.exceptions import RequestException
 
 logger = logging.getLogger("root")
 
@@ -418,7 +419,9 @@ def validate_project_name(self):
     def get_annotation_classes_repo(self, project: ProjectEntity):
         return self._annotation_classes_repo(self._backend_service, project)
 
-    def _copy_annotation_classes(self, annotation_classes_entity_mapping: dict, project: ProjectEntity):
+    def _copy_annotation_classes(
+        self, annotation_classes_entity_mapping: dict, project: ProjectEntity
+    ):
         annotation_classes = self.annotation_classes.get_all()
         for annotation_class in annotation_classes:
             annotation_class_copy = copy.copy(annotation_class)
@@ -448,42 +451,49 @@ def _copy_settings(self, to_project: ProjectEntity):
                     setting_copy.project_id = to_project.uuid
                     new_settings.update(setting_copy)
 
-    def _copy_workflow(self, annotation_classes_entity_mapping: dict, to_project: ProjectEntity):
+    def _copy_workflow(
+        self, annotation_classes_entity_mapping: dict, to_project: ProjectEntity
+    ):
         new_workflows = self._workflows_repo(self._backend_service, to_project)
         for workflow in self.workflows.get_all():
-            existing_workflow_ids = list(
-                map(lambda i: i.uuid, new_workflows.get_all())
-            )
+            existing_workflow_ids = list(map(lambda i: i.uuid, new_workflows.get_all()))
             workflow_data = copy.copy(workflow)
             workflow_data.project_id = to_project.uuid
-            workflow_data.class_id = annotation_classes_entity_mapping[workflow.class_id].uuid
+            workflow_data.class_id = annotation_classes_entity_mapping[
+                workflow.class_id
+            ].uuid
             new_workflows.insert(workflow_data)
             workflows = new_workflows.get_all()
-            new_workflow = next((
-                work_flow
-                for work_flow in workflows
-                if work_flow.uuid not in existing_workflow_ids
-            ), None)
+            new_workflow = next(
+                (
+                    work_flow
+                    for work_flow in workflows
+                    if work_flow.uuid not in existing_workflow_ids
+                ),
+                None,
+            )
             workflow_attributes = []
             for attribute in workflow_data.attribute:
                 for annotation_attribute in annotation_classes_entity_mapping[
                     workflow.class_id
                 ].attribute_groups:
                     if (
-                            attribute["attribute"]["attribute_group"]["name"]
-                            == annotation_attribute["name"]
+                        attribute["attribute"]["attribute_group"]["name"]
+                        == annotation_attribute["name"]
                     ):
                         for annotation_attribute_value in annotation_attribute[
                             "attributes"
                         ]:
                             if (
-                                    annotation_attribute_value["name"]
-                                    == attribute["attribute"]["name"]
+                                annotation_attribute_value["name"]
+                                == attribute["attribute"]["name"]
                             ):
                                 workflow_attributes.append(
                                     {
                                         "workflow_id": new_workflow.uuid,
-                                        "attribute_id": annotation_attribute_value["id"]
+                                        "attribute_id": annotation_attribute_value[
+                                            "id"
+                                        ],
                                     }
                                 )
                                 break
@@ -508,27 +518,32 @@ def execute(self):
                     f"Cloning annotation classes from {self._project.name} to {self._project_to_create.name}."
                 )
                 try:
-                    self._copy_annotation_classes(annotation_classes_entity_mapping, project)
+                    self._copy_annotation_classes(
+                        annotation_classes_entity_mapping, project
+                    )
                     annotation_classes_created = True
-                except AppException:
+                except (AppException, RequestException) as e:
                     self.reporter.log_warning(
                         f"Failed to clone annotation classes from {self._project.name} to {self._project_to_create.name}."
                     )
+                    self.reporter.log_debug(str(e), exc_info=True)
+
             if self._include_settings:
                 self.reporter.log_info(
                     f"Cloning settings from {self._project.name} to {self._project_to_create.name}."
                 )
                 try:
                     self._copy_settings(project)
-                except AppException:
+                except (AppException, RequestException) as e:
                     self.reporter.log_warning(
                         f"Failed to clone settings from {self._project.name} to {self._project_to_create.name}."
                     )
+                    self.reporter.log_debug(str(e), exc_info=True)
 
             if self._include_workflow and self._include_annotation_classes:
                 if self._project.project_type in (
-                        constances.ProjectType.DOCUMENT.value,
-                        constances.ProjectType.VIDEO.value
+                    constances.ProjectType.DOCUMENT.value,
+                    constances.ProjectType.VIDEO.value,
                 ):
                     self.reporter.log_warning(
                         "Workflow copy is deprecated for "
@@ -544,20 +559,22 @@ def execute(self):
                     )
                     try:
                         self._copy_workflow(annotation_classes_entity_mapping, project)
-                    except AppException:
+                    except (AppException, RequestException) as e:
                         self.reporter.log_warning(
                             f"Failed to workflow from {self._project.name} to {self._project_to_create.name}."
                         )
+                        self.reporter.log_debug(str(e), exc_info=True)
             if self._include_contributors:
                 self.reporter.log_info(
                     f"Cloning contributors from {self._project.name} to {self._project_to_create.name}."
                 )
                 try:
                     self._copy_include_contributors(project)
-                except AppException:
+                except (AppException, RequestException) as e:
                     self.reporter.log_warning(
                         f"Failed to clone contributors from {self._project.name} to {self._project_to_create.name}."
                     )
+                    self.reporter.log_debug(str(e), exc_info=True)
             self._response.data = self._projects.get_one(
                 uuid=project.uuid, team_id=project.team_id
             )
@@ -709,10 +726,12 @@ def validate_image_quality(self):
     def validate_project_type(self):
         project = self._projects.get_one(uuid=self._project_id, team_id=self._team_id)
         for attribute in self._to_update:
-            if (
-                attribute.get("attribute", "") == "ImageQuality"
-                and project.project_type in [constances.ProjectType.VIDEO.value, constances.ProjectType.DOCUMENT.value]
-            ):
+            if attribute.get(
+                "attribute", ""
+            ) == "ImageQuality" and project.project_type in [
+                constances.ProjectType.VIDEO.value,
+                constances.ProjectType.DOCUMENT.value,
+            ]:
                 raise AppValidationException(
                     constances.DEPRICATED_DOCUMENT_VIDEO_MESSAGE
                 )