Performance fix

Author: Vaghinak Basentsyan

src/superannotate/lib/app/interface/sdk_interface.py

@@ -1556,7 +1556,7 @@ def upload_images_from_folder_to_project(
     )
     images_to_upload, _ = use_case.images_to_upload
     if use_case.is_valid():
-        with tqdm(total=len(images_to_upload)) as progress_bar:
+        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
             for _ in use_case.execute():
                 progress_bar.update(1)
         return use_case.data
@@ -1859,7 +1859,7 @@ def upload_videos_from_folder_to_project(
     )
 
     uploaded_images, failed_images = [], []
-    for path in tqdm(video_paths):
+    for path in tqdm(video_paths, desc="Uploading videos"):
         with tempfile.TemporaryDirectory() as temp_path:
             res = controller.extract_video_frames(
                 project_name=project_name,
@@ -1953,7 +1953,9 @@ def upload_video_to_project(
         )
         images_to_upload, _ = use_case.images_to_upload
         if use_case.is_valid():
-            with tqdm(total=len(images_to_upload)) as progress_bar:
+            with tqdm(
+                total=len(images_to_upload), desc="Uploading frames."
+            ) as progress_bar:
                 for _ in use_case.execute():
                     progress_bar.update(1)
             return use_case.data[0]
@@ -2488,8 +2490,10 @@ def upload_annotations_from_folder_to_project(
     uploaded_annotations = []
     failed_annotations = []
     missing_annotations = []
-    chunk_size = 10
-    with tqdm(total=len(annotation_paths)) as progress_bar:
+    chunk_size = 50
+    with tqdm(
+        total=len(annotation_paths), desc="Uploading annotations"
+    ) as progress_bar:
         for i in range(0, len(annotation_paths), chunk_size):
             response = controller.upload_annotations_from_folder(
                 project_name=project_name,
@@ -2549,7 +2553,9 @@ def upload_preannotations_from_folder_to_project(
     failed_annotations = []
     missing_annotations = []
     chunk_size = 10
-    with tqdm(total=len(annotation_paths)) as progress_bar:
+    with tqdm(
+        total=len(annotation_paths), desc="Uploading pre annotations"
+    ) as progress_bar:
         for i in range(0, len(annotation_paths), chunk_size):
             response = controller.upload_annotations_from_folder(
                 project_name=project_name,
@@ -3511,7 +3517,7 @@ def _upload_s3_image(image_path: str):
             executor.submit(upload_method, image_path)
             for image_path in images_to_upload
         ]
-        with tqdm(total=len(images_to_upload)) as progress_bar:
+        with tqdm(total=len(images_to_upload), desc="Uploading images") as progress_bar:
             for future in concurrent.futures.as_completed(results):
                 processed_image = future.result()
                 if processed_image.uploaded and processed_image.entity:

src/superannotate/lib/core/usecases.py

@@ -504,7 +504,7 @@ class UploadImageS3UseCase(BaseUseCase):
     def __init__(
         self,
         project: ProjectEntity,
-        project_settings: BaseReadOnlyRepository,
+        project_settings: List[ProjectSettingEntity],
         image_path: str,
         image: io.BytesIO,
         s3_repo: BaseManageableRepository,
@@ -535,7 +535,7 @@ def execute(self):
             huge_image, huge_width, huge_height = image_processor.generate_huge()
             quality = 60
             if not self._image_quality_in_editor:
-                for setting in self._project_settings.get_all():
+                for setting in self._project_settings:
                     if setting.attribute == "ImageQuality":
                         quality = setting.value
             else:
@@ -679,7 +679,11 @@ def execute(self):
             folder_id=self._folder.uuid,
             images=[image.name for image in self._attachments],
         )
-        duplications = [image["name"] for image in duplications]
+        try:
+            duplications = [image["name"] for image in duplications]
+        except Exception:
+            print(duplications)
+            raise
         meta = {}
         to_upload = []
         for image in self._attachments:
@@ -1062,7 +1066,7 @@ def execute(self):
             headers=annotations["annotation_json_path"]["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations.")
+            raise AppException("Couldn't load annotations.")
 
         image_annotations = response.json()
         from_project_annotation_classes = (
@@ -1153,7 +1157,7 @@ def execute(self):
                 headers=annotations["annotation_bluemap_path"]["headers"],
             )
             if not response.ok:
-                raise AppException(f"Couldn't load annotations.")
+                raise AppException("Couldn't load annotations.")
             self.to_project_s3_repo.insert(
                 S3FileEntity(
                     auth_data["annotation_bluemap_path"]["filePath"], response.content
@@ -2082,7 +2086,7 @@ def execute(self):
                 headers=credentials["annotation_json_path"]["headers"],
             )
             if not response.ok:
-                logger.warning(f"Couldn't load annotations.")
+                logger.warning("Couldn't load annotations.")
                 self._response.data = data
                 return self._response
             data["annotation_json"] = response.json()
@@ -2177,7 +2181,7 @@ def execute(self):
             url=annotation_json_creds["url"], headers=annotation_json_creds["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations.")
+            raise AppException("Couldn't load annotations.")
         data["preannotation_json"] = response.json()
         data["preannotation_json_filename"] = f"{self._image_name}{file_postfix}"
         if self._project.project_type == constances.ProjectType.PIXEL.value:
@@ -2254,7 +2258,7 @@ def execute(self):
                 headers=annotation_json_creds["headers"],
             )
             if not response.ok:
-                logger.warning(f"Couldn't load annotations.")
+                logger.warning("Couldn't load annotations.")
                 self._response.data = (None, None)
                 return self._response
             data["annotation_json"] = response.json()
@@ -2333,7 +2337,7 @@ def execute(self):
             url=annotation_json_creds["url"], headers=annotation_json_creds["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations.")
+            raise AppException("Couldn't load annotations.")
         data["preannotation_json"] = response.json()
         data["preannotation_json_filename"] = f"{self._image_name}{file_postfix}"
         mask_path = None
@@ -3272,10 +3276,11 @@ def __init__(
         self,
         project: ProjectEntity,
         folder: FolderEntity,
-        annotation_classes: BaseReadOnlyRepository,
+        annotation_classes: List[AnnotationClassEntity],
         folder_path: str,
         annotation_paths: List[str],
         backend_service_provider: SuerannotateServiceProvider,
+        templates: List[dict],
         pre_annotation: bool = False,
         client_s3_bucket=None,
     ):
@@ -3288,6 +3293,7 @@ def __init__(
         self._annotation_paths = annotation_paths
         self._client_s3_bucket = client_s3_bucket
         self._pre_annotation = pre_annotation
+        self._templates = templates
 
     @property
     def s3_client(self):
@@ -3296,7 +3302,7 @@ def s3_client(self):
     @property
     def annotation_classes_name_map(self) -> dict:
         classes_data = defaultdict(dict)
-        annotation_classes = self._annotation_classes.get_all()
+        annotation_classes = self._annotation_classes
         for annotation_class in annotation_classes:
             class_info = {"id": annotation_class.uuid}
             if annotation_class.attribute_groups:
@@ -3322,11 +3328,8 @@ def annotation_postfix(self):
         )
 
     def get_templates_mapping(self):
-        templates = self._backend_service.get_templates(
-            team_id=self._project.team_id
-        ).get("data", [])
         templates_map = {}
-        for template in templates:
+        for template in self._templates:
             templates_map[template["name"]] = template["id"]
         return templates_map
 
@@ -4271,7 +4274,7 @@ def __init__(
         self._project = project
         self._folder = folder
         self._folder_path = folder_path
-        self._settings = settings
+        self._settings = settings.get_all()
         self._s3_repo = s3_repo
         self._backend_client = backend_client
         self._image_quality_in_editor = image_quality_in_editor

src/superannotate/lib/infrastructure/controller.py

@@ -30,7 +30,16 @@
 from lib.infrastructure.services import SuperannotateBackendService
 
 
-class BaseController:
+class SingleInstanceMetaClass(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in SingleInstanceMetaClass._instances:
+            SingleInstanceMetaClass._instances[cls] = super().__call__(*args, **kwargs)
+        return SingleInstanceMetaClass._instances[cls]
+
+
+class BaseController(metaclass=SingleInstanceMetaClass):
     def __init__(self, logger, config_path=constances.CONFIG_FILE_LOCATION):
         self._config_path = config_path
         self._logger = logger
@@ -129,6 +138,10 @@ def s3_repo(self):
 
 
 class Controller(BaseController):
+    def __init__(self, logger, config_path=constances.CONFIG_FILE_LOCATION):
+        super().__init__(logger, config_path)
+        self._team = None
+
     def _get_project(self, name: str):
         use_case = usecases.GetProjectByNameUseCase(
             name=name,
@@ -267,7 +280,9 @@ def upload_image_to_s3(
         auth_data = self.get_auth_data(project.uuid, self.team_id, folder.uuid)
         use_case = usecases.UploadImageS3UseCase(
             project=project,
-            project_settings=ProjectSettingsRepository(self._backend_client, project),
+            project_settings=ProjectSettingsRepository(
+                self._backend_client, project
+            ).get_all(),
             image_path=image_path,
             image=image_bytes,
             s3_repo=s3_repo,
@@ -432,8 +447,11 @@ def prepare_export(
         return use_case.execute()
 
     def get_team(self):
-        use_case = usecases.GetTeamUseCase(teams=self.teams, team_id=self.team_id)
-        return use_case.execute()
+        if not self._team:
+            self._team = usecases.GetTeamUseCase(
+                teams=self.teams, team_id=self.team_id
+            ).execute()
+        return self._team
 
     def invite_contributor(self, email: str, is_admin: bool):
         use_case = usecases.InviteContributorUseCase(
@@ -1149,9 +1167,12 @@ def upload_annotations_from_folder(
             backend_service_provider=self._backend_client,
             annotation_classes=AnnotationClassRepository(
                 service=self._backend_client, project=project
-            ),
+            ).get_all(),
             pre_annotation=is_pre_annotations,
             client_s3_bucket=client_s3_bucket,
+            templates=self._backend_client.get_templates(team_id=self.team_id).get(
+                "data", []
+            ),
         )
         return use_case.execute()
 

src/superannotate/lib/infrastructure/services.py

@@ -20,6 +20,7 @@ class BaseBackendService(SuerannotateServiceProvider):
     AUTH_TYPE = "sdk"
     PAGINATE_BY = 100
     MAX_RETRY = 3
+    LIMIT = 100
 
     """
     Base service class
@@ -109,7 +110,10 @@ def _get_page(self, url, offset, params=None, key_field: str = None):
             if isinstance(data, dict):
                 if key_field:
                     data = data[key_field]
-                return data, data.get("count", 0) - offset
+                if data.get("count", 0) < self.LIMIT:
+                    return data, 0
+                else:
+                    return data, data.get("count", 0) - offset
             if isinstance(data, list):
                 return {"data": data}, 0
         return {"data": []}, 0

tests/integration/test_preannotation_upload.py

@@ -8,7 +8,7 @@
 
 
 class TestVectorPreAnnotationImage(BaseTestCase):
-    PROJECT_NAME = "test vector"
+    PROJECT_NAME = "TestVectorPreAnnotationImage"
     PROJECT_DESCRIPTION = "Example Project test vector pre-annotation upload"
     PROJECT_TYPE = "Vector"
     TEST_FOLDER_PATH = "data_set/sample_project_vector"

tests/profiling/profiling.py

@@ -1,23 +1,16 @@
 import src.superannotate as sa
-import time
-#
-# s = time.time()
-#
-# sa.get_project_metadata("dev")
-#
-# e = time.time()
-# print(e-s)
-# #
-import cProfile
-import pstats, io
 
+import cProfile
+import pstats
+import io
+sa.delete_images("dev")
 
 
+sa.upload_images_from_folder_to_project("dev", "/Users/vaghinak.basentsyan/www/superannotate-python-sdk/tests/data_set/sample_project_vector")
 pr = cProfile.Profile()
 pr.enable()
 
-sa.delete_images("dev")
-sa.upload_images_from_folder_to_project("dev", "/Users/vaghinak.basentsyan/www/superannotate-python-sdk/tests/data_set/sample_project_vector")
+sa.upload_annotations_from_folder_to_project("dev", "/Users/vaghinak.basentsyan/www/superannotate-python-sdk/tests/data_set/sample_project_vector")
 
 pr.disable()
 s = io.StringIO()