Merge pull request #175 from superannotateai/re-design-sdk

Re design sdk

Author: dshabin

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-addopts = -n32 --dist=loadscope
+;addopts = -n32 --dist=loadscope

requirements.txt

@@ -1,6 +1,6 @@
 pydicom>=2.0.0
 boto3>=1.14.53
-requests>=2.25.1
+requests=2.26.0
 requests-toolbelt>=0.9.1
 tqdm>=4.48.2
 pillow>=7.2.0

src/superannotate/lib/app/interface/sdk_interface.py

@@ -394,7 +394,7 @@ def delete_folders(project: str, folder_names: List[str]):
 
 
 @Trackable
-@validate_input
+# @validate_input
 def get_project_and_folder_metadata(project: Union[str, dict]):
     """Returns project and folder metadata tuple. If folder part is empty,
     than returned folder part is set to None.
@@ -2454,20 +2454,22 @@ def upload_annotations_from_folder_to_project(
     failed_annotations = []
     missing_annotations = []
     chunk_size = 10
-    for i in tqdm(range(0, len(annotation_paths), chunk_size)):
-        response = controller.upload_annotations_from_folder(
-            project_name=project_name,
-            folder_name=folder_name,
-            folder_path=folder_path,
-            annotation_paths=annotation_paths[i : i + chunk_size],  # noqa: E203
-            client_s3_bucket=from_s3_bucket,
-        )
-        if response.errors:
-            logger.warning(response.errors)
-        if response.data:
-            uploaded_annotations.extend(response.data[0])
-            missing_annotations.extend(response.data[1])
-            failed_annotations.extend(response.data[2])
+    with tqdm(total=len(annotation_paths)) as progress_bar:
+        for i in range(0, len(annotation_paths), chunk_size):
+            response = controller.upload_annotations_from_folder(
+                project_name=project_name,
+                folder_name=folder_name,
+                folder_path=folder_path,
+                annotation_paths=annotation_paths[i : i + chunk_size],  # noqa: E203
+                client_s3_bucket=from_s3_bucket,
+            )
+            if response.errors:
+                logger.warning(response.errors)
+            if response.data:
+                uploaded_annotations.extend(response.data[0])
+                missing_annotations.extend(response.data[1])
+                failed_annotations.extend(response.data[2])
+            progress_bar.update(chunk_size)
     return uploaded_annotations, failed_annotations, missing_annotations
 
 

src/superannotate/lib/core/entities.py

@@ -225,6 +225,10 @@ def __init__(
         folder_id: int = None,
         annotator_id: int = None,
         annotator_name: str = None,
+        qa_id: str = None,
+        qa_name: str = None,
+        entropy_value: int = None,
+        approval_status: bool = None,
         is_pinned: bool = None,
         meta: ImageInfoEntity = ImageInfoEntity(),
     ):
@@ -236,7 +240,11 @@ def __init__(
         self.project_id = project_id
         self.annotation_status_code = annotation_status_code
         self.folder_id = folder_id
+        self.qa_id = qa_id
+        self.qa_name = qa_name
+        self.entropy_value = entropy_value
         self.annotator_id = annotator_id
+        self.approval_status = approval_status
         self.annotator_name = annotator_name
         self.is_pinned = is_pinned
         self.meta = meta
@@ -250,6 +258,10 @@ def to_dict(self):
             "project_id": self.project_id,
             "annotation_status": self.annotation_status_code,
             "folder_id": self.folder_id,
+            "qa_id": self.qa_id,
+            "qa_name": self.qa_name,
+            "entropy_value": self.entropy_value,
+            "approval_status": self.approval_status,
             "annotator_id": self.annotator_id,
             "annotator_name": self.annotator_name,
             "is_pinned": self.is_pinned,

src/superannotate/lib/core/usecases.py

@@ -1020,7 +1020,7 @@ def execute(self):
             headers=annotations["annotation_json_path"]["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations {response.text}")
+            raise AppException(f"Couldn't load annotations.")
 
         image_annotations = response.json()
         from_project_annotation_classes = (
@@ -1111,7 +1111,7 @@ def execute(self):
                 headers=annotations["annotation_bluemap_path"]["headers"],
             )
             if not response.ok:
-                raise AppException(f"Couldn't load annotations {response.text}")
+                raise AppException(f"Couldn't load annotations.")
             self.to_project_s3_repo.insert(
                 S3FileEntity(
                     auth_data["annotation_bluemap_path"]["filePath"], response.content
@@ -2036,7 +2036,7 @@ def execute(self):
                 headers=credentials["annotation_json_path"]["headers"],
             )
             if not response.ok:
-                logger.warning(f"Couldn't load annotations {response.text}")
+                logger.warning(f"Couldn't load annotations.")
                 self._response.data = data
                 return self._response
             data["annotation_json"] = response.json()
@@ -2131,7 +2131,7 @@ def execute(self):
             url=annotation_json_creds["url"], headers=annotation_json_creds["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations {response.text}")
+            raise AppException(f"Couldn't load annotations.")
         data["preannotation_json"] = response.json()
         data["preannotation_json_filename"] = f"{self._image_name}{file_postfix}"
         if self._project.project_type == constances.ProjectType.PIXEL.value:
@@ -2208,7 +2208,7 @@ def execute(self):
                 headers=annotation_json_creds["headers"],
             )
             if not response.ok:
-                logger.warning(f"Couldn't load annotations {response.text}")
+                logger.warning(f"Couldn't load annotations.")
                 self._response.data = (None, None)
                 return self._response
             data["annotation_json"] = response.json()
@@ -2287,7 +2287,7 @@ def execute(self):
             url=annotation_json_creds["url"], headers=annotation_json_creds["headers"],
         )
         if not response.ok:
-            raise AppException(f"Couldn't load annotations {response.text}")
+            raise AppException(f"Couldn't load annotations.")
         data["preannotation_json"] = response.json()
         data["preannotation_json_filename"] = f"{self._image_name}{file_postfix}"
         mask_path = None
@@ -3147,6 +3147,7 @@ def execute(self):
 
 
 class UploadAnnotationsUseCase(BaseUseCase):
+    MAX_WORKERS = 10
     def __init__(
         self,
         project: ProjectEntity,
@@ -3326,42 +3327,9 @@ def execute(self):
         else:
             from_s3 = None
 
-        for image_id, image_info in auth_data["images"].items():
-            if from_s3:
-                file = io.BytesIO()
-                s3_object = from_s3.Object(
-                    self._client_s3_bucket, image_id_name_map[image_id].path
-                )
-                s3_object.download_fileobj(file)
-                file.seek(0)
-                annotation_json = json.load(file)
-            else:
-                annotation_json = json.load(open(image_id_name_map[image_id].path))
-
-            self.fill_classes_data(annotation_json)
-            bucket.put_object(
-                Key=image_info["annotation_json_path"],
-                Body=json.dumps(annotation_json),
-            )
-            if self._project.project_type == constances.ProjectType.PIXEL.value:
-                mask_filename = (
-                    image_id_name_map[image_id].name
-                    + constances.ANNOTATION_MASK_POSTFIX
-                )
-                if from_s3:
-                    file = io.BytesIO()
-                    s3_object = self._client_s3_bucket.Objcect(
-                        self._client_s3_bucket, self._folder_path + mask_filename
-                    )
-                    s3_object.download_file(file)
-                    file.seek(0)
-                else:
-                    with open(
-                        f"{self._folder_path}/{mask_filename}", "rb"
-                    ) as mask_file:
-                        file = io.BytesIO(mask_file.read())
-
-                bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            for image_id, image_info in auth_data["images"].items():
+                executor.submit(self.upload_to_s3, image_id, image_info, bucket, from_s3, image_id_name_map)
 
         uploaded_annotations = [annotation.path for annotation in annotations_to_upload]
         missing_annotations = [annotation.path for annotation in missing_annotations]
@@ -3377,6 +3345,43 @@ def execute(self):
         )
         return self._response
 
+    def upload_to_s3(self, image_id: int, image_info, bucket, from_s3, image_id_name_map):
+        if from_s3:
+            file = io.BytesIO()
+            s3_object = from_s3.Object(
+                self._client_s3_bucket, image_id_name_map[image_id].path
+            )
+            s3_object.download_fileobj(file)
+            file.seek(0)
+            annotation_json = json.load(file)
+        else:
+            annotation_json = json.load(open(image_id_name_map[image_id].path))
+
+        self.fill_classes_data(annotation_json)
+        bucket.put_object(
+            Key=image_info["annotation_json_path"],
+            Body=json.dumps(annotation_json),
+        )
+        if self._project.project_type == constances.ProjectType.PIXEL.value:
+            mask_filename = (
+                    image_id_name_map[image_id].name
+                    + constances.ANNOTATION_MASK_POSTFIX
+            )
+            if from_s3:
+                file = io.BytesIO()
+                s3_object = self._client_s3_bucket.Objcect(
+                    self._client_s3_bucket, self._folder_path + mask_filename
+                )
+                s3_object.download_file(file)
+                file.seek(0)
+            else:
+                with open(
+                        f"{self._folder_path}/{mask_filename}", "rb"
+                ) as mask_file:
+                    file = io.BytesIO(mask_file.read())
+
+            bucket.put_object(Key=image_info["annotation_bluemap_path"], Body=file)
+
 
 class CreateModelUseCase(BaseUseCase):
     def __init__(

src/superannotate/lib/infrastructure/services.py

@@ -8,7 +8,6 @@
 
 import lib.core as constance
 import requests.packages.urllib3
-import requests.adapters
 from lib.core.exceptions import AppException
 from lib.core.serviceproviders import SuerannotateServiceProvider
 from requests.exceptions import HTTPError
@@ -32,14 +31,12 @@ def __init__(self, api_url: str, auth_token: str, logger, paginate_by=None):
         self.logger = logger
         self._paginate_by = paginate_by
         self.team_id = auth_token.split("=")[-1]
-        self._session = None
 
     @timed_lru_cache(seconds=360)
     def get_session(self):
-        if not self._session:
-            self._session = requests.Session()
-            self._session.headers.update(self.default_headers)
-        return self._session
+        session = requests.Session()
+        session.headers.update(self.default_headers)
+        return session
 
     @property
     def default_headers(self):
@@ -79,14 +76,12 @@ def _request(
         self, url, method="get", data=None, headers=None, params=None, retried=0
     ) -> requests.Response:
         kwargs = {"json": data} if data else {}
-        headers_dict = self.default_headers.copy()
         session = self.get_session()
         session.headers.update(headers if headers else {})
-        method = getattr(session, method)
         with self.safe_api():
-            response = method(
-                url, **kwargs, headers=headers_dict, params=params, timeout=60,
-            )
+            req = requests.Request(method=method, url=url, **kwargs, params=params)
+            prepared = self._session.prepare_request(req)
+            response = self._session.send(request=prepared)
         if response.status_code == 404 and retried < 3:
             return self._request(
                 url,

tests/integration/test_interface.py

@@ -1,10 +1,13 @@
 import os
 from os.path import dirname
+import tempfile
+
 import src.superannotate as sa
 from src.superannotate.lib.app.exceptions import AppException
 from tests.integration.base import BaseTestCase
 
 
+
 class TestInterface(BaseTestCase):
     PROJECT_NAME = "Interface test"
     TEST_FOLDER_PATH = "data_set/sample_project_vector"
@@ -52,3 +55,24 @@ def test_get_project_metadata(self):
         self.assertListEqual(metadata.get("contributors", []), [])
         metadata_with_users = sa.get_project_metadata(self.PROJECT_NAME, include_contributors=True)
         self.assertIsNotNone(metadata_with_users.get("contributors"))
+
+    def test_upload_annotations_from_folder_to_project(self):
+        sa.upload_images_from_folder_to_project(
+            self.PROJECT_NAME,
+            self.folder_path,
+            annotation_status="InProgress",
+        )
+        uploaded_annotations, _, _ = sa.upload_annotations_from_folder_to_project(
+            self.PROJECT_NAME, self.folder_path
+        )
+        self.assertEqual(len(uploaded_annotations), 4)
+
+    def test_get_images_metadata(self):
+        sa.upload_images_from_folder_to_project(self.PROJECT_NAME, self.folder_path)
+        metadata = sa.search_images(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1, return_metadata=True)
+        self.assertIn("qa_id", metadata[0])
+
+    def test_download_image_annotations(self):
+        sa.upload_images_from_folder_to_project(self.PROJECT_NAME, self.folder_path)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            sa.download_image_annotations(self.PROJECT_NAME, self.EXAMPLE_IMAGE_1, temp_dir)

tests/profiling/profiling.py

@@ -1,9 +1,27 @@
 import src.superannotate as sa
-
 import time
+#
+# s = time.time()
+#
+# sa.get_project_metadata("dev")
+#
+# e = time.time()
+# print(e-s)
+# #
+import cProfile
+import pstats, io
+
+
+
+pr = cProfile.Profile()
+pr.enable()
+
+sa.delete_images("dev")
+sa.upload_images_from_folder_to_project("dev", "/Users/vaghinak.basentsyan/www/superannotate-python-sdk/tests/data_set/sample_project_vector")
 
+pr.disable()
+s = io.StringIO()
 
-stat = time.time()
-sa.search_annotation_classes("Vector Project")
-end = time.time()
-print(11, end-stat)
+ps = pstats.Stats(pr, stream=s)
+ps.print_stats()
+print(s.getvalue())