Merge branch 're-design-sdk' of https://github.com/superannotateai/superannotate-python-sdk into re-design-sdk

Author: dshabin

README.md

@@ -21,7 +21,7 @@ SDK is available on PyPI:
 pip install superannotate
 ```
 
-The package officially supports Python 3.8+ and was tested under Linux and
+The package officially supports Python 3.6+ and was tested under Linux and
 Windows ([Anaconda](https://www.anaconda.com/products/individual#windows)) platforms.
 
 For more detailed installation steps and package usage please have a look at the 

docs/source/index.rst

@@ -48,7 +48,7 @@ SDK is available on PyPI:
    pip install superannotate
 
 
-The package officially supports Python 3.8+ and was tested under Linux and
+The package officially supports Python 3.6+ and was tested under Linux and
 Windows (`Anaconda <https://www.anaconda.com/products/individual#windows>`_) platforms.
 
 For more detailed installation steps and package usage please have a look at 

docs/source/tutorial.sdk.rst

@@ -17,7 +17,7 @@ SDK is available on PyPI:
 
    pip install superannotate
 
-The package officially supports Python 3.8+ and was tested under Linux and
+The package officially supports Python 3.6+ and was tested under Linux and
 Windows (`Anaconda <https://www.anaconda.com/products/individual#windows>`_) platforms.
 
 For certain video related functions to work, ffmpeg package needs to be installed.

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-;addopts = -n32 --dist=loadscope
+addopts = -n32 --dist=loadscope

setup.py

@@ -42,5 +42,5 @@
     entry_points={
         'console_scripts': ['superannotatecli = superannotate.lib.app.bin.superannotate:main']
     },
-    python_requires='>=3.8'
+    python_requires='>=3.6'
 )

src/superannotate/lib/app/helpers.py

@@ -2,11 +2,8 @@
 from functools import wraps
 from inspect import getfullargspec
 from pathlib import Path
-from typing import get_args
-from typing import get_origin
 from typing import get_type_hints
 from typing import List
-from typing import Literal
 from typing import Optional
 from typing import Tuple
 from typing import Union
@@ -134,23 +131,18 @@ def check_types(obj, **kwargs):
         if attr_name == "return":
             continue
         try:
-            if get_origin(attr_type) is list:
-                if get_args(attr_type):
+            if attr_type.__origin__ is list:
+                if attr_type.__args__:
                     for elem in kwargs[attr_name]:
-                        if type(elem) in get_args(attr_type):
+                        if type(elem) in attr_type.__args__:
                             continue
                         else:
                             errors.append(f"Invalid input {attr_name}")
                 elif not isinstance(kwargs[attr_name], list):
                     errors.append(f"Argument {attr_name} is not of type {attr_type}")
-            if get_origin(attr_type) is Literal:
-                if kwargs[attr_name] not in get_args(attr_type):
-                    errors.append(
-                        f'The value of {attr_name} should be {", ".join(get_args(attr_type))}'
-                    )
-            elif get_origin(attr_type) is Union:
+            elif attr_type.__origin__ is Union:
                 if kwargs.get(attr_name) and not isinstance(
-                    kwargs[attr_name], get_args(attr_type)
+                    kwargs[attr_name], attr_type.__args__
                 ):
                     errors.append(f"Argument {attr_name} is not of type {attr_type}")
             elif kwargs.get(attr_name) and not isinstance(kwargs[attr_name], attr_type):

src/superannotate/lib/app/interface/cli_interface.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 from typing import Any
 from typing import Optional
+from tqdm import tqdm
 
 import lib.core as constances
 import pandas as pd
@@ -152,17 +153,19 @@ def upload_image(image_path: str):
             set(filtered_paths),
             [item for item in duplication_counter if duplication_counter[item] > 1],
         )
-        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-            results = [
-                executor.submit(upload_image, image_path)
-                for image_path in images_to_upload
-            ]
-            for future in concurrent.futures.as_completed(results):
-                processed_image = future.result()
-                if processed_image.uploaded and processed_image.entity:
-                    uploaded_image_entities.append(processed_image.entity)
-                else:
-                    failed_images.append(processed_image.path)
+        with tqdm(total=len(images_to_upload)) as progress_bar:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                results = [
+                    executor.submit(upload_image, image_path)
+                    for image_path in images_to_upload
+                ]
+                for future in concurrent.futures.as_completed(results):
+                    processed_image = future.result()
+                    if processed_image.uploaded and processed_image.entity:
+                        uploaded_image_entities.append(processed_image.entity)
+                    else:
+                        failed_images.append(processed_image.path)
+                progress_bar.update(1)
 
         for i in range(0, len(uploaded_image_entities), 500):
             self.controller.upload_images(
@@ -248,7 +251,7 @@ def _upload_annotations(
         project_name, folder_name = split_project_path(project)
         project = self.controller.get_project_metadata(project_name=project_name).data
         if not format:
-            format = "COCO"
+            format = "SuperAnnotate"
         if not data_set_name and format == "COCO":
             raise Exception("Data-set name is required")
         elif not data_set_name:

src/superannotate/lib/app/interface/sdk_interface.py

@@ -11,7 +11,6 @@
 from io import BytesIO
 from pathlib import Path
 from typing import List
-from typing import Literal
 from typing import Optional
 from typing import Union
 from urllib.parse import urlparse
@@ -481,7 +480,7 @@ def search_folders(
 def get_image_bytes(
     project: Union[str, dict],
     image_name: str,
-    variant: Literal["original", "lores"] = "original",
+    variant: str = "original",
 ):
     """Returns an io.BytesIO() object of the image. Suitable for creating
     PIL.Image out of it.
@@ -597,17 +596,8 @@ def upload_images_from_public_urls_to_project(
     project: Union[str, dict],
     img_urls: List[str],
     img_names: Optional[List[str]] = None,
-    annotation_status: Optional[
-        Literal[
-            "NotStarted",
-            "InProgress",
-            "QualityCheck",
-            "Returned",
-            "Completed",
-            "Skipped",
-        ]
-    ] = "NotStarted",
-    image_quality_in_editor: Optional[Literal["compressed", "original"]] = None,
+    annotation_status: Optional[str] = "NotStarted",
+    image_quality_in_editor: Optional[str] = None,
 ):
     """Uploads all images given in the list of URL strings in img_urls to the project.
     Sets status of all the uploaded images to annotation_status if it is not None.
@@ -823,7 +813,7 @@ def move_images(
     ).data
     moved_count = len(moved_images)
     message_postfix = "{from_path} to {to_path}."
-    message_prefix = "Copied images from "
+    message_prefix = "Moved images from "
     if moved_count > 1 or moved_count == 0:
         message_prefix = f"Moved {moved_count}/{len(image_names)} images from "
     elif moved_count == 1:
@@ -992,7 +982,7 @@ def get_project_default_image_quality_in_editor(project: Union[str, dict]):
 @validate_input
 def set_project_default_image_quality_in_editor(
     project: Union[str, dict],
-    image_quality_in_editor: Optional[Literal["compressed", "original"]],
+    image_quality_in_editor: Optional[str],
 ):
     """Sets project's default image quality in editor setting.
 
@@ -1087,9 +1077,7 @@ def get_image_metadata(project: Union[str, dict], image_name: str, *_, **__):
 def set_images_annotation_statuses(
     project: Union[str, dict],
     image_names: List[str],
-    annotation_status: Literal[
-        "NotStarted", "InProgress", "QualityCheck", "Returned", "Completed", "Skipped",
-    ],
+    annotation_status: str,
 ):
     """Sets annotation statuses of images
 
@@ -1497,7 +1485,7 @@ def upload_images_from_folder_to_project(
     from_s3_bucket=None,
     exclude_file_patterns: Optional[str] = constances.DEFAULT_FILE_EXCLUDE_PATTERNS,
     recursive_subfolders: Optional[bool] = False,
-    image_quality_in_editor: Optional[Literal["compressed", "original"]] = None,
+    image_quality_in_editor: Optional[str] = None,
 ):
     """Uploads all images with given extensions from folder_path to the project.
     Sets status of all the uploaded images to set_status if it is not None.
@@ -1689,7 +1677,7 @@ def upload_images_from_s3_bucket_to_project(
     secretAccessKey: str,
     bucket_name: str,
     folder_path: str,
-    image_quality_in_editor: Optional[Literal["compressed", "original"]] = None,
+    image_quality_in_editor: Optional[str] = None,
 ):
     """Uploads all images from AWS S3 bucket to the project.
 

src/superannotate/lib/core/usecases.py

@@ -12,7 +12,6 @@
 from abc import abstractmethod
 from collections import defaultdict
 from collections import namedtuple
-from functools import cached_property
 from pathlib import Path
 from typing import Iterable
 from typing import List
@@ -4102,6 +4101,8 @@ def __init__(
         super().__init__()
 
         self._auth_data = None
+        self._s3_repo_instance = None
+        self._images_to_upload = None
 
         self._project = project
         self._folder = folder
@@ -4134,22 +4135,25 @@ def validate_project_type(self):
                 "The function does not support projects containing videos attached with URLs"
             )
 
-    @cached_property
+    @property
     def auth_data(self):
-        return self._backend_client.get_s3_upload_auth_token(
+        if not self._auth_data:
+            self._auth_data = self._backend_client.get_s3_upload_auth_token(
             team_id=self._project.team_id,
             folder_id=self._folder.uuid,
-            project_id=self._project.uuid,
-        )
+            project_id=self._project.uuid, )
+        return self._auth_data
 
-    @cached_property
+    @property
     def s3_repository(self):
-        return self._s3_repo(
+        if not self._s3_repo_instance:
+            self._s3_repo_instance =  self._s3_repo(
             self.auth_data["accessKeyId"],
             self.auth_data["secretAccessKey"],
             self.auth_data["sessionToken"],
             self.auth_data["bucket"],
         )
+        return self._s3_repo_instance
 
     def _upload_image(self, image_path: str):
         ProcessedImage = namedtuple("ProcessedImage", ["uploaded", "path", "entity"])
@@ -4229,30 +4233,32 @@ def paths(self):
             and "___pixel" not in path
         ]
 
-    @cached_property
+    @property
     def images_to_upload(self):
-        paths = self.paths
-        filtered_paths = []
-        duplicated_paths = []
-        images = self._backend_client.get_bulk_images(
-            project_id=self._project.uuid,
-            team_id=self._project.team_id,
-            folder_id=self._folder.uuid,
-            images=[Path(image).name for image in paths],
-        )
-        image_names = [image["name"] for image in images]
-
-        for path in paths:
-            not_in_exclude_list = [
-                x not in Path(path).name for x in self.exclude_file_patterns
-            ]
-            non_in_service_list = [x not in Path(path).name for x in image_names]
-            if all(not_in_exclude_list) and all(non_in_service_list):
-                filtered_paths.append(path)
-            if not all(non_in_service_list):
-                duplicated_paths.append(path)
+        if not self._images_to_upload:
+            paths = self.paths
+            filtered_paths = []
+            duplicated_paths = []
+            images = self._backend_client.get_bulk_images(
+                project_id=self._project.uuid,
+                team_id=self._project.team_id,
+                folder_id=self._folder.uuid,
+                images=[Path(image).name for image in paths],
+            )
+            image_names = [image["name"] for image in images]
 
-        return list(set(filtered_paths)), duplicated_paths
+            for path in paths:
+                not_in_exclude_list = [
+                    x not in Path(path).name for x in self.exclude_file_patterns
+                ]
+                non_in_service_list = [x not in Path(path).name for x in image_names]
+                if all(not_in_exclude_list) and all(non_in_service_list):
+                    filtered_paths.append(path)
+                if not all(non_in_service_list):
+                    duplicated_paths.append(path)
+
+            self._images_to_upload = list(set(filtered_paths)), duplicated_paths
+        return self._images_to_upload
 
     def execute(self):
         images_to_upload, duplications = self.images_to_upload

src/superannotate/lib/infrastructure/controller.py

@@ -1,6 +1,5 @@
 import copy
 import io
-from functools import cached_property
 from typing import Iterable
 from typing import List
 from typing import Optional
@@ -51,6 +50,12 @@ def __init__(self, logger, config_path=constances.CONFIG_FILE_LOCATION):
             logger=logger,
         )
         self._s3_upload_auth_data = None
+        self._projects = None
+        self._folders = None
+        self._teams = None
+        self._images = None
+        self._ml_models = None
+        self._team_id = None
 
     def set_token(self, token):
         self.configs.insert(ConfigEntity("token", token))
@@ -60,33 +65,45 @@ def set_token(self, token):
             logger=self._logger,
         )
 
-    @cached_property
+    @property
     def projects(self):
-        return ProjectRepository(self._backend_client)
+        if not self._projects:
+            self._projects = ProjectRepository(self._backend_client)
+        return self._projects
 
-    @cached_property
+    @property
     def folders(self):
-        return FolderRepository(self._backend_client)
+        if not self._folders:
+            self._folders = FolderRepository(self._backend_client)
+        return self._folders
 
-    @cached_property
+    @property
     def ml_models(self):
-        return MLModelRepository(self._backend_client, self.team_id)
+        if not self._ml_models:
+            self._ml_models = MLModelRepository(self._backend_client, self.team_id)
+        return self._ml_models
 
-    @cached_property
+    @property
     def teams(self):
-        return TeamRepository(self._backend_client)
+        if not self._teams:
+            self._teams = TeamRepository(self._backend_client)
+        return self._teams
 
-    @cached_property
+    @property
     def images(self):
-        return ImageRepository(self._backend_client)
+        if not self._images:
+            self._images = ImageRepository(self._backend_client)
+        return self._images
 
     @property
     def configs(self):
         return ConfigRepository(self._config_path)
 
-    @cached_property
+    @property
     def team_id(self) -> int:
-        return int(self.configs.get_one("token").value.split("=")[-1])
+        if not self._team_id:
+            self._team_id = int(self.configs.get_one("token").value.split("=")[-1])
+        return self._team_id
 
     @timed_lru_cache(seconds=3600)
     def get_auth_data(self, project_id: int, team_id: int, folder_id: int):