Many fixes to UUID

Author: hovnatan

superannotate/common.py

@@ -65,13 +65,15 @@ def model_training_status_str_to_int(project_status):
 
 def image_path_to_annotation_paths(image_path, project_type):
     image_path = Path(image_path)
-    postfix_json = '___objects.json' if project_type == "Vector" else '___pixel.json'
-    postfix_mask = '___save.png'
     if project_type == "Vector":
-        return (image_path.parent / (image_path.name + postfix_json), )
+        return (
+            image_path.parent /
+            get_annotation_json_name(image_path.name, project_type),
+        )
     return (
-        image_path.parent / (image_path.name + postfix_json),
-        image_path.parent / (image_path.name + postfix_mask)
+        image_path.parent /
+        get_annotation_json_name(image_path.name, project_type),
+        image_path.parent / get_annotation_png_name(image_path.name)
     )
 
 
@@ -316,3 +318,14 @@ def tqdm_converter(
             else:
                 pbar.update(total_num - pbar.n)
                 break
+
+
+def get_annotation_json_name(image_name, project_type):
+    if project_type == "Vector":
+        return image_name + "___objects.json"
+    else:
+        return image_name + "___pixel.json"
+
+
+def get_annotation_png_name(image_name):
+    return image_name + "___save.png"

superannotate/db/images.py

@@ -17,15 +17,15 @@
     add_annotation_polyline_to_json, add_annotation_template_to_json
 )
 from ..api import API
+from ..common import process_api_response
 from ..exceptions import SABaseException
+from ..parameter_decorators import project_metadata
 from .annotation_classes import (
     fill_class_and_attribute_ids, fill_class_and_attribute_names,
     get_annotation_classes_id_to_name, get_annotation_classes_name_to_id,
     search_annotation_classes
 )
 from .project_api import get_project_metadata_bare
-from ..common import process_api_response
-from ..parameter_decorators import project_metadata
 
 logger = logging.getLogger("superannotate-python-sdk")
 
@@ -632,7 +632,14 @@ def get_image_bytes(project, image_name, variant='original'):
     res = response.json()
     url = res[variant]["url"]
     headers = res[variant]["headers"]
+    print(params)
+    print(url)
+    print(headers)
     response = requests.get(url=url, headers=headers)
+    if not response.ok:
+        raise SABaseException(
+            response.status_code, "Couldn't download image" + response.text
+        )
     img = io.BytesIO(response.content)
     return img
 
@@ -762,61 +769,50 @@ def get_image_annotations(project, image_name, project_type=None):
         params=params
     )
     if not response.ok:
-        raise SABaseException(response.status_code, response.text)
+        raise SABaseException(
+            response.status_code,
+            "Couldn't get annotation download token" + response.text
+        )
     res = response.json()
+    # print(json.dumps(res, indent=2))
 
     annotation_classes = search_annotation_classes(project)
     annotation_classes_dict = get_annotation_classes_id_to_name(
         annotation_classes
     )
-    if project_type == "Vector":
-        url = res["objects"]["url"]
-        annotation_json_filename = url.rsplit('/', 1)[-1]
-        headers = res["objects"]["headers"]
-        response = requests.get(url=url, headers=headers)
-        if response.ok:
-            res_json = response.json()
-            fill_class_and_attribute_names(res_json, annotation_classes_dict)
-            return {
-                "annotation_json_filename": annotation_json_filename,
-                "annotation_json": res_json
-            }
-        if not response.ok and response.status_code == 403:
-            return {"annotation_json": None, "annotation_json_filename": None}
-        raise SABaseException(response.status_code, response.text)
-    else:  # pixel
-        url = res["pixelObjects"]["url"]
-        annotation_json_filename = url.rsplit('/', 1)[-1]
-        headers = res["pixelObjects"]["headers"]
-        response = requests.get(url=url, headers=headers)
-        if not response.ok and response.status_code == 403:
-            return {
-                "annotation_json": None,
-                "annotation_json_filename": None,
-                "annotation_mask": None,
-                "annotation_mask_filename": None
-            }
-        elif not response.ok:
-            raise SABaseException(response.status_code, response.text)
-        res_json = response.json()
-        fill_class_and_attribute_names(res_json, annotation_classes_dict)
-        if len(res_json["instances"]) != 0:
-            url = res["pixelSave"]["url"]
-            annotation_mask_filename = url.rsplit('/', 1)[-1]
-            headers = res["pixelSave"]["headers"]
-            response = requests.get(url=url, headers=headers)
-            if not response.ok:
-                raise SABaseException(response.status_code, response.text)
-            mask = io.BytesIO(response.content)
-        else:
-            mask = None
-            annotation_mask_filename = None
-        return {
-            "annotation_json": res_json,
-            "annotation_json_filename": annotation_json_filename,
-            "annotation_mask": mask,
-            "annotation_mask_filename": annotation_mask_filename
-        }
+    main_annotations = res["annotations"]["MAIN"][0]
+    response = requests.get(
+        url=main_annotations["annotation_json_path"]["url"],
+        headers=main_annotations["annotation_json_path"]["headers"]
+    )
+    if not response.ok:
+        raise SABaseException(
+            response.status_code, "Couldn't load annotations" + response.text
+        )
+    res_json = response.json()
+    fill_class_and_attribute_names(res_json, annotation_classes_dict)
+    result = {
+        "annotation_json":
+            response.json(),
+        "annotation_json_filename":
+            common.get_annotation_json_name(image_name, project_type)
+    }
+    if project_type == "Pixel":
+        response = requests.get(
+            url=main_annotations["annotation_bluemap_path"]["url"],
+            headers=main_annotations["annotation_bluemape_path"]["headers"]
+        )
+        if not response.ok:
+            raise SABaseException(
+                response.status_code,
+                "Couldn't load annotations" + response.text
+            )
+        mask = io.BytesIO(response.content)
+        result["annotation_mask"] = mask
+        result["annotation_mask_filename"] = common.get_annotation_png_name(
+            image_name
+        )
+    return result
 
 
 def download_image_annotations(project, image_name, local_dir_path):
@@ -956,17 +952,16 @@ def upload_image_annotations(
             response.status_code, "Couldn't upload annotation. " + response.text
         )
     res = response.json()
-    if project_type == "Vector":
-        res = res['objects']
-        s3_session = boto3.Session(
-            aws_access_key_id=res['accessKeyId'],
-            aws_secret_access_key=res['secretAccessKey'],
-            aws_session_token=res['sessionToken']
-        )
-        s3_resource = s3_session.resource('s3')
-        bucket = s3_resource.Bucket(res["bucket"])
-        bucket.put_object(Key=res['filePath'], Body=json.dumps(annotation_json))
-    else:  # pixel
+    res = res['annotation_json_path']
+    s3_session = boto3.Session(
+        aws_access_key_id=res['accessKeyId'],
+        aws_secret_access_key=res['secretAccessKey'],
+        aws_session_token=res['sessionToken']
+    )
+    s3_resource = s3_session.resource('s3')
+    bucket = s3_resource.Bucket(res["bucket"])
+    bucket.put_object(Key=res['filePath'], Body=json.dumps(annotation_json))
+    if project_type == "Pixel":
         if mask is None:
             raise SABaseException(0, "Pixel annotation should have mask.")
         if not isinstance(mask, io.BytesIO):

superannotate/db/project_images.py

@@ -96,13 +96,12 @@ def upload_image_to_project(
         path=f'/project/{project_id}/sdkImageUploadToken',
         params=params
     )
-    if response.ok:
-        res = response.json()
-        prefix = res['filePath']
-    else:
+    if not response.ok:
         raise SABaseException(
             response.status_code, "Couldn't get upload token " + response.text
         )
+    res = response.json()
+    prefix = res['filePath']
     s3_session = boto3.Session(
         aws_access_key_id=res['accessKeyId'],
         aws_secret_access_key=res['secretAccessKey'],
@@ -120,7 +119,7 @@ def upload_image_to_project(
 
     __create_image(
         [img_name], [key], project, annotation_status, prefix,
-        images_info_and_array[2]
+        [images_info_and_array[2]]
     )
 
     while True:

superannotate/db/projects.py

@@ -658,11 +658,11 @@ def __upload_images_to_aws_thread(
     bucket = s3_resource.Bucket(res["bucket"])
     prefix = res['filePath']
     uploaded_imgs = []
+    uploaded_imgs_info = ([], [], [])
     for i in range(start_index, end_index):
         if i >= len_img_paths:
             break
         path = img_paths[i]
-        key = prefix + f'{Path(path).name}'
         try:
             if from_s3_bucket is not None:
                 file = io.BytesIO()
@@ -678,28 +678,34 @@ def __upload_images_to_aws_thread(
                 with open(path, "rb") as f:
                     file = io.BytesIO(f.read())
             images_array = get_image_array_to_upload(
-                file, image_quality_in_editor, project["type"]
-            )
-            upload_image_array_to_s3(
-                bucket, *images_array, key, project["type"]
+                Path(path).name, file, image_quality_in_editor, project["type"]
             )
+            key = upload_image_array_to_s3(bucket, *images_array, prefix)
         except Exception as e:
             logger.warning("Unable to upload image %s. %s", path, e)
             couldnt_upload[thread_id].append(path)
             continue
         else:
             uploaded[thread_id].append(path)
             uploaded_imgs.append(path)
+            uploaded_imgs_info[0].append(Path(path).name)
+            uploaded_imgs_info[1].append(key)
+            uploaded_imgs_info[2].append(images_array[2])
             if len(uploaded_imgs) >= 100:
                 __create_image(
-                    uploaded_imgs, project, annotation_status, prefix
+                    uploaded_imgs_info[0], uploaded_imgs_info[1], project,
+                    annotation_status, prefix, uploaded_imgs_info[2]
                 )
                 uploaded_imgs = []
-    __create_image(uploaded_imgs, project, annotation_status, prefix)
+                uploaded_imgs_info = ([], [], [])
+    __create_image(
+        uploaded_imgs_info[0], uploaded_imgs_info[1], project,
+        annotation_status, prefix, uploaded_imgs_info[2]
+    )
 
 
 def __create_image(
-    img_names, img_paths, project, annotation_status, remote_dir, size
+    img_names, img_paths, project, annotation_status, remote_dir, sizes
 ):
     if len(img_paths) == 0:
         return
@@ -714,7 +720,7 @@ def __create_image(
         "annotation_status": annotation_status,
         "meta": {}
     }
-    for img_name, img_path in zip(img_names, img_paths):
+    for img_name, img_path, size in zip(img_names, img_paths, sizes):
         img_name_uuid = Path(img_path).name
         remote_path = remote_dir + f"{img_name_uuid}"
         data["images"].append({"name": img_name, "path": remote_path})
@@ -1572,6 +1578,7 @@ def upload_preannotations_from_folder_to_project(
     )
     #TODO: new endpoint to be delivered: use new endpoint to create annotation json for preannotations
 
+
 def _upload_preannotations_from_folder_to_project(
     project, folder_path, from_s3_bucket=None, recursive_subfolders=False
 ):