Fix CLI preannotation thread

Author: hovnatan

superannotate/db/projects.py

@@ -1283,6 +1283,11 @@ def _upload_annotations_from_folder_to_project(
     finish_event.set()
     tqdm_thread.join()
     logger.info("Number of annotations uploaded %s.", sum(num_uploaded))
+    if sum(num_uploaded) != len_annotations_paths:
+        logger.warning(
+            "%s annotations were not uploaded.",
+            len_annotations_paths - sum(num_uploaded)
+        )
 
     for ac_upl in actually_uploaded:
         return_result += [str(p) for p in ac_upl]
@@ -1512,36 +1517,38 @@ def _upload_preannotations_from_folder_to_project(
     annotation_classes_dict = get_annotation_classes_name_to_id(
         annotation_classes
     )
-    while True:
-        if sum(num_uploaded) == len_preannotations_paths:
-            break
-        response = _api.send_request(
-            req_type='GET',
-            path=f'/project/{project_id}/preannotation',
-            params=params
+    response = _api.send_request(
+        req_type='GET',
+        path=f'/project/{project_id}/preannotation',
+        params=params
+    )
+    if not response.ok:
+        raise SABaseException(response.status_code, response.text)
+    aws_creds = response.json()
+
+    threads = []
+    for thread_id in range(_NUM_THREADS):
+        t = threading.Thread(
+            target=__upload_preannotations_thread,
+            args=(
+                aws_creds, project_type, preannotations_filenames, folder_path,
+                annotation_classes_dict, thread_id, chunksize, num_uploaded,
+                already_uploaded, from_s3_bucket
+            ),
+            daemon=True
         )
-        if not response.ok:
-            raise SABaseException(response.status_code, response.text)
-        aws_creds = response.json()
-
-        threads = []
-        for thread_id in range(_NUM_THREADS):
-            t = threading.Thread(
-                target=__upload_preannotations_thread,
-                args=(
-                    aws_creds, project_type, preannotations_filenames,
-                    folder_path, annotation_classes_dict, thread_id, chunksize,
-                    num_uploaded, already_uploaded, from_s3_bucket
-                ),
-                daemon=True
-            )
-            threads.append(t)
-            t.start()
-        for t in threads:
-            t.join()
+        threads.append(t)
+        t.start()
+    for t in threads:
+        t.join()
     finish_event.set()
     tqdm_thread.join()
     logger.info("Number of preannotations uploaded %s.", sum(num_uploaded))
+    if sum(num_uploaded) != len_preannotations_paths:
+        logger.warning(
+            "%s preannotations were not uploaded.",
+            len_preannotations_paths - sum(num_uploaded)
+        )
     return return_result + [str(p) for p in preannotations_paths]
 
 

tests/test_preannotation_upload_cli.py

@@ -66,24 +66,21 @@ def test_annotation_folder_upload_download_cli_vector_COCO(tmpdir):
     sa.upload_images_from_folder_to_project(
         project, from_folder, annotation_status="InProgress"
     )
-    # sa.create_annotation_classes_from_classes_json(
-    #     project, from_folder / "classes" / "classes.json"
-    # )
     subprocess.run(
         f'superannotatecli upload-annotations --project "{name}" --folder "{from_folder}" --format COCO --task {task} --dataset-name {dataset_name}',
         check=True,
         shell=True
     )
-    # time.sleep(5)
-    # count_in = len(list(from_folder.glob("*.json")))
+    time.sleep(5)
+    count_in = 2
 
-    # images = sa.search_images(project)
-    # for image_name in images:
-    #     sa.download_image_preannotations(project, image_name, tmpdir)
+    images = sa.search_images(project)
+    for image_name in images:
+        sa.download_image_annotations(project, image_name, tmpdir)
 
-    # count_out = len(list(Path(tmpdir).glob("*.json")))
+    count_out = len(list(Path(tmpdir).glob("*.json")))
 
-    # assert count_in == count_out
+    assert count_in == count_out
 
 
 def test_preannotation_folder_upload_download_cli_vector_COCO(tmpdir):
@@ -102,24 +99,21 @@ def test_preannotation_folder_upload_download_cli_vector_COCO(tmpdir):
     sa.upload_images_from_folder_to_project(
         project, from_folder, annotation_status="InProgress"
     )
-    # sa.create_annotation_classes_from_classes_json(
-    #     project, from_folder / "classes" / "classes.json"
-    # )
     subprocess.run(
         f'superannotatecli upload-preannotations --project "{name}" --folder "{from_folder}" --format COCO --task {task} --dataset-name {dataset_name}',
         check=True,
         shell=True
     )
-    # time.sleep(5)
-    # count_in = len(list(from_folder.glob("*.json")))
+    time.sleep(5)
+    count_in = 2
 
-    # images = sa.search_images(project)
-    # for image_name in images:
-    #     sa.download_image_preannotations(project, image_name, tmpdir)
+    images = sa.search_images(project)
+    for image_name in images:
+        sa.download_image_preannotations(project, image_name, tmpdir)
 
-    # count_out = len(list(Path(tmpdir).glob("*.json")))
+    count_out = len(list(Path(tmpdir).glob("*.json")))
 
-    # assert count_in == count_out
+    assert count_in == count_out
 
 
 def test_preannotation_folder_upload_download_cli_vector_object_COCO(tmpdir):
@@ -138,14 +132,21 @@ def test_preannotation_folder_upload_download_cli_vector_object_COCO(tmpdir):
     sa.upload_images_from_folder_to_project(
         project, from_folder, annotation_status="InProgress"
     )
-    # sa.create_annotation_classes_from_classes_json(
-    #     project, from_folder / "classes" / "classes.json"
-    # )
     subprocess.run(
         f'superannotatecli upload-preannotations --project "{name}" --folder "{from_folder}" --format COCO --task {task} --dataset-name {dataset_name}',
         check=True,
         shell=True
     )
+    time.sleep(5)
+    count_in = 4
+
+    images = sa.search_images(project)
+    for image_name in images:
+        sa.download_image_preannotations(project, image_name, tmpdir)
+
+    count_out = len(list(Path(tmpdir).glob("*.json")))
+
+    assert count_in == count_out
 
 
 def test_preannotation_folder_upload_download_cli_pixel_object_COCO(tmpdir):
@@ -164,11 +165,18 @@ def test_preannotation_folder_upload_download_cli_pixel_object_COCO(tmpdir):
     sa.upload_images_from_folder_to_project(
         project, from_folder, annotation_status="InProgress"
     )
-    # sa.create_annotation_classes_from_classes_json(
-    #     project, from_folder / "classes" / "classes.json"
-    # )
     subprocess.run(
         f'superannotatecli upload-preannotations --project "{name}" --folder "{from_folder}" --format COCO --task {task} --dataset-name {dataset_name}',
         check=True,
         shell=True
     )
+    time.sleep(5)
+    count_in = 6
+
+    images = sa.search_images(project)
+    for image_name in images:
+        sa.download_image_preannotations(project, image_name, tmpdir)
+
+    count_out = len(list(Path(tmpdir).glob("*.json")))
+
+    assert count_in == count_out