Added cli version command

Add consensus benchmark

Author: Vaghinak Basentsyan

.gitignore

@@ -143,4 +143,5 @@ _mask.c
 tags
 temp
 debug_*
-*.idea
+*.idea
+*.DS_Store

pytest.ini

@@ -2,4 +2,4 @@
 minversion = 3.0
 log_cli=true
 python_files = test_*.py
-;addopts = -n24 --dist=loadscope
+addopts = -n32 --dist=loadscope

sonar-project.properties

@@ -8,3 +8,5 @@ sonar.python.version=3.8
 #sonar.sources=.
 # Encoding of the source code. Default is default system encoding
 #sonar.sourceEncoding=UTF-8
+sonar.python.coverage.reportPaths = ./coverage.xml
+

src/superannotate/lib/app/interface/sdk_interface.py

@@ -2565,15 +2565,30 @@ def benchmark(
     project_name = project
     if isinstance(project, dict):
         project_name = project["name"]
-    response = controller.benchmark(
-        project_name=project_name,
-        ground_truth_folder_name=gt_folder,
-        folder_names=folder_names,
-        export_root=export_root,
-        image_list=image_list,
-        annot_type=annot_type,
-        show_plots=show_plots,
-    )
+
+    if export_root is None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            export_root = temp_dir
+            response = controller.benchmark(
+                project_name=project_name,
+                ground_truth_folder_name=gt_folder,
+                folder_names=folder_names,
+                export_root=export_root,
+                image_list=image_list,
+                annot_type=annot_type,
+                show_plots=show_plots,
+            )
+
+    else:
+        response = controller.benchmark(
+            project_name=project_name,
+            ground_truth_folder_name=gt_folder,
+            folder_names=folder_names,
+            export_root=export_root,
+            image_list=image_list,
+            annot_type=annot_type,
+            show_plots=show_plots,
+        )
     return response.data
 
 
@@ -2603,14 +2618,28 @@ def consensus(
     :return: Pandas DateFrame with columns (creatorEmail, QA, imageName, instanceId, className, area, attribute, folderName, score)
     :rtype: pandas DataFrame
     """
-    response = controller.consensus(
-        project_name=project,
-        folder_names=folder_names,
-        export_root=export_root,
-        image_list=image_list,
-        annot_type=annot_type,
-        show_plots=show_plots,
-    )
+
+    if export_root is None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            export_root = temp_dir
+            response = controller.consensus(
+                project_name=project,
+                folder_names=folder_names,
+                export_path=export_root,
+                image_list=image_list,
+                annot_type=annot_type,
+                show_plots=show_plots,
+            )
+
+    else:
+        response = controller.consensus(
+            project_name=project,
+            folder_names=folder_names,
+            export_path=export_root,
+            image_list=image_list,
+            annot_type=annot_type,
+            show_plots=show_plots,
+        )
     return response.data
 
 

src/superannotate/lib/core/usecases.py

@@ -3627,6 +3627,9 @@ def attribute_to_list(attribute_df):
         if self._show_plots:
             consensus_plot(consensus_df, self._folder_names)
 
+        self._response.data = consensus_df
+        return self._response
+
 
 class RunSegmentationUseCase(BaseUseCase):
     def __init__(

src/superannotate/lib/infrastructure/controller.py

@@ -1,6 +1,5 @@
 import copy
 import io
-import tempfile
 from functools import cached_property
 from typing import Iterable
 from typing import List
@@ -1224,20 +1223,25 @@ def benchmark(
         show_plots: bool,
     ):
         project = self._get_project(project_name)
-        if export_root is None:
-            with tempfile.TemporaryDirectory() as export_dir:
-                # TODO fix prepare_export & download_export calls
-                export = self.prepare_export(project.name)
-                self.download_export(
-                    project_name=project.name,
-                    export_name=export["name"],
-                    folder_path=export_dir,
-                )
+        export_response = self.prepare_export(
+            project.name,
+            folder_names=folder_names,
+            include_fuse=False,
+            only_pinned=False,
+        )
+        self.download_export(
+            project_name=project.name,
+            export_name=export_response.data["name"],
+            folder_path=export_root,
+            extract_zip_contents=True,
+            to_s3_bucket=False,
+        )
+
         use_case = usecases.BenchmarkUseCase(
             project=project,
             ground_truth_folder_name=ground_truth_folder_name,
             folder_names=folder_names,
-            export_dir=export_dir,
+            export_dir=export_root,
             image_list=image_list,
             annotation_type=annot_type,
             show_plots=show_plots,
@@ -1248,24 +1252,30 @@ def consensus(
         self,
         project_name: str,
         folder_names: list,
-        export_root: str,
+        export_path: str,
         image_list: list,
         annot_type: str,
         show_plots: bool,
     ):
         project = self._get_project(project_name)
-        if export_root is None:
-            with tempfile.TemporaryDirectory() as export_dir:
-                export = self.prepare_export(project.name)
-                self.download_export(
-                    project_name=project.name,
-                    export_name=export["name"],
-                    folder_path=export_dir,
-                )
+
+        export_response = self.prepare_export(
+            project.name,
+            folder_names=folder_names,
+            include_fuse=False,
+            only_pinned=False,
+        )
+        self.download_export(
+            project_name=project.name,
+            export_name=export_response.data["name"],
+            folder_path=export_path,
+            extract_zip_contents=True,
+            to_s3_bucket=False,
+        )
         use_case = usecases.ConsensusUseCase(
             project=project,
             folder_names=folder_names,
-            export_dir=export_dir,
+            export_dir=export_path,
             image_list=image_list,
             annotation_type=annot_type,
             show_plots=show_plots,

tests/convertors/test_consensus.py

@@ -0,0 +1,85 @@
+import os
+import time
+from os.path import dirname
+
+import src.superannotate as sa
+from tests.integration.base import BaseTestCase
+
+
+class TestConsensus(BaseTestCase):
+    PROJECT_NAME = "consensus"
+    PROJECT_DESCRIPTION = "Desc"
+    PROJECT_TYPE = "Vector"
+    NEW_PROJECT_NAME = "new"
+    TEST_FOLDER_PTH = "data_set"
+    TEST_FOLDER_PATH = "data_set/sample_project_vector"
+    MODEL_NAME = "Instance segmentation (trained on COCO)"
+    TEST_EXPORT_ROOT = "data_set/consensus_benchmark/consensus_test_data"
+    CONSENSUS_PREFIX = "consensus_"
+
+    @property
+    def folder_path(self):
+        return os.path.join(dirname(dirname(__file__)), self.TEST_FOLDER_PATH)
+
+    @property
+    def export_path(self):
+        return os.path.join(dirname(dirname(__file__)), self.TEST_EXPORT_ROOT)
+
+    def test_consensus(self):
+        annot_types = ['polygon', 'bbox', 'point']
+        folder_names = ['consensus_1', 'consensus_2', 'consensus_3']
+        df_column_names = [
+            'creatorEmail', 'imageName', 'instanceId', 'area', 'className',
+            'attributes', 'folderName', 'score'
+        ]
+        export_path = self.export_path
+        for i in range(1, 4):
+            sa.create_folder(self.PROJECT_NAME, "consensus_" + str(i))
+        sa.create_annotation_classes_from_classes_json(
+            self.PROJECT_NAME, self.export_path + "/classes/classes.json"
+        )
+        sa.upload_images_from_folder_to_project(
+            self.PROJECT_NAME, self.export_path + "/images", annotation_status="Completed"
+        )
+        for i in range(1, 4):
+            sa.upload_images_from_folder_to_project(
+                self.PROJECT_NAME + f'/{self.CONSENSUS_PREFIX}' + str(i),
+                self.export_path + "/images",
+                annotation_status="Completed"
+            )
+        time.sleep(2)
+        sa.upload_annotations_from_folder_to_project(self.PROJECT_NAME, self.export_path)
+        for i in range(1, 4):
+            sa.upload_annotations_from_folder_to_project(
+                self.PROJECT_NAME + f'/{self.CONSENSUS_PREFIX}' + str(i),
+                self.export_path + f'/{self.CONSENSUS_PREFIX}' + str(i)
+            )
+
+        for annot_type in annot_types:
+            res_df = sa.consensus(self.PROJECT_NAME, folder_names, annot_type=annot_type)
+            #test content of projectName column
+            assert sorted(res_df['folderName'].unique()) == folder_names
+
+            #test structure of resulting DataFrame
+            assert sorted(res_df.columns) == sorted(df_column_names)
+
+            #test lower bound of the score
+            assert (res_df['score'] >= 0).all()
+
+            #test upper bound of the score
+            assert (res_df['score'] <= 1).all()
+
+        image_names = [
+            'bonn_000000_000019_leftImg8bit.png',
+            'bielefeld_000000_000321_leftImg8bit.png'
+        ]
+
+        #test filtering images with given image names list
+        res_images = sa.consensus(
+            self.PROJECT_NAME,
+            folder_names,
+            export_root=export_path,
+            image_list=image_names
+        )
+
+        assert sorted(res_images['imageName'].unique()) == sorted(image_names)

tests/data_set/consensus_benchmark/consensus_test_data/classes/classes.json

@@ -1 +1 @@
-[{"id":466281,"project_id":61619,"name":"person","color":"#96963a","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z","attribute_groups":[]},{"id":466282,"project_id":61619,"name":"car","color":"#b01cb4","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z","attribute_groups":[{"id":219243,"class_id":466282,"name":"color","is_multiselect":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z","attributes":[{"id":853751,"group_id":219243,"project_id":61619,"name":"red","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z"},{"id":853752,"group_id":219243,"project_id":61619,"name":"blue","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z"},{"id":853753,"group_id":219243,"project_id":61619,"name":"green","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z"},{"id":853754,"group_id":219243,"project_id":61619,"name":"yellow","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z"}]}]},{"id":466283,"project_id":61619,"name":"sign","color":"#fed339","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z","attribute_groups":[]},{"id":466284,"project_id":61619,"name":"tree","color":"#249e9e","count":0,"createdAt":"2021-03-09T10:46:38.000Z","updatedAt":"2021-03-09T10:46:38.000Z","attribute_groups":[]}]
+[{"id":772150,"project_id":106166,"name":"person","color":"#96963a","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z","attribute_groups":[]},{"id":772151,"project_id":106166,"name":"car","color":"#b01cb4","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z","attribute_groups":[{"id":296037,"class_id":772151,"name":"color","is_multiselect":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z","attributes":[{"id":1074666,"group_id":296037,"project_id":106166,"name":"red","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z"},{"id":1074667,"group_id":296037,"project_id":106166,"name":"blue","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z"},{"id":1074668,"group_id":296037,"project_id":106166,"name":"green","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z"},{"id":1074669,"group_id":296037,"project_id":106166,"name":"yellow","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z"}]}]},{"id":772152,"project_id":106166,"name":"sign","color":"#fed339","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z","attribute_groups":[]},{"id":772153,"project_id":106166,"name":"tree","color":"#249e9e","count":0,"createdAt":"2021-08-25T05:53:40.000Z","updatedAt":"2021-08-25T05:53:40.000Z","attribute_groups":[]}]