Merge pull request #560 from superannotateai/1708_implimentation

1708 implimentation

Author: VaghinakDev

docs/source/api_reference/helpers.rst

@@ -15,8 +15,6 @@ _________________________________________________________________
 .. autofunction:: superannotate.convert_project_type
 .. autofunction:: superannotate.convert_json_version
 
-
-
 ----------
 
 Working with annotations
@@ -28,15 +26,7 @@ ________________________
 
 ----------
 
-Aggregating class distribution from annotations
-_____________________________________________________________
-
-.. autofunction:: superannotate.class_distribution
-
-----------
-
 Utility functions
 --------------------------------
 
-.. autofunction:: superannotate.SAClient.consensus
-.. autofunction:: superannotate.SAClient.benchmark
+.. autofunction:: superannotate.SAClient.consensus

docs/source/userguide/quickstart.rst

@@ -23,8 +23,8 @@ It can be installed on Ubuntu with:
 
    sudo apt-get install ffmpeg
 
-For Windows and Mac OS based installations to use :py:obj:`benchmark` and :py:obj:`consensus`
-functions you might also need to install beforehand :py:obj:`shapely` package,
+For Windows and Mac OS based installations to use :py:obj:`consensus`
+function you might also need to install beforehand :py:obj:`shapely` package,
 which we found to work properly only under Anaconda distribution, with:
 
 .. code-block:: bash

docs/source/userguide/utilities.rst

@@ -188,10 +188,3 @@ The points are colored according to class name. Each annotator is represented wi
 
 .. image:: images/consensus_scatter.png
 
-
-Computing benchmark scores for instances between ground truth project and given project list
-============
-
-
-Benchmark is a tool to compare the quallity of the annotations of the same image that is present in several projects with
-the ground truth annotation of the same image that is in a separate project.

requirements_extra.txt

@@ -1,13 +1,14 @@
-Sphinx==3.5.4
-Jinja2==3.0.3
-tox==3.24.2
-pytest==6.2.4
-pytest-xdist==2.3.0
-pytest-parallel==0.1.0
-pytest-rerunfailures==10.2
-sphinx_rtd_theme==1.0.0
-sphinx_inline_tabs==2022.1.2b11
+Sphinx==6.1.3
+Jinja2==3.1.2
+tox==4.4.5
+sphinx_rtd_theme==1.2.0
+furo==2022.12.7
 jaraco.tidelift==1.5.0
 sphinx-notfound-page==0.8.3
-furo==2022.12.7
-pytest-cov
+sphinx_inline_tabs==2022.1.2b11
+
+pytest==7.2.1
+pytest-xdist==3.2.0
+pytest-parallel==0.1.1
+pytest-cov==4.0.0
+pytest-rerunfailures==10.2

src/superannotate/__init__.py

@@ -12,7 +12,6 @@
 from packaging.version import parse  # noqa
 from superannotate.lib.app.input_converters import convert_json_version  # noqa
 from superannotate.lib.app.input_converters import convert_project_type  # noqa
-from superannotate.lib.app.analytics.class_analytics import class_distribution  # noqa
 from superannotate.lib.app.exceptions import AppException  # noqa
 from superannotate.lib.app.input_converters import convert_json_version  # noqa
 from superannotate.lib.app.input_converters import convert_project_type  # noqa
@@ -43,8 +42,6 @@ def create_app(apps: typing.List[str] = None) -> SAServer:
     # Utils
     "enums",
     "AppException",
-    # analytics
-    "class_distribution",
     # converters
     "convert_json_version",
     "import_annotation",

src/superannotate/lib/app/analytics/class_analytics.py

@@ -449,7 +449,7 @@ def consensus(df, item_name, annot_type):
         from shapely.geometry import Point, Polygon, box
     except ImportError:
         raise ImportError(
-            "To use superannotate.benchmark or superannotate.consensus functions please install shapely package."
+            "To use superannotate.consensus function please install shapely package."
         )
 
     image_df = df[df["itemName"] == item_name]

src/superannotate/lib/app/analytics/common.py

@@ -4,7 +4,6 @@
 import json
 import os
 import sys
-import tempfile
 import warnings
 from pathlib import Path
 from typing import Callable
@@ -1794,74 +1793,6 @@ def download_model(self, model: MLModel, output_dir: Union[str, Path]):
         else:
             return BaseSerializer(res.data).serialize()
 
-    def benchmark(
-        self,
-        project: Union[NotEmptyStr, dict],
-        gt_folder: str,
-        folder_names: List[NotEmptyStr],
-        export_root: Optional[Union[str, Path]] = None,
-        image_list=None,
-        annot_type: Optional[ANNOTATION_TYPE] = "bbox",
-        show_plots=False,
-    ):
-        """Computes benchmark score for each instance of given images that are present both gt_project_name project
-            and projects in folder_names list:
-
-        :param project: project name or metadata of the project
-        :type project: str or dict
-        :param gt_folder: project folder name that contains the ground truth annotations
-        :type gt_folder: str
-        :param folder_names: list of folder names in the project for which the scores will be computed
-        :type folder_names: list of str
-        :param export_root: root export path of the projects
-        :type export_root: Path-like (str or Path)
-        :param image_list: List of image names from the projects list that must be used. If None,
-                then all images from the projects list will be used. Default: None
-        :type image_list: list
-        :param annot_type: Type of annotation instances to consider.
-                Available candidates are: ["bbox", "polygon", "point"]
-        :type annot_type: str
-        :param show_plots: If True, show plots based on results of consensus computation. Default: False
-        :type show_plots: bool
-
-        :return: Pandas DateFrame with columns
-                (creatorEmail, QA, imageName, instanceId, className, area, attribute, folderName, score)
-        :rtype: pandas DataFrame
-        """
-        project_name = project
-        if isinstance(project, dict):
-            project_name = project["name"]
-
-        project = self.controller.projects.get_by_name(project_name).data
-        if project.type not in constants.ProjectType.images:
-            raise AppException(LIMITED_FUNCTIONS[project.type])
-
-        if not export_root:
-            with tempfile.TemporaryDirectory() as temp_dir:
-                response = self.controller.benchmark(
-                    project_name=project_name,
-                    ground_truth_folder_name=gt_folder,
-                    folder_names=folder_names,
-                    export_root=temp_dir,
-                    image_list=image_list,
-                    annot_type=annot_type,
-                    show_plots=show_plots,
-                )
-
-        else:
-            response = self.controller.benchmark(
-                project_name=project_name,
-                ground_truth_folder_name=gt_folder,
-                folder_names=folder_names,
-                export_root=export_root,
-                image_list=image_list,
-                annot_type=annot_type,
-                show_plots=show_plots,
-            )
-            if response.errors:
-                raise AppException(response.errors)
-        return response.data
-
     def consensus(
         self,
         project: NotEmptyStr,

src/superannotate/lib/app/interface/sdk_interface.py

@@ -15,9 +15,7 @@
 import requests
 from botocore.exceptions import ClientError
 from lib.app.analytics.aggregators import DataAggregator
-from lib.app.analytics.common import aggregate_image_annotations_as_df
 from lib.app.analytics.common import consensus
-from lib.app.analytics.common import consensus_plot
 from lib.core.conditions import Condition
 from lib.core.conditions import CONDITION_EQ as EQ
 from lib.core.entities import FolderEntity
@@ -320,99 +318,6 @@ def execute(self):
         return self._response
 
 
-class BenchmarkUseCase(BaseUseCase):
-    def __init__(
-        self,
-        project: ProjectEntity,
-        ground_truth_folder_name: str,
-        folder_names: list,
-        export_dir: str,
-        image_list: list,
-        annotation_type: str,
-        show_plots: bool,
-    ):
-        super().__init__()
-        self._project = project
-        self._ground_truth_folder_name = ground_truth_folder_name
-        self._folder_names = folder_names
-        self._export_dir = export_dir
-        self._image_list = image_list
-        self._annotation_type = annotation_type
-        self._show_plots = show_plots
-
-    def execute(self):
-        project_df = aggregate_image_annotations_as_df(self._export_dir)
-        gt_project_df = project_df[
-            project_df["folderName"] == self._ground_truth_folder_name
-        ]
-        benchmark_dfs = []
-        for folder_name in self._folder_names:
-            folder_df = project_df[project_df["folderName"] == folder_name]
-            project_gt_df = pd.concat([folder_df, gt_project_df])
-            project_gt_df = project_gt_df[project_gt_df["instanceId"].notna()]
-
-            if self._image_list is not None:
-                project_gt_df = project_gt_df.loc[
-                    project_gt_df["itemName"].isin(self._image_list)
-                ]
-
-            project_gt_df.query("type == '" + self._annotation_type + "'", inplace=True)
-
-            project_gt_df = project_gt_df.groupby(
-                ["itemName", "instanceId", "folderName"]
-            )
-
-            def aggregate_attributes(instance_df):
-                def attribute_to_list(attribute_df):
-                    attribute_names = list(attribute_df["attributeName"])
-                    attribute_df["attributeNames"] = len(attribute_df) * [
-                        attribute_names
-                    ]
-                    return attribute_df
-
-                attributes = None
-                if not instance_df["attributeGroupName"].isna().all():
-                    attrib_group_name = instance_df.groupby("attributeGroupName")[
-                        ["attributeGroupName", "attributeName"]
-                    ].apply(attribute_to_list)
-                    attributes = dict(
-                        zip(
-                            attrib_group_name["attributeGroupName"],
-                            attrib_group_name["attributeNames"],
-                        )
-                    )
-
-                instance_df.drop(
-                    ["attributeGroupName", "attributeName"], axis=1, inplace=True
-                )
-                instance_df.drop_duplicates(
-                    subset=["itemName", "instanceId", "folderName"], inplace=True
-                )
-                instance_df["attributes"] = [attributes]
-                return instance_df
-
-            project_gt_df = project_gt_df.apply(aggregate_attributes).reset_index(
-                drop=True
-            )
-            unique_images = set(project_gt_df["itemName"])
-            all_benchmark_data = []
-            for image_name in unique_images:
-                image_data = image_consensus(
-                    project_gt_df, image_name, self._annotation_type
-                )
-                all_benchmark_data.append(pd.DataFrame(image_data))
-            benchmark_project_df = pd.concat(all_benchmark_data, ignore_index=True)
-            benchmark_project_df = benchmark_project_df[
-                benchmark_project_df["folderName"] == folder_name
-            ]
-            benchmark_dfs.append(benchmark_project_df)
-        benchmark_df = pd.concat(benchmark_dfs, ignore_index=True)
-        if self._show_plots:
-            consensus_plot(benchmark_df, self._folder_names)
-        self._response.data = benchmark_df
-        return self._response
-
-
 class ConsensusUseCase(BaseUseCase):
     def __init__(
         self,