Merge pull request #433 from superannotateai/friday

Friday

Author: VaghinakDev

.github/workflows/pre_release.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
+          pip install setuptools wheel twine packaging
       - name: Create distribution files
         run: python setup.py sdist
       - name: Publish distribution to PyPI

docs/source/superannotate.sdk.rst

@@ -29,7 +29,6 @@ ________
 .. _ref_get_project_metadata:
 .. autofunction:: superannotate.get_project_metadata
 .. autofunction:: superannotate.get_project_image_count
-.. autofunction:: superannotate.get_project_and_folder_metadata
 .. autofunction:: superannotate.search_folders
 .. autofunction:: superannotate.get_folder_metadata
 .. autofunction:: superannotate.create_folder
@@ -74,7 +73,11 @@ ______
 
 .. autofunction:: superannotate.query
 .. autofunction:: superannotate.search_items
+.. autofunction:: superannotate.attach_items
+.. autofunction:: superannotate.copy_items
+.. autofunction:: superannotate.move_items
 .. autofunction:: superannotate.get_item_metadata
+.. autofunction:: superannotate.set_annotation_statuses
 
 ----------
 
@@ -83,9 +86,6 @@ ______
 
 
 .. _ref_search_images:
-.. autofunction:: superannotate.search_images
-.. autofunction:: superannotate.search_images_all_folders
-.. autofunction:: superannotate.get_image_metadata
 .. autofunction:: superannotate.download_image
 .. autofunction:: superannotate.set_image_annotation_status
 .. autofunction:: superannotate.set_images_annotation_statuses

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-minversion = 3.0
+minversion = 3.7
 log_cli=true
 python_files = test_*.py
-;addopts = -n auto --dist=loadscope
+addopts = -n auto --dist=loadscope

requirements_dev.txt

@@ -1,2 +1,2 @@
-superannotate_schemas>=1.0.41b1
+superannotate_schemas>=v1.0.42dev2
 

src/superannotate/__init__.py

@@ -25,6 +25,7 @@
     attach_document_urls_to_project,
 )
 from superannotate.lib.app.interface.sdk_interface import attach_image_urls_to_project
+from superannotate.lib.app.interface.sdk_interface import attach_items
 from superannotate.lib.app.interface.sdk_interface import (
     attach_items_from_integrated_storage,
 )
@@ -34,6 +35,7 @@
 from superannotate.lib.app.interface.sdk_interface import consensus
 from superannotate.lib.app.interface.sdk_interface import copy_image
 from superannotate.lib.app.interface.sdk_interface import copy_images
+from superannotate.lib.app.interface.sdk_interface import copy_items
 from superannotate.lib.app.interface.sdk_interface import create_annotation_class
 from superannotate.lib.app.interface.sdk_interface import (
     create_annotation_classes_from_classes_json,
@@ -57,12 +59,8 @@
 from superannotate.lib.app.interface.sdk_interface import get_annotations_per_frame
 from superannotate.lib.app.interface.sdk_interface import get_exports
 from superannotate.lib.app.interface.sdk_interface import get_folder_metadata
-from superannotate.lib.app.interface.sdk_interface import get_image_metadata
 from superannotate.lib.app.interface.sdk_interface import get_integrations
 from superannotate.lib.app.interface.sdk_interface import get_item_metadata
-from superannotate.lib.app.interface.sdk_interface import (
-    get_project_and_folder_metadata,
-)
 from superannotate.lib.app.interface.sdk_interface import get_project_image_count
 from superannotate.lib.app.interface.sdk_interface import get_project_metadata
 from superannotate.lib.app.interface.sdk_interface import get_project_settings
@@ -71,19 +69,19 @@
 from superannotate.lib.app.interface.sdk_interface import init
 from superannotate.lib.app.interface.sdk_interface import invite_contributors_to_team
 from superannotate.lib.app.interface.sdk_interface import move_images
+from superannotate.lib.app.interface.sdk_interface import move_items
 from superannotate.lib.app.interface.sdk_interface import pin_image
 from superannotate.lib.app.interface.sdk_interface import prepare_export
 from superannotate.lib.app.interface.sdk_interface import query
 from superannotate.lib.app.interface.sdk_interface import rename_project
 from superannotate.lib.app.interface.sdk_interface import run_prediction
 from superannotate.lib.app.interface.sdk_interface import search_annotation_classes
 from superannotate.lib.app.interface.sdk_interface import search_folders
-from superannotate.lib.app.interface.sdk_interface import search_images
-from superannotate.lib.app.interface.sdk_interface import search_images_all_folders
 from superannotate.lib.app.interface.sdk_interface import search_items
 from superannotate.lib.app.interface.sdk_interface import search_models
 from superannotate.lib.app.interface.sdk_interface import search_projects
 from superannotate.lib.app.interface.sdk_interface import search_team_contributors
+from superannotate.lib.app.interface.sdk_interface import set_annotation_statuses
 from superannotate.lib.app.interface.sdk_interface import set_auth_token
 from superannotate.lib.app.interface.sdk_interface import set_image_annotation_status
 from superannotate.lib.app.interface.sdk_interface import set_images_annotation_statuses
@@ -161,29 +159,29 @@
     "rename_project",
     "upload_priority_scores",
     # Images Section
-    "search_images",
     "copy_image",
     # Folders Section
     "create_folder",
     "get_folder_metadata",
     "delete_folders",
-    "get_project_and_folder_metadata",
     "search_folders",
     "assign_folder",
     "unassign_folder",
     # Items Section
     "get_item_metadata",
     "search_items",
     "query",
+    "attach_items",
+    "copy_items",
+    "move_items",
+    "set_annotation_statuses",
     # Image Section
     "copy_images",
     "move_images",
     "delete_images",
     "download_image",
     "pin_image",
-    "get_image_metadata",
     "get_project_image_count",
-    "search_images_all_folders",
     "assign_images",
     "unassign_images",
     "download_image_annotations",

src/superannotate/lib/app/analytics/aggregators.py

@@ -56,6 +56,34 @@ class VideoRawData:
     attributeName: str = None
 
 
+class DocumentRawData:
+    docName: str = None
+    folderName: str = None
+    docStatus: str = None
+    docUrl: str = None
+    docAnnotator: str = None
+    docQA: str = None
+    # tag
+    tagId: int = None
+    tag: str = None
+    # instance
+    instanceId: int = None
+    instanceStart: int = None
+    instanceEnd: int = None
+    type: str = None
+    className: str = None
+    createdAt: str = None
+    createdBy: str = None
+    creatorRole: str = None
+    updatedAt: str = None
+    updatedBy: str = None
+    updatorRole: str = None
+    # attribute
+    attributeId: int = None
+    attributeGroupName: str = None
+    attributeName: str = None
+
+
 class DataAggregator:
     def __init__(
         self,
@@ -124,6 +152,8 @@ def aggregate_annotations_as_df(self):
             return self.aggregate_image_annotations_as_df(annotation_paths)
         elif self.project_type == constances.ProjectType.VIDEO.name:
             return self.aggregate_video_annotations_as_df(annotation_paths)
+        elif self.project_type == constances.ProjectType.DOCUMENT.name:
+            return self.aggregate_document_annotations_as_df(annotation_paths)
 
     def aggregate_video_annotations_as_df(self, annotation_paths: List[str]):
         raws = []
@@ -205,7 +235,61 @@ def aggregate_video_annotations_as_df(self, annotation_paths: List[str]):
                     raws.append(instance_raw)
             if not instances:
                 raws.append(raw_data)
-        return pd.DataFrame([raw.__dict__ for raw in raws], dtype=object)
+        df = pd.DataFrame([raw.__dict__ for raw in raws], dtype=object)
+        return df.where(pd.notnull(df), None)
+
+    def aggregate_document_annotations_as_df(self, annotation_paths: List[str]):
+        raws = []
+        for annotation_path in annotation_paths:
+            annotation_path = Path(annotation_path)
+            annotation_data = json.load(open(annotation_path))
+            raw_data = DocumentRawData()
+            # metadata
+            raw_data.docName = annotation_data["metadata"]["name"]
+            raw_data.folderName = (
+                annotation_path.parent.name
+                if annotation_path.parent != self.project_root
+                else None
+            )
+            raw_data.docStatus = annotation_data["metadata"].get("status")
+            raw_data.docUrl = annotation_data["metadata"].get("url")
+            raw_data.docAnnotator = annotation_data["metadata"].get("annotatorEmail")
+            raw_data.docQA = annotation_data["metadata"].get("qaEmail")
+            # append tags
+            for idx, tag in enumerate(annotation_data.get("tags", [])):
+                tag_row = copy.copy(raw_data)
+                tag_row.tagId = idx
+                tag_row.tag = tag
+                raws.append(tag_row)
+            # append instances
+            instances = annotation_data.get("instances", [])
+            for idx, instance in enumerate(instances):
+                instance_raw = copy.copy(raw_data)
+                instance_raw.instanceId = int(idx)
+                instance_raw.instanceStart = instance.get("start")
+                instance_raw.instanceEnd = instance.get("end")
+                instance_raw.type = instance.get("type")
+                instance_raw.className = instance.get("className")
+                instance_raw.createdAt = instance.get("createdAt")
+                instance_raw.createdBy = instance.get("createdBy", {}).get("email")
+                instance_raw.creatorRole = instance.get("createdBy", {}).get("role")
+                instance_raw.updatedAt = instance.get("updatedAt")
+                instance_raw.updatedBy = instance.get("updatedBy", {}).get("email")
+                instance_raw.updatorRole = instance.get("updatedBy", {}).get("role")
+                attributes = instance.get("attributes", [])
+                # append attributes
+                for attribute_id, attribute in enumerate(attributes):
+                    attribute_raw = copy.copy(instance_raw)
+                    attribute_raw.attributeId = attribute_id
+                    attribute_raw.attributeGroupName = attribute.get("groupName")
+                    attribute_raw.attributeName = attribute.get("name")
+                    raws.append(attribute_raw)
+                if not attributes:
+                    raws.append(instance_raw)
+            if not instances:
+                raws.append(raw_data)
+        df = pd.DataFrame([raw.__dict__ for raw in raws], dtype=object)
+        return df.where(pd.notnull(df), None)
 
     def aggregate_image_annotations_as_df(self, annotations_paths: List[str]):
         annotation_data = {

src/superannotate/lib/app/helpers.py

@@ -8,6 +8,7 @@
 
 import boto3
 import pandas as pd
+from superannotate.lib.app.exceptions import AppException
 from superannotate.lib.app.exceptions import PathError
 from superannotate.lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX
 from superannotate.lib.core import PIXEL_ANNOTATION_POSTFIX
@@ -168,3 +169,33 @@ def get_paths_and_duplicated_from_csv(csv_path):
         else:
             duplicate_images.append(temp)
     return images_to_upload, duplicate_images
+
+
+def get_name_url_duplicated_from_csv(csv_path):
+    image_data = pd.read_csv(csv_path, dtype=str)
+    if "url" not in image_data.columns:
+        raise AppException("Column 'url' is required")
+    image_data = image_data[~image_data["url"].isnull()]
+    if "name" in image_data.columns:
+        image_data["name"] = (
+            image_data["name"]
+            .fillna("")
+            .apply(lambda cell: cell if str(cell).strip() else str(uuid.uuid4()))
+        )
+    else:
+        image_data["name"] = [str(uuid.uuid4()) for _ in range(len(image_data.index))]
+
+    image_data = pd.DataFrame(image_data, columns=["name", "url"])
+    img_names_urls = image_data.to_dict(orient="records")
+    duplicate_images = []
+    seen = []
+    images_to_upload = []
+    for i in img_names_urls:
+        temp = i["name"]
+        i["name"] = i["name"].strip()
+        if i["name"] not in seen:
+            seen.append(i["name"])
+            images_to_upload.append(i)
+        else:
+            duplicate_images.append(temp)
+    return images_to_upload, duplicate_images

src/superannotate/lib/app/interface/cli_interface.py

@@ -207,7 +207,7 @@ def _upload_annotations(
                     dataset_format=format,
                     dataset_name=dataset_name,
                     project_type=constances.ProjectType.get_name(
-                        project["project"].project_type
+                        project["project"].type
                     ),
                     task=task,
                 )