json converter

Author: agyurjin

docs/source/superannotate.sdk.rst

@@ -274,6 +274,7 @@ _____________________________________________________________
 .. autofunction:: superannotate.export_annotation
 .. autofunction:: superannotate.convert_project_type
 .. autofunction:: superannotate.coco_split_dataset
+.. autofunction:: superannotate.convert_json_version
 
 
 

superannotate/__init__.py

@@ -90,7 +90,7 @@ def consensus(*args, **kwargs):
 )
 from .input_converters.conversion import (
     coco_split_dataset, convert_project_type, export_annotation,
-    import_annotation
+    import_annotation, convert_json_version
 )
 from .old_to_new_format_convertor import update_json_format
 from .version import __version__

superannotate/input_converters/conversion.py

@@ -7,7 +7,9 @@
 from ..exceptions import SABaseException
 from .export_from_sa_conversions import export_from_sa
 from .import_to_sa_conversions import import_to_sa
-from .sa_conversion import (sa_convert_project_type, split_coco)
+from .sa_conversion import (
+    degrade_json, sa_convert_project_type, split_coco, upgrade_json
+)
 
 ALLOWED_TASK_TYPES = [
     'panoptic_segmentation', 'instance_segmentation', 'keypoint_detection',
@@ -154,9 +156,9 @@ def export_annotation(
     ==============  ======================
 
     :param input_dir: Path to the dataset folder that you want to convert.
-    :type input_dir: str
+    :type input_dir: Pathlike(str or Path)
     :param output_dir: Path to the folder, where you want to have converted dataset.
-    :type output_dir: str
+    :type output_dir: Pathlike(str or Path)
     :param dataset_format: One of the formats that are possible to convert. Available candidates are: ["COCO"]
     :type dataset_format: str
     :param dataset_name: Will be used to create json file in the output_dir.
@@ -309,9 +311,9 @@ def import_annotation(
     ==============  ======================
 
     :param input_dir: Path to the dataset folder that you want to convert.
-    :type input_dir: str
+    :type input_dir: Pathlike(str or Path)
     :param output_dir: Path to the folder, where you want to have converted dataset.
-    :type output_dir: str
+    :type output_dir: Pathlike(str or Path)
     :param dataset_format: Annotation format to convert SuperAnnotate annotation format. Available candidates are: ["COCO", "VOC", "LabelBox", "DataLoop",
                         "Supervisely", 'VGG', 'YOLO', 'SageMake', 'VoTT', 'GoogleCloud']
     :type dataset_format: str
@@ -361,9 +363,9 @@ def convert_project_type(input_dir, output_dir):
     """ Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse.
 
     :param input_dir: Path to the dataset folder that you want to convert.
-    :type input_dir: str or PathLike
+    :type input_dir: Pathlike(str or Path)
     :param output_dir: Path to the folder where you want to have converted files.
-    :type output_dir: str or PathLike
+    :type output_dir: Pathlike(str or Path)
 
     """
     param_info = [
@@ -387,11 +389,11 @@ def coco_split_dataset(
     """ Splits COCO dataset to few datsets.
 
     :param coco_json_path: Path to main COCO JSON dataset, which should be splitted.
-    :type coco_json_path: str or PathLike
+    :type coco_json_path: Pathlike(str or Path)
     :param image_dir: Path to all images in the original dataset.
-    :type coco_json_path: str or PathLike
+    :type coco_json_path: str or Pathlike
     :param coco_json_path: Path to the folder where you want to output splitted COCO JSON files.
-    :type coco_json_path: str or PathLike
+    :type coco_json_path: str or Pathlike
     :param dataset_list_name: List of dataset names.
     :type dataset_list_name: list
     :param ratio_list: List of ratios for each splitted dataset.
@@ -448,3 +450,40 @@ def _type_sanity(var, var_name, var_type):
                 var_name, var_type, type(var)
             )
         )
+
+
+def convert_json_version(input_dir, output_dir, version=2):
+    """
+    Converts SuperAnnotate JSON versions. Newest JSON version is 2.
+
+    :param input_dir: Path to the dataset folder that you want to convert.
+    :type input_dir: Pathlike(str or Path)
+    :param output_dir: Path to the folder, where you want to have converted dataset.
+    :type output_dir: Pathlike(str or Path)
+    :param version: Output version number. Currently is either 1 or 2. Default value is 2. It will upgrade version 1 to  version 2. Set 1 to degrade from version 2 to version 1.
+    :type version: int
+
+    :return: List of converted files
+    :rtype: list
+    """
+    param_info = [
+        (input_dir, 'input_dir', (str, Path)),
+        (output_dir, 'output_dir', (str, Path)), (version, 'version', int)
+    ]
+    for param in param_info:
+        _type_sanity(param[0], param[1], param[2])
+
+    if isinstance(input_dir, str):
+        input_dir = Path(input_dir)
+    if isinstance(output_dir, str):
+        output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if version == 2:
+        converted_files = upgrade_json(input_dir, output_dir)
+    elif version == 1:
+        converted_files = degrade_json(input_dir, output_dir)
+    else:
+        raise SABaseException(0, "'version' is either 1 or 2.")
+
+    return converted_files

superannotate/input_converters/sa_conversion.py

@@ -8,7 +8,7 @@
 import cv2
 import numpy as np
 
-from ..common import (blue_color_generator, hex_to_rgb, write_to_json)
+from ..common import blue_color_generator, hex_to_rgb, write_to_json
 from ..exceptions import SABaseException
 
 logger = logging.getLogger("superannotate-python-sdk")
@@ -181,7 +181,7 @@ def sa_convert_project_type(input_dir, output_dir):
     else:
         raise SABaseException(
             0,
-            "'input_dir' should contain JSON files with '[IMAGE_NAME]___objects.json'  or '[IMAGE_NAME]___pixel.json' names structure."
+            "'input_dir' should contain JSON files with '[IMAGE_NAME]___objects.json' or '[IMAGE_NAME]___pixel.json' names structure."
         )
 
     for img_name in img_names:
@@ -239,3 +239,148 @@ def split_coco(
     for file_name, value in groups.items():
         with open(output_dir / (file_name + '.json'), 'w') as fw:
             json.dump(value, fw, indent=2)
+
+
+def upgrade_json(input_dir, output_dir):
+    files_list = list(input_dir.glob('*.json'))
+    ptype = 'Vector'
+    if '___pixel' in str(files_list[0].name):
+        ptype = "Pixel"
+
+    converted_files = []
+    failed_files = []
+    for file in files_list:
+        file_name = file.name
+        try:
+            output_json = _update_json_format(file, ptype)
+            converted_files.append(file_name)
+            write_to_json(output_dir / file_name, output_json)
+        except Exception as e:
+            failed_files.append(file_name)
+
+    return converted_files
+
+
+def degrade_json(input_dir, output_dir):
+    files_list = list(input_dir.glob('*.json'))
+
+    converted_files = []
+    failed_files = []
+    for file in files_list:
+        file_name = file.name
+        try:
+            output_json = _degrade_json_format(file)
+            converted_files.append(output_dir / file_name)
+            write_to_json(output_dir / file_name, output_json)
+        except Exception as e:
+            failed_files.append(file_name)
+
+    return converted_files
+
+
+def _update_json_format(old_json_path, project_type):
+    old_json_data = json.load(open(old_json_path))
+    new_json_data = {
+        "metadata": {},
+        "instances": [],
+        "tags": [],
+        "comments": []
+    }
+
+    meta_keys = [
+        "name", "width", "height", "status", "pinned", "isPredicted",
+        "projectId", "annotatorEmail", "qaEmail"
+    ]
+    if project_type == "Pixel":
+        meta_keys.append("isSegmented")
+
+    new_json_data["metadata"] = dict.fromkeys(meta_keys)
+
+    #set image name
+    suffix = "___objects.json" if project_type == "Vector" else "___pixel.json"
+    # image_name = os.path.basename(old_json_path).split(suffix)[0]
+    image_name = str(old_json_path.name).split(suffix)[0]
+    metadata = new_json_data["metadata"]
+    metadata["name"] = image_name
+
+    for item in old_json_data:
+        object_type = item.get("type")
+        #add metadata
+        if object_type == "meta":
+            meta_name = item["name"]
+            if meta_name == "imageAttributes":
+                metadata["height"] = item.get("height")
+                metadata["width"] = item.get("width")
+                metadata["status"] = item.get("status")
+                metadata["pinned"] = item.get("pinned")
+            if meta_name == "lastAction":
+                metadata["lastAction"] = dict.fromkeys(["email", "timestamp"])
+                metadata["lastAction"]["email"] = item.get("userId")
+                metadata["lastAction"]["timestamp"] = item.get("timestamp")
+        #add tags
+        elif object_type == "tag":
+            new_json_data["tags"].append(item.get("name"))
+        #add comments
+        elif object_type == "comment":
+            item.pop("type")
+            item["correspondence"] = item["comments"]
+            for comment in item["correspondence"]:
+                comment["email"] = comment["id"]
+                comment.pop("id")
+            item.pop("comments")
+            new_json_data["comments"].append(item)
+        #add instances
+        else:
+            new_json_data["instances"].append(item)
+
+    return new_json_data
+
+
+def _degrade_json_format(new_json_path):
+    sa_loader = []
+    new_json_data = json.load(open(new_json_path))
+
+    meta = {'type': 'meta', 'name': 'imageAttributes'}
+    meta_keys = ['height', 'width', 'status', 'pinned']
+    for meta_key in meta_keys:
+        if meta_key in new_json_data['metadata']:
+            meta[meta_key] = new_json_data['metadata'][meta_key]
+    sa_loader.append(meta)
+
+    if 'lastAction' in new_json_data['metadata']:
+        meta = {
+            'type': 'meta',
+            'name': 'lastAction',
+            'userId': new_json_data['metadata']['lastAction']['email'],
+            'timestamp': new_json_data['metadata']['lastAction']['timestamp']
+        }
+        sa_loader.append(meta)
+
+    for item in new_json_data['instances']:
+        sa_loader.append(item)
+
+    for item in new_json_data['comments']:
+        comments = []
+        for item2 in item['correspondence']:
+            comments.append({'text': item2['text'], 'id': item2['email']})
+            item['comments'] = comments
+        item['createdAt'] = item['correspondence'][0]['timestamp']
+        item['createdBy'] = {
+            'email': item['correspondence'][0]['email'],
+            'role': item['correspondence'][0]['role']
+        }
+        item['updatedAt'] = item['correspondence'][-1]['timestamp']
+        item['updatedBy'] = {
+            'email': item['correspondence'][-1]['email'],
+            'role': item['correspondence'][-1]['role']
+        }
+        item.pop('correspondence')
+        item['type'] = 'comment'
+        item['comments'] = comments
+        sa_loader.append(item)
+
+    for item in new_json_data['tags']:
+        tag = {'type': 'tag', 'name': item}
+        sa_loader.append(tag)
+
+    return sa_loader