Merge pull request #617 from superannotateai/1950_download_annotations

added download_annotations path expanduser logic, changed convert_pro…

Author: nareksa

docs/source/api_reference/helpers.rst

@@ -13,7 +13,6 @@ _________________________________________________________________
 .. autofunction:: superannotate.import_annotation
 .. autofunction:: superannotate.export_annotation
 .. autofunction:: superannotate.convert_project_type
-.. autofunction:: superannotate.convert_json_version
 
 ----------
 

src/superannotate/__init__.py

@@ -9,10 +9,8 @@
 import logging.config  # noqa
 import requests  # noqa
 from packaging.version import parse  # noqa
-from superannotate.lib.app.input_converters import convert_json_version  # noqa
 from superannotate.lib.app.input_converters import convert_project_type  # noqa
 from superannotate.lib.app.exceptions import AppException  # noqa
-from superannotate.lib.app.input_converters import convert_json_version  # noqa
 from superannotate.lib.app.input_converters import convert_project_type  # noqa
 from superannotate.lib.app.input_converters import export_annotation  # noqa
 from superannotate.lib.app.input_converters import import_annotation  # noqa
@@ -32,7 +30,6 @@
     "enums",
     "AppException",
     # converters
-    "convert_json_version",
     "import_annotation",
     "export_annotation",
     "convert_project_type",

src/superannotate/lib/app/input_converters/__init__.py

@@ -1,11 +1,9 @@
-from .conversion import convert_json_version
 from .conversion import convert_project_type
 from .conversion import export_annotation
 from .conversion import import_annotation
 
 
 __all__ = [
-    "convert_json_version",
     "convert_project_type",
     "export_annotation",
     "import_annotation",

src/superannotate/lib/app/input_converters/conversion.py

@@ -6,18 +6,18 @@
 import tempfile
 from argparse import Namespace
 from pathlib import Path
+from typing import Union
 
 from lib.app.exceptions import AppException
 from lib.app.interface.base_interface import Tracker
-from lib.core import DEPRICATED_DOCUMENT_VIDEO_MESSAGE
 from lib.core import LIMITED_FUNCTIONS
 from lib.core.enums import ProjectType
+from typing_extensions import Literal
 
 from .export_from_sa_conversions import export_from_sa
 from .import_to_sa_conversions import import_to_sa
-from .sa_conversion import degrade_json
 from .sa_conversion import sa_convert_project_type
-from .sa_conversion import upgrade_json
+
 
 ALLOWED_TASK_TYPES = [
     "panoptic_segmentation",
@@ -430,63 +430,26 @@ def import_annotation(
 
 
 @Tracker
-def convert_project_type(input_dir, output_dir):
+def convert_project_type(
+    input_dir: Union[str, Path],
+    output_dir: Union[str, Path],
+    convert_to: Literal["Vector", "Pixel"],
+):
     """Converts SuperAnnotate 'Vector' project type to 'Pixel' or reverse.
 
     :param input_dir: Path to the dataset folder that you want to convert.
     :type input_dir: Pathlike(str or Path)
     :param output_dir: Path to the folder where you want to have converted files.
     :type output_dir: Pathlike(str or Path)
-
+    :param convert_to: the project type to which the current project should be converted.
+    :type convert_to: str
     """
     params_info = [
         (input_dir, "input_dir", (str, Path)),
         (output_dir, "output_dir", (str, Path)),
     ]
     _passes_type_sanity(params_info)
-    json_paths = list(Path(str(input_dir)).glob("*.json"))
-    if (
-        json_paths
-        and "___pixel.json" not in json_paths[0].name
-        and "___objects.json" not in json_paths[0].name
-    ):
-        raise AppException(DEPRICATED_DOCUMENT_VIDEO_MESSAGE)
-
-    input_dir, output_dir = _change_type(input_dir, output_dir)
-
-    sa_convert_project_type(input_dir, output_dir)
-
-
-@Tracker
-def convert_json_version(input_dir, output_dir, version=2):
-    """
-    Converts SuperAnnotate JSON versions. Newest JSON version is 2.
-
-    :param input_dir: Path to the dataset folder that you want to convert.
-    :type input_dir: Pathlike(str or Path)
-    :param output_dir: Path to the folder, where you want to have converted dataset.
-    :type output_dir: Pathlike(str or Path)
-    :param version: Output version number. Currently is either 1 or 2. Default value is 2. It will upgrade version 1 to  version 2. Set 1 to degrade from version 2 to version 1.
-    :type version: int
 
-    :return: List of converted files
-    :rtype: list
-    """
-
-    params_info = [
-        (input_dir, "input_dir", (str, Path)),
-        (output_dir, "output_dir", (str, Path)),
-        (version, "version", int),
-    ]
-    _passes_type_sanity(params_info)
     input_dir, output_dir = _change_type(input_dir, output_dir)
 
-    output_dir.mkdir(parents=True, exist_ok=True)
-    if version == 2:
-        converted_files = upgrade_json(input_dir, output_dir)
-    elif version == 1:
-        converted_files = degrade_json(input_dir, output_dir)
-    else:
-        raise AppException("'version' is either 1 or 2.")
-
-    return converted_files
+    sa_convert_project_type(input_dir, output_dir, convert_to)

src/superannotate/lib/app/input_converters/sa_conversion.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import shutil
+from pathlib import Path
 
 import cv2
 import numpy as np
@@ -23,12 +24,13 @@ def from_pixel_to_vector(json_paths, output_dir):
     img_names = []
 
     for json_path in json_paths:
-        file_name = str(json_path.name).replace("___pixel.json", "___objects.json")
-
-        mask_name = str(json_path).replace("___pixel.json", "___save.png")
+        file_name = str(json_path)
+        pixel_postfix = "___pixel.json"
+        postfix = pixel_postfix if file_name.endswith(pixel_postfix) else ".json"
+        mask_name = file_name.replace(postfix, "___save.png")
         img = cv2.imread(mask_name)
         H, W, _ = img.shape
-        sa_json = json.load(open(json_path))
+        sa_json = json.load(open(file_name))
         instances = sa_json["instances"]
         new_instances = []
         global_idx = itertools.count()
@@ -78,21 +80,24 @@ def from_pixel_to_vector(json_paths, output_dir):
                     new_instances.append(temp)
 
         sa_json["instances"] = new_instances
-        write_to_json(output_dir / file_name, sa_json)
-        img_names.append(file_name.replace("___objects.json", ""))
+        write_to_json(
+            str(output_dir / Path(file_name).name.replace(postfix, ".json")), sa_json
+        )
+        img_names.append(file_name.replace(postfix, ""))
     return img_names
 
 
 def from_vector_to_pixel(json_paths, output_dir):
     img_names = []
     for json_path in json_paths:
-        file_name = str(json_path.name).replace("___objects.json", "___pixel.json")
-
-        img_name = str(json_path).replace("___objects.json", "")
+        file_name = str(json_path)
+        vector_postfix = "___objects.json"
+        postfix = vector_postfix if file_name.endswith(vector_postfix) else ".json"
+        img_name = file_name.replace(postfix, "")
         img = cv2.imread(img_name)
         H, W, _ = img.shape
 
-        sa_json = json.load(open(json_path))
+        sa_json = json.load(open(file_name))
         instances = sa_json["instances"]
         mask = np.zeros((H, W, 4))
 
@@ -155,16 +160,18 @@ def from_vector_to_pixel(json_paths, output_dir):
                 instance["parts"] = parts
                 sa_instances.append(instance.copy())
 
-        mask_name = file_name.replace("___pixel.json", "___save.png")
-        cv2.imwrite(str(output_dir.joinpath(mask_name)), mask)
+        mask_name = file_name.replace(postfix, "___save.png")
+        cv2.imwrite(str(output_dir.joinpath(Path(mask_name).name)), mask)
 
         sa_json["instances"] = sa_instances
-        write_to_json(output_dir / file_name, sa_json)
-        img_names.append(file_name.replace("___pixel.json", ""))
+        write_to_json(
+            str(output_dir / Path(file_name).name.replace(postfix, ".json")), sa_json
+        )
+        img_names.append(img_name.replace(".json", ""))
     return img_names
 
 
-def sa_convert_project_type(input_dir, output_dir):
+def sa_convert_project_type(input_dir, output_dir, convert_to):
     json_paths = list(input_dir.glob("*.json"))
 
     output_dir.joinpath("classes").mkdir(parents=True)
@@ -173,158 +180,12 @@ def sa_convert_project_type(input_dir, output_dir):
         output_dir.joinpath("classes", "classes.json"),
     )
 
-    if "___pixel.json" in json_paths[0].name:
+    if convert_to == "Vector":
         img_names = from_pixel_to_vector(json_paths, output_dir)
-    elif "___objects.json" in json_paths[0].name:
+    elif convert_to == "Pixel":
         img_names = from_vector_to_pixel(json_paths, output_dir)
-    elif ".json" in json_paths[0].name:
-        raise AppException(DEPRICATED_DOCUMENT_VIDEO_MESSAGE)
     else:
-        raise AppException(
-            "'input_dir' should contain JSON files with '[IMAGE_NAME]___objects.json' or '[IMAGE_NAME]___pixel.json' names structure.",
-        )
+        raise AppException(DEPRICATED_DOCUMENT_VIDEO_MESSAGE)
 
     for img_name in img_names:
-        copy_file(input_dir.joinpath(img_name), output_dir.joinpath(img_name))
-
-
-def upgrade_json(input_dir, output_dir):
-    files_list = list(input_dir.glob("*.json"))
-    ptype = "Vector"
-    if "___pixel" in str(files_list[0].name):
-        ptype = "Pixel"
-
-    converted_files = []
-    failed_files = []
-    for file in files_list:
-        file_name = file.name
-        try:
-            output_json = _update_json_format(file, ptype)
-            converted_files.append(file_name)
-            write_to_json(output_dir / file_name, output_json)
-        except Exception as e:
-            logger.debug(str(e), exc_info=True)
-            failed_files.append(file_name)
-
-    return converted_files
-
-
-def degrade_json(input_dir, output_dir):
-    files_list = list(input_dir.glob("*.json"))
-
-    converted_files = []
-    failed_files = []
-    for file in files_list:
-        file_name = file.name
-        try:
-            output_json = _degrade_json_format(file)
-            converted_files.append(output_dir / file_name)
-            write_to_json(output_dir / file_name, output_json)
-        except Exception as e:
-            failed_files.append(file_name)
-
-    return converted_files
-
-
-def _update_json_format(old_json_path, project_type):
-    old_json_data = json.load(open(old_json_path))
-    new_json_data = {"metadata": {}, "instances": [], "tags": [], "comments": []}
-
-    meta_keys = [
-        "name",
-        "width",
-        "height",
-        "status",
-        "pinned",
-        "isPredicted",
-        "projectId",
-        "annotatorEmail",
-        "qaEmail",
-    ]
-    if project_type == "Pixel":
-        meta_keys.append("isSegmented")
-
-    new_json_data["metadata"] = dict.fromkeys(meta_keys)
-
-    suffix = "___objects.json" if project_type == "Vector" else "___pixel.json"
-    image_name = str(old_json_path.name).split(suffix)[0]
-    metadata = new_json_data["metadata"]
-    metadata["name"] = image_name
-
-    for item in old_json_data:
-        object_type = item.get("type")
-        if object_type == "meta":
-            meta_name = item["name"]
-            if meta_name == "imageAttributes":
-                metadata["height"] = item.get("height")
-                metadata["width"] = item.get("width")
-                metadata["status"] = item.get("status")
-                metadata["pinned"] = item.get("pinned")
-            if meta_name == "lastAction":
-                metadata["lastAction"] = dict.fromkeys(["email", "timestamp"])
-                metadata["lastAction"]["email"] = item.get("userId")
-                metadata["lastAction"]["timestamp"] = item.get("timestamp")
-        elif object_type == "tag":
-            new_json_data["tags"].append(item.get("name"))
-        elif object_type == "comment":
-            item.pop("type")
-            item["correspondence"] = item["comments"]
-            for comment in item["correspondence"]:
-                comment["email"] = comment["id"]
-                comment.pop("id")
-            item.pop("comments")
-            new_json_data["comments"].append(item)
-        else:
-            new_json_data["instances"].append(item)
-
-    return new_json_data
-
-
-def _degrade_json_format(new_json_path):
-    sa_loader = []
-    new_json_data = json.load(open(new_json_path))
-
-    meta = {"type": "meta", "name": "imageAttributes"}
-    meta_keys = ["height", "width", "status", "pinned"]
-    for meta_key in meta_keys:
-        if meta_key in new_json_data["metadata"]:
-            meta[meta_key] = new_json_data["metadata"][meta_key]
-    sa_loader.append(meta)
-
-    if "lastAction" in new_json_data["metadata"]:
-        meta = {
-            "type": "meta",
-            "name": "lastAction",
-            "userId": new_json_data["metadata"]["lastAction"]["email"],
-            "timestamp": new_json_data["metadata"]["lastAction"]["timestamp"],
-        }
-        sa_loader.append(meta)
-
-    for item in new_json_data["instances"]:
-        sa_loader.append(item)
-
-    for item in new_json_data["comments"]:
-        comments = []
-        for item2 in item["correspondence"]:
-            comments.append({"text": item2["text"], "id": item2["email"]})
-            item["comments"] = comments
-        item["createdAt"] = item["correspondence"][0]["timestamp"]
-        item["createdBy"] = {
-            "email": item["correspondence"][0]["email"],
-            "role": item["correspondence"][0]["role"],
-        }
-        item["updatedAt"] = item["correspondence"][-1]["timestamp"]
-        item["updatedBy"] = {
-            "email": item["correspondence"][-1]["email"],
-            "role": item["correspondence"][-1]["role"],
-        }
-        item.pop("correspondence")
-        item["type"] = "comment"
-        item["comments"] = comments
-        sa_loader.append(item)
-
-    for item in new_json_data["tags"]:
-        tag = {"type": "tag", "name": item}
-        sa_loader.append(tag)
-
-    return sa_loader
+        copy_file(img_name, output_dir / Path(img_name).name)