Added attach_items

Author: VaghinakDev

src/superannotate/__init__.py

@@ -25,6 +25,7 @@
     attach_document_urls_to_project,
 )
 from superannotate.lib.app.interface.sdk_interface import attach_image_urls_to_project
+from superannotate.lib.app.interface.sdk_interface import attach_items
 from superannotate.lib.app.interface.sdk_interface import (
     attach_items_from_integrated_storage,
 )
@@ -175,6 +176,7 @@
     "get_item_metadata",
     "search_items",
     "query",
+    "attach_items",
     # Image Section
     "copy_images",
     "move_images",

src/superannotate/lib/app/helpers.py

@@ -8,6 +8,7 @@
 
 import boto3
 import pandas as pd
+from superannotate.lib.app.exceptions import AppException
 from superannotate.lib.app.exceptions import PathError
 from superannotate.lib.core import ATTACHED_VIDEO_ANNOTATION_POSTFIX
 from superannotate.lib.core import PIXEL_ANNOTATION_POSTFIX
@@ -168,3 +169,33 @@ def get_paths_and_duplicated_from_csv(csv_path):
         else:
             duplicate_images.append(temp)
     return images_to_upload, duplicate_images
+
+
+def get_name_url_duplicated_from_csv(csv_path):
+    image_data = pd.read_csv(csv_path, dtype=str)
+    if "url" not in image_data.columns:
+        raise AppException("Column 'url' is required")
+    image_data = image_data[~image_data["url"].isnull()]
+    if "name" in image_data.columns:
+        image_data["name"] = (
+            image_data["name"]
+            .fillna("")
+            .apply(lambda cell: cell if str(cell).strip() else str(uuid.uuid4()))
+        )
+    else:
+        image_data["name"] = [str(uuid.uuid4()) for _ in range(len(image_data.index))]
+
+    image_data = pd.DataFrame(image_data, columns=["name", "url"])
+    img_names_urls = image_data.to_dict(orient="records")
+    duplicate_images = []
+    seen = []
+    images_to_upload = []
+    for i in img_names_urls:
+        temp = i["name"]
+        i["name"] = i["name"].strip()
+        if i["name"] not in seen:
+            seen.append(i["name"])
+            images_to_upload.append(i)
+        else:
+            duplicate_images.append(temp)
+    return images_to_upload, duplicate_images

src/superannotate/lib/app/interface/sdk_interface.py

@@ -1,3 +1,4 @@
+import collections
 import io
 import json
 import os
@@ -17,10 +18,13 @@
 from lib.app.annotation_helpers import add_annotation_point_to_json
 from lib.app.helpers import extract_project_folder
 from lib.app.helpers import get_annotation_paths
+from lib.app.helpers import get_name_url_duplicated_from_csv
 from lib.app.helpers import get_paths_and_duplicated_from_csv
 from lib.app.interface.types import AnnotationStatuses
 from lib.app.interface.types import AnnotationType
 from lib.app.interface.types import AnnotatorRole
+from lib.app.interface.types import AttachmentArg
+from lib.app.interface.types import AttachmentDict
 from lib.app.interface.types import ClassType
 from lib.app.interface.types import EmailStr
 from lib.app.interface.types import ImageQualityChoices
@@ -36,6 +40,7 @@
 from lib.app.serializers import SettingsSerializer
 from lib.app.serializers import TeamSerializer
 from lib.core import LIMITED_FUNCTIONS
+from lib.core.entities import AttachmentEntity
 from lib.core.entities.integrations import IntegrationEntity
 from lib.core.entities.project_entities import AnnotationClassEntity
 from lib.core.enums import ImageQuality
@@ -297,6 +302,7 @@ def search_images(
         "We're deprecating the search_images function. Please use search_items instead. Learn more."
         "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.search_items"
     )
+    logger.warning(warning_msg)
     warnings.warn(warning_msg, DeprecationWarning)
     project_name, folder_name = extract_project_folder(project)
     project = Controller.get_default()._get_project(project_name)
@@ -1810,6 +1816,12 @@ def attach_image_urls_to_project(
     :return: list of linked image names, list of failed image names, list of duplicate image names
     :rtype: tuple
     """
+    warning_msg = (
+        "We're deprecating the attach_image_urls_to_project function. Please use attach_items instead. Learn more."
+        "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items"
+    )
+    logger.warning(warning_msg)
+    warnings.warn(warning_msg, DeprecationWarning)
     project_name, folder_name = extract_project_folder(project)
     project = Controller.get_default().get_project_metadata(project_name).data
     project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
@@ -1877,6 +1889,12 @@ def attach_video_urls_to_project(
     :return: attached videos, failed videos, skipped videos
     :rtype: (list, list, list)
     """
+    warning_msg = (
+        "We're deprecating the attach_video_urls_to_project function. Please use attach_items instead. Learn more."
+        "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items"
+    )
+    logger.warning(warning_msg)
+    warnings.warn(warning_msg, DeprecationWarning)
     project_name, folder_name = extract_project_folder(project)
     project = Controller.get_default().get_project_metadata(project_name).data
     project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
@@ -2479,8 +2497,10 @@ def search_images_all_folders(
 
     :param project: project name
     :type project: str
+
     :param image_name_prefix: image name prefix for search
     :type image_name_prefix: str
+
     :param annotation_status: if not None, annotation statuses of images to filter,
                               should be one of NotStarted InProgress QualityCheck Returned Completed Skipped
     :type annotation_status: str
@@ -2735,6 +2755,12 @@ def attach_document_urls_to_project(
     :return: list of attached documents, list of not attached documents, list of skipped documents
     :rtype: tuple
     """
+    warning_msg = (
+        "We're deprecating the attach_document_urls_to_project function. Please use attach_items instead. Learn more."
+        "https://superannotate.readthedocs.io/en/stable/superannotate.sdk.html#superannotate.attach_items"
+    )
+    logger.warning(warning_msg)
+    warnings.warn(warning_msg, DeprecationWarning)
     project_name, folder_name = extract_project_folder(project)
     project = Controller.get_default().get_project_metadata(project_name).data
     project_folder_name = project_name + (f"/{folder_name}" if folder_name else "")
@@ -3087,3 +3113,64 @@ def search_items(
     if response.errors:
         raise AppException(response.errors)
     return BaseSerializer.serialize_iterable(response.data)
+
+
+@Trackable
+@validate_arguments
+def attach_items(
+        project: Union[NotEmptyStr, dict],
+        attachments: AttachmentArg,
+        annotation_status: Optional[AnnotationStatuses] = "NotStarted"
+):
+    """Link items from external storage to SuperAnnotate using URLs.
+
+    :param project: project name or folder path (e.g., “project1/folder1”)
+    :type project: str
+
+    :param attachments: path to CSV file or list of dicts containing attachments URLs.
+    :type attachments: path-like (str or Path) or list of dicts
+
+    :param annotation_status: value to set the annotation statuses of the
+                            linked items:
+                                “NotStarted”
+                                “InProgress”
+                                “QualityCheck”
+                                “Returned”
+                                “Completed”
+                                “Skipped”
+    :type annotation_status: str
+
+    :return: list of attached item names, list of not attached item names, list of duplicate item names
+     that are already in SuperAnnotate.
+    :rtype: tuple
+    """
+    attachments = attachments.__root__
+    project_name, folder_name = extract_project_folder(project)
+    if attachments and isinstance(attachments[0], AttachmentDict):
+        unique_attachments = set(attachments)
+        duplicate_attachments = [item for item, count in collections.Counter(attachments).items() if count > 1]
+    else:
+        unique_attachments, duplicate_attachments = get_name_url_duplicated_from_csv(attachments)
+
+    if duplicate_attachments:
+        logger.info("Dropping duplicates.")
+    unique_attachments = parse_obj_as(List[AttachmentEntity], unique_attachments)
+    if unique_attachments:
+        logger.info(f"Attaching  {len(unique_attachments)} file(s) to project {project}.")
+        response = Controller.get_default().attach_items(
+            project_name=project_name,
+            folder_name=folder_name,
+            attachments=unique_attachments,
+            annotation_status=annotation_status,
+        )
+        if response.errors:
+            raise AppException(response.errors)
+
+        uploaded, duplicated = response.data
+        uploaded = [i["name"] for i in uploaded]
+        fails = [
+            attachment.name
+            for attachment in unique_attachments
+            if attachment.name not in uploaded and attachment.name not in duplicated
+        ]
+        return uploaded, fails, duplicated

src/superannotate/lib/app/interface/types.py

@@ -1,4 +1,7 @@
+import uuid
 from functools import wraps
+from pathlib import Path
+from typing import Optional
 from typing import Union
 
 from lib.core.enums import AnnotationStatus
@@ -8,7 +11,13 @@
 from lib.core.enums import UserRole
 from lib.core.exceptions import AppException
 from lib.infrastructure.validators import wrap_error
+from pydantic import BaseModel
+from pydantic import conlist
 from pydantic import constr
+from pydantic import Extra
+from pydantic import Field
+from pydantic import parse_obj_as
+from pydantic import root_validator
 from pydantic import StrictStr
 from pydantic import validate_arguments as pydantic_validate_arguments
 from pydantic import ValidationError
@@ -22,7 +31,9 @@ class EmailStr(StrictStr):
     def validate(cls, value: Union[str]) -> Union[str]:
         try:
             constr(
-                regex=r"^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$"
+                regex=r"^(?=.{1,254}$)(?=.{1,64}@)[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)"
+                      r"*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}"
+                      r"[a-zA-Z0-9])?)*$"
             ).validate(value)
         except StrRegexError:
             raise ValueError("Invalid email")
@@ -79,6 +90,40 @@ def validate(cls, value: Union[str]) -> Union[str]:
         return value
 
 
+class AttachmentDict(BaseModel):
+    url: StrictStr
+    name: Optional[StrictStr] = Field(default_factory=lambda: str(uuid.uuid4()))
+
+    class Config:
+        extra = Extra.ignore
+
+    def __hash__(self):
+        return hash(self.name)
+
+    def __eq__(self, other):
+        return self.url == other.url and self.name.strip() == other.name.strip()
+
+
+AttachmentArgType = Union[NotEmptyStr, Path, conlist(AttachmentDict, min_items=1)]
+
+
+class AttachmentArg(BaseModel):
+    __root__: AttachmentArgType
+
+    def __getitem__(self, index):
+        return self.__root__[index]
+
+    @root_validator(pre=True)
+    def validate_root(cls, values):
+        try:
+            parse_obj_as(AttachmentArgType, values["__root__"])
+        except ValidationError:
+            raise ValueError(
+                "The value must be str, path, or list of dicts with the required 'url' and optional 'name' keys"
+            )
+        return values
+
+
 class ImageQualityChoices(StrictStr):
     VALID_CHOICES = ["compressed", "original"]
 

src/superannotate/lib/core/__init__.py

@@ -61,6 +61,7 @@
 ALREADY_EXISTING_FILES_WARNING = (
     "{} already existing file(s) found that won't be uploaded."
 )
+
 ATTACHING_FILES_MESSAGE = "Attaching {} file(s) to project {}."
 
 ATTACHING_UPLOAD_STATE_ERROR = "You cannot attach URLs in this type of project. Please attach it in an external storage project."

src/superannotate/lib/core/entities/__init__.py

@@ -1,3 +1,4 @@
+from lib.core.entities.base import AttachmentEntity
 from lib.core.entities.base import BaseEntity as TmpBaseEntity
 from lib.core.entities.integrations import IntegrationEntity
 from lib.core.entities.items import DocumentEntity
@@ -33,6 +34,8 @@
     "Entity",
     "VideoEntity",
     "DocumentEntity",
+    # Utils
+    "AttachmentEntity",
     # project
     "ProjectEntity",
     "ProjectSettingEntity",

src/superannotate/lib/core/entities/base.py

@@ -1,3 +1,4 @@
+import uuid
 from datetime import datetime
 from typing import Optional
 
@@ -25,3 +26,11 @@ class BaseEntity(TimedBaseModel):
 
     class Config:
         extra = Extra.allow
+
+
+class AttachmentEntity(BaseModel):
+    name: Optional[str] = Field(default_factory=lambda: str(uuid.uuid4()))
+    url: str
+
+    class Config:
+        extra = Extra.ignore