fixes to support video privacy mode (#410)

Author: jean-lucas

CHANGELOG.md

@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.16.9](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.9) - 2023-11-17
+
+### Fixes
+
+- Minor fixes to video scene upload on privacy moce
+
 ## [0.16.8](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.8) - 2023-11-16
 
 ### Added

nucleus/constants.py

@@ -152,6 +152,8 @@
 TYPE_KEY = "type"
 UPDATED_ITEMS = "updated_items"
 UPDATE_KEY = "update"
+# avoid using this, as it will be deprecated
+UPLOAD_TO_SCALE_KEY = "upload_to_scale"
 URL_KEY = "url"
 VERTICES_KEY = "vertices"
 VIDEO_LOCATION_KEY = "video_location"

nucleus/dataset.py

@@ -785,7 +785,6 @@ def _append_video_scenes(
         update: Optional[bool] = False,
         asynchronous: Optional[bool] = False,
     ) -> Union[dict, AsyncJob]:
-        # TODO: make private in favor of Dataset.append invocation
         if not self.is_scene:
             raise Exception(
                 "Your dataset is not a scene dataset but only supports single dataset items. "
@@ -795,6 +794,7 @@ def _append_video_scenes(
             )
 
         for scene in scenes:
+            scene.use_privacy_mode = self.use_privacy_mode
             scene.validate()
 
         if not asynchronous:

nucleus/scene.py

@@ -14,6 +14,7 @@
     POINTCLOUD_LOCATION_KEY,
     REFERENCE_ID_KEY,
     TRACKS_KEY,
+    UPLOAD_TO_SCALE_KEY,
     VIDEO_LOCATION_KEY,
     VIDEO_URL_KEY,
 )
@@ -507,6 +508,7 @@ class VideoScene(ABC):
     metadata: Optional[dict] = field(default_factory=dict)
     attachment_type: Optional[str] = None
     tracks: List[Track] = field(default_factory=list)
+    use_privacy_mode: bool = False
 
     def __post_init__(self):
         if self.attachment_type:
@@ -534,7 +536,6 @@ def length(self) -> int:
         return len(self.items)
 
     def validate(self):
-        # TODO: make private
         assert (
             self.items or self.video_location
         ), "Please upload either a video_location or an array of dataset items representing frames"
@@ -545,9 +546,10 @@ def validate(self):
             assert (
                 self.length > 0
             ), "When uploading an array of items scene must have a list of items of length at least 1"
-            assert (
-                not self.video_location
-            ), "No video location is accepted when uploading an array of items unless you are using privacy mode"
+            if not self.use_privacy_mode:
+                assert (
+                    not self.video_location
+                ), "No video location is accepted when uploading an array of items unless you are using privacy mode"
             for item in self.items:
                 assert isinstance(
                     item, DatasetItem
@@ -577,9 +579,10 @@ def add_item(
             update: Whether to overwrite the item at the specified index, if it
               exists. Default is False.
         """
-        assert (
-            not self.video_location
-        ), "Cannot add item to a video without items"
+        if not self.use_privacy_mode:
+            assert (
+                not self.video_location
+            ), "Cannot add item to a video without items"
         if index is None:
             index = len(self.items)
         assert (
@@ -598,9 +601,10 @@ def get_item(self, index: int) -> DatasetItem:
 
         Return:
             :class:`DatasetItem`: DatasetItem at the specified index."""
-        assert (
-            not self.video_location
-        ), "Cannot add item to a video without items"
+        if not self.use_privacy_mode:
+            assert (
+                not self.video_location
+            ), "Cannot add item to a video without items"
         if index < 0 or index > len(self.items):
             raise ValueError(
                 f"This scene does not have an item at index {index}"
@@ -613,9 +617,10 @@ def get_items(self) -> List[DatasetItem]:
         Returns:
             List[:class:`DatasetItem`]: List of DatasetItems, sorted by index ascending.
         """
-        assert (
-            not self.video_location
-        ), "Cannot add item to a video without items"
+        if not self.use_privacy_mode:
+            assert (
+                not self.video_location
+            ), "Cannot add item to a video without items"
         return self.items
 
     def info(self):
@@ -679,6 +684,9 @@ def to_payload(self) -> dict:
             payload[METADATA_KEY] = self.metadata
         if self.video_location:
             payload[VIDEO_URL_KEY] = self.video_location
+        # needed in order for the backed validation to work
+        if self.use_privacy_mode is not None:
+            payload[UPLOAD_TO_SCALE_KEY] = not self.use_privacy_mode
         if self.items:
             items_payload = [
                 item.to_payload(is_scene=True) for item in self.items

pyproject.toml

@@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"]  # Easy ignore for getting it running
 
 [tool.poetry]
 name = "scale-nucleus"
-version = "0.16.8"
+version = "0.16.9"
 description = "The official Python client library for Nucleus, the Data Platform for AI"
 license =  "MIT"
 authors = ["Scale AI Nucleus Team <nucleusapi@scaleapi.com>"]

tests/test_video_scene.py

@@ -12,6 +12,7 @@
     REFERENCE_ID_KEY,
     SCENES_KEY,
     TYPE_KEY,
+    UPLOAD_TO_SCALE_KEY,
     URL_KEY,
     VIDEO_URL_KEY,
 )
@@ -106,6 +107,7 @@ def test_video_scene_add_item():
     assert scene.to_payload() == {
         REFERENCE_ID_KEY: scene_ref_id,
         FRAME_RATE_KEY: frame_rate,
+        UPLOAD_TO_SCALE_KEY: True,
         FRAMES_KEY: [
             {
                 URL_KEY: TEST_VIDEO_ITEMS[2].image_location,