From 530ca760de9a1f3add7e61e60a61705049506af8 Mon Sep 17 00:00:00 2001
From: Griswald Brooks <griswald.brooks@gmail.com>
Date: Wed, 4 Mar 2026 18:03:59 -0500
Subject: [PATCH] Migrate lab_sim CLIPSeg objectives to SAM3

Replace GetMasks2DFromTextQuery (CLIPSeg) with GetMasks2DFromExemplar
(SAM3) across 5 lab_sim objectives and 2 reusable subtrees. This
simplifies the grasping pipeline from a two-stage CLIPSeg+SAM2
refinement loop to a single SAM3 call, and removes the dependency on
moveit_pro_clipseg models.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../objectives/addbottlestoplanningscene.xml  | 20 ++--
 src/lab_sim/objectives/ctb2.png               |  3 +
 .../get_candidate_grasps_subtree.xml          | 86 ++++++----------
 .../get_grasp_from_text_prompt_subtree.xml    | 13 +--
 src/lab_sim/objectives/iss1.png               |  3 +
 ...d_bottles_on_table_from_image_exemplar.xml | 71 +++++++-------
 .../ml_find_ctb_from_bounding_box.xml         | 56 +++++++++++
 .../ml_find_ctb_from_image_exemplar.xml       | 63 ++++++++++++
 ...from_file.xml => ml_segment_from_file.xml} | 12 +--
 src/lab_sim/objectives/ml_segment_image.xml   | 17 ++--
 .../objectives/ml_segment_point_cloud.xml     | 26 ++---
 src/lab_sim/objectives/pick_1_pill_bottle.xml | 24 ++---
 .../objectives/pick_all_pill_bottles.xml      |  5 +-
 .../objectives/sam3_segment_image_subtree.xml | 65 +++++++++++++
 .../sam3_segment_point_cloud_subtree.xml      | 97 +++++++++++++++++++
 15 files changed, 417 insertions(+), 144 deletions(-)
 create mode 100644 src/lab_sim/objectives/ctb2.png
 create mode 100644 src/lab_sim/objectives/iss1.png
 create mode 100644 src/lab_sim/objectives/ml_find_ctb_from_bounding_box.xml
 create mode 100644 src/lab_sim/objectives/ml_find_ctb_from_image_exemplar.xml
 rename src/lab_sim/objectives/{ml_segment_bottles_from_file.xml => ml_segment_from_file.xml} (71%)
 create mode 100644 src/lab_sim/objectives/sam3_segment_image_subtree.xml
 create mode 100644 src/lab_sim/objectives/sam3_segment_point_cloud_subtree.xml

diff --git a/src/lab_sim/objectives/addbottlestoplanningscene.xml b/src/lab_sim/objectives/addbottlestoplanningscene.xml
index 269e1a462..cf10c9d84 100644
--- a/src/lab_sim/objectives/addbottlestoplanningscene.xml
+++ b/src/lab_sim/objectives/addbottlestoplanningscene.xml
@@ -15,18 +15,20 @@
       />
       <SubTree ID="Take Wrist Camera Snapshot" _collapsed="true" />
       <SubTree
-        ID="Segment Image from Text Prompt Subtree"
+        ID="SAM3 Segment Image Subtree"
         _collapsed="true"
-        clip_model_path="models/clip.onnx"
-        clipseg_model_path="models/clipseg.onnx"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="0.5"
         masks_visualization_topic="/masks_visualization"
-        masks2d="{masks2d}"
-        model_package="moveit_pro_clipseg"
-        prompts="conical flask"
-        threshold="0.15"
+        text_prompt="glass conical flask"
         image_topic_name="/wrist_camera/color"
-        negative_prompts="box"
-        erosion_size="10"
+        masks2d="{masks2d}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
       />
       <Action
         ID="GetPointCloud"
diff --git a/src/lab_sim/objectives/ctb2.png b/src/lab_sim/objectives/ctb2.png
new file mode 100644
index 000000000..a662806c0
--- /dev/null
+++ b/src/lab_sim/objectives/ctb2.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:000a36dcaa60da62d387ba895656ed97e85a9c430be805b12681aba2fbf85056
+size 4059688
diff --git a/src/lab_sim/objectives/get_candidate_grasps_subtree.xml b/src/lab_sim/objectives/get_candidate_grasps_subtree.xml
index 310875518..220a3b0be 100644
--- a/src/lab_sim/objectives/get_candidate_grasps_subtree.xml
+++ b/src/lab_sim/objectives/get_candidate_grasps_subtree.xml
@@ -15,6 +15,29 @@
         message_out="{image}"
         publisher_timeout_sec="5.000000"
       />
+      <Action
+        ID="GetMasks2DFromExemplar"
+        target_image="{image}"
+        text_prompt="{text_prompt}"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="{confidence_threshold}"
+        masks2d="{masks2d}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
+      />
+      <Action
+        ID="PublishMask2D"
+        image="{image}"
+        masks="{masks2d}"
+        masks_visualization_topic="/masks_visualization"
+        opacity="0.500000"
+        bounding_box_labels="{confidence_scores_str}"
+        name="Show SAM3 masks"
+      />
       <Action
         ID="GetPointCloud"
         topic_name="{camera_points_topic}"
@@ -29,64 +52,11 @@
         message_timeout_sec="5.000000"
         publisher_timeout_sec="5.000000"
       />
-      <Action
-        ID="GetMasks2DFromTextQuery"
-        image="{image}"
-        masks2d="{masks2d}"
-        prompts="{object_prompt}"
-        threshold="{mask_threshold}"
-        clip_model_path="models/clip.onnx"
-        clipseg_model_path="models/clipseg.onnx"
-        model_package="moveit_pro_clipseg"
-        erosion_size="{mask_erosion}"
-      />
-      <Action
-        ID="PublishMask2D"
-        image="{image}"
-        masks="{masks2d}"
-        masks_visualization_topic="/masks_visualization"
-        opacity="0.500000"
-        bounding_box_detection_class="{object_prompt}"
-        name="Show ClipSeg masks"
-      />
-      <Decorator
-        ID="ForEachUntilSuccess"
-        index="{index}"
-        out="{input_mask}"
-        vector_in="{masks2d}"
-        name="Refine the first successful mask from ClipSeg"
-      >
-        <Control ID="Sequence">
-          <Action
-            ID="GetCenterFromMask2D"
-            center="{center2d}"
-            mask="{input_mask}"
-          />
-          <Action
-            ID="GetMasks2DFromPointQuery"
-            image="{image}"
-            masks2d="{refined_masks}"
-            pixel_coords="{center2d}"
-            decoder_model_path="models/decoder.onnx"
-            encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-            model_package="lab_sim"
-          />
-        </Control>
-      </Decorator>
-      <Action
-        ID="PublishMask2D"
-        image="{image}"
-        masks="{refined_masks}"
-        masks_visualization_topic="/masks_visualization"
-        opacity="0.500000"
-        bounding_box_detection_class="{object_prompt}"
-        name="Show SAM2 mask"
-      />
       <Control ID="Sequence" name="Convert 2D masks to segmented pointcloud">
         <Action
           ID="GetMasks3DFromMasks2D"
           camera_info="{camera_info}"
-          masks2d="{refined_masks}"
+          masks2d="{masks2d}"
           point_cloud="{point_cloud}"
           masks3d="{masks3d}"
         />
@@ -133,10 +103,12 @@
       <inout_port name="camera_image_topic" default="{camera_image_topic}" />
       <inout_port name="camera_info_topic" default="{camera_info_topic}" />
       <inout_port name="camera_points_topic" default="{camera_points_topic}" />
+      <inout_port
+        name="confidence_threshold"
+        default="{confidence_threshold}"
+      />
       <inout_port name="grasps" default="{grasps}" />
-      <inout_port name="mask_erosion" default="{mask_erosion}" />
-      <inout_port name="mask_threshold" default="{mask_threshold}" />
-      <inout_port name="object_prompt" default="{object_prompt}" />
+      <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>
 </root>
diff --git a/src/lab_sim/objectives/get_grasp_from_text_prompt_subtree.xml b/src/lab_sim/objectives/get_grasp_from_text_prompt_subtree.xml
index 06934256e..b14f20c13 100644
--- a/src/lab_sim/objectives/get_grasp_from_text_prompt_subtree.xml
+++ b/src/lab_sim/objectives/get_grasp_from_text_prompt_subtree.xml
@@ -16,10 +16,9 @@
         camera_image_topic="{camera_image_topic}"
         camera_info_topic="{camera_info_topic}"
         camera_points_topic="{camera_points_topic}"
-        object_prompt="{object_prompt}"
+        text_prompt="{text_prompt}"
         grasps="{grasps}"
-        mask_erosion="{mask_erosion}"
-        mask_threshold="{mask_threshold}"
+        confidence_threshold="{confidence_threshold}"
         name="Infer grasp poses from wrist camera"
       />
       <SubTree
@@ -48,11 +47,13 @@
       <inout_port name="camera_image_topic" default="{camera_image_topic}" />
       <inout_port name="camera_info_topic" default="{camera_info_topic}" />
       <inout_port name="camera_points_topic" default="{camera_points_topic}" />
-      <inout_port name="mask_erosion" default="{mask_erosion}" />
-      <inout_port name="mask_threshold" default="{mask_threshold}" />
-      <inout_port name="object_prompt" default="{object_prompt}" />
+      <inout_port
+        name="confidence_threshold"
+        default="{confidence_threshold}"
+      />
       <inout_port name="output_grasp" default="{output_grasp}" />
       <inout_port name="planning_group" default="{planning_group}" />
+      <inout_port name="text_prompt" default="{text_prompt}" />
     </SubTree>
   </TreeNodesModel>
 </root>
diff --git a/src/lab_sim/objectives/iss1.png b/src/lab_sim/objectives/iss1.png
new file mode 100644
index 000000000..bef32db0a
--- /dev/null
+++ b/src/lab_sim/objectives/iss1.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0b32bf1b130aaa48b66e9c364e7aec80bc2023b0fb68b85a03765bb04e9e3d7
+size 7504277
diff --git a/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml b/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
index da55644de..455931ae9 100644
--- a/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
+++ b/src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
@@ -9,25 +9,28 @@
     _favorite="false"
   >
     <Control ID="Sequence" name="TopLevelSequence">
-      <SubTree ID="CreateVector" _collapsed="true" vector="{exemplar_parts}" />
-      <Action
-        ID="LoadImageFromFile"
-        file_path="~/user_ws/src/lab_sim/objectives/square_bottle1.png"
-        image="{square_bottle_exemplar}"
-      />
-      <Action
-        ID="CreateBoundingBoxFromOffset"
-        exemplar_image="{square_bottle_exemplar}"
-        bounding_boxes="{exemplar_bboxes}"
-        padding_percent="0.07"
-      />
-      <SubTree
-        ID="Move to Waypoint"
-        _collapsed="true"
-        waypoint_name="Look at Bottles Left"
-        velocity_scale_factor="1.0"
-      />
-      <SubTree ID="Take Wrist Camera Image" _collapsed="true" />
+      <Control ID="Sequence" name="Move robot and capture image">
+        <SubTree
+          ID="Move to Waypoint"
+          _collapsed="true"
+          waypoint_name="Look at Bottles Left"
+          velocity_scale_factor="1.0"
+        />
+        <SubTree ID="Take Wrist Camera Image" _collapsed="true" />
+      </Control>
+      <Control ID="Sequence" name="Prepare image exemplar">
+        <Action
+          ID="LoadImageFromFile"
+          file_path="~/user_ws/src/lab_sim/objectives/square_bottle1.png"
+          image="{square_bottle_exemplar}"
+        />
+        <Action
+          ID="CreateBoundingBoxFromOffset"
+          exemplar_image="{square_bottle_exemplar}"
+          bounding_boxes="{exemplar_bboxes}"
+          padding_percent="0.07"
+        />
+      </Control>
       <Action
         ID="GetMasks2DFromExemplar"
         model_package="lab_sim"
@@ -42,20 +45,22 @@
         exemplar_bboxes="{exemplar_bboxes}"
         exemplar_image="{square_bottle_exemplar}"
       />
-      <Action
-        ID="SwitchUIPrimaryView"
-        primary_view_name="/masks_visualization"
-      />
-      <Action
-        ID="PublishMask2D"
-        bounding_box_labels="{confidence_scores_str}"
-        image="{wrist_camera_image}"
-      />
-      <Action
-        ID="PublishBoundingBoxes2D"
-        image="{square_bottle_exemplar}"
-        bounding_boxes="{exemplar_bboxes}"
-      />
+      <Control ID="Sequence" name="Update UI with result">
+        <Action
+          ID="SwitchUIPrimaryView"
+          primary_view_name="/masks_visualization"
+        />
+        <Action
+          ID="PublishMask2D"
+          bounding_box_labels="{confidence_scores_str}"
+          image="{wrist_camera_image}"
+        />
+        <Action
+          ID="PublishBoundingBoxes2D"
+          image="{square_bottle_exemplar}"
+          bounding_boxes="{exemplar_bboxes}"
+        />
+      </Control>
     </Control>
   </BehaviorTree>
   <TreeNodesModel>
diff --git a/src/lab_sim/objectives/ml_find_ctb_from_bounding_box.xml b/src/lab_sim/objectives/ml_find_ctb_from_bounding_box.xml
new file mode 100644
index 000000000..af2ac24ad
--- /dev/null
+++ b/src/lab_sim/objectives/ml_find_ctb_from_bounding_box.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<root BTCPP_format="4" main_tree_to_execute="ML Find CTB from Bounding Box">
+  <BehaviorTree
+    ID="ML Find CTB from Bounding Box"
+    _description="Uses the bounding box feature of SAM3 to find and segment in image items from scene. Publishes masks on masks_visualization topic and exemplar with bounding box on bboxes_visualization for introspection. Combine image exemplar with text prompt to improve performance."
+    _favorite="false"
+  >
+    <Control ID="Sequence" name="TopLevelSequence">
+      <Action
+        ID="CreateBoundingBoxes2D"
+        bounding_boxes="{target_bboxes}"
+        values="800;940;150;75"
+      />
+      <Action
+        ID="LoadImageFromFile"
+        file_path="~/user_ws/src/lab_sim/objectives/iss1.png"
+        image="{target_image}"
+      />
+      <Action
+        ID="GetMasks2DFromExemplar"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="0.45"
+        confidence_scores_str="{confidence_scores_str}"
+        target_image="{target_image}"
+        confidence_scores="{confidence_scores}"
+        target_bboxes="{target_bboxes}"
+      />
+      <Action
+        ID="SwitchUIPrimaryView"
+        primary_view_name="/masks_visualization"
+      />
+      <Action
+        ID="PublishMask2D"
+        bounding_box_labels="{confidence_scores_str}"
+        image="{target_image}"
+      />
+      <Action
+        ID="PublishBoundingBoxes2D"
+        image="{target_image}"
+        bounding_boxes="{target_bboxes}"
+      />
+    </Control>
+  </BehaviorTree>
+  <TreeNodesModel>
+    <SubTree ID="ML Find CTB from Bounding Box">
+      <MetadataFields>
+        <Metadata runnable="true" />
+        <Metadata subcategory="Application - ML (GPU Recommended)" />
+      </MetadataFields>
+    </SubTree>
+  </TreeNodesModel>
+</root>
diff --git a/src/lab_sim/objectives/ml_find_ctb_from_image_exemplar.xml b/src/lab_sim/objectives/ml_find_ctb_from_image_exemplar.xml
new file mode 100644
index 000000000..fc2addc96
--- /dev/null
+++ b/src/lab_sim/objectives/ml_find_ctb_from_image_exemplar.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<root BTCPP_format="4" main_tree_to_execute="ML Find CTB from Image Exemplar">
+  <BehaviorTree
+    ID="ML Find CTB from Image Exemplar"
+    _description="Uses the image exemplar feature of SAM3 to find and segment the cargo transfer bag from the scene. Publishes masks on masks_visualization topic and exemplar with bounding box on bboxes_visualization for introspection. Combine image exemplar with text prompt to improve performance."
+    _favorite="false"
+  >
+    <Control ID="Sequence" name="TopLevelSequence">
+      <Action
+        ID="LoadImageFromFile"
+        file_path="~/user_ws/src/lab_sim/objectives/ctb2.png"
+        image="{image_exemplar}"
+      />
+      <Action
+        ID="CreateBoundingBoxFromOffset"
+        exemplar_image="{image_exemplar}"
+        bounding_boxes="{exemplar_bboxes}"
+        padding_percent="0.05"
+      />
+      <Action
+        ID="LoadImageFromFile"
+        file_path="~/user_ws/src/lab_sim/objectives/iss1.png"
+        image="{target_image}"
+      />
+      <Action
+        ID="GetMasks2DFromExemplar"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="0.45"
+        confidence_scores_str="{confidence_scores_str}"
+        target_image="{target_image}"
+        confidence_scores="{confidence_scores}"
+        exemplar_bboxes="{exemplar_bboxes}"
+        exemplar_image="{image_exemplar}"
+      />
+      <Action
+        ID="SwitchUIPrimaryView"
+        primary_view_name="/masks_visualization"
+      />
+      <Action
+        ID="PublishMask2D"
+        bounding_box_labels="{confidence_scores_str}"
+        image="{target_image}"
+      />
+      <Action
+        ID="PublishBoundingBoxes2D"
+        image="{image_exemplar}"
+        bounding_boxes="{exemplar_bboxes}"
+      />
+    </Control>
+  </BehaviorTree>
+  <TreeNodesModel>
+    <SubTree ID="ML Find CTB from Image Exemplar">
+      <MetadataFields>
+        <Metadata runnable="true" />
+        <Metadata subcategory="Application - ML (GPU Recommended)" />
+      </MetadataFields>
+    </SubTree>
+  </TreeNodesModel>
+</root>
diff --git a/src/lab_sim/objectives/ml_segment_bottles_from_file.xml b/src/lab_sim/objectives/ml_segment_from_file.xml
similarity index 71%
rename from src/lab_sim/objectives/ml_segment_bottles_from_file.xml
rename to src/lab_sim/objectives/ml_segment_from_file.xml
index df16da5ab..fa9cad262 100644
--- a/src/lab_sim/objectives/ml_segment_bottles_from_file.xml
+++ b/src/lab_sim/objectives/ml_segment_from_file.xml
@@ -1,14 +1,14 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<root BTCPP_format="4" main_tree_to_execute="ML Segment Bottles from File">
+<root BTCPP_format="4" main_tree_to_execute="ML Segment from File">
   <BehaviorTree
-    ID="ML Segment Bottles from File"
-    _description="Loads image of lab from file and uses SAM3 to find bottles. Useful when you want to file tune or experiment is exemplars before testing in robot scene."
+    ID="ML Segment from File"
+    _description="Loads image from file and uses SAM3 to find requested object. Useful when you want to file tune or experiment is exemplars before testing in robot scene."
     _favorite="false"
   >
     <Control ID="Sequence" name="TopLevelSequence">
       <Action
         ID="LoadImageFromFile"
-        file_path="~/user_ws/src/lab_sim/objectives/real_lab1.png"
+        file_path="~/user_ws/src/lab_sim/objectives/iss1.png"
       />
       <Action
         ID="GetMasks2DFromExemplar"
@@ -20,7 +20,7 @@
         confidence_threshold="0.5"
         confidence_scores_str="{confidence_scores_str}"
         target_image="{image}"
-        text_prompt="bottles"
+        text_prompt="white cargo bag"
         mask_count="{mask_count}"
         masks2d="{masks2d}"
       />
@@ -31,7 +31,7 @@
     </Control>
   </BehaviorTree>
   <TreeNodesModel>
-    <SubTree ID="ML Segment Bottles from File">
+    <SubTree ID="ML Segment from File">
       <MetadataFields>
         <Metadata runnable="true" />
         <Metadata subcategory="Application - ML (GPU Recommended)" />
diff --git a/src/lab_sim/objectives/ml_segment_image.xml b/src/lab_sim/objectives/ml_segment_image.xml
index 6e6734695..785e66a58 100644
--- a/src/lab_sim/objectives/ml_segment_image.xml
+++ b/src/lab_sim/objectives/ml_segment_image.xml
@@ -12,17 +12,20 @@
         primary_view_name="/masks_visualization"
       />
       <SubTree
-        ID="Segment Image from No Negative Text Prompt Subtree"
+        ID="SAM3 Segment Image Subtree"
         _collapsed="true"
-        clip_model_path="models/clip.onnx"
-        clipseg_model_path="models/clipseg.onnx"
-        erosion_size="2"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="0.5"
         masks_visualization_topic="/masks_visualization"
-        model_package="moveit_pro_clipseg"
-        threshold="0.15"
-        prompts="an object"
+        text_prompt="an object"
         image_topic_name="/wrist_camera/color"
         masks2d="{masks2d}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
       />
     </Control>
   </BehaviorTree>
diff --git a/src/lab_sim/objectives/ml_segment_point_cloud.xml b/src/lab_sim/objectives/ml_segment_point_cloud.xml
index c878b5211..3cebcd12a 100644
--- a/src/lab_sim/objectives/ml_segment_point_cloud.xml
+++ b/src/lab_sim/objectives/ml_segment_point_cloud.xml
@@ -8,23 +8,25 @@
   >
     <Control ID="Sequence">
       <SubTree
-        ID="Segment Point Cloud from Text Prompt Subtree"
-        _collapsed="true"
-        camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-        image_topic_name="/wrist_camera/color"
+        ID="SAM3 Segment Point Cloud Subtree"
+        _collapsed="false"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="0.5"
         masks_visualization_topic="/masks_visualization"
-        model_package="moveit_pro_clipseg"
+        text_prompt="an object"
+        image_topic_name="/wrist_camera/color"
         points_topic_name="/wrist_camera/points"
-        clip_model_path="models/clip.onnx"
-        clipseg_model_path="models/clipseg.onnx"
-        erosion_size="2"
-        prompts="an object"
-        threshold="0.28"
+        camera_topic_name="/wrist_camera/camera_info"
+        masks2d="{masks2d}"
         masks3d="{masks3d}"
         point_cloud="{point_cloud}"
         point_cloud_vector="{point_cloud_vector}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
       />
     </Control>
   </BehaviorTree>
diff --git a/src/lab_sim/objectives/pick_1_pill_bottle.xml b/src/lab_sim/objectives/pick_1_pill_bottle.xml
index 19c048cf2..1ccb6e9c5 100644
--- a/src/lab_sim/objectives/pick_1_pill_bottle.xml
+++ b/src/lab_sim/objectives/pick_1_pill_bottle.xml
@@ -16,23 +16,25 @@
         waypoint_name="Above Pick Cube"
       />
       <SubTree
-        ID="Segment Point Cloud from Text Prompt Subtree"
+        ID="SAM3 Segment Point Cloud Subtree"
         _collapsed="false"
-        camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-        image_topic_name="/wrist_camera/color"
+        model_package="lab_sim"
+        encoder_model_path="models/sam3_vision_encoder.onnx"
+        decoder_model_path="models/sam3_decoder.onnx"
+        geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
+        text_encoder_model_path="models/sam3_text_encoder.onnx"
+        confidence_threshold="0.5"
         masks_visualization_topic="/masks_visualization"
-        model_package="moveit_pro_clipseg"
+        text_prompt="a cylindrical object with a dark gray and silver metallic body, featuring a circular blue cap on top"
+        image_topic_name="/wrist_camera/color"
         points_topic_name="/wrist_camera/points"
-        clip_model_path="models/clip.onnx"
-        clipseg_model_path="models/clipseg.onnx"
-        erosion_size="2"
-        prompts="a cylindrical object with a dark gray and silver metallic body, featuring a circular blue cap on top"
-        threshold="0.28"
+        camera_topic_name="/wrist_camera/camera_info"
+        masks2d="{masks2d}"
         masks3d="{masks3d}"
         point_cloud="{point_cloud}"
         point_cloud_vector="{point_cloud_vector}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
       />
       <Decorator
         ID="ForEach"
diff --git a/src/lab_sim/objectives/pick_all_pill_bottles.xml b/src/lab_sim/objectives/pick_all_pill_bottles.xml
index d6bce1fcb..d7f04382c 100644
--- a/src/lab_sim/objectives/pick_all_pill_bottles.xml
+++ b/src/lab_sim/objectives/pick_all_pill_bottles.xml
@@ -58,9 +58,8 @@
               camera_points_topic="/wrist_camera/points"
               planning_group="manipulator"
               output_grasp="{output_grasp}"
-              object_prompt="a cylindrical object with a dark gray and silver metallic body, featuring a circular blue cap on top"
-              mask_threshold=".15"
-              mask_erosion="15"
+              text_prompt="a cylindrical object with a dark gray and silver metallic body, featuring a circular blue cap on top"
+              confidence_threshold="0.5"
               name="Get grasp pose for a pill bottle"
             />
             <SubTree ID="Add Table to Planning Scene" _collapsed="true" />
diff --git a/src/lab_sim/objectives/sam3_segment_image_subtree.xml b/src/lab_sim/objectives/sam3_segment_image_subtree.xml
new file mode 100644
index 000000000..c4c69558b
--- /dev/null
+++ b/src/lab_sim/objectives/sam3_segment_image_subtree.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<root BTCPP_format="4" main_tree_to_execute="SAM3 Segment Image Subtree">
+  <!--//////////-->
+  <BehaviorTree
+    ID="SAM3 Segment Image Subtree"
+    _description="Captures an image and runs SAM3 to produce 2D masks from a text prompt."
+    _favorite="false"
+  >
+    <Control ID="Sequence">
+      <Action
+        ID="GetImage"
+        message_timeout_sec="5.000000"
+        topic_name="{image_topic_name}"
+        message_out="{image}"
+      />
+      <Action
+        ID="GetMasks2DFromExemplar"
+        target_image="{image}"
+        text_prompt="{text_prompt}"
+        model_package="{model_package}"
+        encoder_model_path="{encoder_model_path}"
+        decoder_model_path="{decoder_model_path}"
+        geometry_encoder_model_path="{geometry_encoder_model_path}"
+        text_encoder_model_path="{text_encoder_model_path}"
+        confidence_threshold="{confidence_threshold}"
+        masks2d="{masks2d}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
+        mask_count="{mask_count}"
+      />
+      <Action
+        ID="PublishMask2D"
+        image="{image}"
+        masks="{masks2d}"
+        masks_visualization_topic="{masks_visualization_topic}"
+        opacity="0.500000"
+        bounding_box_labels="{confidence_scores_str}"
+      />
+    </Control>
+  </BehaviorTree>
+  <TreeNodesModel>
+    <SubTree ID="SAM3 Segment Image Subtree">
+      <MetadataFields>
+        <Metadata runnable="false" />
+        <Metadata subcategory="Perception - ML" />
+      </MetadataFields>
+      <input_port name="image_topic_name" />
+      <input_port name="text_prompt" />
+      <input_port name="model_package" />
+      <input_port name="encoder_model_path" />
+      <input_port name="decoder_model_path" />
+      <input_port name="geometry_encoder_model_path" />
+      <input_port name="text_encoder_model_path" />
+      <input_port name="confidence_threshold" />
+      <input_port name="masks_visualization_topic" />
+      <output_port name="masks2d" default="{masks2d}" />
+      <output_port name="confidence_scores" default="{confidence_scores}" />
+      <output_port
+        name="confidence_scores_str"
+        default="{confidence_scores_str}"
+      />
+      <output_port name="mask_count" default="{mask_count}" />
+    </SubTree>
+  </TreeNodesModel>
+</root>
diff --git a/src/lab_sim/objectives/sam3_segment_point_cloud_subtree.xml b/src/lab_sim/objectives/sam3_segment_point_cloud_subtree.xml
new file mode 100644
index 000000000..c95966a9a
--- /dev/null
+++ b/src/lab_sim/objectives/sam3_segment_point_cloud_subtree.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<root BTCPP_format="4" main_tree_to_execute="SAM3 Segment Point Cloud Subtree">
+  <!--//////////-->
+  <BehaviorTree
+    ID="SAM3 Segment Point Cloud Subtree"
+    _description="Captures a point cloud and segments objects using SAM3 text prompting."
+    _favorite="false"
+  >
+    <Control ID="Sequence">
+      <Action ID="ClearSnapshot" />
+      <SubTree
+        ID="SAM3 Segment Image Subtree"
+        _collapsed="true"
+        image_topic_name="{image_topic_name}"
+        text_prompt="{text_prompt}"
+        model_package="{model_package}"
+        encoder_model_path="{encoder_model_path}"
+        decoder_model_path="{decoder_model_path}"
+        geometry_encoder_model_path="{geometry_encoder_model_path}"
+        text_encoder_model_path="{text_encoder_model_path}"
+        confidence_threshold="{confidence_threshold}"
+        masks_visualization_topic="{masks_visualization_topic}"
+        masks2d="{masks2d}"
+        confidence_scores="{confidence_scores}"
+        confidence_scores_str="{confidence_scores_str}"
+        mask_count="{mask_count}"
+      />
+      <Action
+        ID="GetPointCloud"
+        topic_name="{points_topic_name}"
+        message_timeout_sec="10.000000"
+        message_out="{point_cloud}"
+      />
+      <Action
+        ID="GetCameraInfo"
+        topic_name="{camera_topic_name}"
+        message_out="{camera_info}"
+        message_timeout_sec="5.000000"
+      />
+      <Action
+        ID="GetMasks3DFromMasks2D"
+        camera_info="{camera_info}"
+        masks2d="{masks2d}"
+        point_cloud="{point_cloud}"
+        masks3d="{masks3d}"
+      />
+      <SubTree
+        ID="Create Point Cloud Vector From Masks"
+        _collapsed="false"
+        masks3d="{masks3d}"
+        point_cloud="{point_cloud}"
+        point_cloud_vector="{point_cloud_vector}"
+      />
+      <Decorator
+        ID="ForEach"
+        index="{index}"
+        out="{point_cloud_fragment}"
+        vector_in="{point_cloud_vector}"
+      >
+        <Action
+          ID="SendPointCloudToUI"
+          pcd_topic="/pcd_pointcloud_captures"
+          point_cloud="{point_cloud_fragment}"
+        />
+      </Decorator>
+    </Control>
+  </BehaviorTree>
+  <TreeNodesModel>
+    <SubTree ID="SAM3 Segment Point Cloud Subtree">
+      <MetadataFields>
+        <Metadata runnable="false" />
+        <Metadata subcategory="Perception - ML" />
+      </MetadataFields>
+      <input_port name="image_topic_name" />
+      <input_port name="text_prompt" />
+      <input_port name="model_package" />
+      <input_port name="encoder_model_path" />
+      <input_port name="decoder_model_path" />
+      <input_port name="geometry_encoder_model_path" />
+      <input_port name="text_encoder_model_path" />
+      <input_port name="confidence_threshold" />
+      <input_port name="masks_visualization_topic" />
+      <input_port name="points_topic_name" />
+      <input_port name="camera_topic_name" />
+      <output_port name="masks2d" default="{masks2d}" />
+      <output_port name="masks3d" default="{masks3d}" />
+      <output_port name="point_cloud" default="{point_cloud}" />
+      <output_port name="point_cloud_vector" default="{point_cloud_vector}" />
+      <output_port name="confidence_scores" default="{confidence_scores}" />
+      <output_port
+        name="confidence_scores_str"
+        default="{confidence_scores_str}"
+      />
+      <output_port name="mask_count" default="{mask_count}" />
+    </SubTree>
+  </TreeNodesModel>
+</root>