Skip to content

Migrate lab_sim CLIPSeg objectives to SAM3 #524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
griswaldbrooks wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Migrate lab_sim CLIPSeg objectives to SAM3 #524

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions src/lab_sim/objectives/addbottlestoplanningscene.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,20 @@
/>
<SubTree ID="Take Wrist Camera Snapshot" _collapsed="true" />
<SubTree
ID="Segment Image from Text Prompt Subtree"
ID="SAM3 Segment Image Subtree"
_collapsed="true"
clip_model_path="models/clip.onnx"
clipseg_model_path="models/clipseg.onnx"
model_package="lab_sim"
encoder_model_path="models/sam3_vision_encoder.onnx"
decoder_model_path="models/sam3_decoder.onnx"
geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
text_encoder_model_path="models/sam3_text_encoder.onnx"
confidence_threshold="0.5"
masks_visualization_topic="/masks_visualization"
masks2d="{masks2d}"
model_package="moveit_pro_clipseg"
prompts="conical flask"
threshold="0.15"
text_prompt="glass conical flask"
image_topic_name="/wrist_camera/color"
negative_prompts="box"
erosion_size="10"
masks2d="{masks2d}"
confidence_scores="{confidence_scores}"
confidence_scores_str="{confidence_scores_str}"
/>
<Action
ID="GetPointCloud"
Expand Down
3 changes: 3 additions & 0 deletions src/lab_sim/objectives/ctb2.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
86 changes: 29 additions & 57 deletions src/lab_sim/objectives/get_candidate_grasps_subtree.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,29 @@
message_out="{image}"
publisher_timeout_sec="5.000000"
/>
<Action
ID="GetMasks2DFromExemplar"
target_image="{image}"
text_prompt="{text_prompt}"
model_package="lab_sim"
encoder_model_path="models/sam3_vision_encoder.onnx"
decoder_model_path="models/sam3_decoder.onnx"
geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
text_encoder_model_path="models/sam3_text_encoder.onnx"
confidence_threshold="{confidence_threshold}"
masks2d="{masks2d}"
confidence_scores="{confidence_scores}"
confidence_scores_str="{confidence_scores_str}"
/>
<Action
ID="PublishMask2D"
image="{image}"
masks="{masks2d}"
masks_visualization_topic="/masks_visualization"
opacity="0.500000"
bounding_box_labels="{confidence_scores_str}"
name="Show SAM3 masks"
/>
<Action
ID="GetPointCloud"
topic_name="{camera_points_topic}"
Expand All @@ -29,64 +52,11 @@
message_timeout_sec="5.000000"
publisher_timeout_sec="5.000000"
/>
<Action
ID="GetMasks2DFromTextQuery"
image="{image}"
masks2d="{masks2d}"
prompts="{object_prompt}"
threshold="{mask_threshold}"
clip_model_path="models/clip.onnx"
clipseg_model_path="models/clipseg.onnx"
model_package="moveit_pro_clipseg"
erosion_size="{mask_erosion}"
/>
<Action
ID="PublishMask2D"
image="{image}"
masks="{masks2d}"
masks_visualization_topic="/masks_visualization"
opacity="0.500000"
bounding_box_detection_class="{object_prompt}"
name="Show ClipSeg masks"
/>
<Decorator
ID="ForEachUntilSuccess"
index="{index}"
out="{input_mask}"
vector_in="{masks2d}"
name="Refine the first successful mask from ClipSeg"
>
<Control ID="Sequence">
<Action
ID="GetCenterFromMask2D"
center="{center2d}"
mask="{input_mask}"
/>
<Action
ID="GetMasks2DFromPointQuery"
image="{image}"
masks2d="{refined_masks}"
pixel_coords="{center2d}"
decoder_model_path="models/decoder.onnx"
encoder_model_path="models/sam2_hiera_large_encoder.onnx"
model_package="lab_sim"
/>
</Control>
</Decorator>
<Action
ID="PublishMask2D"
image="{image}"
masks="{refined_masks}"
masks_visualization_topic="/masks_visualization"
opacity="0.500000"
bounding_box_detection_class="{object_prompt}"
name="Show SAM2 mask"
/>
<Control ID="Sequence" name="Convert 2D masks to segmented pointcloud">
<Action
ID="GetMasks3DFromMasks2D"
camera_info="{camera_info}"
masks2d="{refined_masks}"
masks2d="{masks2d}"
point_cloud="{point_cloud}"
masks3d="{masks3d}"
/>
Expand Down Expand Up @@ -133,10 +103,12 @@
<inout_port name="camera_image_topic" default="{camera_image_topic}" />
<inout_port name="camera_info_topic" default="{camera_info_topic}" />
<inout_port name="camera_points_topic" default="{camera_points_topic}" />
<inout_port
name="confidence_threshold"
default="{confidence_threshold}"
/>
<inout_port name="grasps" default="{grasps}" />
<inout_port name="mask_erosion" default="{mask_erosion}" />
<inout_port name="mask_threshold" default="{mask_threshold}" />
<inout_port name="object_prompt" default="{object_prompt}" />
<inout_port name="text_prompt" default="{text_prompt}" />
</SubTree>
</TreeNodesModel>
</root>
13 changes: 7 additions & 6 deletions src/lab_sim/objectives/get_grasp_from_text_prompt_subtree.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@
camera_image_topic="{camera_image_topic}"
camera_info_topic="{camera_info_topic}"
camera_points_topic="{camera_points_topic}"
object_prompt="{object_prompt}"
text_prompt="{text_prompt}"
grasps="{grasps}"
mask_erosion="{mask_erosion}"
mask_threshold="{mask_threshold}"
confidence_threshold="{confidence_threshold}"
name="Infer grasp poses from wrist camera"
/>
<SubTree
Expand Down Expand Up @@ -48,11 +47,13 @@
<inout_port name="camera_image_topic" default="{camera_image_topic}" />
<inout_port name="camera_info_topic" default="{camera_info_topic}" />
<inout_port name="camera_points_topic" default="{camera_points_topic}" />
<inout_port name="mask_erosion" default="{mask_erosion}" />
<inout_port name="mask_threshold" default="{mask_threshold}" />
<inout_port name="object_prompt" default="{object_prompt}" />
<inout_port
name="confidence_threshold"
default="{confidence_threshold}"
/>
<inout_port name="output_grasp" default="{output_grasp}" />
<inout_port name="planning_group" default="{planning_group}" />
<inout_port name="text_prompt" default="{text_prompt}" />
</SubTree>
</TreeNodesModel>
</root>
3 changes: 3 additions & 0 deletions src/lab_sim/objectives/iss1.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
71 changes: 38 additions & 33 deletions src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,28 @@
_favorite="false"
>
<Control ID="Sequence" name="TopLevelSequence">
<SubTree ID="CreateVector" _collapsed="true" vector="{exemplar_parts}" />
<Action
ID="LoadImageFromFile"
file_path="~/user_ws/src/lab_sim/objectives/square_bottle1.png"
image="{square_bottle_exemplar}"
/>
<Action
ID="CreateBoundingBoxFromOffset"
exemplar_image="{square_bottle_exemplar}"
bounding_boxes="{exemplar_bboxes}"
padding_percent="0.07"
/>
<SubTree
ID="Move to Waypoint"
_collapsed="true"
waypoint_name="Look at Bottles Left"
velocity_scale_factor="1.0"
/>
<SubTree ID="Take Wrist Camera Image" _collapsed="true" />
<Control ID="Sequence" name="Move robot and capture image">
<SubTree
ID="Move to Waypoint"
_collapsed="true"
waypoint_name="Look at Bottles Left"
velocity_scale_factor="1.0"
/>
<SubTree ID="Take Wrist Camera Image" _collapsed="true" />
</Control>
<Control ID="Sequence" name="Prepare image exemplar">
<Action
ID="LoadImageFromFile"
file_path="~/user_ws/src/lab_sim/objectives/square_bottle1.png"
image="{square_bottle_exemplar}"
/>
<Action
ID="CreateBoundingBoxFromOffset"
exemplar_image="{square_bottle_exemplar}"
bounding_boxes="{exemplar_bboxes}"
padding_percent="0.07"
/>
</Control>
<Action
ID="GetMasks2DFromExemplar"
model_package="lab_sim"
Expand All @@ -42,20 +45,22 @@
exemplar_bboxes="{exemplar_bboxes}"
exemplar_image="{square_bottle_exemplar}"
/>
<Action
ID="SwitchUIPrimaryView"
primary_view_name="/masks_visualization"
/>
<Action
ID="PublishMask2D"
bounding_box_labels="{confidence_scores_str}"
image="{wrist_camera_image}"
/>
<Action
ID="PublishBoundingBoxes2D"
image="{square_bottle_exemplar}"
bounding_boxes="{exemplar_bboxes}"
/>
<Control ID="Sequence" name="Update UI with result">
<Action
ID="SwitchUIPrimaryView"
primary_view_name="/masks_visualization"
/>
<Action
ID="PublishMask2D"
bounding_box_labels="{confidence_scores_str}"
image="{wrist_camera_image}"
/>
<Action
ID="PublishBoundingBoxes2D"
image="{square_bottle_exemplar}"
bounding_boxes="{exemplar_bboxes}"
/>
</Control>
</Control>
</BehaviorTree>
<TreeNodesModel>
Expand Down
56 changes: 56 additions & 0 deletions src/lab_sim/objectives/ml_find_ctb_from_bounding_box.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8" ?>
<root BTCPP_format="4" main_tree_to_execute="ML Find CTB from Bounding Box">
<BehaviorTree
ID="ML Find CTB from Bounding Box"
_description="Uses the bounding box feature of SAM3 to find and segment in image items from scene. Publishes masks on masks_visualization topic and exemplar with bounding box on bboxes_visualization for introspection. Combine image exemplar with text prompt to improve performance."
_favorite="false"
>
<Control ID="Sequence" name="TopLevelSequence">
<Action
ID="CreateBoundingBoxes2D"
bounding_boxes="{target_bboxes}"
values="800;940;150;75"
/>
<Action
ID="LoadImageFromFile"
file_path="~/user_ws/src/lab_sim/objectives/iss1.png"
image="{target_image}"
/>
<Action
ID="GetMasks2DFromExemplar"
model_package="lab_sim"
encoder_model_path="models/sam3_vision_encoder.onnx"
decoder_model_path="models/sam3_decoder.onnx"
geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
text_encoder_model_path="models/sam3_text_encoder.onnx"
confidence_threshold="0.45"
confidence_scores_str="{confidence_scores_str}"
target_image="{target_image}"
confidence_scores="{confidence_scores}"
target_bboxes="{target_bboxes}"
/>
<Action
ID="SwitchUIPrimaryView"
primary_view_name="/masks_visualization"
/>
<Action
ID="PublishMask2D"
bounding_box_labels="{confidence_scores_str}"
image="{target_image}"
/>
<Action
ID="PublishBoundingBoxes2D"
image="{target_image}"
bounding_boxes="{target_bboxes}"
/>
</Control>
</BehaviorTree>
<TreeNodesModel>
<SubTree ID="ML Find CTB from Bounding Box">
<MetadataFields>
<Metadata runnable="true" />
<Metadata subcategory="Application - ML (GPU Recommended)" />
</MetadataFields>
</SubTree>
</TreeNodesModel>
</root>
63 changes: 63 additions & 0 deletions src/lab_sim/objectives/ml_find_ctb_from_image_exemplar.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8" ?>
<root BTCPP_format="4" main_tree_to_execute="ML Find CTB from Image Exemplar">
<BehaviorTree
ID="ML Find CTB from Image Exemplar"
_description="Uses the image exemplar feature of SAM3 to find and segment the cargo transfer bag from the scene. Publishes masks on masks_visualization topic and exemplar with bounding box on bboxes_visualization for introspection. Combine image exemplar with text prompt to improve performance."
_favorite="false"
>
<Control ID="Sequence" name="TopLevelSequence">
<Action
ID="LoadImageFromFile"
file_path="~/user_ws/src/lab_sim/objectives/ctb2.png"
image="{image_exemplar}"
/>
<Action
ID="CreateBoundingBoxFromOffset"
exemplar_image="{image_exemplar}"
bounding_boxes="{exemplar_bboxes}"
padding_percent="0.05"
/>
<Action
ID="LoadImageFromFile"
file_path="~/user_ws/src/lab_sim/objectives/iss1.png"
image="{target_image}"
/>
<Action
ID="GetMasks2DFromExemplar"
model_package="lab_sim"
encoder_model_path="models/sam3_vision_encoder.onnx"
decoder_model_path="models/sam3_decoder.onnx"
geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"
text_encoder_model_path="models/sam3_text_encoder.onnx"
confidence_threshold="0.45"
confidence_scores_str="{confidence_scores_str}"
target_image="{target_image}"
confidence_scores="{confidence_scores}"
exemplar_bboxes="{exemplar_bboxes}"
exemplar_image="{image_exemplar}"
/>
<Action
ID="SwitchUIPrimaryView"
primary_view_name="/masks_visualization"
/>
<Action
ID="PublishMask2D"
bounding_box_labels="{confidence_scores_str}"
image="{target_image}"
/>
<Action
ID="PublishBoundingBoxes2D"
image="{image_exemplar}"
bounding_boxes="{exemplar_bboxes}"
/>
</Control>
</BehaviorTree>
<TreeNodesModel>
<SubTree ID="ML Find CTB from Image Exemplar">
<MetadataFields>
<Metadata runnable="true" />
<Metadata subcategory="Application - ML (GPU Recommended)" />
</MetadataFields>
</SubTree>
</TreeNodesModel>
</root>
Loading
Loading