From b882be1fbbb79367d539d7c2b918b7bb912de36b Mon Sep 17 00:00:00 2001 From: SnowMasaya Date: Fri, 19 May 2023 11:30:14 +0900 Subject: [PATCH] Add city segformer example --- ...ge_segmentation_ros_node_city_segformer.py | 264 ++++++++++++++++++ .../city_segformer/tutorial-city-segformer.md | 117 ++++++++ 2 files changed, 381 insertions(+) create mode 100644 scripts/city_segformer/image_segmentation_ros_node_city_segformer.py create mode 100644 scripts/city_segformer/tutorial-city-segformer.md diff --git a/scripts/city_segformer/image_segmentation_ros_node_city_segformer.py b/scripts/city_segformer/image_segmentation_ros_node_city_segformer.py new file mode 100644 index 0000000..e445217 --- /dev/null +++ b/scripts/city_segformer/image_segmentation_ros_node_city_segformer.py @@ -0,0 +1,264 @@ +""" +Performance test for apriltag_ros. + +The graph consists of the following: +- Preprocessors: + 1. PrepResizeNode: resizes images to HD +- Graph under Test: + 1. AprilTagNode: detects Apriltags + +Required: +- Packages: + - apriltag_ros +- Datasets: + - assets/datasets/r2b_dataset/r2b_storage +""" + +from launch_ros.actions import ComposableNodeContainer +from launch_ros.descriptions import ComposableNode +import launch +from launch.actions import DeclareLaunchArgument +from launch.substitutions import LaunchConfiguration + +from ros2_benchmark import ImageResolution +from ros2_benchmark import ROS2BenchmarkConfig, ROS2BenchmarkTest + +IMAGE_RESOLUTION = ImageResolution.HD +ROSBAG_PATH = 'datasets/r2b_dataset/r2b_storage' + + +def launch_setup(container_prefix, container_sigterm_timeout): + """Launch the DNN Image encoder, Triton node and UNet decoder node.""" + + launch_args = [ + DeclareLaunchArgument( + 'network_image_width', + default_value='1820', + description='The input image width that the network expects'), + DeclareLaunchArgument( + 'network_image_height', + default_value='1024', + description='The input image height that the network expects'), + DeclareLaunchArgument( + 'encoder_image_mean', + default_value='[0.5, 0.5, 0.5]', + description='The mean for image normalization'), + DeclareLaunchArgument( + 'encoder_image_stddev', + default_value='[0.5, 0.5, 0.5]', + description='The standard deviation for image normalization'), + DeclareLaunchArgument( + 'model_name', + default_value='citysemsegformer', + description='The name of the model'), + DeclareLaunchArgument( + 'model_repository_paths', + default_value='["/workspaces/isaac_ros-dev/models/"]', + description='The absolute path to the repository of models'), + DeclareLaunchArgument( + 'max_batch_size', + default_value='0', + description='The maximum allowed batch size of the model'), + DeclareLaunchArgument( + 'input_tensor_names', + default_value='["input_tensor"]', + description='A list of tensor names to bound to the specified input binding names'), + DeclareLaunchArgument( + 'input_binding_names', + default_value='["input"]', + description='A list of input tensor binding names (specified by model)'), + DeclareLaunchArgument( + 'input_tensor_formats', + default_value='["nitros_tensor_list_nchw_rgb_f32"]', + description='The nitros format of the input tensors'), + DeclareLaunchArgument( + 'output_tensor_names', + default_value='["output_tensor"]', + description='A list of tensor names to bound to the specified output binding names'), + DeclareLaunchArgument( + 'output_binding_names', + default_value='["output"]', + description='A list of output tensor binding names (specified by model)'), + DeclareLaunchArgument( + 'output_tensor_formats', + default_value='["nitros_tensor_list_nhwc_rgb_f32"]', + description='The nitros format of the output tensors'), + DeclareLaunchArgument( + 'network_output_type', + default_value='argmax', + description='The output type that the network provides (softmax, sigmoid or argmax)'), + DeclareLaunchArgument( + 'color_segmentation_mask_encoding', + default_value='rgb8', + description='The image encoding of the colored segmentation mask (rgb8 or bgr8)'), + DeclareLaunchArgument( + 'mask_width', + default_value='1820', + description='The width of the segmentation mask'), + DeclareLaunchArgument( + 'mask_height', + default_value='1024', + description='The height of the segmentation mask'), + ] + + # DNN Image Encoder parameters + network_image_width = LaunchConfiguration('network_image_width') + network_image_height = LaunchConfiguration('network_image_height') + encoder_image_mean = LaunchConfiguration('encoder_image_mean') + encoder_image_stddev = LaunchConfiguration('encoder_image_stddev') + + # Triton parameters + model_name = LaunchConfiguration('model_name') + model_repository_paths = LaunchConfiguration('model_repository_paths') + max_batch_size = LaunchConfiguration('max_batch_size') + input_tensor_names = LaunchConfiguration('input_tensor_names') + input_binding_names = LaunchConfiguration('input_binding_names') + input_tensor_formats = LaunchConfiguration('input_tensor_formats') + output_tensor_names = LaunchConfiguration('output_tensor_names') + output_binding_names = LaunchConfiguration('output_binding_names') + output_tensor_formats = LaunchConfiguration('output_tensor_formats') + + # U-Net Decoder parameters + network_output_type = LaunchConfiguration('network_output_type') + color_segmentation_mask_encoding = LaunchConfiguration('color_segmentation_mask_encoding') + mask_width = LaunchConfiguration('mask_width') + mask_height = LaunchConfiguration('mask_height') + + data_loader_node = ComposableNode( + name='DataLoaderNode', + namespace=TestImageSegmentationNode.generate_namespace(), + package='ros2_benchmark', + plugin='ros2_benchmark::DataLoaderNode', + remappings=[ + ('hawk_0_left_rgb_image', 'data_loader/image'), + ('hawk_0_left_rgb_camera_info', 'data_loader/camera_info')] + ) + + # Parameters preconfigured for PeopleSemSegNet. + encoder_node = ComposableNode( + name='dnn_image_encoder', + namespace=TestImageSegmentationNode.generate_namespace(), + package='isaac_ros_dnn_encoders', + plugin='nvidia::isaac_ros::dnn_inference::DnnImageEncoderNode', + parameters=[{ + 'network_image_width': network_image_width, + 'network_image_height': network_image_height, + 'image_mean': encoder_image_mean, + 'image_stddev': encoder_image_stddev, + }], + remappings=[ + ('image', 'data_loader/image'), + ('encoded_tensor', 'tensor_pub') + ] + ) + + playback_node = ComposableNode( + name='PlaybackNode', + namespace=TestImageSegmentationNode.generate_namespace(), + package='ros2_benchmark', + plugin='ros2_benchmark::PlaybackNode', + parameters=[{ + 'data_formats': [ + 'sensor_msgs/msg/Image', + ], + }], + remappings=[ + ('buffer/input0', 'data_loader/image'), + ('input0', 'image'), + ] + ) + + triton_node = ComposableNode( + name='triton_node', + namespace=TestImageSegmentationNode.generate_namespace(), + package='isaac_ros_triton', + plugin='nvidia::isaac_ros::dnn_inference::TritonNode', + parameters=[{ + 'model_name': model_name, + 'model_repository_paths': model_repository_paths, + 'max_batch_size': max_batch_size, + 'input_tensor_names': input_tensor_names, + 'input_binding_names': input_binding_names, + 'input_tensor_formats': input_tensor_formats, + 'output_tensor_names': output_tensor_names, + 'output_binding_names': output_binding_names, + 'output_tensor_formats': output_tensor_formats, + }], + ) + + unet_decoder_node = ComposableNode( + name='unet_decoder_node', + namespace=TestImageSegmentationNode.generate_namespace(), + package='isaac_ros_unet', + plugin='nvidia::isaac_ros::unet::UNetDecoderNode', + parameters=[{ + 'network_output_type': network_output_type, + 'color_segmentation_mask_encoding': color_segmentation_mask_encoding, + 'mask_width': mask_width, + 'mask_height': mask_height, + 'color_palette': [0x556B2F, 0x800000, 0x008080, 0x000080, 0x9ACD32, 0xFF0000, 0xFF8C00, + 0xFFD700, 0x00FF00, 0xBA55D3, 0x00FA9A, 0x00FFFF, 0x0000FF, 0xF08080, + 0xFF00FF, 0x1E90FF, 0xDDA0DD, 0xFF1493, 0x87CEFA, 0xFFDEAD], + }], + ) + + monitor_node = ComposableNode( + name='MonitorNode', + namespace=TestImageSegmentationNode.generate_namespace(), + package='ros2_benchmark', + plugin='ros2_benchmark::MonitorNode', + parameters=[{ + 'monitor_data_format': 'sensor_msgs/msg/Image', + }], + remappings=[ + ('output', 'unet/colored_segmentation_mask') + ] + ) + + composable_node_container = ComposableNodeContainer( + name='container', + namespace=TestImageSegmentationNode.generate_namespace(), + package='rclcpp_components', + executable='component_container_mt', + prefix=container_prefix, + sigterm_timeout=container_sigterm_timeout, + composable_node_descriptions=[ + data_loader_node, + encoder_node, + playback_node, + monitor_node, + triton_node, + unet_decoder_node + ], + output='screen' + ) + + + final_launch_description = launch_args + [composable_node_container] + return [launch.LaunchDescription(final_launch_description)] + +def generate_test_description(): + return TestImageSegmentationNode.generate_test_description_with_nsys(launch_setup) + +class TestImageSegmentationNode(ROS2BenchmarkTest): + """Performance test for CitySegfirner.""" + + # Custom configurations + config = ROS2BenchmarkConfig( + benchmark_name='image_segmentation_ros ImageSegmentationNode Benchmark', + input_data_path=ROSBAG_PATH, + # The slice of the rosbag to use + # Upper and lower bounds of peak throughput search window + publisher_upper_frequency=100.0, + publisher_lower_frequency=10.0, + # The number of frames to be buffered + playback_message_buffer_size=300, + start_recording_service_timeout_sec=60, + start_recording_service_future_timeout_sec=60, + custom_report_info={'data_resolution': IMAGE_RESOLUTION}, + benchmark_mode='TIMELINE', + assets_root='/workspaces/isaac_ros-dev/src/ros2_benchmark/assets', + ) + + def test_benchmark(self): + self.run_benchmark() diff --git a/scripts/city_segformer/tutorial-city-segformer.md b/scripts/city_segformer/tutorial-city-segformer.md new file mode 100644 index 0000000..cf9990f --- /dev/null +++ b/scripts/city_segformer/tutorial-city-segformer.md @@ -0,0 +1,117 @@ +# ros2_benchmark city_segoformer + +## Overview + +Measuring benchmark [CitySegformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/citysemsegformer) performance + + +## Quickstart + +1. Set up your development environment by following the instructions [here](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common/blob/main/docs/dev-env-setup.md). +2. Clone this repository and its dependencies under `~/workspaces/isaac_ros-dev/src`. + + ```bash + cd ~/workspaces/isaac_ros-dev/src + ``` + + ```bash + git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common + ``` + + ```bash + git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_nitros + ``` + + ```bash + git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_image_segmentation + ``` + + ```bash + git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_dnn_inference + ``` + + ```bash + git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_image_pipeline + ``` + + ```bash + git clone https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark + ``` + +3. Pull down r2b dataset 2023 by following the instructions [here](https://github.com/SnowMasaya/ros2_benchmark#datasets) or fetch just the rosbag used in this Quickstart with the following command. + + ```bash + mkdir -p $R2B_WS_HOME/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage && \ + cd $R2B_WS_HOME/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage && \ + wget --content-disposition 'https://api.ngc.nvidia.com/v2/resources/nvidia/isaac/r2bdataset2023/versions/1/files/r2b_storage/metadata.yaml' && \ + wget --content-disposition 'https://api.ngc.nvidia.com/v2/resources/nvidia/isaac/r2bdataset2023/versions/1/files/r2b_storage/r2b_storage_0.db3' + ``` + +4. Launch the Docker container using the `run_dev.sh` script: + + ```bash + cd ~/workspaces/isaac_ros-dev/src/isaac_ros_common && \ + ./scripts/run_dev.sh + ``` + +5. Download the `City Segoformer` ETLT file: + + [City Segformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/citysemsegformer) + + ```bash + mkdir -p /workspaces/isaac_ros-dev/models/citysemsegformer/1 + cd /workspaces/isaac_ros-dev/models/citysemsegformer + wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/tao/citysemsegformer/versions/deployable_v1.0/zip -O citysemsegformer_deployable_v1.0.zip + unzip citysemsegformer_deployable_v1.0.zip + + ``` + +6. Convert the ETLT file to a TensorRT plan file: + + ```bash + /opt/nvidia/tao/tao-converter -k tlt_encode -d 3,1024,1820 -p input,1x3x1024x1820,1x3x1024x1820,1x3x1024x1820 -t fp16 -e /workspaces/isaac_ros-dev/models/citysemsegformer/1/model.plan ./citysemsegformer.etlt + ``` + +7. Create a file called `/workspaces/isaac_ros-dev/models/citysemsegformer/config.pbtxt` by copying the sample Triton config file: + + ```bash + name: "citysemsegformer" + platform: "tensorrt_plan" + max_batch_size: 0 + input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ 1, 3, 1024, 1820 ] + } + ] + output [ + { + name: "output" + data_type: TYPE_INT32 + dims: [ 1, 1024, 1820, 1 ] + } + ] + version_policy: { + specific { + versions: [ 1 ] + } + } + + ``` + +8. Inside the container, build and source the workspace: + + ```bash + cd /workspaces/isaac_ros-dev && \ + colcon build --symlink-install && \ + source install/setup.bash + ``` + +9. Run the following launch files to spin up a demo of this package: + + ```bash + launch_test src/ros2_benchmark/scripts/city_segformer/image_segmentation_ros_node_city_segformer.py + ``` +Once the benchmark is finished, the final performance measurements are displayed in the terminal. Additionally, the final results and benchmark metadata (e.g., system information, benchmark configurations) are also exported as a JSON file. +