From 30de3faf7981a4bffd76d999dc8e611b7b0981a5 Mon Sep 17 00:00:00 2001 From: XZK <785202311@qq.com> Date: Sat, 31 Dec 2022 21:49:13 +0800 Subject: [PATCH] =?UTF-8?q?add=20feature=EF=BC=9Adump=20record=20to=20exce?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/tools/dump_record_to_excel/BUILD | 13 ++ .../tools/dump_record_to_excel/dump_record.py | 152 ++++++++++++++++++ .../dump_record_to_excel/dump_record.yaml | 10 ++ modules/tools/dump_record_to_excel/readme.txt | 36 +++++ 4 files changed, 211 insertions(+) create mode 100644 modules/tools/dump_record_to_excel/BUILD create mode 100644 modules/tools/dump_record_to_excel/dump_record.py create mode 100644 modules/tools/dump_record_to_excel/dump_record.yaml create mode 100644 modules/tools/dump_record_to_excel/readme.txt diff --git a/modules/tools/dump_record_to_excel/BUILD b/modules/tools/dump_record_to_excel/BUILD new file mode 100644 index 0000000000..798955c1fe --- /dev/null +++ b/modules/tools/dump_record_to_excel/BUILD @@ -0,0 +1,13 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "dump_record", + srcs = ["dump_record.py"], + deps = [ + "//cyber/python/cyber_py3:cyber", + "//cyber/python/cyber_py3:record", + "//modules/tools/common:message_manager", + ], +) diff --git a/modules/tools/dump_record_to_excel/dump_record.py b/modules/tools/dump_record_to_excel/dump_record.py new file mode 100644 index 0000000000..292dd8abb0 --- /dev/null +++ b/modules/tools/dump_record_to_excel/dump_record.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +############################################################################### +# Copyright 2017 The Apollo Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +""" +This program can dump a cyber record into separate text files that contains +the pb messages +""" +import os +os.system("pip install pandas") +os.system("pip install openpyxl") +import argparse +import sys +import yaml +import pandas as pd +from openpyxl import load_workbook +from cyber.python.cyber_py3 import cyber +from cyber.python.cyber_py3 import record +from modules.tools.common.message_manager import PbMessageManager + +g_message_manager = PbMessageManager() + + +def parse_attrib(attrib_name, topic, keyword_list): + if ('Repeated' in str(type(eval(attrib_name)))): # Dont support repeated component + return + if (isinstance(eval(attrib_name), str) or isinstance(eval(attrib_name), int) or isinstance(eval(attrib_name), float)): + if not keyword_list or (keyword_list and [True for i in keyword_list if i in attrib_name]): + attrib_list[topic][attrib_name] = [] + else: + for j in [(attrib_name + '.' + i) for i in dir(eval(attrib_name)) if ord('a') <= ord(i[0]) <= ord('z') and i != "yield"]: + parse_attrib(j, topic, keyword_list) + + +def dump_record(in_record, out_dir, start_time, duration, topic_list): + global msg + global attrib_list + Flag_initial = True + + topic_name_list = [i["name"] for i in topic_list if i["name"]] + attrib_list = {i:{} for i in topic_name_list} + timestamp_list = {i:[] for i in topic_name_list} + for i, topic in enumerate(topic_name_list): + meta_msg = g_message_manager.get_msg_meta_by_topic(topic) + if meta_msg is None: + print('Unknown topic: %s' % topic) + continue + msg = meta_msg.msg_type() + parse_attrib("msg", topic, topic_list[i]["keyword"]) + + # print(topic_list) + # print(topic_name_list) + # print(attrib_list) + + try: + freader = record.RecordReader(in_record) + except Exception: + print('Cannot open record file %s' % in_record) + else: + for BagMessage in freader.read_messages(): + if Flag_initial: + initial_time = BagMessage.timestamp + Flag_initial = False + + t_sec = BagMessage.timestamp + if start_time and t_sec / 1e9 < start_time: + # print('Not yet reached the start time') + continue + if duration: + if start_time: + if t_sec / 1e9 >= start_time + duration: + print('Done') + break + elif t_sec / 1e9 >= initial_time / 1e9 + duration: + print('Done') + break + + topic = BagMessage.topic + if topic == '/apollo/sensor/mobileye': + continue + if topic in topic_name_list: + meta_msg = g_message_manager.get_msg_meta_by_topic(topic) + if meta_msg is None: + print('Unknown topic: %s' % topic) + continue + msg = meta_msg.msg_type() + msg.ParseFromString(BagMessage.message) + for key, value in attrib_list[topic].items(): + value.append(eval(key)) + timestamp_list[topic].append(msg.header.timestamp_sec) + + writer = pd.ExcelWriter(out_dir + '/dump_record_result.xlsx') + for i, topic in enumerate(topic_name_list): + Raw_Data = pd.DataFrame(attrib_list[topic], timestamp_list[topic], attrib_list[topic].keys()) + Raw_Data.to_excel(writer, sheet_name=topic.replace('/','_')) + writer.save() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="A tool to dump the protobuf messages in a cyber record into text files" + ) + parser.add_argument( + "in_record", + action="store", + type=str, + help="the input cyber record") + parser.add_argument( + "-s", + "--start_time", + action="store", + type=float, + help="start time to parse record, eg: 1635927111.6681252") + parser.add_argument( + "-d", + "--duration", + action="store", + type=float, + help="duration of parse record, unit:s") + parser.add_argument( + "out_dir", + action="store", + type=str, + help="the output directory for the dumped file") + + args = parser.parse_args() + if not os.path.exists(args.out_dir): + print('%s does not exist' % args.out_dir) + sys.exit(1) + + yaml_file = '/apollo/modules/tools/dump_record/dump_record.yaml' + with open(yaml_file, 'r') as f: + params = yaml.safe_load(f) + topic_list = params['Topic'] + # print(topic_list) + + dump_record(args.in_record, args.out_dir, args.start_time, args.duration, topic_list) + + # print(attrib_list) \ No newline at end of file diff --git a/modules/tools/dump_record_to_excel/dump_record.yaml b/modules/tools/dump_record_to_excel/dump_record.yaml new file mode 100644 index 0000000000..79e42ef6f2 --- /dev/null +++ b/modules/tools/dump_record_to_excel/dump_record.yaml @@ -0,0 +1,10 @@ +Topic: + - name: /apollo/planning + keyword: + - path_point + + - name: + keyword: + + - name: + keyword: \ No newline at end of file diff --git a/modules/tools/dump_record_to_excel/readme.txt b/modules/tools/dump_record_to_excel/readme.txt new file mode 100644 index 0000000000..f701daf5d1 --- /dev/null +++ b/modules/tools/dump_record_to_excel/readme.txt @@ -0,0 +1,36 @@ +工具使用步骤： +1, 在yaml文件中定义需要解析的topic以及具体的属性 +2, 运行脚本 + +详细说明： +第一步： + # 在/apollo/modules/tools/dump_record/dump_record.yaml中添加需要解析的topic信息 + # name即topic的名称，与cyber monitor中看到的名称一致 + # keyword即想要提取这个topic中具体哪个属性信息，keyword采取模糊匹配，只要属性中包含关键字就会被记录下来。 + # keyword留空则表示记录该topic下的所有信息,但是keyword关键字必须保留 + + # 注意： + # 1, 目前脚本不支持数量可变的属性提取，比如障碍物的数量每次都不一样，即在cyber monitor中可以用右键扩展的属性目前不支持 + # 2, yaml语法要求，冒号后面必须接一个空格，然后再写具体信息，空格不可省略 + + # 示例 ----------------------------------------------------- + + Topic: + - name: /apollo/planning # 提取planning这个topic中包含point和rss关键字的所有属性 + keyword: + - point + - rss + + - name: /apollo/control # 提取control这个topic中包含speed关键字的所有属性 + keyword: + - speed + + - name: /apollo/prediction # 提取prediction这个topic中的所有属性 + keyword: + +第二步： + 进入apollo容器内，运行 ./scripts/dump_record.sh 「-s 1635927111.6681252」 「-d 5」 record文件路径 文件输出目录 + 其中-s和-d是可选参数，分别表示开始时间（start_time）和持续时间（duration）。详情请查看程序或运行 ./scripts/dump_record.sh -h + 开始时间（start_time）可以通过cyber_record play 播包时获取，本程序也会打印出record文件的最早的时间 + 举例： + ./scripts/dump_record.sh data/record/1103.record data/record/test -d 5 -s 1635927115.909