diff --git a/.licenserc.yaml b/.licenserc.yaml index f471698950..d87c350091 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -35,6 +35,8 @@ header: - 'example/*/*.pem' - 'example/*/*.port' - 'example/build_with_bazel_module/.bazelversion' + - 'example/benchmark_fb/test.fbs' + - 'example/benchmark_fb/test_generated.h' - 'src/bthread/offset_inl.list' - 'test/*.crt' - 'test/*.key' diff --git a/BUILD.bazel b/BUILD.bazel index 727af8574a..fbe85abcef 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -36,6 +36,7 @@ DEFINES = [ "__STDC_FORMAT_MACROS", "__STDC_LIMIT_MACROS", "__STDC_CONSTANT_MACROS", + "BRPC_WITH_FLATBUFFERS=0", ] + select({ "//bazel/config:brpc_with_glog": ["BRPC_WITH_GLOG=1"], "//conditions:default": ["BRPC_WITH_GLOG=0"], @@ -547,6 +548,8 @@ cc_library( "src/brpc/policy/thrift_protocol.cpp", "src/brpc/event_dispatcher_epoll.cpp", "src/brpc/event_dispatcher_kqueue.cpp", + "src/brpc/details/flatbuffers_impl.cpp", + "src/brpc/policy/flatbuffers_protocol.cpp", ]) + select({ "//bazel/config:brpc_with_thrift": glob([ "src/brpc/thrift*.cpp", diff --git a/CMakeLists.txt b/CMakeLists.txt index 47c1458625..2ef163fc0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,7 @@ option(WITH_THRIFT "With thrift framed protocol supported" OFF) option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF) option(WITH_SNAPPY "With snappy" OFF) option(WITH_RDMA "With RDMA" OFF) +option(WITH_FLATBUFFERS "With FlatBuffers RPC support" OFF) option(WITH_UBRING "With UB" OFF) option(WITH_DEBUG_BTHREAD_SCHE_SAFETY "With debugging bthread sche safety" OFF) option(WITH_DEBUG_LOCK "With debugging lock" OFF) @@ -115,6 +116,11 @@ if (WITH_BTHREAD_TRACER) list(APPEND BRPC_COMMON_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_PATH}) endif () +set(WITH_FLATBUFFERS_VAL "0") +if(WITH_FLATBUFFERS) + set(WITH_FLATBUFFERS_VAL "1") +endif() + set(WITH_RDMA_VAL "0") if(WITH_RDMA) set(WITH_RDMA_VAL "1") @@ -159,6 +165,7 @@ endif() list(APPEND BRPC_COMMON_DEFINITIONS BRPC_WITH_GLOG=${WITH_GLOG_VAL} BRPC_WITH_RDMA=${WITH_RDMA_VAL} + BRPC_WITH_FLATBUFFERS=${WITH_FLATBUFFERS_VAL} BRPC_WITH_UBRING=${WITH_UBRING_VAL} BRPC_DEBUG_BTHREAD_SCHE_SAFETY=${WITH_DEBUG_BTHREAD_SCHE_SAFETY_VAL} BRPC_DEBUG_LOCK=${WITH_DEBUG_LOCK_VAL} @@ -226,6 +233,28 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") endif() find_package(Protobuf REQUIRED) + +if(WITH_FLATBUFFERS) + find_path( + FLATBUFFERS_INCLUDE_PATH + NAMES flatbuffers/flatbuffers.h + ) + if(NOT FLATBUFFERS_INCLUDE_PATH) + message( + FATAL_ERROR + "WITH_FLATBUFFERS=ON but flatbuffers/flatbuffers.h was not found" + ) + endif() + list( + APPEND + BRPC_COMMON_INCLUDE_DIRS + ${FLATBUFFERS_INCLUDE_PATH} + ) + message( + STATUS + "FlatBuffers include path: ${FLATBUFFERS_INCLUDE_PATH}" + ) +endif() find_package(ZLIB REQUIRED) if(Protobuf_VERSION VERSION_GREATER 4.21) # required by absl @@ -576,6 +605,15 @@ file(GLOB_RECURSE BVAR_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/bvar file(GLOB_RECURSE BTHREAD_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/bthread/*.cpp") file(GLOB_RECURSE JSON2PB_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/json2pb/*.cpp") file(GLOB_RECURSE BRPC_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/*.cpp") + +if(NOT WITH_FLATBUFFERS) + list( + FILTER + BRPC_SOURCES + EXCLUDE + REGEX "/flatbuffers_(impl|protocol)\\.cpp$" + ) +endif() file(GLOB_RECURSE THRIFT_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/thrift*.cpp") file(GLOB_RECURSE EXCLUDE_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/brpc/event_dispatcher_*.cpp") diff --git a/Makefile b/Makefile index 86de388448..4e21e34df8 100644 --- a/Makefile +++ b/Makefile @@ -205,7 +205,12 @@ JSON2PB_OBJS = $(addsuffix .o, $(basename $(JSON2PB_SOURCES))) BRPC_DIRS = src/brpc src/brpc/details src/brpc/builtin src/brpc/policy src/brpc/policy/mysql src/brpc/rdma THRIFT_SOURCES = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/thrift*,$(SRCEXTS)))) +FLATBUFFERS_SOURCES = src/brpc/details/flatbuffers_impl.cpp \ + src/brpc/policy/flatbuffers_protocol.cpp EXCLUDE_SOURCES = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/event_dispatcher_*,$(SRCEXTS)))) +ifneq (BRPC_WITH_FLATBUFFERS=1,$(findstring BRPC_WITH_FLATBUFFERS=1,$(CPPFLAGS))) + EXCLUDE_SOURCES += $(FLATBUFFERS_SOURCES) +endif BRPC_SOURCES_ALL = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/*,$(SRCEXTS)))) BRPC_SOURCES = $(filter-out $(THRIFT_SOURCES) $(EXCLUDE_SOURCES), $(BRPC_SOURCES_ALL)) BRPC_PROTOS = $(filter %.proto,$(BRPC_SOURCES)) diff --git a/config.h.in b/config.h.in index d8de111be9..ece54504ce 100644 --- a/config.h.in +++ b/config.h.in @@ -21,6 +21,11 @@ #endif #cmakedefine BRPC_WITH_GLOG @WITH_GLOG_VAL@ +#ifdef BRPC_WITH_FLATBUFFERS +#undef BRPC_WITH_FLATBUFFERS +#endif +#define BRPC_WITH_FLATBUFFERS @WITH_FLATBUFFERS_VAL@ + #ifdef BUTIL_USE_CPU_FREQUENCY #undef BUTIL_USE_CPU_FREQUENCY #endif diff --git a/config_brpc.sh b/config_brpc.sh index 85692de3cb..62923c0742 100755 --- a/config_brpc.sh +++ b/config_brpc.sh @@ -54,9 +54,10 @@ else LDD=ldd fi -TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-rdma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` +TEMP=`getopt -o v: --long headers:,libs:,cc:,cxx:,with-glog,with-thrift,with-flatbuffers,with-rdma,with-mesalink,with-bthread-tracer,with-debug-bthread-sche-safety,with-debug-lock,with-asan,with-riscv-zvbc,with-riscv-zbc,with-cpu-frequency,nodebugsymbols,werror -n 'config_brpc' -- "$@"` WITH_GLOG=0 WITH_THRIFT=0 +WITH_FLATBUFFERS=0 WITH_RDMA=0 WITH_MESALINK=0 WITH_BTHREAD_TRACER=0 @@ -89,6 +90,7 @@ while true; do --cxx ) CXX=$2; shift 2 ;; --with-glog ) WITH_GLOG=1; shift 1 ;; --with-thrift) WITH_THRIFT=1; shift 1 ;; + --with-flatbuffers) WITH_FLATBUFFERS=1; shift 1 ;; --with-rdma) WITH_RDMA=1; shift 1 ;; --with-mesalink) WITH_MESALINK=1; shift 1 ;; --with-bthread-tracer) WITH_BTHREAD_TRACER=1; shift 1 ;; @@ -481,7 +483,7 @@ append_to_output "STATIC_LINKINGS=$STATIC_LINKINGS" append_to_output "DYNAMIC_LINKINGS=$DYNAMIC_LINKINGS" # CPP means C PreProcessing, not C PlusPlus -CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY" +CPPFLAGS="${CPPFLAGS} -DBRPC_WITH_GLOG=$WITH_GLOG -DBRPC_WITH_FLATBUFFERS=$WITH_FLATBUFFERS -DBRPC_DEBUG_BTHREAD_SCHE_SAFETY=$BRPC_DEBUG_BTHREAD_SCHE_SAFETY -DBRPC_DEBUG_LOCK=$BRPC_DEBUG_LOCK -DBUTIL_USE_CPU_FREQUENCY=$WITH_CPU_FREQUENCY" # Avoid over-optimizations of TLS variables by GCC>=4.8 # See: https://github.com/apache/brpc/issues/1693 @@ -526,6 +528,14 @@ if [ $WITH_THRIFT != 0 ]; then fi fi +if [ $WITH_FLATBUFFERS != 0 ]; then + FLATBUFFERS_HDR=$(find_dir_of_header_or_die flatbuffers/flatbuffers.h) + FLATBUFFERS_LIB=$(find_dir_of_lib_or_die flatbuffers) + append_to_output_headers "$FLATBUFFERS_HDR" + append_to_output_libs "$FLATBUFFERS_LIB" + append_to_output "STATIC_LINKINGS+=-lflatbuffers" +fi + if [ $WITH_RDMA != 0 ]; then RDMA_LIB=$(find_dir_of_lib_or_die ibverbs) RDMA_HDR=$(find_dir_of_header_or_die infiniband/verbs.h) diff --git a/docs/cn/brpc_flatbuffers_progress.md b/docs/cn/brpc_flatbuffers_progress.md new file mode 100644 index 0000000000..95c210a6a1 --- /dev/null +++ b/docs/cn/brpc_flatbuffers_progress.md @@ -0,0 +1,170 @@ +# bRPC FlatBuffers 集成进展说明 + +更新日期：2026-09-10 + +## 1. 当前结论 + +已在个人 fork 中完成 FlatBuffers 消息构造及 RPC 原型移植、可选构建支持、消息与 TCP RPC 自动化测试。当前两个测试程序包含 7 个测试用例，均已通过本地验证。 + +当前阶段是具备基本正确性验证的集成原型。完整的安全视图设计、拷贝路径分析、性能收益证明和社区合入仍需继续推进。 + +本说明依据本地开发过程中记录的 Git 提交、构建输出和测试日志编写，不代表 Apache 社区已接受或合入这些改动。 + +## 2. 代码位置与提交 + +- 仓库：https://github.com/Spicy-cream/brpc +- 当前开发分支：`feature/flatbuffers-rpc-tests` +- 分支地址：https://github.com/Spicy-cream/brpc/tree/feature/flatbuffers-rpc-tests +- 本文记录的代码版本：`0f591739` + +| 提交 | 内容 | +| --- | --- | +| `d14d1002` | 移植社区 FlatBuffers 消息构造实现 | +| `f7255f84` | 移植 FlatBuffers RPC 协议，并完成相关兼容修复与示例调整 | +| `4cad8ef6` | 增加 FlatBuffers RPC 可选构建支持 | +| `dd2c20ae` | 增加消息所有权与校验测试 | +| `0f591739` | 增加 RPC 收发、请求拒绝与拒绝后恢复测试 | + +以上分支和提交已推送到个人 fork；本文编写时尚未据此创建新的 Apache PR。 + +## 3. 与社区原有工作的关系 + +集成基于 Apache bRPC 的已有贡献： + +- [PR #3196：FlatBuffers 消息构造](https://github.com/apache/brpc/pull/3196) +- [PR #3197：FlatBuffers RPC 协议](https://github.com/apache/brpc/pull/3197) + +消息构造与协议的基础设计来源于原有社区工作。本阶段主要完成移植、问题修复、构建开关以及回归验证。后续贡献应保留原作者署名，并与原作者及维护者协调提交范围。 + +当前分支还包含前期设计和 benchmark 文档。面向 Apache 提交时，需要按目标分支重新整理差异，不宜直接提交整个开发分支。 + +## 4. 已完成的实现 + +### 4.1 消息构造与缓冲区所有权 + +- 接入 `MessageBuilder`、`Message` 及相关缓冲区适配实现。 +- 修复 `Message` 的移动构造与移动赋值，通过 `SingleIOBuf::swap` 转移缓冲区及元数据。 +- 验证 Builder 销毁后，已释放的 Message 仍能被校验和读取。 +- 验证移动操作保留数据地址和长度，移动后的源对象不会使目标消息失效。 + +这些测试证明所覆盖的消息生命周期和所有权行为，不等价于证明整个 RPC 链路零拷贝。 + +### 4.2 RPC 接入 + +- 接入 Channel、Server、协议注册及请求响应处理。 +- 提供 `fb_rpc` 调用路径和 `example/benchmark_fb` 示例。 +- 完成本地 TCP 请求响应验证。 +- 示例服务在访问请求字段前调用 `Verify`；示例客户端能够校验响应并比较字段。 + +当前协议响应携带错误码，不传递服务端详细错误文本。客户端收到非零错误码时使用通用文本 `server response error`。自动化测试据此检查错误码，而不要求服务端错误字符串原样返回。 + +### 4.3 可选构建 + +- 增加 `WITH_FLATBUFFERS` CMake 选项，默认关闭。 +- OFF 时排除相应实现及受保护的接口依赖。 +- ON 时启用 FlatBuffers 实现并发现依赖头文件。 +- 导出配置头中记录 `BRPC_WITH_FLATBUFFERS` 的 0/1 值。 +- OFF 时排除 `brpc_flatbuffers_*_unittest.cpp` 测试目标。 + +已验证 ON/OFF 核心库构建通过，ON 导出头文件和静态库可以用于构建外部示例。自动化测试 OFF 配置验证了 FlatBuffers 测试未注册，此项不是一次完整的 OFF 测试套件运行。 + +## 5. 自动化测试覆盖 + +### 5.1 消息测试 + +文件：`test/brpc_flatbuffers_message_unittest.cpp` + +| 测试 | 验证内容 | +| --- | --- | +| `ReleasedMessageSurvivesBuilderDestruction` | Builder 销毁后消息仍有效，校验及各字段读取正确 | +| `MoveConstructorPreservesBuffer` | 移动构造保留缓冲区地址和长度，源对象清理后目标仍有效 | +| `MoveAssignmentReplacesExistingMessage` | 正确替换已有消息，源对象销毁后目标仍有效 | +| `RejectsCorruptRootOffset` | 根偏移被破坏后，Verifier 返回失败 | + +### 5.2 TCP RPC 测试 + +文件：`test/brpc_flatbuffers_rpc_unittest.cpp` + +测试使用 `127.0.0.1:0` 自动分配端口，启动真实 Server 与 Channel，禁用重试，并设置 RPC 超时。 + +| 测试 | 验证内容 | +| --- | --- | +| `ValidRequestRoundTrip` | 完成请求响应，校验响应结构并比较全部业务字段 | +| `CorruptRequestIsRejectedByService` | 服务校验损坏请求后返回 `EREQUEST`，并确认服务端拒绝计数 | +| `ValidRequestSucceedsAfterRejection` | 同一 Server、Channel 先成功、再拒绝损坏请求、随后再次成功 | + +负向测试结合错误码与服务端接受/拒绝计数，避免仅凭 `Failed()` 就把连接失败或超时误判为成功拒绝。 + +这里测试的是服务实现主动调用 `Verify` 的行为，尚未实现协议层对任意业务类型的统一校验。 + +### 5.3 已记录的执行结果 + +```text +configure_status=0 +build_status=0 + +brpc_flatbuffers_message_unittest ... Passed +brpc_flatbuffers_rpc_unittest ....... Passed + +100% tests passed, 0 tests failed out of 2 +``` + +CTest 的两个条目分别对应两个测试程序，合计 4 + 3 = 7 个测试用例。OFF 配置下查询 FlatBuffers 测试得到 `Total Tests: 0`。 + +验证环境为 Ubuntu 24.04 WSL2、GCC 13.3、Protobuf 3.21.12、FlatBuffers 25.12.19，以及本地 GoogleTest 源码。以上结果来自本地运行，尚不等同于跨平台 CI 验证。 + +## 6. 本地复现 + +以下命令从仓库根目录执行。FlatBuffers 安装目录和 GoogleTest 源码目录需按实际环境调整。 + +```bash +cmake -S . -B build-flatbuffers-tests-on -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_FLATBUFFERS=ON \ + -DBUILD_UNIT_TESTS=ON \ + -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest \ + -DCMAKE_PREFIX_PATH=/home/l30084420/workspace/serialization-benchmark/local + +cmake --build build-flatbuffers-tests-on \ + --target brpc_flatbuffers_message_unittest brpc_flatbuffers_rpc_unittest \ + --parallel 8 + +ctest --test-dir build-flatbuffers-tests-on \ + -R '^brpc_flatbuffers_(message|rpc)_unittest$' \ + --timeout 60 --output-on-failure +``` + +OFF 配置及测试注册检查： + +```bash +cmake -S . -B build-flatbuffers-tests-off -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_FLATBUFFERS=OFF \ + -DBUILD_UNIT_TESTS=ON \ + -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest + +ctest --test-dir build-flatbuffers-tests-off -N \ + -R '^brpc_flatbuffers_.*' +``` + +当前 CMake 配置会生成源码目录中的 `src/butil/config.h`。同一 checkout 切换 ON/OFF 配置后，重新构建前应重新配置目标模式；不要在同一源码目录同时进行两种配置的构建。 + +当前测试复用了 `example/benchmark_fb` 中的 schema 生成代码。生成头文件带有 FlatBuffers 版本检查，后续需要整理测试 schema 和生成流程，避免将社区测试长期绑定到本机版本。 + +## 7. 性能数据的适用范围 + +此前在本地 TCP 短时冒烟运行中观察到约 18k–19k QPS、约 50–52 微秒 RPC 延迟。这些结果用于说明链路可运行，不作为 FlatBuffers 相对 Protobuf 的性能收益结论。 + +尚需统一编译模式、负载、并发、校验策略和测量区间，进行可重复的对照测试。消息构造耗时、校验/读取耗时和完整 RPC 延迟应分别统计。 + +## 8. 后续工作 + +1. 与原 PR 作者及 Apache 维护者沟通，整理基于 Apache 的提交范围和贡献归属。 +2. 整理测试 schema、版本依赖和生成流程，并接入适当的 CI 构建与回归验证。 +3. 根据协议审查补充边界、异常报文、附件、并发及生命周期测试，开展内存检查工具验证。 +4. 继续设计校验后的类型化视图，明确缓冲区所有权、可变性及访问边界。 +5. 分析发送与接收路径中的实际拷贝，进行有对照的性能测试。 + +完整安全视图、端到端零拷贝、RDMA/URMA 路径和正式社区合入均不属于当前已完成的验证范围。 diff --git a/docs/cn/flatbuffers_zero_copy_benchmark/complete_benchmark_report.md b/docs/cn/flatbuffers_zero_copy_benchmark/complete_benchmark_report.md new file mode 100644 index 0000000000..d13c4f4fb6 --- /dev/null +++ b/docs/cn/flatbuffers_zero_copy_benchmark/complete_benchmark_report.md @@ -0,0 +1,329 @@ +# Protobuf / FlatBuffers / Cap'n Proto 测试流程与结果汇总 + +## 1. 测试目标 + +本项目比较以下三种序列化方案： + +- Protobuf 3.21.12 +- FlatBuffers 25.12.19 +- Cap'n Proto 1.5.0 + +测试目标是为 bRPC 社区设计一套面向 TCP、RDMA/URMA 的通用免反序列化/低复制数据传输方案，并回答以下问题： + +1. 三种格式单独进行序列化时的成本有什么区别？ +2. Protobuf 反序列化与 FlatBuffers/Cap'n Proto 建立只读视图的成本有什么区别？ +3. 将生产者和消费者分成两个进程后，结果是否仍然成立？ +4. 接入完整 bRPC TCP 请求—响应流程后，额外的数据复制会带来什么影响？ +5. 哪种格式更适合作为 bRPC 免反序列化特性的第一阶段实现？ + +## 2. 测试环境 + +| 项目 | 环境 | +|---|---| +| 操作系统 | Ubuntu 24.04 on WSL2 | +| 内核 | 6.18.33.2-microsoft-standard-WSL2 | +| 编译器 | GCC 13.3.0 | +| Protobuf | 3.21.12 | +| FlatBuffers | 25.12.19 | +| Cap'n Proto | 1.5.0 | +| bRPC commit | `6a1c6bfb496f56b77494de89146eb27c6c9ef0dd` | +| bRPC branch | `pr-22-compile-fix` | + +当前全部测试均在本地 WSL2 中完成。尚未使用真实 RDMA/URMA 设备，也尚未得到跨物理机器网络结果。 + +## 3. 测试数据模型 + +### 3.1 Simple 模型 + +Simple 模型包含： + +- Header：request ID、时间戳、版本、来源； +- SimplePayload：一个字节数组。 + +该模型用于观察以连续大块 Payload 为主、元数据较少的场景。 + +### 3.2 Complex 模型 + +Complex 模型包含： + +- Header； +- 16 个 Record； +- 每个 Record 包含 ID、名称、Metrics、samples、两个 Tag 和一部分 Payload。 + +该模型用于观察多层嵌套结构、字符串、数组和重复字段较多的场景。 + +### 3.3 Payload 范围 + +测试覆盖 18 个 Payload： + +```text +64B, 128B, 256B, 512B, +1KiB, 2KiB, 4KiB, 8KiB, +16KiB, 32KiB, 64KiB, +128KiB, 256KiB, 512KiB, +1MiB, 2MiB, 4MiB, 8MiB +``` + +所有方案使用相同的原始字节序列和 checksum 算法。正式结果中 checksum 失败数均为 0。 + +## 4. 已完成的测试层次 + +| 测试 | 进程模型 | 数据传递方式 | 主要目的 | 状态 | +|---|---|---|---|---| +| 库级测试 | 单进程 | 进程内缓冲区 | 分离测量编码、复制、解析/建视图和访问 | 已完成，共 6 轮 | +| IPC 测试 | 两个独立进程 | POSIX 共享内存单槽位 | 测量生产者和消费者分离后的成本 | 已完成，共 3 轮、35,100 条、0 失败 | +| bRPC 测试 | 客户端 + 服务端 | localhost TCP | 测量完整 RPC 请求—响应路径 | 已完成，共 3 轮、35,100 条、0 失败 | + +## 5. 单进程库级测试 + +### 5.1 测试流程 + +```text +构造对象 +→ 序列化 +→ memcpy 到消费者缓冲区 +→ Protobuf 反序列化，或 FlatBuffers/Cap'n Proto 建立视图 +→ 部分字段访问 +→ 完整数据访问 +→ checksum 校验 +``` + +CSV 分别记录： + +- `serialize_ns` +- `copy_ns` +- `deserialize_or_view_ns` +- `partial_access_ns` +- `full_access_ns` +- `end_to_end_ns` +- `encoded_bytes` +- `checksum` +- `success` + +因此，该测试既能单独比较序列化和反序列化，也能比较完整本地数据处理流水线。 + +### 5.2 主要结果 + +- Protobuf 对复杂小消息的编码结果最紧凑。 +- FlatBuffers 和 Cap'n Proto 可以在编码缓冲区上建立视图，不需要构造完整反序列化对象。 +- FlatBuffers 的连续缓冲区和运行稳定性更适合作为工程集成起点。 +- Cap'n Proto 建立 Reader 很快，但部分大消息和复杂对象测试中的波动较大。 +- Payload 增大后，复制和完整内存扫描逐渐成为主要成本。 + +## 6. 双进程共享内存测试 + +### 6.1 测试流程 + +```text +Serializer/Producer 进程 + 构造 simple/complex 对象 + → 序列化 + → memcpy 到 POSIX 共享内存 + → 将槽位状态设为 READY + +Deserializer/Consumer 进程 + 等待 READY + → Protobuf ParseFromArray + 或 FlatBuffers Verifier + GetRoot + 或 Cap'n Proto FlatArrayMessageReader + → 完整访问 Payload + → checksum 校验 + → 将槽位状态设为 CONSUMED +``` + +两个进程地址空间彼此独立，使用一个共享内存槽位进行严格的生产—消费 ping-pong。 + +CSV 分别记录： + +- `serialize_ns`：生产者序列化时间； +- `publish_ns`：复制到共享内存的时间； +- `consumer_parse_or_view_ns`：消费者解析或建视图时间； +- `consumer_access_ns`：消费者完整访问时间； +- `end_to_end_ns`：发布后至消费者处理完成的时间。 + +用于比较的完整流水线时间为： + +```text +serialize_ns + publish_ns + end_to_end_ns +``` + +每轮第一条记录包含人工/进程启动等待，在汇总统计中予以剔除。 + +### 6.2 完整流水线 P50 + +| 模型 / Payload | Protobuf | FlatBuffers | Cap'n Proto | +|---|---:|---:|---:| +| Simple 64B | 0.56 μs | 0.52 μs | 0.54 μs | +| Simple 4KiB | 1.56 μs | 1.35 μs | 1.88 μs | +| Simple 1MiB | 0.81 ms | 0.73 ms | 0.73 ms | +| Simple 8MiB | 9.35 ms | 8.28 ms | 8.05 ms | +| Complex 64B | 13.21 μs | 4.43 μs | 1.97 μs | +| Complex 4KiB | 15.49 μs | 6.78 μs | 3.55 μs | +| Complex 1MiB | 0.66 ms | 0.43 ms | 0.74 ms | +| Complex 8MiB | 7.27 ms | 5.52 ms | 7.95 ms | + +### 6.3 单独解析/建视图 P50（8MiB Complex） + +| 格式 | 解析/建视图时间 | +|---|---:| +| Protobuf | 620 μs | +| FlatBuffers | 4.4 μs | +| Cap'n Proto | 3.1 μs | + +该结果是免反序列化方向最关键的本地证据：FlatBuffers 和 Cap'n Proto 建立视图的成本比 Protobuf 构造完整对象低约两个数量级。 + +### 6.4 单独序列化 P50（8MiB Complex） + +| 格式 | 序列化时间 | +|---|---:| +| FlatBuffers | 4.01 ms | +| Protobuf | 5.33 ms | +| Cap'n Proto | 6.61 ms | + +Complex 中大消息场景下，FlatBuffers 的序列化和完整流水线性能最好。 + +### 6.5 IPC 测试结论 + +- Simple 小消息差别很小。 +- Complex 小消息中 Cap'n Proto 最快，FlatBuffers 次之，Protobuf 构造和解析成本最高。 +- Complex 中大消息中 FlatBuffers 整体表现最好。 +- 对全部 Payload 进行完整扫描时，三种格式都无法避免真实的内存读取成本。 +- 当前生产者仍先编码到临时缓冲区，再复制到共享内存；尚未实现直接在共享/注册内存中原地构建。 + +## 7. 完整 bRPC localhost TCP 测试 + +### 7.1 测试数据路径 + +Protobuf： + +```text +客户端构造原生 Protobuf RPC message +→ bRPC 内部编码 +→ localhost TCP +→ bRPC 自动解析 +→ 服务方法访问数据并校验 +→ 返回小型 BenchmarkResponse +``` + +FlatBuffers/Cap'n Proto： + +```text +客户端编码连续缓冲区 +→ 复制进 bRPC request_attachment/IOBuf +→ localhost TCP +→ 服务端从 IOBuf 复制到连续 vector +→ 建立视图/Reader +→ 访问数据并校验 +→ 返回小型 BenchmarkResponse +``` + +这是完整的 RPC 请求—响应测试，但属于“大请求 + 小响应”，服务端没有将完整 Payload 原样返回。 + +### 7.2 客户端完整流水线 P50 + +| 模型 / Payload | Protobuf native | FlatBuffers attachment | Cap'n Proto attachment | +|---|---:|---:|---:| +| Simple 64B | 69.0 μs | 68.3 μs | 69.1 μs | +| Simple 64KiB | 100.4 μs | 130.1 μs | 133.1 μs | +| Simple 1MiB | 0.87 ms | 1.08 ms | 1.28 ms | +| Simple 8MiB | 9.12 ms | 11.18 ms | 14.10 ms | +| Complex 64B | 87.4 μs | 76.4 μs | 75.4 μs | +| Complex 64KiB | 126.4 μs | 133.4 μs | 149.6 μs | +| Complex 1MiB | 0.61 ms | 0.93 ms | 1.08 ms | +| Complex 8MiB | 6.69 ms | 9.17 ms | 10.86 ms | + +客户端完整流水线按以下口径计算： + +```text +serialize_ns + rpc_roundtrip_ns +``` + +需要注意：Protobuf 的实际线性编码和服务端解析由 bRPC 内部完成，部分成本包含在 `rpc_roundtrip_ns` 中。 + +### 7.3 8MiB Complex 估算吞吐量 + +| 格式 | 吞吐量 | +|---|---:| +| Protobuf native | 约 1195 MiB/s | +| FlatBuffers attachment | 约 872 MiB/s | +| Cap'n Proto attachment | 约 737 MiB/s | + +### 7.4 bRPC 测试结论 + +- 64B Simple 场景三者约为 68～69 μs，主要由 RPC 固定开销主导。 +- 当前 bRPC 路径下，大消息 Protobuf native 最快，FlatBuffers attachment 次之，Cap'n Proto attachment 最慢。 +- 这不能直接证明 Protobuf 格式本身在大消息上更优，因为三种格式使用了不同的 bRPC 数据路径。 +- FlatBuffers/Cap'n Proto attachment 路径多出了客户端写入 IOBuf和服务端复制出 IOBuf 的成本。 +- 当前 CSV 中 `server_access_ns` 实际更接近 attachment 复制、建视图和访问组成的服务端总处理时间，不应解读成纯字段访问时间。 + +## 8. IPC 与 bRPC 结果的关键对照 + +以 8MiB Complex 为例： + +| 格式 | 共享内存双进程 | bRPC localhost TCP | +|---|---:|---:| +| Protobuf | 7.27 ms | 6.69 ms | +| FlatBuffers | 5.52 ms | 9.17 ms | +| Cap'n Proto | 7.95 ms | 10.86 ms | + +FlatBuffers 在共享内存路径中比 Protobuf 快约 24%，但在当前 bRPC attachment 路径中比 Protobuf 慢约 37%。 + +这表明当前的主要问题不是 FlatBuffers 无法带来收益，而是 attachment 路径中的额外复制掩盖了免反序列化收益。 + +## 9. 对 bRPC 免序列化特性的启发 + +仅仅把 FlatBuffers 或 Cap'n Proto 编码结果放入传统 attachment 并不够。建议为 bRPC 设计统一的可寻址数据区域抽象，例如： + +```text +RemoteRegion / ZeroCopyAttachment / SerializedView +``` + +理想路径： + +```text +发送端在可发送/注册内存中构建编码结果 +→ TCP、RDMA 或 URMA 传输 +→ 接收端直接持有接收内存区域 +→ FlatBuffers/Cap'n Proto 在该区域建立只读视图 +→ 按需访问字段 +``` + +第一阶段建议优先适配 FlatBuffers，原因包括： + +- Complex 中大消息的共享内存流水线性能最好； +- 单一连续缓冲区更容易映射到 IOBuf、RDMA 和 URMA 注册内存； +- 建视图成本很低； +- 运行结果总体比 Cap'n Proto 稳定； +- 对现有 bRPC attachment 接口的改造复杂度相对较低。 + +Cap'n Proto 可作为第二阶段适配对象，需要进一步处理 word 对齐、segment、遍历限制和大消息稳定性。 + +## 10. 当前尚未完成的测试 + +以下内容尚未测试： + +- 两台物理机器之间的普通 TCP； +- 大请求 + 大响应的 Payload Echo； +- 多客户端并发和吞吐量饱和； +- CPU 绑核、NUMA 和内存亲和性控制； +- 多槽位共享内存流水线； +- 发送端直接在目标共享/注册内存中原地构建； +- 真实 RDMA 数据路径； +- 真实 URMA 数据路径； +- 最终 bRPC RemoteRegion/ZeroCopyAttachment 特性的 A/B 对照。 + +## 11. 总结 + +当前已经完成： + +1. 单进程中可分项统计的序列化和反序列化/建视图测试； +2. 单个序列化生产者进程与单个反序列化消费者进程的共享内存测试； +3. 客户端与服务端之间完整的 bRPC localhost TCP 请求—响应测试。 + +全部正式测试均覆盖 Protobuf、FlatBuffers、Cap'n Proto、Simple/Complex 和 64B～8MiB，正确性检查全部通过。 + +当前最重要的实验结论是： + +> FlatBuffers/Cap'n Proto 的建视图确实远快于 Protobuf 反序列化，但如果 bRPC attachment 仍需要额外内存复制，这一优势可能被完全抵消。社区特性应同时解决反序列化和缓冲区复制问题，而不是只替换编码格式。 + +本文档中的结果是 WSL2 本地基线，不能替代真实 RDMA/URMA 环境中的最终实验。 diff --git a/docs/cn/flatbuffers_zero_copy_benchmark/deserialization_only_report.md b/docs/cn/flatbuffers_zero_copy_benchmark/deserialization_only_report.md new file mode 100644 index 0000000000..164ea2a1a2 --- /dev/null +++ b/docs/cn/flatbuffers_zero_copy_benchmark/deserialization_only_report.md @@ -0,0 +1,164 @@ +# Protobuf / FlatBuffers / Cap'n Proto 单独反序列化测试总结 + +> 重新生成日期：2026-09-03 +> 数据来源：`ipc-run1.csv`、`ipc-run2.csv`、`ipc-run3.csv`，共 35,100 条记录，失败 0 条。 +> 统计口径：仅使用 `consumer_parse_or_view_ns`；不包含序列化、发布复制、字段访问、IPC 等待和 RPC。 + +## 1. 测试目的 + +本测试只比较消费者已经获得完整编码缓冲区后，将其转换成可读取消息所需的成本： + +- Protobuf：完整反序列化并构造 C++ 对象； +- FlatBuffers：验证缓冲区并取得根对象视图； +- Cap'n Proto：建立 Reader 并取得根对象视图。 + +本文将该指标统一称为 `parse_or_view_ns`。它不包含生产端序列化、跨进程发布、RPC 或完整 Payload 扫描。 + +## 2. 单独反序列化的定义 + +计时起点是消费者已经持有完整、可访问的编码缓冲区，计时终点是得到可供字段访问的消息对象或只读视图： + +```text +已有编码缓冲区 +→ 开始计时 +→ 解析或建立视图 +→ 得到根消息 +→ 停止计时 +``` + +不包括： + +- 发送端对象构造和序列化； +- 编码缓冲区生成； +- memcpy 到共享内存； +- 生产者/消费者等待； +- 部分字段读取和完整 Payload 扫描； +- TCP、bRPC、RDMA 或 URMA。 + +## 3. 三种格式的计时边界 + +### 3.1 Protobuf + +```text +创建空的 SimpleMessage/ComplexMessage +→ ParseFromArray(encoded_buffer) +→ 得到完整 C++ 对象树 +``` + +Protobuf 必须遍历 wire format、分配嵌套对象和字符串/数组，并把字段填充到新对象中。 + +### 3.2 FlatBuffers + +```text +创建 Verifier +→ VerifyBuffer() +→ GetRoot() +→ 得到指向原缓冲区的只读视图 +``` + +FlatBuffers 不创建完整对象副本，但当前测试把完整缓冲区验证计入 `parse_or_view_ns`。 + +### 3.3 Cap'n Proto + +```text +创建 FlatArrayMessageReader +→ getRoot() +→ 得到指向原缓冲区的 Reader +``` + +Cap'n Proto 数据必须满足 word 对齐要求，并设置足够的 traversal limit。当前计时不包含对整个消息进行与 FlatBuffers Verifier 完全等价的全量验证，因此二者的安全检查口径并不完全相同。 + +## 4. 测试流程 + +双进程测试采用： + +```text +Producer 将编码消息发布到 POSIX 共享内存 +→ 槽位状态变成 READY +→ Consumer 直接在共享区域执行解析/建视图 +→ 停止 parse/view 计时 +→ 另行测量完整访问 +→ checksum 校验 +``` + +本文只使用 CSV 中的 `consumer_parse_or_view_ns`，不把 `consumer_access_ns` 加入反序列化结果。 + +模型和 Payload 与序列化测试一致：Simple/Complex，64B～8MiB。正式测试运行三轮且所有 checksum 正确。 + +## 5. 代表性解析/建视图 P50 + +### 5.1 Simple 模型 + +| Payload | Protobuf 解析 | FlatBuffers 验证+建视图 | Cap'n Proto 建 Reader | +|---|---:|---:|---:| +| 64B | 0.122 μs | 0.082 μs | 0.071 μs | +| 4KiB | 0.427 μs | 0.082 μs | 0.071 μs | +| 64KiB | 3.24 μs | 0.080 μs | 0.090 μs | +| 1MiB | 30.10 μs | 0.085 μs | 0.246 μs | +| 8MiB | 603 μs | 0.511 μs | 3.37 μs | + +### 5.2 Complex 模型 + +| Payload | Protobuf 解析 | FlatBuffers 验证+建视图 | Cap'n Proto 建 Reader | +|---|---:|---:|---:| +| 64B | 3.39 μs | 0.992 μs | 0.070 μs | +| 4KiB | 4.24 μs | 1.71 μs | 0.070 μs | +| 64KiB | 8.72 μs | 1.20 μs | 0.096 μs | +| 1MiB | 37.64 μs | 1.09 μs | 0.235 μs | +| 8MiB | 620 μs | 4.44 μs | 3.12 μs | + +## 6. 主要结论 + +1. Protobuf 解析时间随 Payload 增大而明显增长，因为它需要扫描编码数据并构造完整对象。 +2. FlatBuffers 和 Cap'n Proto 主要建立指向原始缓冲区的视图，建视图成本显著更低。 +3. 8MiB Complex 中，Protobuf 约为 620 μs，FlatBuffers 约为 4.4 μs，Cap'n Proto 约为 3.1 μs；后两者比 Protobuf 低约两个数量级。 +4. FlatBuffers 的 Complex 建视图数据包含 Verifier，因此比只建立 Reader 的 Cap'n Proto 更高。 +5. “建视图很快”不等于“完整处理消息不需要时间”。如果业务读取全部 8MiB Payload，内存扫描成本仍然存在。 + +## 7. 反序列化与数据访问必须分开 + +消费者阶段分为： + +```text +parse_or_view_ns +→ 将缓冲区变成可读消息或视图 + +consumer_access_ns +→ 实际遍历字段和 Payload，计算 checksum +``` + +免反序列化主要优化第一部分。对于只读取少数字段的业务，FlatBuffers/Cap'n Proto 可以避免解析和复制未访问字段，收益可能很大；对于必须完整扫描大 Payload 的业务，访问内存的成本无法通过格式本身消除。 + +## 8. 与 bRPC 测试的关系 + +在当前 bRPC 测试中： + +- Protobuf 由 bRPC 在调用服务方法前自动解析，无法在服务方法中单独计时； +- FlatBuffers/Cap'n Proto 需要先从 bRPC IOBuf 复制到连续 vector，再建立视图； +- 额外复制会掩盖免反序列化收益。 + +共享内存测试能够单独观察解析/建视图成本，因此更清楚地证明免反序列化的潜力；bRPC 测试则衡量现有系统中的真实完整路径。 + +## 9. 对特性设计的意义 + +要让本测试中的低建视图成本在 bRPC、RDMA/URMA 中真正发挥作用，接收端必须能直接访问传输完成后的内存区域： + +```text +传输完成 +→ 接收端获得 RemoteRegion/ZeroCopyAttachment +→ 不复制到新的连续 vector +→ 直接验证并建立 FlatBuffers/Cap'n Proto 视图 +→ 按需访问字段 +``` + +FlatBuffers 适合作为第一阶段：它采用单一连续缓冲区、建视图成本低、验证模型清晰，且比 Cap'n Proto 更容易接入 IOBuf 和注册内存。Cap'n Proto 可在第二阶段处理对齐、segment 和 traversal limit 等问题。 + +## 10. 当前限制 + +- 测试位于 WSL2 本地共享内存，不代表跨机器或 RDMA/URMA 延迟； +- 使用单槽 ping-pong 和忙等待，没有测试并发与流水线饱和； +- 没有进行 CPU 绑核和 NUMA 控制； +- FlatBuffers 与 Cap'n Proto 的验证强度不完全一致； +- 本文的反序列化结果不包含字段访问时间，这是有意的指标隔离。 + +当前结论是：FlatBuffers/Cap'n Proto 的建视图成本确实远低于 Protobuf 完整解析；但最终社区方案还必须同时消除接收路径复制，才能在完整 RPC 中兑现这部分收益。 diff --git a/docs/cn/flatbuffers_zero_copy_benchmark/serialization_only_report.md b/docs/cn/flatbuffers_zero_copy_benchmark/serialization_only_report.md new file mode 100644 index 0000000000..ec5b254ff9 --- /dev/null +++ b/docs/cn/flatbuffers_zero_copy_benchmark/serialization_only_report.md @@ -0,0 +1,153 @@ +# Protobuf / FlatBuffers / Cap'n Proto 单独序列化测试总结 + +> 重新生成日期：2026-09-03 +> 数据来源：`ipc-run1.csv`、`ipc-run2.csv`、`ipc-run3.csv`，共 35,100 条记录，失败 0 条。 +> 统计口径：仅使用 `serialize_ns`；不包含发布复制、反序列化、数据访问、IPC 等待和 RPC。 + +## 1. 测试目的 + +本测试只比较三种方案从统一业务数据生成最终可传输编码缓冲区的成本： + +- Protobuf 3.21.12 +- FlatBuffers 25.12.19 +- Cap'n Proto 1.5.0 + +本文不讨论反序列化、共享内存发布、网络传输或 RPC。 + +## 2. 单独序列化的定义 + +本项目将单独序列化定义为： + +```text +准备好的原始 Payload +→ 开始计时 +→ 构造对应格式的 Simple/Complex 消息 +→ 填充 Header、Record、Metrics、Tag 和 Payload +→ 生成最终编码缓冲区 +→ 停止计时 +``` + +计时结果记录在 `serialize_ns`。它包括对象/Builder 创建、字段填充、内存分配、Payload 写入以及生成最终 wire-format 缓冲区。 + +不包括： + +- 将编码结果复制到另一块缓冲区； +- `publish_ns` 共享内存发布； +- Protobuf `ParseFromArray()`； +- FlatBuffers `Verifier`、`GetRoot()`； +- Cap'n Proto `FlatArrayMessageReader`； +- 部分或完整字段访问； +- IPC 等待、TCP、bRPC、RDMA 或 URMA。 + +## 3. 三种格式的计时边界 + +### 3.1 Protobuf + +```text +创建 SimpleMessage/ComplexMessage +→ 填充所有字段 +→ SerializeToString() +→ 得到 std::string 编码结果 +``` + +### 3.2 FlatBuffers + +```text +创建 FlatBufferBuilder +→ 创建 String、Vector 和 Table +→ Finish() +→ 得到 Builder 中的连续编码缓冲区 +``` + +FlatBuffers 没有与 Protobuf 完全相同的“先构造普通对象，再单独编码”阶段；Builder 构造过程本身就是最终内存布局生成过程。 + +### 3.3 Cap'n Proto + +```text +创建 MallocMessageBuilder +→ 初始化结构体和列表 +→ 填充所有字段 +→ messageToFlatArray() +→ 得到连续 word 数组 +``` + +## 4. 测试数据 + +模型： + +- Simple：Header + 单个连续字节数组； +- Complex：Header + 16 个嵌套 Record，每个 Record 包含名称、Metrics、samples、Tag 和一部分 Payload。 + +Payload 覆盖： + +```text +64B、128B、256B、512B、1KiB、2KiB、4KiB、8KiB、 +16KiB、32KiB、64KiB、128KiB、256KiB、512KiB、 +1MiB、2MiB、4MiB、8MiB +``` + +正式 IPC 数据共运行三轮。以下结果取三轮合并后的中位数 P50；每轮第一条进程启动等待记录不参与汇总。 + +## 5. 代表性结果 + +### 5.1 Simple 模型 + +| Payload | Protobuf | FlatBuffers | Cap'n Proto | +|---|---:|---:|---:| +| 64B | 0.176 μs | 0.110 μs | 0.161 μs | +| 4KiB | 0.324 μs | 0.172 μs | 0.233 μs | +| 64KiB | 18.80 μs | 20.77 μs | 17.91 μs | +| 1MiB | 0.640 ms | 0.592 ms | 0.590 ms | +| 8MiB | 7.26 ms | 6.86 ms | 6.78 ms | + +Simple 模型中三者差距总体有限。小消息中 FlatBuffers 最快；8MiB 时 FlatBuffers 和 Cap'n Proto 接近，均略快于 Protobuf。 + +### 5.2 Complex 模型 + +| Payload | Protobuf | FlatBuffers | Cap'n Proto | +|---|---:|---:|---:| +| 64B | 8.32 μs | 3.04 μs | 1.25 μs | +| 4KiB | 9.15 μs | 3.21 μs | 1.30 μs | +| 64KiB | 15.94 μs | 16.42 μs | 30.28 μs | +| 1MiB | 0.475 ms | 0.294 ms | 0.601 ms | +| 8MiB | 5.33 ms | 4.01 ms | 6.61 ms | + +Complex 小消息中 Cap'n Proto 最快，原因是固定嵌套结构的 Builder 构造成本较低；随着 Payload 增大，Cap'n Proto 的连续化成本上升。Complex 1MiB 和 8MiB 中 FlatBuffers 最快。 + +## 6. 主要结论 + +1. 没有一种格式在所有模型和 Payload 下始终最快。 +2. Simple 小消息：FlatBuffers 略优，但绝对差异只有几十到几百纳秒。 +3. Complex 小消息：Cap'n Proto 明显领先，FlatBuffers 次之，Protobuf 最慢。 +4. Complex 中大消息：FlatBuffers 最有优势；8MiB 比 Protobuf 快约 25%，比 Cap'n Proto 快约 39%。 +5. 大消息序列化时间主要由 Payload 写入、内存分配和最终缓冲区生成决定。 +6. Protobuf 对复杂小消息通常编码更紧凑，但紧凑程度和序列化耗时是不同指标。 + +## 7. 公平性说明 + +`serialize_ns` 是“从统一原始数据得到可传输缓冲区”的业务口径，不是只测一个库函数的微基准。这一口径适合比较真实发送端成本，但应注意： + +- Protobuf 构造普通消息对象后再次执行编码； +- FlatBuffers 直接通过 Builder 构造最终布局； +- Cap'n Proto 通过 Builder 构造消息后又执行 `messageToFlatArray()` 连续化。 + +三种库的编程模型不同，无法完全拆成语义相同的内部步骤。 + +## 8. 当前限制与下一步 + +当前生产端仍然执行： + +```text +生成临时编码缓冲区 +→ 后续再复制到共享内存或 bRPC IOBuf +``` + +因此测试已经隔离出序列化时间，但尚未测量“直接在目标共享内存或 RDMA/URMA 注册内存中原地构建”。下一步应为 FlatBuffers 提供目标内存分配器，比较： + +```text +临时缓冲区构建 + memcpy +vs. +直接在可发送/注册内存中构建 +``` + +该测试结果来自 WSL2 本地 CPU 和内存，不能直接视为远端 RDMA/URMA 性能结果。 diff --git a/docs/cn/flatbuffers_zero_copy_design.md b/docs/cn/flatbuffers_zero_copy_design.md new file mode 100644 index 0000000000..f9167aab67 --- /dev/null +++ b/docs/cn/flatbuffers_zero_copy_design.md @@ -0,0 +1,423 @@ +# bRPC FlatBuffers 零拷贝集成与远程内存演进方案 + +> 文档性质：社区 RFC / Feature Proposal 初稿 +> 建议标题：**FlatBuffers Zero-Copy View and Transport-Aware Buffer Integration for bRPC** +> 目标社区：Apache bRPC +> 实施原则：先完善 FlatBuffers + `SingleIOBuf`，再扩展 RDMA/URMA；不在一个 PR 中同时引入序列化框架、协议和新传输层。 + +## 1. 摘要 + +bRPC 已经合入 `SingleIOBuf`，并正在推进 FlatBuffers 的消息构造和协议接入。因此，本方案不重新发明一套 FlatBuffers RPC，而是补齐以下能力： + +1. 客户端直接在 bRPC 管理的连续缓冲区中构造 FlatBuffer，避免 `FlatBufferBuilder -> vector/string -> IOBuf` 的额外复制。 +2. 服务端把收到的连续消息作为只读 FlatBuffers View 暴露给业务代码，避免 `IOBuf -> vector/string -> GetRoot()` 的额外复制。 +3. 在进入业务方法前完成一次有边界的合法性校验，并把底层 Block 生命周期绑定到请求或异步 Closure。 +4. 为现有 bRPC RDMA SEND/RECV 路径提供注册内存分配策略；无法连续分配或消息过大时安全回退。 +5. 后续以独立实验特性增加 `RemoteRegion`，让大对象可以通过 RDMA/URMA 单边读取按需访问，而不是塞入普通 RPC 消息。 + +该方案的核心不是“完全没有序列化”，而是：FlatBuffers 仍需构造线格式，但接收端无需反序列化重建对象，并尽量让构造、传输和访问共享同一块内存。 + +## 2. 背景与现状 + +### 2.1 社区已有基础 + +- bRPC 1.17.0 已引入 `SingleIOBuf`，用于管理单个连续的 `IOBuf::Block`，这是 FlatBuffers 连续内存要求与 bRPC I/O 缓冲区之间的基础桥梁。 +- 社区 FlatBuffers 系列工作已经规划为三步：`SingleIOBuf`、FlatBuffers 消息构造 API、FlatBuffers 协议处理。 +- 因此新贡献应围绕接收 View、校验、生命周期、内存分配策略、基准测试以及 RDMA 适配展开，而不是另建一套相互竞争的接口。 + +### 2.2 当前实验发现的问题 + +现有测试覆盖 Protobuf、FlatBuffers、Cap'n Proto，包含 simple/complex 两类嵌套结构和 64 B～8 MiB payload。 + +本机 WSL 测试的关键现象： + +- 三轮 bRPC localhost TCP 测试共 35,100 条记录，正确性失败为 0。 +- complex 8 MiB 的 bRPC 路径 P50：Protobuf 约 6.69 ms，FlatBuffers 约 9.17 ms。 +- 同一负载在双进程共享内存路径中，complex 8 MiB P50：Protobuf 约 7.27 ms，FlatBuffers 约 5.52 ms。 +- complex 8 MiB 接收端 parse/view P50：Protobuf 约 620 us，FlatBuffers 约 4.4 us。 + +这说明 FlatBuffers 的只读 View 很快，但现有实验 RPC 路径仍把 FlatBuffer 放进 attachment，并在服务端复制为连续 `vector` 后访问。额外内存复制和大块分配掩盖了免反序列化收益。 + +上述结论是根据当前实验实现作出的工程推断，不能直接当作 bRPC 主干实现的性能结论；正式贡献必须用主干和社区正在评审的 FlatBuffers 分支重新复现。 + +## 3. 要解决的问题 + +### 3.1 功能问题 + +1. FlatBuffers 要求连续字节区，而普通 `IOBuf` 可能由多个 Block 组成。 +2. attachment 只是非结构化字节，并不能提供类型安全的 FlatBuffers RPC 方法签名。 +3. 直接 `GetRoot()` 不代表数据合法；网络输入必须校验。 +4. View 中的指针依赖底层消息内存，异步服务容易产生悬空引用。 +5. RDMA 注册内存、普通堆内存和远程 Region 的所有权与释放方式不同。 + +### 3.2 性能问题 + +需要消除或量化以下复制： + +```text +业务对象 + -> FlatBufferBuilder 内部缓冲区 + -> string/vector + -> IOBuf + -> Socket/RDMA 缓冲区 + -> 接收 IOBuf + -> string/vector + -> FlatBuffers View +``` + +理想的首阶段路径为： + +```text +业务对象 + -> SingleIOBuf-backed MessageBuilder + -> bRPC 协议头 + 同一数据 Block + -> 接收侧 SingleIOBuf + -> Verified FlatBuffers View +``` + +## 4. 范围与非目标 + +### 4.1 首版范围 + +- C++ 客户端和服务端。 +- `baidu_std` 协议或社区当前 FlatBuffers PR 选定的协议路径。 +- TCP localhost、TCP 双机和现有 bRPC RDMA SEND/RECV。 +- FlatBuffers schema 生成的 simple/complex RPC。 +- 同步和异步服务的内存生命周期测试。 +- 64 B～8 MiB 基准与错误输入测试。 + +### 4.2 首版非目标 + +- 不替代 Protobuf；Protobuf 继续承担 IDL、控制面或兼容路径。 +- 不声称发送端“免序列化”；FlatBuffers 构造本身仍有成本。 +- 不在首个 PR 中实现完整 URMA 传输层。 +- 不要求任意分段 `IOBuf` 都可直接成为一个 FlatBuffer。 +- 不把 FlatBuffers、Cap'n Proto、URMA 和 RDMA 同时塞进一个巨型 PR。 + +## 5. 用户接口设计 + +以下接口应尽量复用社区现有 `brpc::flatbuffers::MessageBuilder` 和 `Message`，最终名称以现有 PR 为准。 + +### 5.1 构造与发送 + +```cpp +brpc::flatbuffers::BuilderOptions options; +options.initial_capacity = payload_size; +options.protocol_headroom = 64; +options.storage = brpc::flatbuffers::StoragePolicy::kAuto; + +brpc::flatbuffers::MessageBuilder builder(options); +auto request = CreateRequest(builder, /* fields */); +builder.Finish(request); + +brpc::flatbuffers::Message message = builder.ReleaseMessage(); +stub.Exchange(&controller, &message, &response, nullptr); +``` + +`StoragePolicy::kAuto` 的语义： + +- 普通 TCP：使用适合 `SingleIOBuf` 的连续 Block。 +- RDMA 已启用且容量满足：优先从注册内存池分配。 +- 无法满足时：回退普通内存或现有序列化路径，并暴露统计计数。 + +### 5.2 接收与访问 + +```cpp +void Exchange(google::protobuf::RpcController* cntl_base, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + + auto root = request->GetVerifiedRoot(); + if (!root.ok()) { + static_cast(cntl_base) + ->SetFailed(EINVAL, "invalid FlatBuffers request"); + return; + } + Use(root->payload()); +} +``` + +建议增加的核心抽象： + +```cpp +struct VerifyOptions { + size_t max_message_bytes; + size_t max_depth; + size_t max_tables; +}; + +template +StatusOr> GetVerifiedRoot( + const VerifyOptions& options = {}) const; +``` + +`VerifiedView` 同时持有： + +- `const T*` 根对象； +- 底层 Block 的只读所有权引用； +- 已验证标记； +- 消息大小和可选 schema/type 标识。 + +不应向用户返回一个脱离所有权的裸指针。 + +## 6. 内部实现 + +### 6.1 发送端 + +1. `MessageBuilder` 使用现有 Slab/Block allocator 获取一个连续 Block。 +2. Block 前部预留 bRPC 协议头空间，FlatBuffers 从后续位置构造。 +3. `Finish()` 后冻结可写状态。 +4. `ReleaseMessage()` 转移 Block 引用，不复制 payload。 +5. 协议打包器只追加/引用该 Block，不调用 `to_string()` 或中间 `vector`。 + +必须增加调试断言或计数，确认消息打包期间没有发生 payload 字节复制。 + +### 6.2 接收端 + +1. 协议解析器识别消息类型、长度和可选 schema 标识。 +2. 若 payload 已在单个连续 Block 中，直接建立 `Message`。 +3. 若 payload 分段： + - 小消息可合并到一个连续 Block； + - 大消息默认回退并记录 `flatbuffers_receive_coalesce_bytes`； + - 不允许把不连续内存伪装成连续 FlatBuffer。 +4. 使用 `flatbuffers::Verifier` 做一次有上限校验。 +5. 业务方法得到 `VerifiedView`；请求完成或异步回调释放前，Block 必须存活。 + +### 6.3 生命周期状态 + +```text +Writable Builder + | + Finish + v +Frozen Message ---- send/in-flight ----> Received Message + | + Verify + v + Verified View + | + RPC/Closure 完成后释放 +``` + +约束： + +- Frozen 后不可修改。 +- View 不可跨越其 Block owner 生命周期。 +- 异步保存 View 时必须显式保留 owner，而不是只保存 `const T*`。 +- 同一个未声明线程安全的 builder 不得并发写。 + +### 6.4 协议元数据 + +首版建议只加入最少元数据： + +```text +encoding = flatbuffers +schema/type = stable type id（可选） +payload_size = N +flags = verified / compressed / remote-region +``` + +不要把 C++ RTTI 名字写入线协议。类型 ID 应稳定、跨编译器，并支持版本演进。压缩与零拷贝天然冲突：启用压缩时应明确退化为解压到新缓冲区。 + +## 7. RDMA 与 URMA 演进 + +### 7.1 现有 RDMA SEND/RECV + +bRPC RDMA 已经围绕 `IOBuf::Block` 和注册内存池实现零拷贝能力。FlatBuffers 可先复用该能力，但存在一个关键限制：FlatBuffer 需要一整块连续内存，而现有 RDMA 接收池常用固定大小 Block；8 MiB 消息不一定能由单个现有 Block 承载。 + +建议新增内部策略，而非立刻修改公开 API： + +```cpp +enum class RegisteredAllocationResult { + kRegisteredContiguous, + kNormalContiguous, + kSegmentedFallback, + kRejectedTooLarge, +}; +``` + +并提供： + +- 小/中消息注册连续块池； +- 大消息按需注册或大块池，带容量上限； +- 注册失败、内存压力或超限时回退； +- 指标记录实际走到的路径。 + +### 7.2 后续 RemoteRegion / URMA + +当 payload 很大且业务只访问少量字段时，把整个 8 MiB FlatBuffer主动发送到服务端仍不理想。后续可引入独立的远程区域描述符： + +```cpp +struct RemoteRegionDescriptor { + uint64_t region_id; + uint64_t remote_address; + uint64_t length; + uint32_t access_key; + uint32_t provider_id; // RDMA / URMA + uint64_t lease_id; +}; +``` + +控制面通过普通 bRPC 传递 descriptor，数据面由 provider 执行 RDMA/URMA Read。接收端可按需拉取 FlatBuffer 的索引或数据页，并通过 lease 保证远端内存仍有效。 + +这一阶段需要另行解决： + +- FlatBuffers 偏移访问跨远程页时的读取和缓存； +- lease、撤销、超时和断连清理； +- rkey/token 的认证与越界检查； +- 分页读取与预取策略； +- TCP fallback； +- URMA 设备能力探测和 provider 插件化。 + +因此 RemoteRegion 应是后续 RFC，而不是 FlatBuffers 首次集成的合入条件。 + +## 8. 安全与健壮性 + +必须包含以下保护： + +- 网络输入默认验证，不能只调用 `GetRoot()`。 +- 最大消息大小、最大嵌套深度和对象数量限制。 +- 长度加法、偏移和对齐的溢出检查。 +- schema/type 不匹配时明确失败。 +- fuzz：截断、随机偏移、超大 vector、非法 vtable。 +- Block 只读冻结，防止验证后修改（TOCTOU）。 +- RDMA/URMA descriptor 必须校验权限、长度、租约和连接身份。 +- 记录 fallback，避免“看起来是零拷贝，实际发生了合并复制”。 + +## 9. 可观测性 + +建议加入以下 bvar 或等价指标： + +- `flatbuffers_requests_total` +- `flatbuffers_verify_failures_total` +- `flatbuffers_builder_reallocations_total` +- `flatbuffers_send_copy_bytes` +- `flatbuffers_receive_coalesce_bytes` +- `flatbuffers_contiguous_fast_path_total` +- `flatbuffers_fallback_total{reason}` +- `flatbuffers_registered_block_total` +- `flatbuffers_registered_allocation_failures_total` +- `flatbuffers_remote_read_bytes`（后续） + +只有把复制字节数作为一等指标，基准结果才能说明是真正的零拷贝，而不是仅仅 API 名称如此。 + +## 10. 测试与验收 + +### 10.1 正确性矩阵 + +| 维度 | 取值 | +|---|---| +| Schema | simple、complex nested | +| Payload | 64 B～8 MiB，2 的幂 | +| Format | Protobuf、FlatBuffers；Cap'n Proto 仅作 benchmark 对照 | +| Transport | localhost TCP、双机 TCP、现有 RDMA | +| Invocation | sync、async | +| Buffer path | contiguous、segmented fallback、allocation failure | + +每个组合检查：字段值、checksum、encoded bytes、错误码和生命周期。 + +### 10.2 性能指标 + +分别报告，禁止只给一个模糊的“端到端”： + +- build/serialize latency； +- protocol pack latency； +- copied bytes； +- RPC round-trip latency； +- verify latency； +- first-field、sparse、full-scan access latency； +- QPS、CPU cycles、allocations、峰值内存； +- P50/P95/P99，而不只平均值。 + +建议首版验收目标： + +1. 所有正确性组合零失败。 +2. 连续快路径中不出现 payload 大小级别的 `IOBuf -> vector/string` 复制。 +3. complex 8 MiB FlatBuffers RPC 相比当前 attachment 实验至少降低 20% 的客户端构造至服务端访问总耗时；最终阈值以社区 CI/测试机复测为准。 +4. complex 8 MiB 服务端 view 初始化保持在微秒级，且不包含全量复制。 +5. Protobuf 和普通 attachment 基准无显著回退。 +6. ASan、UBSan、TSan（适用用例）及 fuzz 测试通过。 + +## 11. 社区贡献拆分 + +### PR 0：RFC 与可复现基准 + +- 先在 Issue/RFC 中对齐当前 #3196/#3197 的状态和接口。 +- 提交 simple/complex、64 B～8 MiB benchmark。 +- 增加 copied-bytes、allocation 和 verification 指标。 +- 明确现有 attachment 基准不是 FlatBuffers 原生集成结果。 + +### PR 1：API 加固与接收 View + +- 在现有 `Message` 上增加有界 verifier API。 +- 定义 owner-carrying `VerifiedView`。 +- 补充 null root、错误 schema、截断数据和异步生命周期测试。 +- 修复社区评审已发现的空字段和 descriptor 生命周期问题。 + +### PR 2：连续快路径 + +- `MessageBuilder -> SingleIOBuf -> protocol` 无中间 payload 复制。 +- 接收端连续 Block 直接建立 Message/View。 +- 分段数据合并与明确 fallback 指标。 +- TCP benchmark 和回归测试。 + +### PR 3：现有 RDMA 注册内存适配 + +- transport-aware 内部分配器。 +- 注册连续 Block 池、容量上限和失败回退。 +- RDMA 双机测试；没有 RDMA 设备的 CI 使用 mock allocator。 + +### PR 4：实验性 RemoteRegion provider + +- RDMA provider 和 URMA provider 统一接口。 +- descriptor、lease、权限和远程读状态机。 +- 仅在独立构建开关下启用，成熟后再讨论公共 API 稳定性。 + +## 12. 建议目录布局 + +```text +src/brpc/flatbuffers/ + message.h/.cpp + message_builder.h/.cpp + verified_view.h + verifier_options.h + block_allocator.h/.cpp + +test/flatbuffers/ + message_builder_test.cpp + verified_view_test.cpp + malformed_message_test.cpp + async_lifetime_test.cpp + protocol_roundtrip_test.cpp + +example/flatbuffers_c++/ + echo.fbs + client.cpp + server.cpp + +test/benchmark/ + flatbuffers_rpc_benchmark.cpp +``` + +实际路径应服从 #3196/#3197 已采用的目录，避免在它们合入前制造平行实现。 + +## 13. 向社区提交时的说明模板 + +> bRPC 已有 SingleIOBuf，并正在加入 FlatBuffers message/protocol support。本提案希望在现有实现上补充 verified zero-copy receive view、明确的 buffer lifetime、copy/fallback observability，以及现有 RDMA registered-block integration。我们的初步 benchmark 显示，FlatBuffers 在 8 MiB complex 消息上的 view 初始化只需微秒级，但 attachment 路径中的整块复制会掩盖这一优势。计划先提交可复现 benchmark 和 API/lifetime tests，再分别提交 TCP contiguous fast path、RDMA registered allocator，最后以实验 RFC 讨论 URMA RemoteRegion。 + +## 14. 推荐的近期行动 + +1. 把本地 bRPC 切到最新主干，在独立分支检查 `SingleIOBuf` 实际 API。 +2. 拉取或基于 #3196/#3197 分支构建，不从零复制一套 FlatBuffers service API。 +3. 将现有 benchmark 改成社区 MessageBuilder/Message API，删除服务端 `IOBuf -> vector`。 +4. 增加 copied-bytes 与 allocation 计数后重新跑 TCP 三轮。 +5. 整理最小复现、结果表和 flame graph，先发 Discussion/Issue 征求维护者意见。 +6. 获得接口方向确认后，从 PR 1 开始提交小而独立的改动。 + +## 15. 结论 + +该特性可行，但合适的社区贡献不是笼统的“给 bRPC 加 FlatBuffers”，因为基础工作已经存在。最有价值且可合入的方向是：让现有 FlatBuffers 消息真正贯通 `SingleIOBuf`、协议层和接收端只读 View；用验证、生命周期和可观测性保证它可安全用于生产；然后复用 bRPC RDMA 注册内存，最后再把 URMA/RDMA 单边远程内存作为独立演进层。 + +这一路线既能直接解释并改善当前 benchmark 中暴露的复制瓶颈，也能为后续“Over URMA/RDMA 通用免反序列化方案”提供稳定的消息对象和内存所有权基础。 diff --git a/example/benchmark_fb/CMakeLists.txt b/example/benchmark_fb/CMakeLists.txt new file mode 100644 index 0000000000..3ab1f8745d --- /dev/null +++ b/example/benchmark_fb/CMakeLists.txt @@ -0,0 +1,139 @@ +cmake_minimum_required(VERSION 2.8.10) +project(benchmark_fb C CXX) + +option(LINK_SO "Whether examples are linked dynamically" OFF) +option(WITH_ASAN "With AddressSanitizer" OFF) + +execute_process( + COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'" + OUTPUT_VARIABLE OUTPUT_PATH +) + +set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) + +include(FindThreads) +include(FindProtobuf) + +# include current directory for generated files +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +# Search for libthrift* by best effort. If it is not found and brpc is +# compiled with thrift protocol enabled, a link error would be reported. +find_library(THRIFT_LIB NAMES thrift) +if (NOT THRIFT_LIB) + set(THRIFT_LIB "") +endif() + +find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) +if(LINK_SO) + find_library(BRPC_LIB NAMES brpc) +else() + find_library(BRPC_LIB NAMES libbrpc.a brpc) +endif() +if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) + message(FATAL_ERROR "Fail to find brpc") +endif() +include_directories(${BRPC_INCLUDE_PATH}) + +find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) +find_library(GFLAGS_LIBRARY NAMES gflags libgflags) +if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) + message(FATAL_ERROR "Fail to find gflags") +endif() +include_directories(${GFLAGS_INCLUDE_PATH}) + +# Find FlatBuffers +find_path(FLATBUFFERS_INCLUDE_PATH flatbuffers/flatbuffers.h) +find_library(FLATBUFFERS_LIBRARY NAMES flatbuffers) +if((NOT FLATBUFFERS_INCLUDE_PATH) OR (NOT FLATBUFFERS_LIBRARY)) + message(FATAL_ERROR "Fail to find flatbuffers") +endif() +include_directories(${FLATBUFFERS_INCLUDE_PATH}) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + include(CheckFunctionExists) + CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) + if(NOT HAVE_CLOCK_GETTIME) + set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") + endif() +endif() + +# set(CMAKE_CXX_FLAGS "${DEFINE_CLOCK_GETTIME} -g -O0 -D__const__=__unused__ -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") +set(CMAKE_CXX_FLAGS "${DEFINE_CLOCK_GETTIME} -DNDEBUG -O2 -D__const__=__unused__ -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") + +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") + +if (WITH_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") +endif() + +if(CMAKE_VERSION VERSION_LESS "3.1.3") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() +else() + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +endif() + +find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) +find_library(LEVELDB_LIB NAMES leveldb) +if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) + message(FATAL_ERROR "Fail to find leveldb") +endif() +include_directories(${LEVELDB_INCLUDE_PATH}) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(OPENSSL_ROOT_DIR + "/usr/local/opt/openssl" # Homebrew installed OpenSSL + ) +endif() + +find_package(OpenSSL) +include_directories(${OPENSSL_INCLUDE_DIR}) + +set(DYNAMIC_LIB + ${CMAKE_THREAD_LIBS_INIT} + ${GFLAGS_LIBRARY} + ${PROTOBUF_LIBRARIES} + ${LEVELDB_LIB} + ${OPENSSL_CRYPTO_LIBRARY} + ${OPENSSL_SSL_LIBRARY} + ${FLATBUFFERS_LIBRARY} + ${THRIFT_LIB} + dl + ) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(DYNAMIC_LIB ${DYNAMIC_LIB} + pthread + "-framework CoreFoundation" + "-framework CoreGraphics" + "-framework CoreData" + "-framework CoreText" + "-framework Security" + "-framework Foundation" + "-Wl,-U,_MallocExtension_ReleaseFreeMemory" + "-Wl,-U,_ProfilerStart" + "-Wl,-U,_ProfilerStop" + "-Wl,-U,__Z13GetStackTracePPvii" + "-Wl,-U,_mallctl" + "-Wl,-U,_malloc_stats_print" + ) +endif() + +set(FLATBUFFERS_SOURCES + test.brpc.fb.cpp + test_generated.h + test.brpc.fb.h +) + +add_executable(client client.cpp ${FLATBUFFERS_SOURCES}) +add_executable(server server.cpp ${FLATBUFFERS_SOURCES}) + +target_link_libraries(client ${BRPC_LIB} ${DYNAMIC_LIB}) +target_link_libraries(server ${BRPC_LIB} ${DYNAMIC_LIB}) \ No newline at end of file diff --git a/example/benchmark_fb/client.cpp b/example/benchmark_fb/client.cpp new file mode 100644 index 0000000000..af1a0f97ff --- /dev/null +++ b/example/benchmark_fb/client.cpp @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include "test.brpc.fb.h" + + +DEFINE_int32(thread_num, 1, "Number of threads to send requests"); +DEFINE_int32(attachment_size, 0, "Carry so many byte attachment along with requests"); +DEFINE_int32(request_size, 16, "Bytes of each request"); +DEFINE_string(servers, "0.0.0.0:8002", "IP Address of server"); +DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds"); +DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); +DEFINE_bool(verify_response, true, + "Verify FlatBuffers response and compare all fields"); +DEFINE_bool(corrupt_request, false, + "Corrupt the FlatBuffers root offset for negative testing"); +DEFINE_int32(dummy_port, -1, "Launch dummy server at this port"); + +std::string g_request; +butil::IOBuf g_attachment; + +bvar::LatencyRecorder g_latency_recorder("client"); +bvar::LatencyRecorder g_msg_recorder("msg"); +bvar::Adder g_error_count("client_error_count"); + +static void* sender(void* arg) { + test::BenchmarkServiceStub stub(static_cast(arg)); + int log_id = 0; + while (!brpc::IsAskedToQuit()) { + brpc::Controller cntl; + brpc::flatbuffers::Message response; + + cntl.set_log_id(log_id++); + cntl.request_attachment().append(g_attachment); + + uint64_t msg_begin_ns = butil::cpuwide_time_ns(); + brpc::flatbuffers::MessageBuilder mb; + auto message = mb.CreateString(g_request); + auto req = test::CreateBenchmarkRequest(mb, 123, 333, 1111, 2222, 0, message); + mb.Finish(req); + brpc::flatbuffers::Message request = mb.ReleaseMessage(); + + if (FLAGS_corrupt_request) { + CHECK_GE(request.size(), sizeof(uint32_t)); + uint8_t* data = + static_cast(request.mutable_data()); + + // Destroy the FlatBuffers root-table offset. + data[0] = 0xff; + data[1] = 0xff; + data[2] = 0xff; + data[3] = 0xff; + } + + uint64_t msg_end_ns = butil::cpuwide_time_ns(); + stub.Test(&cntl, &request, &response, NULL); + + if (FLAGS_corrupt_request) { + if (cntl.Failed()) { + LOG(INFO) + << "Corrupt FlatBuffers request rejected: " + << cntl.ErrorText(); + bthread_usleep(50000); + continue; + } + + LOG(FATAL) + << "Corrupt FlatBuffers request was unexpectedly accepted"; + } + + if (!cntl.Failed()) { + if (FLAGS_verify_response) { + CHECK(response.Verify()) + << "Invalid FlatBuffers response"; + + const test::BenchmarkResponse* response_root = + response.GetRoot(); + + CHECK(response_root != nullptr); + CHECK_EQ(123, response_root->opcode()); + CHECK_EQ(333, response_root->echo_attachment()); + CHECK_EQ(1111, response_root->attachment_size()); + CHECK_EQ(2222, response_root->request_id()); + CHECK_EQ(0, response_root->reserved()); + CHECK(response_root->message() != nullptr); + CHECK_EQ(g_request, response_root->message()->str()); + } + + g_latency_recorder << cntl.latency_us(); + g_msg_recorder << (msg_end_ns - msg_begin_ns); + } else { + g_error_count << 1; + CHECK(brpc::IsAskedToQuit()) + << "error=" << cntl.ErrorText() << " latency=" << cntl.latency_us(); + // We can't connect to the server, sleep a while. Notice that this + // is a specific sleeping to prevent this thread from spinning too + // fast. You should continue the business logic in a production + // server rather than sleeping. + bthread_usleep(50000); + } + } + return NULL; +} + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + // Print parameter information in one line + LOG(INFO) << "Parameters - request_size : " << FLAGS_request_size + << ", attachment_size: " << FLAGS_attachment_size + << ", thread_num: " << FLAGS_thread_num; + + // A Channel represents a communication line to a Server. Notice that + // Channel is thread-safe and can be shared by all threads in your program. + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.connection_type = ""; + options.connect_timeout_ms = std::min(FLAGS_timeout_ms / 2, 100); + options.timeout_ms = FLAGS_timeout_ms; + options.max_retry = FLAGS_max_retry; + if (channel.Init(FLAGS_servers.c_str(), &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + if (FLAGS_attachment_size > 0) { + void* _attachment_addr = malloc(FLAGS_attachment_size); + if (!_attachment_addr) { + LOG(ERROR) << "Fail to alloc _attachment from system heap"; + return -1; + } + g_attachment.append(_attachment_addr, FLAGS_attachment_size); + free(_attachment_addr); + } + if (FLAGS_request_size < 0) { + LOG(ERROR) << "Bad request_size=" << FLAGS_request_size; + return -1; + } + g_request.resize(FLAGS_request_size, 'r'); + + if (FLAGS_dummy_port >= 0) { + brpc::StartDummyServerAt(FLAGS_dummy_port); + } + + std::vector bids; + bids.resize(FLAGS_thread_num); + for (int i = 0; i < FLAGS_thread_num; ++i) { + if (bthread_start_background(&bids[i], NULL, sender, &channel) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + } + + while (!brpc::IsAskedToQuit()) { + sleep(1); + LOG(INFO) << "Sending request at qps=" << (g_latency_recorder.qps(1) / 1000) + << "k latency=" << g_latency_recorder.latency(1) << "us" + << " msg latency=" << g_msg_recorder.latency(1) << "ns"; + } + + LOG(INFO) << "Client is going to quit"; + for (int i = 0; i < FLAGS_thread_num; ++i) { + bthread_join(bids[i], NULL); + } + + LOG(INFO) << "Average QPS: " << (g_latency_recorder.qps()/1000) << "k" + << " Average latency: " << g_latency_recorder.latency() << "us" + << " msg latency: " << g_msg_recorder.latency() << "ns"; + + return 0; +} diff --git a/example/benchmark_fb/server.cpp b/example/benchmark_fb/server.cpp new file mode 100644 index 0000000000..4e4d5e5650 --- /dev/null +++ b/example/benchmark_fb/server.cpp @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include "test.brpc.fb.h" + +DEFINE_bool(echo_attachment, true, "Echo attachment as well"); +DEFINE_int32(port, 8080, "TCP Port of this server"); +DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no " + "read/write operations during the last `idle_timeout_s'"); +DEFINE_int32(max_concurrency, 0, "Limit of request processing in parallel"); +DEFINE_int32(internal_port, -1, "Only allow builtin services at this port"); + +namespace test{ +class BenchmarkServiceImpl : public BenchmarkService { +public: + BenchmarkServiceImpl() {} + ~BenchmarkServiceImpl() {} + + void Test(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request_base, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + brpc::Controller* cntl = + static_cast(controller); + if (request_base == nullptr || + !request_base->Verify()) { + controller->SetFailed( + "Invalid FlatBuffers BenchmarkRequest"); + return; + } + + const test::BenchmarkRequest* request = + request_base->GetRoot(); + // Set Response Message + brpc::flatbuffers::MessageBuilder mb_; + const auto* msg = request->message(); + const char* req_str = msg ? msg->c_str() : ""; + auto message = mb_.CreateString(req_str); + auto resp = test::CreateBenchmarkResponse(mb_, request->opcode(), + request->echo_attachment(), request->attachment_size(), + request->request_id(),request->reserved(), message); + mb_.Finish(resp); + *response = mb_.ReleaseMessage(); + if (FLAGS_echo_attachment) { + cntl->response_attachment().append(cntl->request_attachment()); + } + } +}; + +} + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + + // Generally you only need one Server. + brpc::Server server; + + // Instance of your service. + test::BenchmarkServiceImpl benchmark_service_impl; + + if (server.AddService(&benchmark_service_impl, + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + LOG(ERROR) << "Fail to add service"; + return -1; + } + + // Start the server. + brpc::ServerOptions options; + options.idle_timeout_sec = FLAGS_idle_timeout_s; + options.max_concurrency = FLAGS_max_concurrency; + options.internal_port = FLAGS_internal_port; + + if (server.Start(FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to start Server"; + return -1; + } + + // Wait until Ctrl-C is pressed, then Stop() and Join() the server. + server.RunUntilAskedToQuit(); + return 0; + +} diff --git a/example/benchmark_fb/test.brpc.fb.cpp b/example/benchmark_fb/test.brpc.fb.cpp new file mode 100644 index 0000000000..28d040d7ab --- /dev/null +++ b/example/benchmark_fb/test.brpc.fb.cpp @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated by the BRPC C++ plugin. +// If you make any local change, they will be lost. +// source: test.fbs +#include "test.brpc.fb.h" +#include +#include + +namespace test { + +static brpc::flatbuffers::ServiceDescriptor* file_level_service_descriptors_my_2eproto[1] = {NULL}; +struct FileLevelServiceDescriptorsCleanup_my_2eproto { + ~FileLevelServiceDescriptorsCleanup_my_2eproto() { + for (int i = 0; i < 1; ++i) { + delete file_level_service_descriptors_my_2eproto[i]; + file_level_service_descriptors_my_2eproto[i] = NULL; + } + } +}; +static FileLevelServiceDescriptorsCleanup_my_2eproto + file_level_service_descriptors_cleanup_my_2eproto; +static std::once_flag file_level_service_descriptors_BenchmarkService_my_2eproto_once_flag; + +BenchmarkService::~BenchmarkService() {} + +const brpc::flatbuffers::ServiceDescriptor* BenchmarkService::descriptor() { + std::call_once(file_level_service_descriptors_BenchmarkService_my_2eproto_once_flag, []() { + const brpc::flatbuffers::BrpcDescriptorTable desc_table = { + "test.", "BenchmarkService", "Test"}; + if (brpc::flatbuffers::parse_service_descriptors(desc_table, &file_level_service_descriptors_my_2eproto[0])) { + std::cout << "ERROR: " << "Fail to parse_service_descriptors" << std::endl; + } + }); + return file_level_service_descriptors_my_2eproto[0]; +} + +const brpc::flatbuffers::ServiceDescriptor* BenchmarkService::GetDescriptor() { + return descriptor(); +} + +void BenchmarkService::Test(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) { + controller->SetFailed("method Test() not implemented."); + std::cout << "ERROR: " << "method Test() not implemented." << std::endl; +} + +void BenchmarkService::FBCallMethod(const brpc::flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) { + FLATBUFFERS_ASSERT(method->service() == file_level_service_descriptors_my_2eproto[0]); + switch(method->index()) { + case 0: + Test(controller, request, response, done); + break; + default: + std::cout << "ERROR: " << "Bad method index; this should never happen." << std::endl; + break; + } +} +BenchmarkServiceStub::BenchmarkServiceStub(brpc::flatbuffers::RpcChannel* channel) + : channel_(channel), owns_channel_(false) {} + +BenchmarkServiceStub::BenchmarkServiceStub( + brpc::flatbuffers::RpcChannel* channel, + brpc::flatbuffers::Service::ChannelOwnership ownership) + : channel_(channel), + owns_channel_(ownership == brpc::flatbuffers::Service::STUB_OWNS_CHANNEL) {} + +BenchmarkServiceStub::~BenchmarkServiceStub(){ + if (owns_channel_) {delete channel_;} +} + +void BenchmarkServiceStub::Test(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) { + channel_->FBCallMethod(descriptor()->method(0), + controller, request, response, done); +} + +} // namespace test diff --git a/example/benchmark_fb/test.brpc.fb.h b/example/benchmark_fb/test.brpc.fb.h new file mode 100644 index 0000000000..ca28962de2 --- /dev/null +++ b/example/benchmark_fb/test.brpc.fb.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated by the BRPC C++ plugin. +// If you make any local change, they will be lost. +// source: test.fbs +#ifndef BRPC_test__INCLUDED +#define BRPC_test__INCLUDED +#include "test_generated.h" +#include +#include + +namespace test { + +class BenchmarkServiceStub; +class BenchmarkService : public brpc::flatbuffers::Service { +protected: + inline BenchmarkService() {}; +public: + virtual ~BenchmarkService(); + static const brpc::flatbuffers::ServiceDescriptor* descriptor(); + virtual void Test(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done); + + const brpc::flatbuffers::ServiceDescriptor* GetDescriptor(); + void FBCallMethod(const brpc::flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done); +private: + FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(BenchmarkService); +}; + +class BenchmarkServiceStub : public BenchmarkService { +public: + BenchmarkServiceStub(brpc::flatbuffers::RpcChannel* channel); + BenchmarkServiceStub(brpc::flatbuffers::RpcChannel* channel, + brpc::flatbuffers::Service::ChannelOwnership ownership); + ~BenchmarkServiceStub(); + inline brpc::flatbuffers::RpcChannel* channel() { return channel_; } + void Test(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done); + +private: + brpc::flatbuffers::RpcChannel* channel_; + bool owns_channel_; + FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(BenchmarkServiceStub); +}; + +} // namespace test + +#endif // BRPC_test__INCLUDED diff --git a/example/benchmark_fb/test.fbs b/example/benchmark_fb/test.fbs new file mode 100644 index 0000000000..b842b0360d --- /dev/null +++ b/example/benchmark_fb/test.fbs @@ -0,0 +1,24 @@ +namespace test; + +table BenchmarkRequest { + opcode:int; + echo_attachment:int; + attachment_size:long; + request_id:long; + + reserved:long; + message:string; +} + +table BenchmarkResponse { + opcode:int; + echo_attachment:int; + attachment_size:long; + request_id:long; + reserved:long; + message:string; +} + +rpc_service BenchmarkService { + Test(BenchmarkRequest):BenchmarkResponse; +} \ No newline at end of file diff --git a/example/benchmark_fb/test_generated.h b/example/benchmark_fb/test_generated.h new file mode 100644 index 0000000000..e71aaa8dc0 --- /dev/null +++ b/example/benchmark_fb/test_generated.h @@ -0,0 +1,250 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + +#ifndef FLATBUFFERS_GENERATED_TEST_TEST_H_ +#define FLATBUFFERS_GENERATED_TEST_TEST_H_ + +#include "flatbuffers/flatbuffers.h" + +// Ensure the included flatbuffers.h is the same version as when this file was +// generated, otherwise it may not be compatible. +static_assert(FLATBUFFERS_VERSION_MAJOR == 25 && + FLATBUFFERS_VERSION_MINOR == 12 && + FLATBUFFERS_VERSION_REVISION == 19, + "Non-compatible flatbuffers version included"); + +namespace test { + +struct BenchmarkRequest; +struct BenchmarkRequestBuilder; + +struct BenchmarkResponse; +struct BenchmarkResponseBuilder; + +struct BenchmarkRequest FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef BenchmarkRequestBuilder Builder; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_OPCODE = 4, + VT_ECHO_ATTACHMENT = 6, + VT_ATTACHMENT_SIZE = 8, + VT_REQUEST_ID = 10, + VT_RESERVED = 12, + VT_MESSAGE = 14 + }; + int32_t opcode() const { + return GetField(VT_OPCODE, 0); + } + int32_t echo_attachment() const { + return GetField(VT_ECHO_ATTACHMENT, 0); + } + int64_t attachment_size() const { + return GetField(VT_ATTACHMENT_SIZE, 0); + } + int64_t request_id() const { + return GetField(VT_REQUEST_ID, 0); + } + int64_t reserved() const { + return GetField(VT_RESERVED, 0); + } + const ::flatbuffers::String *message() const { + return GetPointer(VT_MESSAGE); + } + template + bool Verify(::flatbuffers::VerifierTemplate &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_OPCODE, 4) && + VerifyField(verifier, VT_ECHO_ATTACHMENT, 4) && + VerifyField(verifier, VT_ATTACHMENT_SIZE, 8) && + VerifyField(verifier, VT_REQUEST_ID, 8) && + VerifyField(verifier, VT_RESERVED, 8) && + VerifyOffset(verifier, VT_MESSAGE) && + verifier.VerifyString(message()) && + verifier.EndTable(); + } +}; + +struct BenchmarkRequestBuilder { + typedef BenchmarkRequest Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_opcode(int32_t opcode) { + fbb_.AddElement(BenchmarkRequest::VT_OPCODE, opcode, 0); + } + void add_echo_attachment(int32_t echo_attachment) { + fbb_.AddElement(BenchmarkRequest::VT_ECHO_ATTACHMENT, echo_attachment, 0); + } + void add_attachment_size(int64_t attachment_size) { + fbb_.AddElement(BenchmarkRequest::VT_ATTACHMENT_SIZE, attachment_size, 0); + } + void add_request_id(int64_t request_id) { + fbb_.AddElement(BenchmarkRequest::VT_REQUEST_ID, request_id, 0); + } + void add_reserved(int64_t reserved) { + fbb_.AddElement(BenchmarkRequest::VT_RESERVED, reserved, 0); + } + void add_message(::flatbuffers::Offset<::flatbuffers::String> message) { + fbb_.AddOffset(BenchmarkRequest::VT_MESSAGE, message); + } + explicit BenchmarkRequestBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateBenchmarkRequest( + ::flatbuffers::FlatBufferBuilder &_fbb, + int32_t opcode = 0, + int32_t echo_attachment = 0, + int64_t attachment_size = 0, + int64_t request_id = 0, + int64_t reserved = 0, + ::flatbuffers::Offset<::flatbuffers::String> message = 0) { + BenchmarkRequestBuilder builder_(_fbb); + builder_.add_reserved(reserved); + builder_.add_request_id(request_id); + builder_.add_attachment_size(attachment_size); + builder_.add_message(message); + builder_.add_echo_attachment(echo_attachment); + builder_.add_opcode(opcode); + return builder_.Finish(); +} + +inline ::flatbuffers::Offset CreateBenchmarkRequestDirect( + ::flatbuffers::FlatBufferBuilder &_fbb, + int32_t opcode = 0, + int32_t echo_attachment = 0, + int64_t attachment_size = 0, + int64_t request_id = 0, + int64_t reserved = 0, + const char *message = nullptr) { + auto message__ = message ? _fbb.CreateString(message) : 0; + return test::CreateBenchmarkRequest( + _fbb, + opcode, + echo_attachment, + attachment_size, + request_id, + reserved, + message__); +} + +struct BenchmarkResponse FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table { + typedef BenchmarkResponseBuilder Builder; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { + VT_OPCODE = 4, + VT_ECHO_ATTACHMENT = 6, + VT_ATTACHMENT_SIZE = 8, + VT_REQUEST_ID = 10, + VT_RESERVED = 12, + VT_MESSAGE = 14 + }; + int32_t opcode() const { + return GetField(VT_OPCODE, 0); + } + int32_t echo_attachment() const { + return GetField(VT_ECHO_ATTACHMENT, 0); + } + int64_t attachment_size() const { + return GetField(VT_ATTACHMENT_SIZE, 0); + } + int64_t request_id() const { + return GetField(VT_REQUEST_ID, 0); + } + int64_t reserved() const { + return GetField(VT_RESERVED, 0); + } + const ::flatbuffers::String *message() const { + return GetPointer(VT_MESSAGE); + } + template + bool Verify(::flatbuffers::VerifierTemplate &verifier) const { + return VerifyTableStart(verifier) && + VerifyField(verifier, VT_OPCODE, 4) && + VerifyField(verifier, VT_ECHO_ATTACHMENT, 4) && + VerifyField(verifier, VT_ATTACHMENT_SIZE, 8) && + VerifyField(verifier, VT_REQUEST_ID, 8) && + VerifyField(verifier, VT_RESERVED, 8) && + VerifyOffset(verifier, VT_MESSAGE) && + verifier.VerifyString(message()) && + verifier.EndTable(); + } +}; + +struct BenchmarkResponseBuilder { + typedef BenchmarkResponse Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_opcode(int32_t opcode) { + fbb_.AddElement(BenchmarkResponse::VT_OPCODE, opcode, 0); + } + void add_echo_attachment(int32_t echo_attachment) { + fbb_.AddElement(BenchmarkResponse::VT_ECHO_ATTACHMENT, echo_attachment, 0); + } + void add_attachment_size(int64_t attachment_size) { + fbb_.AddElement(BenchmarkResponse::VT_ATTACHMENT_SIZE, attachment_size, 0); + } + void add_request_id(int64_t request_id) { + fbb_.AddElement(BenchmarkResponse::VT_REQUEST_ID, request_id, 0); + } + void add_reserved(int64_t reserved) { + fbb_.AddElement(BenchmarkResponse::VT_RESERVED, reserved, 0); + } + void add_message(::flatbuffers::Offset<::flatbuffers::String> message) { + fbb_.AddOffset(BenchmarkResponse::VT_MESSAGE, message); + } + explicit BenchmarkResponseBuilder(::flatbuffers::FlatBufferBuilder &_fbb) + : fbb_(_fbb) { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset CreateBenchmarkResponse( + ::flatbuffers::FlatBufferBuilder &_fbb, + int32_t opcode = 0, + int32_t echo_attachment = 0, + int64_t attachment_size = 0, + int64_t request_id = 0, + int64_t reserved = 0, + ::flatbuffers::Offset<::flatbuffers::String> message = 0) { + BenchmarkResponseBuilder builder_(_fbb); + builder_.add_reserved(reserved); + builder_.add_request_id(request_id); + builder_.add_attachment_size(attachment_size); + builder_.add_message(message); + builder_.add_echo_attachment(echo_attachment); + builder_.add_opcode(opcode); + return builder_.Finish(); +} + +inline ::flatbuffers::Offset CreateBenchmarkResponseDirect( + ::flatbuffers::FlatBufferBuilder &_fbb, + int32_t opcode = 0, + int32_t echo_attachment = 0, + int64_t attachment_size = 0, + int64_t request_id = 0, + int64_t reserved = 0, + const char *message = nullptr) { + auto message__ = message ? _fbb.CreateString(message) : 0; + return test::CreateBenchmarkResponse( + _fbb, + opcode, + echo_attachment, + attachment_size, + request_id, + reserved, + message__); +} + +} // namespace test + +#endif // FLATBUFFERS_GENERATED_TEST_TEST_H_ diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index ff81e521d5..7bf8b8c00d 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -142,7 +142,7 @@ static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { } butil::MurmurHash3_x64_128_Update(&mm_ctx, buf.data(), buf.size()); buf.clear(); - + if (opt.has_ssl_options()) { const CertInfo& cert = opt.ssl_options().client_cert; if (!cert.certificate.empty()) { @@ -209,7 +209,7 @@ int Channel::InitChannelOptions(const ChannelOptions* options) { // Save has_error which will be overriden in later assignments to // connection_type. const bool has_error = _options.connection_type.has_error(); - + if (protocol->supported_connection_type & CONNECTION_TYPE_SINGLE) { _options.connection_type = CONNECTION_TYPE_SINGLE; } else if (protocol->supported_connection_type & CONNECTION_TYPE_POOLED) { @@ -477,11 +477,18 @@ static void HandleBackupRequest(void* arg) { bthread_id_error(correlation_id, EBACKUPREQUEST); } -void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, - google::protobuf::RpcController* controller_base, - const google::protobuf::Message* request, - google::protobuf::Message* response, - google::protobuf::Closure* done) { +template +void Channel::CallMethodInternal(const typename std::conditional::type* method, + google::protobuf::RpcController* controller_base, + const typename std::conditional::type* request, + typename std::conditional::type* response, + google::protobuf::Closure* done) { const int64_t start_send_real_us = butil::gettimeofday_us(); Controller* cntl = static_cast(controller_base); cntl->OnRPCBegin(start_send_real_us); @@ -541,22 +548,40 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, const int64_t start_send_us = butil::cpuwide_time_us(); std::string method_name; if (_get_method_name) { - method_name = butil::EnsureString(_get_method_name(method, cntl)); + if (is_pb) { + auto pb_method = reinterpret_cast(method); + method_name = butil::EnsureString(_get_method_name(pb_method, cntl)); + } else { + // FlatBuffers doesn't support _get_method_name yet + method_name = ""; + } + } else if (method) { - method_name = butil::EnsureString(method->full_name()); + if (is_pb) { + auto pb_method = reinterpret_cast(method); + method_name = butil::EnsureString(pb_method->full_name()); +#if BRPC_WITH_FLATBUFFERS + } else { + auto fb_method = reinterpret_cast(method); + method_name = butil::EnsureString(fb_method->full_name()); +#endif + } + } else { const static std::string NULL_METHOD_STR = "null-method"; method_name = NULL_METHOD_STR; } - std::shared_ptr span = Span::CreateClientSpan( + if (!method_name.empty()) { + std::shared_ptr span = Span::CreateClientSpan( method_name, start_send_real_us - start_send_us); - if (span) { - ControllerPrivateAccessor accessor(cntl); - span->set_log_id(cntl->log_id()); - span->set_base_cid(correlation_id); - span->set_protocol(_options.protocol); - span->set_start_send_us(start_send_us); - accessor.set_span(span); + if (span) { + ControllerPrivateAccessor accessor(cntl); + span->set_log_id(cntl->log_id()); + span->set_base_cid(correlation_id); + span->set_protocol(_options.protocol); + span->set_start_send_us(start_send_us); + accessor.set_span(span); + } } } // Override some options if they haven't been set by Controller @@ -575,11 +600,22 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, if (cntl->connection_type() == CONNECTION_TYPE_UNKNOWN) { cntl->set_connection_type(_options.connection_type); } - cntl->_response = response; + cntl->_done = done; cntl->_pack_request = _pack_request; - cntl->_method = method; cntl->_auth = _options.auth; + // Use reinterpret_cast to avoid template instantiation errors + // The actual type is guaranteed by the is_pb parameter + if (is_pb) { + cntl->_method = reinterpret_cast(method); + cntl->_response = reinterpret_cast(response); +#if BRPC_WITH_FLATBUFFERS + } else { + cntl->_fb_method = reinterpret_cast(method); + cntl->_fb_response = reinterpret_cast(response); + cntl->set_use_flatbuffer(); +#endif + } if (SingleServer()) { cntl->_single_server_id = _server_id; @@ -663,6 +699,24 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, } } +void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done) { + CallMethodInternal(method, controller_base, request, response, done); +} + +#if BRPC_WITH_FLATBUFFERS +void Channel::FBCallMethod(const brpc::flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) { + CallMethodInternal(method, controller_base, request, response, done); +} +#endif + void Channel::Describe(std::ostream& os, const DescribeOptions& opt) const { os << "Channel["; if (SingleServer()) { @@ -692,4 +746,26 @@ int Channel::CheckHealth() { } } +// CallMethodInternal instance for pb and fb +template +void Channel::CallMethodInternal( + const google::protobuf::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done +); + +#if BRPC_WITH_FLATBUFFERS +// CallMethodInternal instance for pb and fb +template +void Channel::CallMethodInternal( + const brpc::flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done +); +#endif + } // namespace brpc diff --git a/src/brpc/channel.h b/src/brpc/channel.h index 47f262f627..c4ba3424b5 100644 --- a/src/brpc/channel.h +++ b/src/brpc/channel.h @@ -22,6 +22,7 @@ // To brpc developers: This is a header included by user, don't depend // on internal structures, use opaque pointers instead. +#include "butil/config.h" #include // std::ostream #include "bthread/errno.h" // Redefine errno #include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr @@ -38,6 +39,16 @@ #include "brpc/naming_service_filter.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/details/flatbuffers_impl.h" +#else +namespace brpc { +namespace flatbuffers { +class Message; +class MethodDescriptor; +} // namespace flatbuffers +} // namespace brpc +#endif namespace brpc { @@ -50,7 +61,7 @@ struct ChannelOptions { // Default: 200 (milliseconds) // Maximum: 0x7fffffff (roughly 30 days) int32_t connect_timeout_ms; - + // Max duration of RPC over this Channel. -1 means wait indefinitely. // Overridable by Controller.set_timeout_ms(). // Default: 500 (milliseconds) @@ -73,9 +84,9 @@ struct ChannelOptions { // Default: 3 // Maximum: INT_MAX int max_retry; - - // When the error rate of a server node is too high, isolate the node. - // Note that this isolation is GLOBAL, the node will become unavailable + + // When the error rate of a server node is too high, isolate the node. + // Note that this isolation is GLOBAL, the node will become unavailable // for all channels running in this process during the isolation. // Default: false bool enable_circuit_breaker; @@ -92,7 +103,7 @@ struct ChannelOptions { // Possible values: "single", "pooled", "short". AdaptiveConnectionType connection_type; - // Channel.Init() succeeds even if there's no server in the NamingService. + // Channel.Init() succeeds even if there's no server in the NamingService. // E.g. the BNS directory is empty. All RPC over the channel will fail before // new nodes being added to the NamingService. // Default: true (false before r32470) @@ -146,7 +157,7 @@ struct ChannelOptions { // Default: "" std::string connection_group; - // Set the health check param according to the channel granularity. + // Set the health check param according to the channel granularity. // Its priority is higher than FLAGS_health_check_path and FLAGS_health_check_timeout_ms. // When it is not set, FLAGS_health_check_path and FLAGS_health_check_timeout_ms will take effect. HealthCheckOption hc_option; @@ -175,7 +186,11 @@ struct ChannelOptions { // channel.Init("bns://rdev.matrix.all", "rr", nullptr/*default options*/); // MyService_Stub stub(&channel); // stub.MyMethod(&controller, &request, &response, nullptr); -class Channel : public ChannelBase { +class Channel : public ChannelBase +#if BRPC_WITH_FLATBUFFERS + , public brpc::flatbuffers::RpcChannel +#endif +{ friend class Controller; friend class SelectiveChannel; public: @@ -195,7 +210,7 @@ friend class SelectiveChannel; // Connect this channel to a group of servers whose addresses can be // accessed via `naming_service_url' according to its protocol. Use the - // method specified by `load_balancer_name' to distribute traffic to + // method specified by `load_balancer_name' to distribute traffic to // servers. Use default options if `options' is nullptr. // Supported naming service("protocol://service_name"): // bns:// # Baidu Naming Service @@ -212,11 +227,11 @@ friend class SelectiveChannel; // "" or nullptr # treat `naming_service_url' as `server_addr_and_port' // # Init(xxx, "", options) and Init(xxx, nullptr, options) // # are exactly same with Init(xxx, options) - int Init(const char* naming_service_url, + int Init(const char* naming_service_url, const char* load_balancer_name, const ChannelOptions* options); - // Call `method' of the remote service with `request' as input, and + // Call `method' of the remote service with `request' as input, and // `response' as output. `controller' contains options and extra data. // If `done' is not nullptr, this method returns after request was sent // and `done->Run()' will be called when the call finishes, otherwise @@ -228,6 +243,14 @@ friend class SelectiveChannel; google::protobuf::Closure* done); // Get current options. +#if BRPC_WITH_FLATBUFFERS +void FBCallMethod(const brpc::flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done); +#endif + const ChannelOptions& options() const { return _options; } void Describe(std::ostream&, const DescribeOptions&) const; @@ -240,7 +263,7 @@ friend class SelectiveChannel; protected: bool SingleServer() const { return _lb.get() == nullptr; } - // Pick a server using `lb' and then send RPC. Wait for response when + // Pick a server using `lb' and then send RPC. Wait for response when // sending synchronous RPC. // NOTE: DO NOT directly use `controller' after this call when // sending asynchronous RPC (controller->_done != nullptr) since @@ -254,6 +277,19 @@ friend class SelectiveChannel; const ChannelOptions* options, int raw_port = -1); + template + inline void CallMethodInternal(const typename std::conditional::type* method, + google::protobuf::RpcController* controller_base, + const typename std::conditional::type* request, + typename std::conditional::type* response, + google::protobuf::Closure* done); + std::string _service_name; std::string _scheme; butil::EndPoint _server_address; diff --git a/src/brpc/channel_base.h b/src/brpc/channel_base.h index ed6ff24e40..3b5a13f8aa 100644 --- a/src/brpc/channel_base.h +++ b/src/brpc/channel_base.h @@ -24,6 +24,7 @@ #include "butil/logging.h" #include // google::protobuf::RpcChannel #include "brpc/describable.h" +#include "brpc/details/flatbuffers_common.h" // To brpc developers: This is a header included by user, don't depend // on internal structures, use opaque pointers instead. diff --git a/src/brpc/controller.cpp b/src/brpc/controller.cpp index 4fff9fd2f4..2536ad5051 100644 --- a/src/brpc/controller.cpp +++ b/src/brpc/controller.cpp @@ -501,6 +501,7 @@ void Controller::ResetPods() { _inheritable.Reset(); _pchan_sub_count = 0; _response = nullptr; + _fb_response = nullptr; _done = nullptr; _sender = nullptr; _request_code = 0; @@ -510,6 +511,7 @@ void Controller::ResetPods() { _accessed = nullptr; _pack_request = nullptr; _method = nullptr; + _fb_method = nullptr; _auth = nullptr; _idl_names = idl_single_req_single_res; _idl_result = IDL_VOID_RESULT; @@ -1481,7 +1483,15 @@ void Controller::IssueRPC(int64_t start_realtime_us) { // Make request butil::IOBuf packet; SocketMessage* user_packet = nullptr; - _pack_request(&packet, &user_packet, cid.value, _method, this, + // Compatibility shim for the FlatBuffers prototype. The FlatBuffers + // packer converts this opaque value back without dereferencing it as a + // protobuf descriptor. + const google::protobuf::MethodDescriptor* method_desc = + is_use_flatbuffer() + ? reinterpret_cast( + _fb_method) + : _method; + _pack_request(&packet, &user_packet, cid.value, method_desc, this, _request_buf, using_auth); // TODO: PackRequest may accept SocketMessagePtr<>? SocketMessagePtr<> user_packet_guard(user_packet); diff --git a/src/brpc/controller.h b/src/brpc/controller.h index c05dbb75a7..7b264cc0d1 100644 --- a/src/brpc/controller.h +++ b/src/brpc/controller.h @@ -48,6 +48,7 @@ #include "brpc/grpc.h" #include "brpc/kvmap.h" #include "brpc/rpc_dump.h" +#include "brpc/details/flatbuffers_common.h" // EAUTH is defined in MAC #ifndef EAUTH @@ -119,7 +120,7 @@ enum BindSockAction { typedef butil::FlatMap UserFieldsMap; // A Controller mediates a single method call. The primary purpose of -// the controller is to provide a way to manipulate settings per RPC-call +// the controller is to provide a way to manipulate settings per RPC-call // and to find out about RPC-level errors. class Controller : public google::protobuf::RpcController/*non-copyable*/ { friend class Channel; @@ -173,6 +174,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Whether set_response_checksum_type()'s checksum also covers the // attachment. See set_response_checksum_attachment(). static const uint32_t FLAGS_RESPONSE_CHECKSUM_WITH_ATTACHMENT = (1 << 24); + static const uint32_t FLAGS_USE_FLATBUFFER = (1 << 25); public: struct Inheritable { @@ -190,7 +192,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); Controller(); Controller(const Inheritable& parent_ctx); ~Controller(); - + // ------------------------------------------------------------------ // Client-side methods // These calls shall be made from the client side only. Their results @@ -249,6 +251,8 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Response of the RPC call (passed to CallMethod) google::protobuf::Message* response() const { return _response; } + brpc::flatbuffers::Message* fb_response() const { return _fb_response; } + // An identifier to send to server along with request. This is widely used // throughout baidu's servers to tag a searching session (a series of // queries following the topology of servers) with a same log_id. @@ -296,7 +300,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); } bool has_request_code() const { return has_flag(FLAGS_REQUEST_CODE); } uint64_t request_code() const { return _request_code; } - + // Mutable header of http request. HttpHeader& http_request() { if (_http_request == nullptr) { @@ -337,14 +341,16 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Get the called method. May-be nullptr for non-pb services. const google::protobuf::MethodDescriptor* method() const { return _method; } + const brpc::flatbuffers::MethodDescriptor* fb_method() const { return _fb_method; } + // Get the controllers for accessing sub channels in combo channels. // Ordinary channel: // sub_count() is 0 and sub() is always nullptr. // ParallelChannel/PartitionChannel: - // sub_count() is #sub-channels and sub(i) is the controller for + // sub_count() is #sub-channels and sub(i) is the controller for // accessing i-th sub channel inside ParallelChannel, if i is outside // [0, sub_count() - 1], sub(i) is nullptr. - // NOTE: You must test sub() against nullptr, ALWAYS. Even if i is inside + // NOTE: You must test sub() against nullptr, ALWAYS. Even if i is inside // range, sub(i) can still be nullptr: // * the rpc call may fail and terminate before accessing the sub channel // * the sub channel was skipped @@ -385,16 +391,16 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // - Any error occurred will destroy the reader by calling r->Destroy(). // - r->Destroy() is guaranteed to be called once and only once. void ReadProgressiveAttachmentBy(ProgressiveReader* r); - + // True if ReadProgressiveAttachmentBy() was ever called successfully. bool has_progressive_reader() const { return has_flag(FLAGS_PROGRESSIVE_READER); } - + // RPC may fail with EOVERCROWDED if the socket to write is too full // (limited by -socket_max_unwritten_bytes). In some scenarios, user // may wish to suppress the error completely. To do this, call this // method before doing the RPC. void ignore_eovercrowded() { add_flag(FLAGS_IGNORE_EOVERCROWDED); } - + // Set if the field of bytes in protobuf message should be encoded // to base64 string in HTTP request. void set_pb_bytes_to_base64(bool f) { set_flag(FLAGS_PB_BYTES_TO_BASE64, f); } @@ -409,14 +415,14 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // of json in HTTP response. void set_pb_jsonify_empty_array(bool f) { set_flag(FLAGS_PB_JSONIFY_EMPTY_ARRAY, f); } bool has_pb_jsonify_empty_array() const { return has_flag(FLAGS_PB_JSONIFY_EMPTY_ARRAY); } - + // Whether to always print primitive fields. By default proto3 primitive // fields with default values will be omitted in JSON output. For example, an // int32 field set to 0 will be omitted. Set this flag to true will override // the default behavior and print primitive fields regardless of their values. void set_always_print_primitive_fields(bool f) { set_flag(FLAGS_ALWAYS_PRINT_PRIMITIVE_FIELDS, f); } bool has_always_print_primitive_fields() const { return has_flag(FLAGS_ALWAYS_PRINT_PRIMITIVE_FIELDS); } - + // Tell RPC that done of the RPC can be run in the same thread where // the RPC is issued, otherwise done is always run in a different thread. @@ -486,7 +492,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); _http_response = nullptr; return tmp; } - + // User attached data or body of http response, which is wired to network // directly instead of being serialized into protobuf messages. butil::IOBuf& response_attachment() { return _response_attachment; } @@ -497,7 +503,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // replaced by ErrorText() and should be managed by user self. void manage_http_body_on_error(bool manage_or_not) { set_flag(FLAGS_MANAGE_HTTP_BODY_ON_ERROR, manage_or_not); } - + bool does_manage_http_body_on_error() const { return has_flag(FLAGS_MANAGE_HTTP_BODY_ON_ERROR); } @@ -523,7 +529,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); void set_response_checksum_attachment(bool with_attachment) { set_flag(FLAGS_RESPONSE_CHECKSUM_WITH_ATTACHMENT, with_attachment); } - + // Non-zero when this RPC call is traced (by rpcz or rig). // NOTE: Only valid at server-side, always zero at client-side. uint64_t trace_id() const; @@ -546,27 +552,27 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Always nullptr at client-side. const Server* server() const { return _server; } - // Get the data attached to current RPC session. The data is created by + // Get the data attached to current RPC session. The data is created by // ServerOptions.session_local_data_factory and reused between different // RPC. If factory is nullptr, this method returns nullptr. void* session_local_data(); // Get the data attached to a mongo session(practically a socket). MongoContext* mongo_session_data() { return _mongo_session_data.get(); } - + // ------------------------------------------------------------------- // Both-side methods. // Following methods can be called from both client and server. But they // may have different or opposite semantics. // ------------------------------------------------------------------- - // Client-side: successful or last server called. Accessible from + // Client-side: successful or last server called. Accessible from // PackXXXRequest() in protocols. // Server-side: returns the client sending the request butil::EndPoint remote_side() const { return _remote_side; } - + // Client-side: the local address for talking with server, undefined until - // this RPC succeeds (because the connection may not be established + // this RPC succeeds (because the connection may not be established // before RPC). // Server-side: the address that clients access. butil::EndPoint local_side() const { return _local_side; } @@ -580,24 +586,24 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); ResetNonPods(); ResetPods(); } - + // Causes Failed() to return true on the client side. "reason" will be // incorporated into the message returned by ErrorText(). // NOTE: Change http_response().status_code() according to `error_code' - // as well if the protocol is HTTP. If you want to overwrite the + // as well if the protocol is HTTP. If you want to overwrite the // status_code, call http_response().set_status_code() after SetFailed() // (rather than before SetFailed) void SetFailed(const std::string& reason) override; void SetFailed(int error_code, const char* reason_fmt, ...) __attribute__ ((__format__ (__printf__, 3, 4))); - + // After a call has finished, returns true if the RPC call failed. // The response to Channel is undefined when Failed() is true. // Calling Failed() before a call has finished is undefined. bool Failed() const override; // If Failed() is true, return description of the errors. - // NOTE: ErrorText() != berror(ErrorCode()). + // NOTE: ErrorText() != berror(ErrorCode()). std::string ErrorText() const override; // Last error code. Equals 0 iff Failed() is false. @@ -615,9 +621,9 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); ChecksumType response_checksum_type() const { return _response_checksum_type; } bool request_checksum_attachment() const { return has_flag(FLAGS_REQUEST_CHECKSUM_WITH_ATTACHMENT); } bool response_checksum_attachment() const { return has_flag(FLAGS_RESPONSE_CHECKSUM_WITH_ATTACHMENT); } - const HttpHeader& http_request() const + const HttpHeader& http_request() const { return _http_request != nullptr ? *_http_request : DefaultHttpHeader(); } - + const HttpHeader& http_response() const { return _http_response != nullptr ? *_http_response : DefaultHttpHeader(); } @@ -627,7 +633,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Get the object to write key/value which will be flushed into // LOG(INFO) when this controller is deleted. KVMap& SessionKV(); - + // Flush SessionKV() into `os' void FlushSessionKV(std::ostream& os); @@ -704,6 +710,9 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // the received time of RPC is not recorded in the controller. int64_t get_rpc_received_us() const { return _rpc_received_us; } + void set_use_flatbuffer() { add_flag(FLAGS_USE_FLATBUFFER); } + bool is_use_flatbuffer() const { return has_flag(FLAGS_USE_FLATBUFFER); } + private: struct CompletionInfo { CallId id; // call_id of the corresponding request @@ -737,7 +746,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); void HandleSendFailed(); static int RunOnCancel(bthread_id_t, void* data, int error_code); - + void set_auth_context(const AuthContext* ctx); // MongoContext is created by ParseMongoRequest when the first msg comes @@ -765,7 +774,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); BackupRequestPolicy* backup_request_policy; int max_retry; int32_t tos; - ConnectionType connection_type; + ConnectionType connection_type; CompressType request_compress_type; ChecksumType request_checksum_type; bool request_checksum_with_attachment; @@ -776,7 +785,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); void SaveClientSettings(ClientSettings*) const; void ApplyClientSettings(const ClientSettings&); - + bool FailedInline() const { return _error_code; } CallId get_id(int nretry) const { @@ -793,7 +802,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); return id; } private: - + // Append server information to `_error_text' void AppendServerIdentiy(); @@ -809,7 +818,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); int nretry; // sent in nretry-th retry. bool need_feedback; // The LB needs feedback. bool enable_circuit_breaker; // The channel enabled circuit_breaker - bool touched_by_stream_creator; + bool touched_by_stream_creator; SocketId peer_id; // main server id int64_t begin_time_us; // sent real time. // The actual `Socket' for sending RPC. It's socket id will be @@ -870,8 +879,8 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); void set_used_by_rpc() { add_flag(FLAGS_USED_BY_RPC); } bool is_used_by_rpc() const { return has_flag(FLAGS_USED_BY_RPC); } - bool has_enabled_circuit_breaker() const { - return has_flag(FLAGS_ENABLED_CIRCUIT_BREAKER); + bool has_enabled_circuit_breaker() const { + return has_flag(FLAGS_ENABLED_CIRCUIT_BREAKER); } bool is_ending_rpc() const { return has_flag(FLAGS_ENDING_RPC); } @@ -890,7 +899,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); std::string _error_text; butil::EndPoint _remote_side; butil::EndPoint _local_side; - + void* _session_local_data; const Server* _server; bthread_id_t _oncancel_id; @@ -911,7 +920,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Used by ParallelChannel int _fail_limit; - + uint32_t _pipelined_count; // [Timeout related] @@ -943,6 +952,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); Inheritable _inheritable; int _pchan_sub_count; google::protobuf::Message* _response; + brpc::flatbuffers::Message* _fb_response; google::protobuf::Closure* _done; RPCSender* _sender; uint64_t _request_code; @@ -951,16 +961,17 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // for passing parameters to created bthread, don't modify it otherwhere. CompletionInfo _tmp_completion_info; - + Call _current_call; Call* _unfinished_call; ExcludedServers* _accessed; - + StreamCreator* _stream_creator; // Fields will be used when making requests Protocol::PackRequest _pack_request; const google::protobuf::MethodDescriptor* _method; + const brpc::flatbuffers::MethodDescriptor* _fb_method; const Authenticator* _auth; butil::IOBuf _request_buf; IdlNames _idl_names; @@ -975,7 +986,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); std::unique_ptr _session_kv; - // Fields with large size but low access frequency + // Fields with large size but low access frequency butil::IOBuf _request_attachment; butil::IOBuf _response_attachment; @@ -1051,13 +1062,13 @@ std::ostream& operator<<(std::ostream& os, const Controller::LogPrefixDummy& p); } // namespace brpc // Print contextual logs prefixed with "@rid=REQUEST_ID" which marks a session -// and eases debugging. The REQUEST_ID is carried in http/rpc request or +// and eases debugging. The REQUEST_ID is carried in http/rpc request or // inherited from another controller. // As a server: // Call CLOG*(cntl) << ... to log instead of LOG(*) << .. // As a client: // Inside a service: -// Use Controller(service_cntl->inheritable()) to create controllers which +// Use Controller(service_cntl->inheritable()) to create controllers which // inherit session info from the service's requests // Standalone brpc client: // Set cntl->set_request_id(REQUEST_ID); diff --git a/src/brpc/details/controller_private_accessor.h b/src/brpc/details/controller_private_accessor.h index 1aad5b2b4e..ab8d17070c 100644 --- a/src/brpc/details/controller_private_accessor.h +++ b/src/brpc/details/controller_private_accessor.h @@ -85,7 +85,7 @@ class ControllerPrivateAccessor { _cntl->_local_side = pt; return *this; } - + ControllerPrivateAccessor& set_auth_context(const AuthContext* ctx) { _cntl->set_auth_context(ctx); return *this; @@ -94,12 +94,12 @@ class ControllerPrivateAccessor { // Overloaded set_span methods to support both shared_ptr and raw pointer ControllerPrivateAccessor& set_span(const std::shared_ptr& span); ControllerPrivateAccessor& set_span(Span* span); - + ControllerPrivateAccessor& set_request_protocol(ProtocolType protocol) { _cntl->_request_protocol = protocol; return *this; } - + std::shared_ptr span() const; uint32_t pipelined_count() const { return _cntl->_pipelined_count; } @@ -126,9 +126,12 @@ class ControllerPrivateAccessor { StreamIds request_streams() { return _cntl->_request_streams; } StreamIds response_streams() { return _cntl->_response_streams; } - void set_method(const google::protobuf::MethodDescriptor* method) + void set_method(const google::protobuf::MethodDescriptor* method) { _cntl->_method = method; } + void set_fb_method(const brpc::flatbuffers::MethodDescriptor* method) + { _cntl->_fb_method = method; } + void set_readable_progressive_attachment(ReadableProgressiveAttachment* s) { _cntl->_rpa.reset(s); } diff --git a/src/brpc/details/flatbuffers_common.h b/src/brpc/details/flatbuffers_common.h new file mode 100644 index 0000000000..de7efcec69 --- /dev/null +++ b/src/brpc/details/flatbuffers_common.h @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_FLATBUFFERS_COMMON_H_ +#define BRPC_FLATBUFFERS_COMMON_H_ + +#include +#undef FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS +#define FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(TypeName) \ + TypeName(const TypeName&) = delete; \ + void operator=(const TypeName&) = delete + +namespace google { +namespace protobuf { + class Closure; + class RpcController; +} // namespace protobuf +} // namespace google + +namespace brpc { +namespace flatbuffers { + +class Message; +class Service; +class ServiceDescriptor; +class MethodDescriptor; + +// Abstract interface for an RPC channel. An RpcChannel represents a +// communication line to a Service which can be used to call that Service's +// methods. The Service may be running on another machine. Normally, you +// should not call an RpcChannel directly, but instead construct a stub Service +// wrapping it. Example: +// RpcChannel* channel = new MyRpcChannel("remotehost.example.com:1234"); +// MyService* service = new MyService::Stub(channel); +// service->MyMethod(request, &response, callback); +class RpcChannel { +public: + inline RpcChannel() {} + virtual ~RpcChannel() {} + + // Call the given method of the remote service. The signature of this + // procedure looks the same as Service::CallMethod(), but the requirements + // are less strict in one important way: the request and response objects + // need not be of any specific class as long as their descriptors are + // method->input_type() and method->output_type(). + virtual void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller_base, const Message* request, + Message* response, google::protobuf::Closure* done) = 0; + +private: + FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(RpcChannel); +}; + +class Service { +public: + inline Service() {} + virtual ~Service() {} + + // When constructing a stub, you may pass STUB_OWNS_CHANNEL as the second + // parameter to the constructor to tell it to delete its RpcChannel when + // destroyed. + enum ChannelOwnership { STUB_OWNS_CHANNEL, STUB_DOESNT_OWN_CHANNEL }; + + // Get the ServiceDescriptor describing this service and its methods. + virtual const ServiceDescriptor* GetDescriptor() = 0; + + // Call a method of the service specified by MethodDescriptor. This is + // normally implemented as a simple switch() that calls the standard + // definitions of the service's methods. + // + // Preconditions: + // * method->service() == GetDescriptor() + // * request and response are of the exact same classes as the objects + // returned by GetRequestPrototype(method) and + // GetResponsePrototype(method). + // * After the call has started, the request must not be modified and the + // response must not be accessed at all until "done" is called. + // * "controller" is of the correct type for the RPC implementation being + // used by this Service. For stubs, the "correct type" depends on the + // RpcChannel which the stub is using. Server-side Service + // implementations are expected to accept whatever type of RpcController + // the server-side RPC implementation uses. + // + // Postconditions: + // * "done" will be called when the method is complete. This may be + // before CallMethod() returns or it may be at some point in the future. + // * If the RPC succeeded, "response" contains the response returned by + // the server. + // * If the RPC failed, "response"'s contents are undefined. The + // RpcController can be queried to determine if an error occurred and + // possibly to get more information about the error. + virtual void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, const Message* request, + Message* response, google::protobuf::Closure* done) = 0; + + private: + FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(Service); +}; + +} // namespace flatbuffers +} // namespace brpc + +#endif // BRPC_FLATBUFFERS_COMMON_H_ diff --git a/src/brpc/details/flatbuffers_impl.cpp b/src/brpc/details/flatbuffers_impl.cpp new file mode 100644 index 0000000000..cd48b372c7 --- /dev/null +++ b/src/brpc/details/flatbuffers_impl.cpp @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include "butil/iobuf.h" +#include "butil/logging.h" +#include "butil/thread_local.h" +#include "butil/third_party/murmurhash3/murmurhash3.h" +#include "brpc/details/flatbuffers_impl.h" + +namespace brpc { +namespace flatbuffers { + +#define METHOD_SPLIT " " +#define PREFIX_SPLIT "." +#define DEFAULT_RESERVE_SIZE 64 // Reserve space for rpc header and meta + +uint8_t *SlabAllocator::allocate(size_t size) { + _old_size_param = size; + size_t real_size = size + DEFAULT_RESERVE_SIZE; + + _full_buf_head = (uint8_t *)_iobuf.allocate(real_size); + if (_full_buf_head == nullptr) { + _fb_begin_head = nullptr; + return nullptr; + } + _fb_begin_head = _full_buf_head + DEFAULT_RESERVE_SIZE; + return _fb_begin_head; +} + +uint8_t *SlabAllocator::reallocate_downward(uint8_t *old_p, + size_t old_size, + size_t new_size, + size_t in_use_back, + size_t in_use_front) { + _old_size_param = new_size; + size_t new_real_size = new_size + DEFAULT_RESERVE_SIZE; + + // inc_ref to prevent old_block from being freed inside reallocate_downward + // when its internal dec_ref reduces the count, ensuring old_p stays valid + // for the subsequent memcpy_downward. + void* old_block = _iobuf.get_cur_block(); + butil::SingleIOBuf::target_block_inc_ref(old_block); + _full_buf_head = (uint8_t *)_iobuf.reallocate_downward(new_real_size, 0, 0); + if (_full_buf_head == nullptr) { + butil::SingleIOBuf::target_block_dec_ref(old_block); + return nullptr; + } + _fb_begin_head = _full_buf_head + DEFAULT_RESERVE_SIZE; + memcpy_downward(old_p, old_size, _fb_begin_head, new_size, in_use_back, in_use_front); + butil::SingleIOBuf::target_block_dec_ref(old_block); + return _fb_begin_head; +} + +void SlabAllocator::memcpy_downward(uint8_t *old_p, size_t old_size, uint8_t *new_p, + size_t new_size, size_t in_use_back, + size_t in_use_front) { + memcpy(new_p + new_size - in_use_back, old_p + old_size - in_use_back, + in_use_back); + memcpy(new_p, old_p, in_use_front); +} + +int ServiceDescriptor::init(const BrpcDescriptorTable& table) { + if (table.service_name.empty() || table.prefix.empty() || + table.method_name_list.empty()) { + errno = EINVAL; + return -1; + } + std::vector res; + std::string strs = table.method_name_list + METHOD_SPLIT; + size_t pos = strs.find(METHOD_SPLIT); + while(pos != strs.npos) { + std::string tmp = strs.substr(0, pos); + if (!tmp.empty() && tmp != METHOD_SPLIT && tmp != "\n") { + res.push_back(tmp); + } + strs = strs.substr(pos + 1, strs.size()); + pos = strs.find(METHOD_SPLIT); + } + method_count_ = res.size(); + if (method_count_ == 0) { + errno = EINVAL; + return -1; + } + name_ = table.service_name; + full_name_ = table.prefix + std::string(PREFIX_SPLIT) + name_; + butil::MurmurHash3_x86_32(full_name_.c_str(), full_name_.size(), 1, &index_); + methods_ = new MethodDescriptor*[method_count_]; + if (!methods_) { + return -1; + } + memset(methods_, 0, method_count_ * sizeof(MethodDescriptor*)); + for (int i = 0; i < method_count_; ++i) { + methods_[i] = new MethodDescriptor(table.prefix.c_str(), res[i].c_str(), this, i); + if (!methods_[i]) { + release_descriptor(); + return -1; + } + } + return 0; +} + +void ServiceDescriptor::release_descriptor() { + if (methods_ != NULL ) { + for (int i = 0; i < method_count_; ++i) { + if (methods_[i] != NULL) { + delete methods_[i]; + } + methods_[i] = NULL; + } + delete[] methods_; + methods_ = NULL; + } +} + +ServiceDescriptor::~ServiceDescriptor() { + release_descriptor(); +} + +const MethodDescriptor* ServiceDescriptor::method(int index) const { + if (index < 0 || index >= method_count_) { + errno = EINVAL; + return NULL; + } + if (!methods_) { + errno = EPERM; + return NULL; + } + return methods_[index]; +} + +MethodDescriptor::MethodDescriptor(const char* prefix, + const char* name, + const ServiceDescriptor* service, + int index) { + name_ = name; + full_name_ = std::string(prefix) + std::string(PREFIX_SPLIT) + std::string(name); + service_ = service; + index_ = index; +} + +Message MessageBuilder::ReleaseMessage() { + const uint8_t *msg_data = buf_.data(); // pointer to msg + uint32_t msg_size = static_cast(buf_.size()); + butil::SingleIOBuf& iobuf = slab_allocator_.get_iobuf(); + const uint8_t *buf_data = (const uint8_t *) iobuf.get_begin(); + // Calculate offsets from the buffer start + // reserve the memory space for rpc header and meta + const uint8_t *data_with_rpc = msg_data - DEFAULT_RESERVE_SIZE; + uint32_t new_msg_size = msg_size; + + new_msg_size += DEFAULT_RESERVE_SIZE; + uint32_t begin = data_with_rpc - buf_data; + + const butil::IOBuf::BlockRef& ref = iobuf.get_cur_ref(); + butil::IOBuf::BlockRef sub_ref = {ref.offset + begin, new_msg_size, ref.block}; + Message msg(sub_ref, DEFAULT_RESERVE_SIZE, msg_size); + Reset(); + return msg; +} + +Message::Message(const butil::SingleIOBuf &iobuf, + uint32_t meta_size, + uint32_t msg_size) + : _iobuf(iobuf) + , _meta_size(meta_size) + , _msg_size(msg_size) { + if (msg_size == 0) { + _msg_size = _iobuf.get_length() - meta_size; + } +} + +Message::Message(const butil::IOBuf::BlockRef &ref, + uint32_t meta_size, + uint32_t msg_size) + : _iobuf(ref) + , _meta_size(meta_size) + , _msg_size(msg_size) { + if (msg_size == 0) { + _msg_size = _iobuf.get_length() - meta_size; + } +} + +bool Message::parse_msg_from_iobuf(const butil::IOBuf& buf, size_t msg_size, size_t meta_size) { + size_t buf_size = buf.size(); + if (buf_size != (msg_size + meta_size)) { + return false; + } + if (!_iobuf.assign(buf, buf_size)) { + return false; + } + _meta_size = meta_size; + _msg_size = msg_size; + + return true; +} + +bool Message::append_msg_to_iobuf(butil::IOBuf& buf) { + _iobuf.append_to(&buf); + return true; +} + +void *Message::reduce_meta_size_and_get_buf(uint32_t new_size) { + if (new_size < _meta_size) { + uint32_t off = _meta_size - new_size; + const butil::IOBuf::BlockRef& ref = _iobuf.get_cur_ref(); + butil::IOBuf::BlockRef sub_ref = {ref.offset + off, ref.length - off, ref.block}; + _iobuf = butil::SingleIOBuf(sub_ref); + _meta_size = new_size; + return const_cast(_iobuf.get_begin()); + } else if (new_size == _meta_size) { + return const_cast(_iobuf.get_begin()); + } + LOG(WARNING) << "You should change DEFAULT_RESERVE_SIZE, new_size=" << new_size << " > _meta_size=" << _meta_size; + return NULL; +} + +int parse_service_descriptors(const BrpcDescriptorTable& descriptor_table, + ServiceDescriptor** descriptor_out) { + if(descriptor_out == NULL) { + errno = EINVAL; + return -1; + } + ServiceDescriptor *out = new ServiceDescriptor; + if (!out) { + return -1; + } + int ret = out->init(descriptor_table); + if (ret < 0) { + delete out; + return -1; + } + *descriptor_out = out; + return 0; +} + +bool ParseFbFromIOBUF(Message* msg, size_t msg_size, const butil::IOBuf& buf, size_t meta_size) { + // buf total size must equal meta_size (RPC header) + msg_size (FlatBuffers payload). + if (!msg || msg_size == 0 || buf.size() != (msg_size + meta_size)) { + return false; + } + return msg->parse_msg_from_iobuf(buf, msg_size, meta_size); +} + +bool SerializeFbToIOBUF(Message* msg, butil::IOBuf& buf) { + if (!msg) { + return false; + } + return msg->append_msg_to_iobuf(buf); +} + +} // namespace flatbuffers +} // namespace brpc diff --git a/src/brpc/details/flatbuffers_impl.h b/src/brpc/details/flatbuffers_impl.h new file mode 100644 index 0000000000..ebeb772d95 --- /dev/null +++ b/src/brpc/details/flatbuffers_impl.h @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_FLATBUFFERS_IMPL_H_ +#define BRPC_FLATBUFFERS_IMPL_H_ + +#include +#include +#include +#include +#include "butil/iobuf.h" +#include "butil/single_iobuf.h" +#include "brpc/details/flatbuffers_common.h" +#include "brpc/nonreflectable_message.h" +#include "brpc/pb_compat.h" + +namespace brpc { +namespace flatbuffers { + +class MessageBuilder; + +// Custom allocator for FlatBuffers that uses IOBuf as underlying storage. +// This allocator manages memory allocation for FlatBuffers messages within +// brpc's zero-copy buffer system, enabling efficient serialization and +// deserialization without unnecessary memory copies. +class SlabAllocator : public ::flatbuffers::Allocator { +public: + SlabAllocator() + : _full_buf_head(nullptr) + , _fb_begin_head(nullptr) + , _old_size_param(0) {} + + SlabAllocator(const SlabAllocator &other) = delete; + + SlabAllocator &operator=(const SlabAllocator &other) = delete; + + SlabAllocator(SlabAllocator&& other) + : _full_buf_head(nullptr) + , _fb_begin_head(nullptr) + , _old_size_param(0) { + swap(other); + } + + SlabAllocator &operator=(SlabAllocator &&other) { + // move-construct and swap idiom + SlabAllocator temp(std::move(other)); + swap(temp); + return *this; + } + + void swap(SlabAllocator &other) { + _iobuf.swap(other._iobuf); + std::swap(_full_buf_head, other._full_buf_head); + std::swap(_fb_begin_head, other._fb_begin_head); + std::swap(_old_size_param, other._old_size_param); + } + + virtual ~SlabAllocator() {} + /* + * Allocate memory from the slab allocator. + * buffer struct: fb header + fb message + */ + virtual uint8_t *allocate(size_t size); + + virtual void deallocate(uint8_t *p, size_t size) override { + if (p == _fb_begin_head) { + _iobuf.deallocate((void*)_full_buf_head); + } + } + + void deallocate(void *p) { + _iobuf.deallocate(p); + } + + virtual uint8_t *reallocate_downward(uint8_t *old_p, size_t old_size, + size_t new_size, size_t in_use_back, + size_t in_use_front); +protected: + void memcpy_downward(uint8_t *old_p, size_t old_size, uint8_t *new_p, + size_t new_size, size_t in_use_back, + size_t in_use_front); +private: + + butil::SingleIOBuf &get_iobuf() { + return _iobuf; + } + uint8_t *_full_buf_head; + uint8_t *_fb_begin_head; + size_t _old_size_param; + butil::SingleIOBuf _iobuf; + friend class MessageBuilder; +}; + +// SlabAllocatorMember is a hack to ensure that the MessageBuilder's +// slab_allocator_ member is constructed before the FlatBufferBuilder, since +// the allocator is used in the FlatBufferBuilder ctor. +struct SlabAllocatorMember { + SlabAllocator slab_allocator_; +}; + +// Represents a FlatBuffers message in brpc's zero-copy buffer system. +// This class wraps a FlatBuffers message stored in an IOBuf +// The message is move-only and cannot be copied. +class Message : public NonreflectableMessage { +public: + Message() : _meta_size(0), _msg_size(0) {} + + // NonreflectableMessage requires MergeFrom to be implemented. + // FlatBuffers Message is move-only and does not support merging. + void MergeFrom(const Message& /*from*/) override { + LOG(FATAL) << "FlatBuffers Message does not support MergeFrom. " + << "Use move semantics instead."; + } + +private: + Message(const butil::SingleIOBuf &iobuf, + uint32_t meta_size = 0, + uint32_t msg_size = 0); + + Message(const butil::IOBuf::BlockRef &ref, + uint32_t meta_size = 0, + uint32_t msg_size = 0); + friend class MessageBuilder; +public: + Message(const Message &other) = delete; + Message &operator=(const Message &other) = delete; + + Message(Message&& other) noexcept + : NonreflectableMessage(), + _iobuf(), + _meta_size(0), + _msg_size(0) { + swap(other); + } + + Message& operator=(Message&& other) noexcept { + if (this != &other) { + Clear(); + swap(other); + } + return *this; + } + + void swap(Message& other) noexcept { + _iobuf.swap(other._iobuf); + + const uint32_t meta_size = _meta_size; + _meta_size = other._meta_size; + other._meta_size = meta_size; + + const uint32_t msg_size = _msg_size; + _msg_size = other._msg_size; + other._msg_size = msg_size; + } + + void *mutable_data() { + return (void *)const_cast(data()); + } + + const uint8_t *data() const { + const uint8_t *buf = (const uint8_t *)_iobuf.get_begin(); + if (buf == nullptr) { + return nullptr; + } + return buf + _meta_size; + } + + void *mutable_buf_begin() const { + return const_cast(_iobuf.get_begin()); + } + + void *reduce_meta_size_and_get_buf(uint32_t new_size); + + uint32_t get_meta_size() const { + return _meta_size; + } + + size_t size() const { + return _msg_size; + } + + template + bool Verify() const { + ::flatbuffers::Verifier verifier(data(), size()); + return verifier.VerifyBuffer(nullptr); + } + + template T *GetMutableRoot() { return ::flatbuffers::GetMutableRoot(mutable_data()); } + template const T *GetRoot() const { return ::flatbuffers::GetRoot(data()); } + + bool parse_msg_from_iobuf(const butil::IOBuf& buf, size_t msg_size, size_t meta_size); + + bool append_msg_to_iobuf(butil::IOBuf& buf); + + void Clear() override { + _iobuf.reset(); + _meta_size = 0; + _msg_size = 0; + } +private: + butil::SingleIOBuf _iobuf; + uint32_t _meta_size; + uint32_t _msg_size; +}; + +// MessageBuilder is a BRPC-specific FlatBufferBuilder that uses SlabAllocator +// to allocate BRPC buffers. +class MessageBuilder : private SlabAllocatorMember, + public ::flatbuffers::FlatBufferBuilder { +public: + explicit MessageBuilder(::flatbuffers::uoffset_t initial_size = 1024) + : ::flatbuffers::FlatBufferBuilder(initial_size, &slab_allocator_, false) {} + + MessageBuilder(const MessageBuilder &other) = delete; + + MessageBuilder &operator=(const MessageBuilder &other) = delete; + + MessageBuilder(MessageBuilder &&other) + : ::flatbuffers::FlatBufferBuilder(1024, &slab_allocator_, false) { + // Default construct and swap idiom. + Swap(other); + } + + /// Create a MessageBuilder from a FlatBufferBuilder. + explicit MessageBuilder(::flatbuffers::FlatBufferBuilder &&src, + void (*dealloc)(void *) = NULL) + : ::flatbuffers::FlatBufferBuilder(1024, &slab_allocator_, false) { + src.Swap(*this); + src.SwapBufAllocator(*this); + if (buf_.capacity()) { + uint8_t *buf = buf_.scratch_data(); // pointer to memory + size_t capacity = buf_.capacity(); // size of memory + slab_allocator_._iobuf.assign_user_data((void*)buf, capacity, dealloc); + } else { + slab_allocator_._iobuf.reset(); + } + } + + /// Move-assign a FlatBufferBuilder to a MessageBuilder. + /// Only FlatBufferBuilder with default allocator (basically, nullptr) is + /// supported. + MessageBuilder &operator=(::flatbuffers::FlatBufferBuilder &&src) { + // Move construct a temporary and swap + MessageBuilder temp(std::move(src)); + Swap(temp); + return *this; + } + + MessageBuilder &operator=(MessageBuilder &&other) { + // Move construct a temporary and swap + MessageBuilder temp(std::move(other)); + Swap(temp); + return *this; + } + + void Swap(MessageBuilder &other) { + slab_allocator_.swap(other.slab_allocator_); + ::flatbuffers::FlatBufferBuilder::Swap(other); + // After swapping the FlatBufferBuilder, we swap back the allocator, which + // restores the original allocator back in place. This is necessary because + // MessageBuilder's allocator is its own member (SlabAllocatorMember). The + // allocator passed to FlatBufferBuilder::vector_downward must point to this + // member. + buf_.swap_allocator(other.buf_); + } + + ~MessageBuilder() {} + + // GetMessage extracts the subslab of the buffer corresponding to the + // flatbuffers-encoded region and wraps it in a `Message` to handle buffer + // ownership. + // Message GetMessage() = delete; + + Message ReleaseMessage(); +}; + +struct BrpcDescriptorTable { + const std::string prefix; + const std::string service_name; + // Method names separated by EXACTLY ONE space character ' '. + // e.g. "Method1 Method2 Method3" + // Tab, newline, or multiple consecutive spaces are NOT supported. + const std::string method_name_list; +}; + +class ServiceDescriptor { +public: + ServiceDescriptor() : name_("") + ,full_name_("") + ,methods_(NULL) + ,method_count_(0) + ,index_(0) {} + ~ServiceDescriptor(); + int init(const BrpcDescriptorTable& table); + // The name of the service, not including its containing scope. + const std::string& name() const {return name_;} + // The fully-qualified name of the service, scope delimited by periods. + const std::string& full_name() const {return full_name_;} + // Index of this service within the file's services array. + uint32_t index() const {return index_;} + // The number of methods this service defines. + int method_count() const {return method_count_;} + // Gets a MethodDescriptor by index, where 0 <= index < method_count(). + // These are returned in the order they were defined in the .proto file. + const MethodDescriptor* method(int index) const; +private: + void release_descriptor(); + std::string name_; + std::string full_name_; + MethodDescriptor** methods_; + int method_count_; + uint32_t index_; + FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(ServiceDescriptor); +}; + +int parse_service_descriptors(const BrpcDescriptorTable& descriptor_table, + ServiceDescriptor** descriptor_out); + +class MethodDescriptor { +public: + MethodDescriptor(const char* prefix, + const char* name, + const ServiceDescriptor* service, + int index); + // Name of this method, not including containing scope. + const std::string& name() const {return name_;} + // The fully-qualified name of the method, scope delimited by periods. + const std::string& full_name() const {return full_name_;} + // Index within the service's Descriptor. + int index() const {return index_;} + const ServiceDescriptor* service() const { return service_;} +private: + std::string name_; + std::string full_name_; + const ServiceDescriptor* service_; + int index_; + FB_BRPC_DISALLOW_EVIL_CONSTRUCTORS(MethodDescriptor); +}; + +// Parse a FlatBuffers message from |buf| into |msg|. +// |msg_size|: byte size of the FlatBuffers payload (excluding meta). +// |meta_size|: byte size of the RPC header / meta prefix in |buf|. +// Requires: buf.size() == msg_size + meta_size. +bool ParseFbFromIOBUF(Message* msg, size_t msg_size, const butil::IOBuf& buf, size_t meta_size = 0); + +bool SerializeFbToIOBUF(Message* msg, butil::IOBuf& buf); + +} // namespace flatbuffers +} // namespace brpc + +#endif // BRPC_FLATBUFFERS_IMPL_H_ diff --git a/src/brpc/details/server_private_accessor.h b/src/brpc/details/server_private_accessor.h index 0e6e4fbba8..3902240c86 100644 --- a/src/brpc/details/server_private_accessor.h +++ b/src/brpc/details/server_private_accessor.h @@ -86,6 +86,15 @@ class ServerPrivateAccessor { return _server->FindServicePropertyByName(name); } +#if BRPC_WITH_FLATBUFFERS + const Server::FlatBuffersMethodProperty* + FindFlatBuffersMethodPropertyByIndex( + uint32_t server_index, int method_index) const { + return _server->FindFlatBuffersMethodPropertyByIndex( + server_index, method_index); + } +#endif + const Server::ServiceProperty* FindServicePropertyAdaptively(const butil::StringPiece& service_name) const { if (service_name.find('.') == butil::StringPiece::npos) { diff --git a/src/brpc/global.cpp b/src/brpc/global.cpp index 0a0837e096..87a20b49a3 100644 --- a/src/brpc/global.cpp +++ b/src/brpc/global.cpp @@ -87,6 +87,9 @@ #include "brpc/policy/rtmp_protocol.h" #include "brpc/policy/esp_protocol.h" #include "brpc/policy/mysql/mysql_protocol.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/policy/flatbuffers_protocol.h" +#endif #ifdef ENABLE_THRIFT_FRAMED_PROTOCOL # include "brpc/policy/thrift_protocol.h" #endif @@ -456,6 +459,25 @@ static void GlobalInitializeOrDieImpl() { exit(1); } +#if BRPC_WITH_FLATBUFFERS + Protocol fb_protocol = { + ParseFlatBuffersMessage, + SerializeFlatBuffersRequest, + PackFlatBuffersRequest, + ProcessFlatBuffersRequest, + ProcessFlatBuffersResponse, + nullptr, + nullptr, + nullptr, + CONNECTION_TYPE_SINGLE, + "fb_rpc" + }; + if (RegisterProtocol( + PROTOCOL_FLATBUFFERS_RPC, fb_protocol) != 0) { + exit(1); + } +#endif + Protocol streaming_protocol = { ParseStreamingMessage, nullptr, nullptr, ProcessStreamingMessage, ProcessStreamingMessage, diff --git a/src/brpc/options.proto b/src/brpc/options.proto index 13b8b682e6..23f28c0a9e 100644 --- a/src/brpc/options.proto +++ b/src/brpc/options.proto @@ -67,6 +67,7 @@ enum ProtocolType { PROTOCOL_H2 = 27; PROTOCOL_COUCHBASE = 28; PROTOCOL_MYSQL = 29; // Client side only + PROTOCOL_FLATBUFFERS_RPC = 30; } enum CompressType { diff --git a/src/brpc/policy/flatbuffers_protocol.cpp b/src/brpc/policy/flatbuffers_protocol.cpp new file mode 100644 index 0000000000..9e03b27f9e --- /dev/null +++ b/src/brpc/policy/flatbuffers_protocol.cpp @@ -0,0 +1,480 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "butil/logging.h" // LOG() +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/single_iobuf.h" // butil::SingleIOBuf +#include "butil/time.h" + +#include "butil/raw_pack.h" // RawPacker RawUnpacker + +#include "brpc/controller.h" // Controller +#include "brpc/socket.h" // Socket +#include "brpc/server.h" // Server +#include "brpc/stream_impl.h" +#include "brpc/rpc_dump.h" // SampledRequest +#include "brpc/policy/most_common_message.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/server_private_accessor.h" +#include "brpc/policy/flatbuffers_protocol.h" + +namespace brpc { +namespace policy { + +struct FBRpcRequestMeta { + struct { + uint32_t service_index; + int32_t method_index; + } request; + int32_t message_size; + int32_t attachment_size; + int64_t correlation_id; +}__attribute__((packed)); + +struct FBRpcResponseMeta { + struct { + int32_t error_code; + } response; + int32_t message_size; + int32_t attachment_size; + int64_t correlation_id; +}__attribute__((packed)); + +struct FBRpcRequestHeader { + char header[12]; + struct FBRpcRequestMeta meta; +}__attribute__((packed)); + +struct FBRpcResponseHeader { + char header[12]; + struct FBRpcResponseMeta meta; +}__attribute__((packed)); + +bool inline ParseFbFromIOBuf(brpc::flatbuffers::Message* msg, size_t msg_size, const butil::IOBuf& buf) { + return brpc::flatbuffers::ParseFbFromIOBUF(msg, msg_size, buf); +} + +// Notes: +// 1. 12-byte header [FRPC][body_size][meta_size] +// 2. body_size and meta_size are in network byte order +// 3. Use service->service_index + method_index to specify the method to call +// 4. `attachment_size' is set iff request/response has attachment +// 5. Not supported: chunk_info + +// Pack header into `buf' + +static inline void PackFlatbuffersRpcHeader(char* rpc_header, int meta_size, int payload_size) { + // supress strict-aliasing warning. + uint32_t* dummy = (uint32_t*)rpc_header; + *dummy = *(uint32_t*)"FRPC"; + butil::RawPacker(rpc_header + 4) + .pack32(meta_size + payload_size) + .pack32(meta_size); +} + +static inline bool ParseMetaBufferFromIOBUF(butil::SingleIOBuf* dest, + const butil::IOBuf& source, uint32_t msg_size) { + return dest->assign(source, msg_size); +} + +ParseResult ParseFlatBuffersMessage(butil::IOBuf* source, Socket* socket, + bool /*read_eof*/, const void*) { + char header_buf[12]; + const size_t n = source->copy_to(header_buf, sizeof(header_buf)); + if (n >= 4) { + void* dummy = header_buf; + if (*(const uint32_t*)dummy != *(const uint32_t*)"FRPC") { + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + + } else { + if (memcmp(header_buf, "FRPC", n) != 0) { + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + } + + if (n < sizeof(header_buf)) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + uint32_t body_size; + uint32_t meta_size; + butil::RawUnpacker(header_buf + 4).unpack32(body_size).unpack32(meta_size); + if (body_size > FLAGS_max_body_size) { + // We need this log to report the body_size to give users some clues + // which is not printed in InputMessenger. + LOG(ERROR) << "body_size=" << body_size << " from " + << socket->remote_side() << " is too large"; + return MakeParseError(PARSE_ERROR_TOO_BIG_DATA); + } else if (source->length() < sizeof(header_buf) + body_size) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + if (meta_size > body_size) { + LOG(ERROR) << "meta_size=" << meta_size << " is bigger than body_size=" + << body_size; + // Pop the message + source->pop_front(sizeof(header_buf) + body_size); + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + source->pop_front(sizeof(header_buf)); + MostCommonMessage* msg = MostCommonMessage::Get(); + source->cutn(&msg->meta, meta_size); + source->cutn(&msg->payload, body_size - meta_size); + return MakeMessage(msg); +} + +static void SendFlatBuffersRpcResponse(int64_t correlation_id, + Controller* cntl, + brpc::flatbuffers::Message* req, + brpc::flatbuffers::Message* res, + const Server* server, + MethodStatus* method_status_raw, + int64_t received_us) { + ControllerPrivateAccessor accessor(cntl); + Socket* sock = accessor.get_sending_socket(); + ConcurrencyRemover concurrency_remover(method_status_raw, cntl, received_us); + std::unique_ptr recycle_cntl(cntl); + std::unique_ptr recycle_req(req); + std::unique_ptr recycle_res(res); + if (cntl->IsCloseConnection()) { + sock->SetFailed(); + return; + } + bool append_body = false; + butil::IOBuf res_body; + // `res' can be NULL here, in which case we don't serialize it + // If user calls `SetFailed' on Controller, we don't serialize + // response either + struct FBRpcResponseHeader *rpc_header = NULL; + uint32_t reserve_size = sizeof(struct FBRpcResponseHeader); + + if (res != NULL && !cntl->Failed()) { + rpc_header = static_cast(res->reduce_meta_size_and_get_buf(sizeof(struct FBRpcResponseHeader))); + if (BAIDU_UNLIKELY(rpc_header == NULL)) { + cntl->SetFailed(ERESPONSE, "Fail to reduce meta size and get buf"); + } else { + if (!brpc::flatbuffers::SerializeFbToIOBUF(res, res_body)) { + cntl->SetFailed(ERESPONSE, "Fail to serialize response"); + } else { + append_body = true; + } + } + } + + // Don't use res->ByteSize() since it may be compressed + size_t res_size = 0; + size_t attached_size = 0; + size_t meta_size = sizeof(struct FBRpcResponseMeta); + if (append_body && rpc_header != NULL) { + res_size = res_body.length() - reserve_size; + attached_size = cntl->response_attachment().length(); + PackFlatbuffersRpcHeader(rpc_header->header, + meta_size, res_size + attached_size); + rpc_header->meta.message_size = res_size; + rpc_header->meta.attachment_size = attached_size; + rpc_header->meta.response.error_code = cntl->ErrorCode(); + rpc_header->meta.correlation_id = correlation_id; + if (attached_size > 0) { + res_body.append(cntl->response_attachment().movable()); + } + } else { // error response + struct FBRpcResponseHeader tmp_header; + tmp_header.meta.message_size = 0; + tmp_header.meta.attachment_size = 0; + tmp_header.meta.response.error_code = cntl->ErrorCode(); + tmp_header.meta.correlation_id = correlation_id; + PackFlatbuffersRpcHeader(tmp_header.header, + meta_size, 0); + res_body.clear(); + res_body.append((void const*) &tmp_header, + sizeof(struct FBRpcResponseHeader)); + } + Socket::WriteOptions wopt; + wopt.ignore_eovercrowded = true; + if (sock->Write(&res_body, &wopt) != 0) { + const int errcode = errno; + PLOG_IF(WARNING, errcode != EPIPE) << "Fail to write into " << *sock; + cntl->SetFailed(errcode, "Fail to write into %s", + sock->description().c_str()); + return; + } +} + +void ProcessFlatBuffersRequest(InputMessageBase* msg_base) { + DestroyingPtr msg(static_cast(msg_base)); + SocketUniquePtr socket_guard(msg->ReleaseSocket()); + Socket* socket = socket_guard.get(); + const Server* server = static_cast(msg_base->arg()); + ScopedNonServiceError non_service_error(server); + butil::SingleIOBuf meta_buf; + if (!ParseMetaBufferFromIOBUF(&meta_buf, + msg->meta, sizeof(struct FBRpcRequestMeta))) { + LOG(WARNING) << "Fail to parse RpcMeta from " << *socket; + socket->SetFailed(EREQUEST, "Fail to parse RpcMeta from %s", + socket->description().c_str()); + return; + } + const struct FBRpcRequestMeta* meta = + static_cast(meta_buf.get_begin()); + if (!meta) { + LOG(WARNING) << "RpcMeta from " << *socket << " is NULL"; + socket->SetFailed(EREQUEST, "Fail to parse RpcMeta from %s", + socket->description().c_str()); + return; + } + + std::unique_ptr cntl; + cntl.reset(new (std::nothrow) Controller); + if (NULL == cntl.get()) { + LOG(WARNING) << "Fail to new Controller"; + return; + } + + std::unique_ptr req; + std::unique_ptr res; + + ServerPrivateAccessor server_accessor(server); + + ControllerPrivateAccessor accessor(cntl.get()); + accessor.set_server(server) + .set_peer_id(socket->id()) + .set_remote_side(socket->remote_side()) + .set_local_side(socket->local_side()) + .set_request_protocol(PROTOCOL_FLATBUFFERS_RPC) + .move_in_server_receiving_sock(socket_guard); + MethodStatus* method_status = NULL; + do { + if (!server->IsRunning()) { + cntl->SetFailed(ELOGOFF, "Server is stopping"); + break; + } + + if (socket->is_overcrowded()) { + cntl->SetFailed(EOVERCROWDED, "Connection to %s is overcrowded", + butil::endpoint2str(socket->remote_side()).c_str()); + break; + } + + if (!server_accessor.AddConcurrency(cntl.get())) { + cntl->SetFailed(ELIMIT, "Reached server's max_concurrency=%d", + server->options().max_concurrency); + break; + } + + const Server::FlatBuffersMethodProperty* mp = + server_accessor.FindFlatBuffersMethodPropertyByIndex(meta->request.service_index, + meta->request.method_index); + if (NULL == mp) { + cntl->SetFailed(ENOMETHOD, "Fail to find method_index=%d service_index=%u ", + meta->request.method_index, + meta->request.service_index); + break; + } + // Switch to service-specific error. + non_service_error.release(); + if (mp->status) { + method_status = mp->status; + if (!method_status->OnRequested()) { + cntl->SetFailed(ELIMIT, "Reached %s's MaxConcurrency=%d", + mp->method->full_name().c_str(), + method_status->MaxConcurrency()); + break; + } + } + brpc::flatbuffers::Service* svc = mp->service; + const brpc::flatbuffers::MethodDescriptor* method = mp->method; + accessor.set_fb_method(method); + const int reqsize = static_cast(msg->payload.size()); + butil::IOBuf req_buf; + butil::IOBuf* req_buf_ptr = &msg->payload; + if (meta->attachment_size > 0) { + if (reqsize < meta->attachment_size) { + cntl->SetFailed(EREQUEST, + "attachment_size=%d is larger than request_size=%d", + meta->attachment_size, reqsize); + break; + } + int body_without_attachment_size = reqsize - meta->attachment_size; + msg->payload.cutn(&req_buf, body_without_attachment_size); + req_buf_ptr = &req_buf; + cntl->request_attachment().swap(msg->payload); + } + + req.reset(new brpc::flatbuffers::Message()); + if (!brpc::flatbuffers::ParseFbFromIOBUF(req.get(), meta->message_size, *req_buf_ptr)) { + cntl->SetFailed(EREQUEST, "Fail to parse request message, " + "request_size=%d", reqsize); + break; + } + res.reset(new brpc::flatbuffers::Message()); + // `socket' will be held until response has been sent + google::protobuf::Closure* done = ::brpc::NewCallback< + int64_t, Controller*, brpc::flatbuffers::Message*, + brpc::flatbuffers::Message*, const Server*, + MethodStatus*, int64_t>( + &SendFlatBuffersRpcResponse, meta->correlation_id, cntl.get(), + req.get(), res.get(), server, + method_status, msg->received_us()); + + msg.reset(); + req_buf.clear(); + svc->FBCallMethod(method, cntl.release(), + req.release(), res.release(), done); + return; + } while (false); + // `cntl', `req' and `res' will be deleted inside `SendFlatBuffersRpcResponse' + // `socket' will be held until response has been sent + SendFlatBuffersRpcResponse(meta->correlation_id, cntl.release(), + req.release(), res.release(), server, + method_status, -1); +} + +void ProcessFlatBuffersResponse(InputMessageBase* msg_base) { + DestroyingPtr msg(static_cast(msg_base)); + butil::SingleIOBuf meta_buf; + if (!ParseMetaBufferFromIOBUF(&meta_buf, + msg->meta, sizeof(struct FBRpcResponseMeta))) { + LOG(WARNING) << "Fail to parse from response meta"; + return; + } + const struct FBRpcResponseMeta* meta = + static_cast(meta_buf.get_begin()); + if (!meta) { + LOG(WARNING) << "Fail to parse from response meta: meta is NULL"; + return; + } + + const bthread_id_t cid = { static_cast(meta->correlation_id) }; + Controller* cntl = NULL; + const int rc = bthread_id_lock(cid, (void**)&cntl); + if (rc != 0) { + LOG_IF(ERROR, rc != EINVAL && rc != EPERM) + << "Fail to lock correlation_id=" << cid << ": " << berror(rc); + return; + } + + ControllerPrivateAccessor accessor(cntl); + const int saved_error = cntl->ErrorCode(); + do { + if (meta->response.error_code != 0) { + // If error_code is unset, default is 0 = success. + cntl->SetFailed(meta->response.error_code, + "server response error"); + break; + } + // Parse response message if error code from meta is 0 + butil::IOBuf res_buf; + const int res_size = msg->payload.length(); + butil::IOBuf* res_buf_ptr = &msg->payload; + if (meta->attachment_size > 0) { + if (meta->attachment_size > res_size) { + cntl->SetFailed( + ERESPONSE, + "attachment_size=%d is larger than response_size=%d", + meta->attachment_size, res_size); + break; + } + int body_without_attachment_size = res_size - meta->attachment_size; + msg->payload.cutn(&res_buf, body_without_attachment_size); + res_buf_ptr = &res_buf; + cntl->response_attachment().swap(msg->payload); + } + + if (cntl->fb_response()) { + if (!brpc::flatbuffers::ParseFbFromIOBUF(cntl->fb_response(), + meta->message_size, *res_buf_ptr)) { + cntl->SetFailed( + ERESPONSE, "Fail to parse response message, " + " response_size=%d", res_size); + } + } // else silently ignore the response. + } while (0); + // Unlocks correlation_id inside. Revert controller's + // error code if it version check of `cid' fails + msg.reset(); // optional, just release resourse ASAP + accessor.OnResponse(cid, saved_error); +} + +void PackFlatBuffersRequest(butil::IOBuf* req_buf, + SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* cntl, + const butil::IOBuf& request_body, + const Authenticator* /*auth*/) { + // FlatBuffers does not use protobuf service definitions. The caller passes + // a brpc::flatbuffers::MethodDescriptor* disguised as a + // google::protobuf::MethodDescriptor*, so we reinterpret_cast it back. + const brpc::flatbuffers::MethodDescriptor* fb_method = + reinterpret_cast(method); + struct FBRpcRequestHeader *rpc_header = NULL; + size_t req_size = request_body.length(); + rpc_header = (struct FBRpcRequestHeader*)const_cast(request_body.fetch1()); + if (BAIDU_UNLIKELY(rpc_header == NULL)) { + return cntl->SetFailed(ERESPONSE, "fail to get fb request rpc header"); + } + req_size -= sizeof(struct FBRpcRequestHeader); + + //ControllerPrivateAccessor accessor(cntl); + if (fb_method) { + rpc_header->meta.request.service_index = fb_method->service()->index(); + rpc_header->meta.request.method_index = fb_method->index(); + } else { + return cntl->SetFailed(ENOMETHOD, "%s.method is NULL", __FUNCTION__); + } + + rpc_header->meta.correlation_id = correlation_id; + + size_t meta_size = sizeof(struct FBRpcRequestMeta); + rpc_header->meta.message_size = req_size; + const size_t attached_size = cntl->request_attachment().length(); + if (attached_size > 0) { + rpc_header->meta.attachment_size = attached_size; + } else { + rpc_header->meta.attachment_size = 0; + } + PackFlatbuffersRpcHeader(rpc_header->header, meta_size, req_size + attached_size); + + req_buf->append(request_body); + + if (attached_size > 0) { + req_buf->append(cntl->request_attachment()); + } +} + +void SerializeFlatBuffersRequest(butil::IOBuf* buf, + Controller* cntl, + const google::protobuf::Message* request) { + if (!request) { + return cntl->SetFailed(EREQUEST, "`request' is NULL"); + } + if (request->GetDescriptor() != + brpc::flatbuffers::Message::descriptor()) { + return cntl->SetFailed(EREQUEST, "request is not a flatbuffers::Message"); + } + brpc::flatbuffers::Message* fb_request = + const_cast( + static_cast(request)); + uint32_t reserve_size = sizeof(struct FBRpcRequestHeader); + fb_request->reduce_meta_size_and_get_buf(reserve_size); + if (!brpc::flatbuffers::SerializeFbToIOBUF(fb_request, *buf)) { + return cntl->SetFailed(EREQUEST, "Fail to serialize request"); + } +} + +} // namespace policy +} // namespace brpc diff --git a/src/brpc/policy/flatbuffers_protocol.h b/src/brpc/policy/flatbuffers_protocol.h new file mode 100644 index 0000000000..2795c59f20 --- /dev/null +++ b/src/brpc/policy/flatbuffers_protocol.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_POLICY_FLATBUFFERS_PROTOCOL_H +#define BRPC_POLICY_FLATBUFFERS_PROTOCOL_H + +#include "brpc/protocol.h" +#include "brpc/details/flatbuffers_impl.h" + +namespace brpc { +namespace policy { + +// Parse binary format of flatbuffers-pbrpc. +ParseResult ParseFlatBuffersMessage(butil::IOBuf* source, Socket *socket, bool read_eof, const void *arg); + +// Actions to a (client) request in flatbuffers-pbrpc format. +void ProcessFlatBuffersRequest(InputMessageBase* msg_base); + +// Actions to a (server) response in flatbuffers-pbrpc format. +void ProcessFlatBuffersResponse(InputMessageBase* msg); + +// The serialize_request implementation used by flatbuffers protocol. +void SerializeFlatBuffersRequest(butil::IOBuf* buf, + Controller* cntl, + const google::protobuf::Message* request); + +// Pack `request' to `method' into `buf'. +void PackFlatBuffersRequest(butil::IOBuf* buf, + SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* controller, + const butil::IOBuf& request, + const Authenticator* auth); + +} // namespace policy +} // namespace brpc + +#endif // BRPC_POLICY_FLATBUFFERS_PROTOCOL_H \ No newline at end of file diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index baf8b8fa94..dd4a3119b8 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -169,6 +169,57 @@ ServerSSLOptions* ServerOptions::mutable_ssl_options() { return _ssl_options.get(); } +#if BRPC_WITH_FLATBUFFERS +Server::FlatBuffersMethodProperty::FlatBuffersMethodProperty() + : service(NULL) + , method(NULL) + , status(NULL) { +} + +Server::FlatBuffersServiceProperty::FlatBuffersServiceProperty() + :service(NULL) + ,method_count(0) + ,methods_list(NULL){ +} + +Server::FlatBuffersServiceProperty::~FlatBuffersServiceProperty() { + if (methods_list) { + for (int i = 0; i < method_count; ++i) { + if (methods_list[i]) { + delete methods_list[i]->status; + delete methods_list[i]; + } + } + delete[] methods_list; + methods_list = NULL; + } +} + +Server::FlatBuffersServiceProperty::FlatBuffersServiceProperty( + FlatBuffersServiceProperty&& other) + : service(other.service) + , method_count(other.method_count) + , methods_list(other.methods_list) { + other.service = NULL; + other.method_count = 0; + other.methods_list = NULL; +} + +Server::FlatBuffersServiceProperty& +Server::FlatBuffersServiceProperty::operator=(FlatBuffersServiceProperty&& other) { + if (this != &other) { + this->~FlatBuffersServiceProperty(); + service = other.service; + method_count = other.method_count; + methods_list = other.methods_list; + other.service = NULL; + other.method_count = 0; + other.methods_list = NULL; + } + return *this; +} +#endif + Server::MethodProperty::OpaqueParams::OpaqueParams() : is_tabbed(false) , allow_default_url(false) @@ -424,6 +475,16 @@ const std::string Server::ServiceProperty::service_name() const { return s_unknown_name; } +#if BRPC_WITH_FLATBUFFERS +const std::string& Server::FlatBuffersServiceProperty::service_name() const { + if (service) { + return service->GetDescriptor()->full_name(); + } + const static std::string s_unknown_name = ""; + return s_unknown_name; +} +#endif + Server::Server(ProfilerLinker) : _session_local_data_pool(nullptr) , _status(UNINITIALIZED) @@ -691,7 +752,7 @@ int Server::InitALPNOptions(const ServerSSLOptions* options) { if (options == nullptr) { LOG(ERROR) << "Fail to init alpn options, ssl options is nullptr."; return -1; - } + } std::string raw_protocol; const std::string& alpns = options->alpns; @@ -1599,6 +1660,74 @@ int Server::AddServiceInternal(google::protobuf::Service* service, return 0; } +#if BRPC_WITH_FLATBUFFERS +int Server::AddServiceInternal(brpc::flatbuffers::Service* service, + bool is_builtin_service, + const ServiceOptions& options) { + if (is_builtin_service) { + LOG(ERROR) << "builtin_service of flatbuffers rpc is not supported"; + return -1; + } + if (NULL == service) { + LOG(ERROR) << "Parameter[service] is NULL!"; + return -1; + } + const brpc::flatbuffers::ServiceDescriptor* sd = service->GetDescriptor(); + int method_count = sd->method_count(); + if (method_count <= 0) { + LOG(ERROR) << "service=" << sd->full_name() + << " does not have any method."; + return -1; + } + if (InitializeOnce() != 0) { + LOG(ERROR) << "Fail to initialize Server[" << version() << ']'; + return -1; + } + if (status() != READY) { + LOG(ERROR) << "Can't add service=" << sd->full_name() << " to Server[" + << version() << "] which is " << status_str(status()); + return -1; + } + // Check service conflict using service's index + FlatBuffersServiceProperty* c_ss = _fb_server_index_map.seek(sd->index()); + if (c_ss != NULL) { + LOG(ERROR) << "service:" << sd->full_name() + << " with index:"<< sd->index() + << " conflicts with registed service:" << c_ss->service->GetDescriptor()->full_name() + << " Try to change your service name."; + return -1; + } + + // Register ServiceProperty + FlatBuffersServiceProperty ss; + ss.service = service; + ss.method_count = method_count; + ss.methods_list = new FlatBuffersMethodProperty*[method_count]; + if (!ss.methods_list) { + LOG(ERROR) << "Fail to alloc methods_list"; + return -1; + } + memset(ss.methods_list, 0, method_count * sizeof(FlatBuffersMethodProperty*)); + + // Register MethodProperty + for (int i = 0; i < method_count; ++i) { + const brpc::flatbuffers::MethodDescriptor* md = sd->method(i); + FlatBuffersMethodProperty* mp = new FlatBuffersMethodProperty(); + if (!mp) { + LOG(ERROR) << "Fail to alloc FlatBuffersMethodProperty"; + return -1; + } + mp->service = service; + mp->method = md; + mp->status = new MethodStatus; + ss.methods_list[i] = mp; + } + _fb_server_index_map[sd->index()] = std::move(ss); + + return 0; +} +#endif + ServiceOptions::ServiceOptions() : ownership(SERVER_DOESNT_OWN_SERVICE) , allow_default_url(false) @@ -1636,6 +1765,22 @@ int Server::AddService(google::protobuf::Service* service, return AddServiceInternal(service, false, options); } +#if BRPC_WITH_FLATBUFFERS +int Server::AddService(brpc::flatbuffers::Service* service, + ServiceOwnership ownership) { + ServiceOptions options; + options.ownership = ownership; + return AddServiceInternal(service, false, options); +} +#endif + +#if BRPC_WITH_FLATBUFFERS +int Server::AddService(brpc::flatbuffers::Service* service, + const ServiceOptions& options) { + return AddServiceInternal(service, false, options); +} +#endif + int Server::AddBuiltinService(google::protobuf::Service* service) { ServiceOptions options; options.ownership = SERVER_OWNS_SERVICE; @@ -1769,6 +1914,9 @@ void Server::ClearServices() { } delete it->second.http_url; } +#if BRPC_WITH_FLATBUFFERS + _fb_server_index_map.clear(); +#endif _fullname_service_map.clear(); _service_map.clear(); _method_map.clear(); @@ -2035,6 +2183,28 @@ Server::FindServicePropertyByName(const butil::StringPiece& name) const { return _service_map.seek(name); } +#if BRPC_WITH_FLATBUFFERS +const Server::FlatBuffersServiceProperty* +Server::FindFlatBuffersServicePropertyByIndex(uint32_t service_index) const { + return _fb_server_index_map.seek(service_index); +} +#endif + +#if BRPC_WITH_FLATBUFFERS +const Server::FlatBuffersMethodProperty* +Server::FindFlatBuffersMethodPropertyByIndex(uint32_t service_index, int method_index) const { + const Server::FlatBuffersServiceProperty* sp = + FindFlatBuffersServicePropertyByIndex(service_index); + if (NULL == sp || NULL == sp->methods_list) { + return NULL; + } + if (method_index < 0 || method_index >= sp->method_count) { + return NULL; + } + return sp->methods_list[method_index]; +} +#endif + int Server::AddCertificate(const CertInfo& cert) { if (!_options.has_ssl_options()) { LOG(ERROR) << "ServerOptions.ssl_options is not configured yet"; diff --git a/src/brpc/server.h b/src/brpc/server.h index 6e7d2b2b17..6ddd6ddd9b 100644 --- a/src/brpc/server.h +++ b/src/brpc/server.h @@ -22,6 +22,7 @@ // To brpc developers: This is a header included by user, don't depend // on internal structures, use opaque pointers instead. +#include "butil/config.h" #include "bthread/errno.h" // Redefine errno #include "bthread/bthread.h" // Server may need some bthread functions, // e.g. bthread_usleep @@ -46,6 +47,9 @@ #include "brpc/baidu_master_service.h" #include "brpc/rpc_pb_message_factory.h" #include "brpc/socket_mode.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/details/flatbuffers_impl.h" +#endif namespace brpc { @@ -436,6 +440,33 @@ class Server { }; typedef butil::FlatMap MethodMap; +#if BRPC_WITH_FLATBUFFERS + struct FlatBuffersMethodProperty { + brpc::flatbuffers::Service* service; + const brpc::flatbuffers::MethodDescriptor* method; + MethodStatus* status; + FlatBuffersMethodProperty(); + }; + + struct FlatBuffersServiceProperty { + brpc::flatbuffers::Service* service; + int method_count; + FlatBuffersMethodProperty** methods_list; + bool is_user_service() const { return false; } + + const std::string& service_name() const; + FlatBuffersServiceProperty(); + ~FlatBuffersServiceProperty(); + FlatBuffersServiceProperty( + const FlatBuffersServiceProperty&) = delete; + FlatBuffersServiceProperty& operator=( + const FlatBuffersServiceProperty&) = delete; + FlatBuffersServiceProperty(FlatBuffersServiceProperty&& other); + FlatBuffersServiceProperty& operator=( + FlatBuffersServiceProperty&& other); + }; +#endif + struct ThreadLocalOptions { bthread_key_t tls_key; const DataFactory* thread_local_data_factory; @@ -501,7 +532,12 @@ class Server { bool allow_default_url = false); int AddService(google::protobuf::Service* service, const ServiceOptions& options); - +#if BRPC_WITH_FLATBUFFERS + int AddService(brpc::flatbuffers::Service* service, + ServiceOwnership ownership); + int AddService(brpc::flatbuffers::Service* service, + const ServiceOptions& options); +#endif // Remove a service from this server. // NOTE: removing a service while server is running is forbidden. // Returns 0 on success, -1 otherwise. @@ -636,6 +672,12 @@ friend class Controller; bool is_builtin_service, const ServiceOptions& options); +#if BRPC_WITH_FLATBUFFERS + int AddServiceInternal(brpc::flatbuffers::Service* service, + bool is_builtin_service, + const ServiceOptions& options); +#endif + int AddBuiltinService(google::protobuf::Service* service); // Remove all methods of `service' from internal structures. @@ -688,6 +730,14 @@ friend class Controller; const ServiceProperty* FindServicePropertyByName(const butil::StringPiece& name) const; +#if BRPC_WITH_FLATBUFFERS + const FlatBuffersServiceProperty* + FindFlatBuffersServicePropertyByIndex(uint32_t service_index) const; + + const FlatBuffersMethodProperty* + FindFlatBuffersMethodPropertyByIndex(uint32_t service_index, int method_index) const; +#endif + std::string ServerPrefix() const; // Mapping from hostname to corresponding SSL_CTX @@ -762,6 +812,13 @@ friend class Controller; // uses service->name() to designate an RPC service ServiceMap _service_map; +#if BRPC_WITH_FLATBUFFERS + // Used by FlatBuffers services. + typedef butil::FlatMap + FlatBuffersServiceIDMap; + FlatBuffersServiceIDMap _fb_server_index_map; +#endif + // The only non-builtin service in _service_map, otherwise nullptr. google::protobuf::Service* _first_service; diff --git a/test/BUILD.bazel b/test/BUILD.bazel index 8706f63f2a..f1634ac126 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -232,6 +232,9 @@ generate_unittests( name = "brpc_unittests", srcs = glob([ "brpc_*_unittest.cpp", + ], exclude = [ + "brpc_flatbuffers_message_unittest.cpp", + "brpc_flatbuffers_rpc_unittest.cpp", ]), deps = [ ":gperftools_helper", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6aebe271f4..bb190e9752 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -269,9 +269,16 @@ endforeach() # brpc tests file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp") +if(NOT WITH_FLATBUFFERS) + list(FILTER BRPC_UNITTESTS EXCLUDE REGEX "brpc_flatbuffers_.*_unittest\\.cpp$") +endif() foreach(BRPC_UT ${BRPC_UNITTESTS}) get_filename_component(BRPC_UT_WE ${BRPC_UT} NAME_WE) add_executable(${BRPC_UT_WE} ${BRPC_UT} $) + if(BRPC_UT_WE STREQUAL "brpc_flatbuffers_rpc_unittest") + target_sources(${BRPC_UT_WE} PRIVATE + ${PROJECT_SOURCE_DIR}/example/benchmark_fb/test.brpc.fb.cpp) + endif() target_link_libraries(${BRPC_UT_WE} PRIVATE brpc-shared-debug gtest_main diff --git a/test/Makefile b/test/Makefile index f2348e7fa3..2e33196a0b 100644 --- a/test/Makefile +++ b/test/Makefile @@ -180,6 +180,11 @@ TEST_BTHREAD_SOURCES = $(wildcard bthread_*unittest.cpp) TEST_BTHREAD_OBJS = $(addsuffix .o, $(basename $(TEST_BTHREAD_SOURCES))) TEST_BRPC_SOURCES = $(wildcard brpc_*unittest.cpp) +ifneq (BRPC_WITH_FLATBUFFERS=1,$(findstring BRPC_WITH_FLATBUFFERS=1,$(CPPFLAGS))) + TEST_BRPC_SOURCES := $(filter-out \ + brpc_flatbuffers_message_unittest.cpp \ + brpc_flatbuffers_rpc_unittest.cpp,$(TEST_BRPC_SOURCES)) +endif TEST_BRPC_OBJS = $(addsuffix .o, $(basename $(TEST_BRPC_SOURCES))) TEST_PROTO_SOURCES = $(wildcard *.proto) diff --git a/test/brpc_flatbuffers_message_unittest.cpp b/test/brpc_flatbuffers_message_unittest.cpp new file mode 100644 index 0000000000..3836453d76 --- /dev/null +++ b/test/brpc_flatbuffers_message_unittest.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include "brpc/details/flatbuffers_impl.h" +#include "../example/benchmark_fb/test_generated.h" + +namespace { + +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; + +Message BuildRequest(const char* value) { + MessageBuilder builder; + const auto text = builder.CreateString(value); + const auto root = test::CreateBenchmarkRequest( + builder, 7, 1, 64, 123, 9, text); + builder.Finish(root); + return builder.ReleaseMessage(); +} + +TEST(FlatBuffersMessageTest, BuilderMoveConstructorPreservesBuffer) { + MessageBuilder source; + const auto text = source.CreateString("builder move constructor"); + const auto root = test::CreateBenchmarkRequest( + source, 7, 1, 64, 123, 9, text); + source.Finish(root); + + MessageBuilder target(std::move(source)); + Message message = target.ReleaseMessage(); + + ASSERT_TRUE(message.Verify()); + ASSERT_NE(nullptr, message.GetRoot()->message()); + EXPECT_EQ("builder move constructor", + message.GetRoot()->message()->str()); +} + +TEST(FlatBuffersMessageTest, BuilderMoveAssignmentReplacesBuffer) { + MessageBuilder target; + const auto old_text = target.CreateString("old builder"); + target.Finish(test::CreateBenchmarkRequest( + target, 1, 0, 8, 10, 0, old_text)); + + MessageBuilder source; + const auto new_text = source.CreateString("replacement builder"); + source.Finish(test::CreateBenchmarkRequest( + source, 7, 1, 64, 123, 9, new_text)); + + target = std::move(source); + Message message = target.ReleaseMessage(); + + ASSERT_TRUE(message.Verify()); + const auto* result = message.GetRoot(); + EXPECT_EQ(7, result->opcode()); + ASSERT_NE(nullptr, result->message()); + EXPECT_EQ("replacement builder", result->message()->str()); +} + +TEST(FlatBuffersMessageTest, ReleasedMessageSurvivesBuilderDestruction) { + Message message = BuildRequest("hello"); + ASSERT_TRUE(message.Verify()); + const auto* root = message.GetRoot(); + EXPECT_EQ(7, root->opcode()); + EXPECT_EQ(1, root->echo_attachment()); + EXPECT_EQ(64, root->attachment_size()); + EXPECT_EQ(123, root->request_id()); + EXPECT_EQ(9, root->reserved()); + ASSERT_NE(nullptr, root->message()); + EXPECT_EQ("hello", root->message()->str()); +} + +TEST(FlatBuffersMessageTest, MoveConstructorPreservesBuffer) { + Message source = BuildRequest("move constructor"); + const auto* data = source.data(); + const auto size = source.size(); + Message target(std::move(source)); + EXPECT_EQ(data, target.data()); + EXPECT_EQ(size, target.size()); + EXPECT_EQ(0u, source.size()); + source.Clear(); + ASSERT_TRUE(target.Verify()); + ASSERT_NE(nullptr, target.GetRoot()->message()); + EXPECT_EQ("move constructor", + target.GetRoot()->message()->str()); +} + +TEST(FlatBuffersMessageTest, MoveAssignmentReplacesExistingMessage) { + Message target = BuildRequest("old contents"); + const uint8_t* data = nullptr; + size_t size = 0; + { + Message source = BuildRequest("replacement"); + data = source.data(); + size = source.size(); + target = std::move(source); + EXPECT_EQ(0u, source.size()); + } + EXPECT_EQ(data, target.data()); + EXPECT_EQ(size, target.size()); + ASSERT_TRUE(target.Verify()); + ASSERT_NE(nullptr, target.GetRoot()->message()); + EXPECT_EQ("replacement", + target.GetRoot()->message()->str()); +} + +TEST(FlatBuffersMessageTest, RejectsCorruptRootOffset) { + Message message = BuildRequest("corrupt"); + ASSERT_TRUE(message.Verify()); + ASSERT_GE(message.size(), sizeof(::flatbuffers::uoffset_t)); + std::memset(message.mutable_data(), 0xff, + sizeof(::flatbuffers::uoffset_t)); + EXPECT_FALSE(message.Verify()); +} + +} // namespace diff --git a/test/brpc_flatbuffers_rpc_unittest.cpp b/test/brpc_flatbuffers_rpc_unittest.cpp new file mode 100644 index 0000000000..b0819cccf1 --- /dev/null +++ b/test/brpc_flatbuffers_rpc_unittest.cpp @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/errno.pb.h" +#include "brpc/server.h" +#include "../example/benchmark_fb/test.brpc.fb.h" + +namespace { + +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; + +class ValidatingService : public test::BenchmarkService { +public: + std::atomic accepted{0}; + std::atomic rejected{0}; + + void Test(google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + if (request == nullptr || !request->Verify()) { + ++rejected; + static_cast(controller)->SetFailed( + brpc::EREQUEST, "Invalid FlatBuffers BenchmarkRequest"); + return; + } + const auto* root = request->GetRoot(); + MessageBuilder builder; + const auto text = builder.CreateString( + root->message() ? root->message()->str() : std::string()); + const auto result = test::CreateBenchmarkResponse( + builder, root->opcode(), root->echo_attachment(), + root->attachment_size(), root->request_id(), root->reserved(), text); + builder.Finish(result); + *response = builder.ReleaseMessage(); + ++accepted; + } +}; + +class FlatBuffersRpcTest : public ::testing::Test { +protected: + void SetUp() override { + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.timeout_ms = 3000; + options.max_retry = 0; + ASSERT_EQ(0, channel.Init(server.listen_address(), &options)); + } + + void TearDown() override { + if (server.IsRunning()) { + server.Stop(0); + server.Join(); + } + } + + Message MakeRequest() { + MessageBuilder builder; + const auto text = builder.CreateString("rpc round trip"); + const auto root = test::CreateBenchmarkRequest( + builder, 7, 0, 0, 123, 9, text); + builder.Finish(root); + return builder.ReleaseMessage(); + } + + void CheckValidCall() { + Message request = MakeRequest(); + Message response; + brpc::Controller controller; + test::BenchmarkServiceStub stub(&channel); + stub.Test(&controller, &request, &response, nullptr); + ASSERT_FALSE(controller.Failed()) << controller.ErrorText(); + ASSERT_TRUE(response.Verify()); + const auto* root = response.GetRoot(); + EXPECT_EQ(7, root->opcode()); + EXPECT_EQ(0, root->echo_attachment()); + EXPECT_EQ(0, root->attachment_size()); + EXPECT_EQ(123, root->request_id()); + EXPECT_EQ(9, root->reserved()); + ASSERT_NE(nullptr, root->message()); + EXPECT_EQ("rpc round trip", root->message()->str()); + } + + void CheckRejectedCall() { + Message request = MakeRequest(); + ASSERT_GE(request.size(), sizeof(::flatbuffers::uoffset_t)); + std::memset(request.mutable_data(), 0xff, sizeof(::flatbuffers::uoffset_t)); + Message response; + brpc::Controller controller; + test::BenchmarkServiceStub stub(&channel); + stub.Test(&controller, &request, &response, nullptr); + ASSERT_TRUE(controller.Failed()); + // The protocol transmits the error code, not the service error text. + EXPECT_EQ(brpc::EREQUEST, controller.ErrorCode()) << controller.ErrorText(); + } + + // Keep the non-owned service alive until the server has stopped. + ValidatingService service; + brpc::Server server; + brpc::Channel channel; +}; + +TEST_F(FlatBuffersRpcTest, ValidRequestRoundTrip) { + ASSERT_NO_FATAL_FAILURE(CheckValidCall()); + EXPECT_EQ(1, service.accepted.load()); + EXPECT_EQ(0, service.rejected.load()); +} + +TEST_F(FlatBuffersRpcTest, CorruptRequestIsRejectedByService) { + ASSERT_NO_FATAL_FAILURE(CheckRejectedCall()); + EXPECT_EQ(0, service.accepted.load()); + EXPECT_EQ(1, service.rejected.load()); +} + +TEST_F(FlatBuffersRpcTest, ValidRequestSucceedsAfterRejection) { + ASSERT_NO_FATAL_FAILURE(CheckValidCall()); + ASSERT_NO_FATAL_FAILURE(CheckRejectedCall()); + ASSERT_NO_FATAL_FAILURE(CheckValidCall()); + EXPECT_EQ(2, service.accepted.load()); + EXPECT_EQ(1, service.rejected.load()); +} + +} // namespace