From ef7b67e03c696167d8d703150546199dcc37bc28 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 15:04:52 +0800 Subject: [PATCH 1/2] perf(build): compile the C++ module in parallel cmake --build runs the native tool one file at a time unless a jobs count is given, and the CMAKE_BUILD_PARALLEL_LEVEL cache variable in cpp/CMakeLists.txt had no effect: only the environment variable of that name is read. Every Maven C++ build therefore compiled serially, which dominates CI time in the Release + ASan job. Pass --parallel to the compile execution and take the level from CMAKE_BUILD_PARALLEL_LEVEL, which ./mvnw and mvnw.cmd now export with the host core count. A plain mvn falls back to 4 jobs, and -Dcmake.build.parallel.level=N overrides both. build.sh gains -j/--jobs for the same default, and the inert CMakeLists block is removed. --- cpp/CLAUDE.md | 5 +++ cpp/CMakeLists.txt | 14 ++------- cpp/README-zh.md | 2 ++ cpp/README.md | 5 +++ cpp/build.sh | 28 ++++++++++++++++- cpp/pom.xml | 34 +++++++++++++++++++++ mvnw | 16 ++++++++++ mvnw.cmd | 5 +++ skills/tsfile/references/cpp-build-write.md | 5 +++ 9 files changed, 102 insertions(+), 12 deletions(-) diff --git a/cpp/CLAUDE.md b/cpp/CLAUDE.md index ca9e33b72c..867070a2c1 100644 --- a/cpp/CLAUDE.md +++ b/cpp/CLAUDE.md @@ -51,6 +51,11 @@ make -j$(nproc) # Via Maven from repo root ./mvnw clean verify -P with-cpp +# Maven and build.sh compile in parallel. ./mvnw and build.sh use one job per +# CPU core; a plain `mvn` falls back to 4. Override with +# -Dcmake.build.parallel.level=N, CMAKE_BUILD_PARALLEL_LEVEL, or build.sh -j=N +./mvnw -P with-cpp verify -Dcmake.build.parallel.level=8 + # Run all tests (fast iteration only: runs the built binary and skips the # Maven compile/link/package steps that CI performs) ./build/Release/lib/TsFile_Test diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e45cd87f3b..ffbad4a604 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -166,17 +166,9 @@ if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE) endif () -if (NOT DEFINED CMAKE_BUILD_PARALLEL_LEVEL) - include(ProcessorCount) - ProcessorCount(N) - if (N EQUAL 0) - set(N 1) - endif () - set(CMAKE_BUILD_PARALLEL_LEVEL ${N} CACHE STRING "Number of parallel build jobs") - message("CMAKE BUILD PARALLEL LEVEL: ${CMAKE_BUILD_PARALLEL_LEVEL} (auto-detected)") -else () - message("CMAKE BUILD PARALLEL LEVEL: ${CMAKE_BUILD_PARALLEL_LEVEL} (from environment)") -endif () +# Build parallelism is set by the driver, not here: `cmake --build` honors only the +# CMAKE_BUILD_PARALLEL_LEVEL environment variable (never a cache variable of that +# name), so the Maven build passes --parallel and build.sh passes -j instead. message("CMAKE BUILD TYPE " ${CMAKE_BUILD_TYPE}) # Tune Release builds for the build-host CPU (-march=native). OFF by default: diff --git a/cpp/README-zh.md b/cpp/README-zh.md index 3ea8821976..8b127e07bf 100644 --- a/cpp/README-zh.md +++ b/cpp/README-zh.md @@ -113,6 +113,8 @@ mvn clean verify -P with-cpp `build.sh` 默认使用 Release,构建完成后可在 `./build/Release` 目录下找到生成的共享库文件。 +`build.sh` 和 `./mvnw` 默认按 CPU 核数并行编译(`mvnw` 会导出 `CMAKE_BUILD_PARALLEL_LEVEL`);直接使用 `mvn` 时默认 4 个并行任务。可用 `-j=N`(`build.sh`)、`-Dcmake.build.parallel.level=N`(Maven)或环境变量 `CMAKE_BUILD_PARALLEL_LEVEL` 调整并行度。 + 在向 GitHub 提交代码之前,请确保 `mvn` 编译通过。 --- diff --git a/cpp/README.md b/cpp/README.md index af97b8aabb..30ca0461eb 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -115,6 +115,11 @@ To build tsfile, use Maven which automatically detects and uses the appropriate mvn clean verify -P with-cpp ``` +Compilation runs one job per CPU core under `./mvnw` (the wrapper exports +`CMAKE_BUILD_PARALLEL_LEVEL`); a plain `mvn` invocation falls back to 4 jobs. +Set `-Dcmake.build.parallel.level=N` or `CMAKE_BUILD_PARALLEL_LEVEL` to choose +the job count, and `bash build.sh -j=N` for the direct CMake build. + **Toolchain Selection:** Maven will automatically select the compiler based on your platform: diff --git a/cpp/build.sh b/cpp/build.sh index fb0ff1cbfe..d8f74a9a23 100644 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -36,6 +36,7 @@ enable_lz4=ON enable_lzokay=ON enable_zlib=ON enable_zstd=ON +build_jobs="" # Keep the default build compatible with CMake 3.11-era industrial toolchains. # Bundled XZ Utils requires CMake 3.20+, so LZMA2 must be enabled explicitly. enable_lzma2=OFF @@ -47,6 +48,17 @@ get_key_value() { echo "${1#*=}" } +detect_build_jobs() +{ + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v sysctl >/dev/null 2>&1 && sysctl -n hw.ncpu >/dev/null 2>&1; then + sysctl -n hw.ncpu + else + echo 1 + fi +} + usage() { cat < Enable or disable code coverage. + -j=, --jobs= Number of parallel compile jobs (default: detected + processor count). --build-shared= Build libtsfile as a shared library (default: ON). --build-static Build libtsfile as a static library. @@ -102,6 +116,7 @@ function print_config() echo "enable_zlib=$enable_zlib" echo "enable_zstd=$enable_zstd" echo "enable_lzma2=$enable_lzma2" + echo "build_jobs=$build_jobs" } function run_test_for_cov() @@ -138,6 +153,11 @@ parse_options() -c) shift enable_cov=$(get_key_value "$1");; + -j=* | --jobs=*) + build_jobs=$(get_key_value "$1");; + -j | --jobs) + shift + build_jobs=$(get_key_value "$1");; --build-shared=*) build_shared=$(get_key_value "$1");; --build-static) @@ -183,6 +203,12 @@ parse_options() } parse_options "$@" + +if [ -z "${build_jobs}" ] +then + build_jobs=$(detect_build_jobs) +fi + print_config if [[ ${run_cov_only} -eq 1 ]] @@ -243,7 +269,7 @@ cmake ../../ \ -DENABLE_ZSTD=$enable_zstd \ -DENABLE_LZMA2=$enable_lzma2 -VERBOSE=1 make +VERBOSE=1 make -j"${build_jobs}" if [ ${do_install} -eq 1 ] then VERBOSE=1 make install diff --git a/cpp/pom.xml b/cpp/pom.xml index 97428f1948..ab967160cd 100644 --- a/cpp/pom.xml +++ b/cpp/pom.xml @@ -42,6 +42,14 @@ --> address,undefined ON + + 4 ON BUNDLED OFF @@ -163,6 +171,15 @@ generators such as the Makefile generators. --> ${build.type} + + + + + @@ -231,6 +248,23 @@ + + + build-parallelism-from-env + + + env.CMAKE_BUILD_PARALLEL_LEVEL + + + + ${env.CMAKE_BUILD_PARALLEL_LEVEL} + + jenkins-build diff --git a/mvnw b/mvnw index 5643201c7d..2dcc99db7d 100755 --- a/mvnw +++ b/mvnw @@ -32,6 +32,8 @@ # e.g. to debug Maven itself, use # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 # MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# CMAKE_BUILD_PARALLEL_LEVEL - parallel jobs for the C++ (CMake) build; set to +# the host core count when unset # ---------------------------------------------------------------------------- if [ -z "$MAVEN_SKIP_RC" ] ; then @@ -300,6 +302,20 @@ if $cygwin; then MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` fi +# Let the native (C++) build use every core unless the caller pinned a level. +if [ -z "$CMAKE_BUILD_PARALLEL_LEVEL" ]; then + if [ -n "$NUMBER_OF_PROCESSORS" ]; then + CMAKE_BUILD_PARALLEL_LEVEL="$NUMBER_OF_PROCESSORS" + elif command -v nproc >/dev/null 2>&1; then + CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) + elif command -v sysctl >/dev/null 2>&1; then + CMAKE_BUILD_PARALLEL_LEVEL=$(sysctl -n hw.ncpu 2>/dev/null) + fi + if [ -n "$CMAKE_BUILD_PARALLEL_LEVEL" ]; then + export CMAKE_BUILD_PARALLEL_LEVEL + fi +fi + # Provide a "standardized" way to retrieve the CLI args that will # work with both Windows and non-Windows executions. MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" diff --git a/mvnw.cmd b/mvnw.cmd index 8a15b7f311..00f053083d 100755 --- a/mvnw.cmd +++ b/mvnw.cmd @@ -31,6 +31,8 @@ @REM e.g. to debug Maven itself, use @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM CMAKE_BUILD_PARALLEL_LEVEL - parallel jobs for the C++ (CMake) build; set to +@REM the host core count when unset @REM ---------------------------------------------------------------------------- @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' @@ -154,6 +156,9 @@ if exist %WRAPPER_JAR% ( ) @REM End of extension +@REM Let the native (C++) build use every core unless the caller pinned a level. +if not defined CMAKE_BUILD_PARALLEL_LEVEL set "CMAKE_BUILD_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS%" + @REM Provide a "standardized" way to retrieve the CLI args that will @REM work with both Windows and non-Windows executions. set MAVEN_CMD_LINE_ARGS=%* diff --git a/skills/tsfile/references/cpp-build-write.md b/skills/tsfile/references/cpp-build-write.md index 52b9dd7c8f..7d9a8bb9b7 100644 --- a/skills/tsfile/references/cpp-build-write.md +++ b/skills/tsfile/references/cpp-build-write.md @@ -37,6 +37,11 @@ cmake --build cpp/build/static --target tsfile cd cpp && bash build.sh --build-static ``` +Maven builds and `build.sh` compile in parallel: `./mvnw` and `build.sh` use one +job per CPU core, while a plain `mvn` falls back to 4. Control the count with +`-Dcmake.build.parallel.level=N`, the `CMAKE_BUILD_PARALLEL_LEVEL` environment +variable, or `bash build.sh -j=N`. + Use a separate build directory when switching library kind. A direct MSVC consumer of the static archive must define `TSFILE_STATIC`; consumers of the exported CMake target inherit the required definition. Verify the artifact in From f97029a55faf36849fc09b74157e52be5385ecd4 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 16 Nov 2026 15:47:34 +0800 Subject: [PATCH 2/2] fix(build): harden the parallel level probes and -j parsing Verify that each core-count probe actually returns a value before trusting it, and add a getconf _NPROCESSORS_ONLN fallback: sysctl exists on Linux but hw.ncpu does not. Keep CMAKE_BUILD_PARALLEL_LEVEL undefined on Windows when NUMBER_OF_PROCESSORS is missing, and accept -j in build.sh. --- cpp/build.sh | 5 ++++- mvnw | 13 ++++++++++--- mvnw.cmd | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index d8f74a9a23..f57c43c612 100644 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -74,7 +74,8 @@ Options: Checks passed to -fsanitize= when ASan is enabled (default: address,undefined). -c= Enable or disable code coverage. - -j=, --jobs= Number of parallel compile jobs (default: detected + -j=, -j, --jobs= + Number of parallel compile jobs (default: detected processor count). --build-shared= Build libtsfile as a shared library (default: ON). @@ -155,6 +156,8 @@ parse_options() enable_cov=$(get_key_value "$1");; -j=* | --jobs=*) build_jobs=$(get_key_value "$1");; + -j[0-9]*) + build_jobs=${1#-j};; -j | --jobs) shift build_jobs=$(get_key_value "$1");; diff --git a/mvnw b/mvnw index 2dcc99db7d..9af9c87649 100755 --- a/mvnw +++ b/mvnw @@ -303,12 +303,19 @@ if $cygwin; then fi # Let the native (C++) build use every core unless the caller pinned a level. +# Each probe is only trusted when it actually returns a value; `sysctl` in +# particular exists on Linux, where `hw.ncpu` does not. if [ -z "$CMAKE_BUILD_PARALLEL_LEVEL" ]; then if [ -n "$NUMBER_OF_PROCESSORS" ]; then CMAKE_BUILD_PARALLEL_LEVEL="$NUMBER_OF_PROCESSORS" - elif command -v nproc >/dev/null 2>&1; then - CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) - elif command -v sysctl >/dev/null 2>&1; then + fi + if [ -z "$CMAKE_BUILD_PARALLEL_LEVEL" ] && command -v nproc >/dev/null 2>&1; then + CMAKE_BUILD_PARALLEL_LEVEL=$(nproc 2>/dev/null) + fi + if [ -z "$CMAKE_BUILD_PARALLEL_LEVEL" ]; then + CMAKE_BUILD_PARALLEL_LEVEL=$(getconf _NPROCESSORS_ONLN 2>/dev/null) + fi + if [ -z "$CMAKE_BUILD_PARALLEL_LEVEL" ]; then CMAKE_BUILD_PARALLEL_LEVEL=$(sysctl -n hw.ncpu 2>/dev/null) fi if [ -n "$CMAKE_BUILD_PARALLEL_LEVEL" ]; then diff --git a/mvnw.cmd b/mvnw.cmd index 00f053083d..ba6c8569b9 100755 --- a/mvnw.cmd +++ b/mvnw.cmd @@ -157,7 +157,8 @@ if exist %WRAPPER_JAR% ( @REM End of extension @REM Let the native (C++) build use every core unless the caller pinned a level. -if not defined CMAKE_BUILD_PARALLEL_LEVEL set "CMAKE_BUILD_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS%" +@REM Guard on NUMBER_OF_PROCESSORS so the variable is never defined but empty. +if not defined CMAKE_BUILD_PARALLEL_LEVEL if defined NUMBER_OF_PROCESSORS set "CMAKE_BUILD_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS%" @REM Provide a "standardized" way to retrieve the CLI args that will @REM work with both Windows and non-Windows executions.