From 485d4ca556a909ca01bf510e7ae3faac03c16668 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Thu, 22 Jan 2026 15:22:49 +0800 Subject: [PATCH 01/11] add 0.49 ci --- .github/workflows/build_image.yml | 82 +++++++++++++++++++++++-------- scripts/release-docker.sh | 8 +-- 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index e62ed77..ca8a19b 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -48,6 +48,28 @@ jobs: name: vector-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu path: target/x86_64-unknown-linux-gnu/release/vector + build-x86_64-unknown-linux-gnu-nextgen-binary: + name: Build x86_64-unknown-linux-gnu-nextgen + runs-on: ubuntu-latest + needs: + - generate-build-meta + env: + VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} + VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} + steps: + - uses: actions/checkout@v2 + - name: Set up toolchains + uses: actions-rs/toolchain@v1 + - name: Install cross + run: cargo install cross + - name: Build x86_64-unknown-linux-gnu-nextgen + run: make build-x86_64-unknown-linux-gnu-nextgen + - name: Upload x86_64-unknown-linux-gnu-nextgen + uses: actions/upload-artifact@v2 + with: + name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu + path: target/x86_64-unknown-linux-gnu/release/vector-nextgen + build-aarch64-unknown-linux-gnu-binary: name: Build aarch64-unknown-linux-gnu runs-on: ubuntu-latest @@ -71,8 +93,8 @@ jobs: name: vector-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu path: target/aarch64-unknown-linux-gnu/release/vector - build-armv7-unknown-linux-gnueabihf-binary: - name: Build armv7-unknown-linux-gnueabihf + build-aarch64-unknown-linux-gnu-nextgen-binary: + name: Build aarch64-unknown-linux-gnu-nextgen runs-on: ubuntu-latest needs: - generate-build-meta @@ -86,22 +108,23 @@ jobs: uses: actions-rs/toolchain@v1 - name: Install cross run: cargo install cross - - name: Build armv7-unknown-linux-gnueabihf - run: make build-armv7-unknown-linux-gnueabihf - - name: Upload armv7-unknown-linux-gnueabihf + - name: Build aarch64-unknown-linux-gnu-nextgen + run: make build-aarch64-unknown-linux-gnu-nextgen + - name: Upload aarch64-unknown-linux-gnu-nextgen uses: actions/upload-artifact@v2 with: - name: vector-${{ env.VECTOR_BUILD_SHA }}-armv7-unknown-linux-gnueabihf - path: target/armv7-unknown-linux-gnueabihf/release/vector + name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu + path: target/aarch64-unknown-linux-gnu/release/vector-nextgen push-image: - name: Push Docker Image + name: Push Docker Images runs-on: ubuntu-latest needs: - generate-build-meta - build-x86_64-unknown-linux-gnu-binary + - build-x86_64-unknown-linux-gnu-nextgen-binary - build-aarch64-unknown-linux-gnu-binary - - build-armv7-unknown-linux-gnueabihf-binary + - build-aarch64-unknown-linux-gnu-nextgen-binary env: VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} @@ -111,11 +134,14 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Configure AWS Credentials for DBaaS Dev - uses: aws-actions/configure-aws-credentials@v1 + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v1 with: - role-to-assume: arn:aws:iam::385595570414:role/vector-cicd - aws-region: us-west-2 + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + - name: Configure Docker for GCR + run: gcloud auth configure-docker --quiet - name: Download staged binary (x86_64-unknown-linux-gnu) uses: actions/download-artifact@v2 with: @@ -126,15 +152,29 @@ jobs: with: name: vector-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu path: target/aarch64-unknown-linux-gnu/release - - name: Download staged binary (armv7-unknown-linux-gnueabihf) + - name: Download staged binary (x86_64-unknown-linux-gnu-nextgen) uses: actions/download-artifact@v2 with: - name: vector-${{ env.VECTOR_BUILD_SHA }}-armv7-unknown-linux-gnueabihf - path: target/armv7-unknown-linux-gnueabihf/release + name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu + path: target/x86_64-unknown-linux-gnu/release + - name: Download staged binary (aarch64-unknown-linux-gnu-nextgen) + uses: actions/download-artifact@v2 + with: + name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu + path: target/aarch64-unknown-linux-gnu/release - name: Set Release Meta run: | - echo "TAG=385595570414.dkr.ecr.us-west-2.amazonaws.com/tidbcloud/vector:nightly-$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - name: Push to DBaaS Dev ECR - run: | - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 385595570414.dkr.ecr.us-west-2.amazonaws.com - make release-docker + SHORT_SHA=$(git rev-parse --short HEAD) + echo "TAG=gcr.io/pingcap-public/dbaas/vector:nightly-${SHORT_SHA}" >> $GITHUB_ENV + echo "TAG_NG=gcr.io/pingcap-public/dbaas/vector:nightly-${SHORT_SHA}-ng" >> $GITHUB_ENV + - name: Build and push standard image + env: + REPO: gcr.io/pingcap-public/dbaas/vector + TAG: ${{ env.TAG }} + run: make release-docker + - name: Build and push nextgen image + env: + REPO: gcr.io/pingcap-public/dbaas/vector + TAG: ${{ env.TAG_NG }} + NEXTGEN: "true" + run: make release-docker-nextgen diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index d04d482..41eb6be 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -31,9 +31,11 @@ trap cleanup EXIT # linux/amd64 -> amd64 # linux/arm64 -> arm64 # linux/arm/v7 -> arm -cp target/x86_64-unknown-linux-gnu/release/vector "$WORK_DIR"/vector-amd64 -cp target/aarch64-unknown-linux-gnu/release/vector "$WORK_DIR"/vector-arm64 -# cp target/armv7-unknown-linux-gnueabihf/release/vector "$WORK_DIR"/vector-arm +BINARY_NAME="${NEXTGEN:+vector-nextgen}" +BINARY_NAME="${BINARY_NAME:-vector}" +cp target/x86_64-unknown-linux-gnu/release/${BINARY_NAME} "$WORK_DIR"/vector-amd64 +cp target/aarch64-unknown-linux-gnu/release/${BINARY_NAME} "$WORK_DIR"/vector-arm64 +# cp target/armv7-unknown-linux-gnueabihf/release/${BINARY_NAME} "$WORK_DIR"/vector-arm # cp config/vector.toml "$WORK_DIR" VERSION="${VECTOR_VERSION:-"$(scripts/version.sh)"}" From c5777a6eda33abe955deb462d9882789cd09af80 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Thu, 22 Jan 2026 15:30:27 +0800 Subject: [PATCH 02/11] add 0.49 ci --- .github/workflows/build_image.yml | 251 +++++++++++------------------- 1 file changed, 87 insertions(+), 164 deletions(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index ca8a19b..37e9564 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -10,171 +10,94 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + GCR_REPO: gcr.io/pingcap-public/dbaas/vector jobs: - generate-build-meta: - name: Generate Build Meta + build-and-push-images: + name: Build Images & Push To GCR runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - - uses: actions/checkout@v2 - - name: Generate Build Meta - id: build_meta - run: | - echo "vector_build_desc=$(git rev-parse --short HEAD) $(date +%Y-%m-%d)" >> $GITHUB_OUTPUT - echo "vector_build_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - outputs: - vector_build_desc: ${{ steps.build_meta.outputs.vector_build_desc }} - vector_build_sha: ${{ steps.build_meta.outputs.vector_build_sha }} - - build-x86_64-unknown-linux-gnu-binary: - name: Build x86_64-unknown-linux-gnu - runs-on: ubuntu-latest - needs: - - generate-build-meta - env: - VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} - VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} - steps: - - uses: actions/checkout@v2 - - name: Set up toolchains - uses: actions-rs/toolchain@v1 - - name: Install cross - run: cargo install cross - - name: Build x86_64-unknown-linux-gnu - run: make build-x86_64-unknown-linux-gnu - - name: Upload x86_64-unknown-linux-gnu - uses: actions/upload-artifact@v2 - with: - name: vector-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu - path: target/x86_64-unknown-linux-gnu/release/vector - - build-x86_64-unknown-linux-gnu-nextgen-binary: - name: Build x86_64-unknown-linux-gnu-nextgen - runs-on: ubuntu-latest - needs: - - generate-build-meta - env: - VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} - VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} - steps: - - uses: actions/checkout@v2 - - name: Set up toolchains - uses: actions-rs/toolchain@v1 - - name: Install cross - run: cargo install cross - - name: Build x86_64-unknown-linux-gnu-nextgen - run: make build-x86_64-unknown-linux-gnu-nextgen - - name: Upload x86_64-unknown-linux-gnu-nextgen - uses: actions/upload-artifact@v2 - with: - name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu - path: target/x86_64-unknown-linux-gnu/release/vector-nextgen - - build-aarch64-unknown-linux-gnu-binary: - name: Build aarch64-unknown-linux-gnu - runs-on: ubuntu-latest - needs: - - generate-build-meta - env: - VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} - VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} - JEMALLOC_SYS_WITH_LG_PAGE: 16 - steps: - - uses: actions/checkout@v2 - - name: Set up toolchains - uses: actions-rs/toolchain@v1 - - name: Install cross - run: cargo install cross - - name: Build aarch64-unknown-linux-gnu - run: make build-aarch64-unknown-linux-gnu - - name: Upload aarch64-unknown-linux-gnu - uses: actions/upload-artifact@v2 - with: - name: vector-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu - path: target/aarch64-unknown-linux-gnu/release/vector - - build-aarch64-unknown-linux-gnu-nextgen-binary: - name: Build aarch64-unknown-linux-gnu-nextgen - runs-on: ubuntu-latest - needs: - - generate-build-meta - env: - VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} - VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} - JEMALLOC_SYS_WITH_LG_PAGE: 16 - steps: - - uses: actions/checkout@v2 - - name: Set up toolchains - uses: actions-rs/toolchain@v1 - - name: Install cross - run: cargo install cross - - name: Build aarch64-unknown-linux-gnu-nextgen - run: make build-aarch64-unknown-linux-gnu-nextgen - - name: Upload aarch64-unknown-linux-gnu-nextgen - uses: actions/upload-artifact@v2 - with: - name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu - path: target/aarch64-unknown-linux-gnu/release/vector-nextgen - - push-image: - name: Push Docker Images - runs-on: ubuntu-latest - needs: - - generate-build-meta - - build-x86_64-unknown-linux-gnu-binary - - build-x86_64-unknown-linux-gnu-nextgen-binary - - build-aarch64-unknown-linux-gnu-binary - - build-aarch64-unknown-linux-gnu-nextgen-binary - env: - VECTOR_BUILD_DESC: ${{ needs.generate-build-meta.outputs.vector_build_desc }} - VECTOR_BUILD_SHA: ${{ needs.generate-build-meta.outputs.vector_build_sha }} - steps: - - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 - - name: Configure Docker for GCR - run: gcloud auth configure-docker --quiet - - name: Download staged binary (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v2 - with: - name: vector-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu - path: target/x86_64-unknown-linux-gnu/release - - name: Download staged binary (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v2 - with: - name: vector-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu - path: target/aarch64-unknown-linux-gnu/release - - name: Download staged binary (x86_64-unknown-linux-gnu-nextgen) - uses: actions/download-artifact@v2 - with: - name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-x86_64-unknown-linux-gnu - path: target/x86_64-unknown-linux-gnu/release - - name: Download staged binary (aarch64-unknown-linux-gnu-nextgen) - uses: actions/download-artifact@v2 - with: - name: vector-nextgen-${{ env.VECTOR_BUILD_SHA }}-aarch64-unknown-linux-gnu - path: target/aarch64-unknown-linux-gnu/release - - name: Set Release Meta - run: | - SHORT_SHA=$(git rev-parse --short HEAD) - echo "TAG=gcr.io/pingcap-public/dbaas/vector:nightly-${SHORT_SHA}" >> $GITHUB_ENV - echo "TAG_NG=gcr.io/pingcap-public/dbaas/vector:nightly-${SHORT_SHA}-ng" >> $GITHUB_ENV - - name: Build and push standard image - env: - REPO: gcr.io/pingcap-public/dbaas/vector - TAG: ${{ env.TAG }} - run: make release-docker - - name: Build and push nextgen image - env: - REPO: gcr.io/pingcap-public/dbaas/vector - TAG: ${{ env.TAG_NG }} - NEXTGEN: "true" - run: make release-docker-nextgen + - name: Check out code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.git-ref || github.sha }} + + - name: Cache cargo dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Set up Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + rustup component add rustfmt clippy + + - name: Install cross + run: cargo install cross + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set build date and tags + id: set_tags + run: | + BUILD_DATE=$(date +%Y%m%d) + SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7) + echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT + echo "sha_short=${SHA_SHORT}" >> $GITHUB_OUTPUT + echo "tag=${{ env.GCR_REPO }}:nightly-${SHA_SHORT}" >> $GITHUB_OUTPUT + echo "tag_ng=${{ env.GCR_REPO }}:nightly-${SHA_SHORT}-ng" >> $GITHUB_OUTPUT + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + + - name: Configure Gcloud + run: | + gcloud auth configure-docker gcr.io --quiet + + - name: Build x86_64 and aarch64 binaries (standard) + run: | + make build-x86_64-unknown-linux-gnu + JEMALLOC_SYS_WITH_LG_PAGE=16 make build-aarch64-unknown-linux-gnu + + - name: Build and push standard image + env: + REPO: ${{ env.GCR_REPO }} + TAG: ${{ steps.set_tags.outputs.tag }} + run: make release-docker + + - name: Build x86_64 and aarch64 binaries (nextgen) + run: | + make build-x86_64-unknown-linux-gnu-nextgen + JEMALLOC_SYS_WITH_LG_PAGE=16 make build-aarch64-unknown-linux-gnu-nextgen + + - name: Build and push nextgen image + env: + REPO: ${{ env.GCR_REPO }} + TAG: ${{ steps.set_tags.outputs.tag_ng }} + NEXTGEN: "true" + run: make release-docker-nextgen + + - name: Build Summary + run: | + echo "āœ… Vector images built and pushed successfully!" + echo "" + echo "šŸ“¦ **Built Images:**" + echo "- **Standard:** \`${{ steps.set_tags.outputs.tag }}\`" + echo "- **Nextgen:** \`${{ steps.set_tags.outputs.tag_ng }}\`" + echo "" + echo "šŸ” **Git SHA:** ${{ github.sha }}" + echo "šŸ“… **Build Date:** ${{ steps.set_tags.outputs.build_date }}" From 829fbf0eb8ab8a4860522af45413ff18ee9bcf77 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Fri, 23 Jan 2026 11:37:36 +0800 Subject: [PATCH 03/11] add 0.49 ci --- .github/workflows/build_image.yml | 25 +++++++++++-------------- src/sources/mocked_topsql/shutdown.rs | 2 ++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 37e9564..1ef2530 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -1,7 +1,6 @@ name: build_image permissions: - id-token: write contents: read on: @@ -10,15 +9,14 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - GCR_REPO: gcr.io/pingcap-public/dbaas/vector + DOCKER_REPO: slggamer/vector jobs: build-and-push-images: - name: Build Images & Push To GCR + name: Build Images & Push To Docker Hub runs-on: ubuntu-latest permissions: contents: read - id-token: write steps: - name: Check out code uses: actions/checkout@v3 @@ -58,15 +56,14 @@ jobs: SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-7) echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT echo "sha_short=${SHA_SHORT}" >> $GITHUB_OUTPUT - echo "tag=${{ env.GCR_REPO }}:nightly-${SHA_SHORT}" >> $GITHUB_OUTPUT - echo "tag_ng=${{ env.GCR_REPO }}:nightly-${SHA_SHORT}-ng" >> $GITHUB_OUTPUT + echo "tag=${{ env.DOCKER_REPO }}:nightly-${SHA_SHORT}" >> $GITHUB_OUTPUT + echo "tag_ng=${{ env.DOCKER_REPO }}:nightly-${SHA_SHORT}-ng" >> $GITHUB_OUTPUT - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - - - name: Configure Gcloud - run: | - gcloud auth configure-docker gcr.io --quiet + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: slggamer + password: ${{ secrets.DOCKERHUBTOKEN }} - name: Build x86_64 and aarch64 binaries (standard) run: | @@ -75,7 +72,7 @@ jobs: - name: Build and push standard image env: - REPO: ${{ env.GCR_REPO }} + REPO: ${{ env.DOCKER_REPO }} TAG: ${{ steps.set_tags.outputs.tag }} run: make release-docker @@ -86,7 +83,7 @@ jobs: - name: Build and push nextgen image env: - REPO: ${{ env.GCR_REPO }} + REPO: ${{ env.DOCKER_REPO }} TAG: ${{ steps.set_tags.outputs.tag_ng }} NEXTGEN: "true" run: make release-docker-nextgen diff --git a/src/sources/mocked_topsql/shutdown.rs b/src/sources/mocked_topsql/shutdown.rs index cf9ef12..8d6ea8b 100644 --- a/src/sources/mocked_topsql/shutdown.rs +++ b/src/sources/mocked_topsql/shutdown.rs @@ -48,6 +48,7 @@ impl ShutdownSubscriber { } } + #[allow(dead_code)] pub fn extend(&self) -> (ShutdownNotifier, ShutdownSubscriber) { let (tx, rx) = watch::channel(()); ( @@ -64,6 +65,7 @@ impl ShutdownSubscriber { self.done().await } + #[allow(dead_code)] pub fn subscribe(&self) -> watch::Receiver<()> { self.rx.clone() } From 2526827c24ca6a5bc55a9f53f479720179161888 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Fri, 23 Jan 2026 14:04:21 +0800 Subject: [PATCH 04/11] add 0.49 ci --- .github/workflows/build_image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 1ef2530..de5f577 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -66,6 +66,10 @@ jobs: password: ${{ secrets.DOCKERHUBTOKEN }} - name: Build x86_64 and aarch64 binaries (standard) + env: + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 + CARGO_PROFILE_RELEASE_LTO: "thin" + CARGO_BUILD_JOBS: 2 run: | make build-x86_64-unknown-linux-gnu JEMALLOC_SYS_WITH_LG_PAGE=16 make build-aarch64-unknown-linux-gnu @@ -77,6 +81,10 @@ jobs: run: make release-docker - name: Build x86_64 and aarch64 binaries (nextgen) + env: + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 + CARGO_PROFILE_RELEASE_LTO: "thin" + CARGO_BUILD_JOBS: 2 run: | make build-x86_64-unknown-linux-gnu-nextgen JEMALLOC_SYS_WITH_LG_PAGE=16 make build-aarch64-unknown-linux-gnu-nextgen From c3b4b8675a7f41afcc8626617ae950630fccad09 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Fri, 23 Jan 2026 15:02:26 +0800 Subject: [PATCH 05/11] add 0.49 ci --- .github/workflows/build_image.yml | 89 ++++++++++++++++++++++++++++--- .gitignore | 4 ++ Makefile | 15 ++++++ 3 files changed, 101 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index de5f577..8add643 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -29,10 +29,19 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo- + ${{ runner.os }}-cargo-registry- + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false - name: Set up Rust toolchain run: | @@ -40,6 +49,12 @@ jobs: echo "$HOME/.cargo/bin" >> $GITHUB_PATH rustup component add rustfmt clippy + - name: Check disk space + run: | + df -h + echo "Available disk space before build:" + df -h . | tail -1 + - name: Install cross run: cargo install cross @@ -65,14 +80,30 @@ jobs: username: slggamer password: ${{ secrets.DOCKERHUBTOKEN }} - - name: Build x86_64 and aarch64 binaries (standard) + - name: Build x86_64 binary (standard) env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 CARGO_PROFILE_RELEASE_LTO: "thin" CARGO_BUILD_JOBS: 2 + CARGO_INCREMENTAL: 0 run: | make build-x86_64-unknown-linux-gnu - JEMALLOC_SYS_WITH_LG_PAGE=16 make build-aarch64-unknown-linux-gnu + # Clean up intermediate files to save disk space + find target/x86_64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true + find target/x86_64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true + + - name: Build aarch64 binary (standard) + env: + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 + CARGO_PROFILE_RELEASE_LTO: "thin" + CARGO_BUILD_JOBS: 2 + CARGO_INCREMENTAL: 0 + JEMALLOC_SYS_WITH_LG_PAGE: 16 + run: | + make build-aarch64-unknown-linux-gnu + # Clean up intermediate files to save disk space + find target/aarch64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true + find target/aarch64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true - name: Build and push standard image env: @@ -80,14 +111,40 @@ jobs: TAG: ${{ steps.set_tags.outputs.tag }} run: make release-docker - - name: Build x86_64 and aarch64 binaries (nextgen) + - name: Clean up standard build artifacts + run: | + # Remove standard build artifacts after Docker image is built + rm -rf target/x86_64-unknown-linux-gnu/release/build + rm -rf target/aarch64-unknown-linux-gnu/release/build + find target/x86_64-unknown-linux-gnu/release/deps -type f ! -name "*.rlib" -delete 2>/dev/null || true + find target/aarch64-unknown-linux-gnu/release/deps -type f ! -name "*.rlib" -delete 2>/dev/null || true + # Keep only the final binaries + echo "Cleaned up intermediate build artifacts" + + - name: Build x86_64 binary (nextgen) env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 CARGO_PROFILE_RELEASE_LTO: "thin" CARGO_BUILD_JOBS: 2 + CARGO_INCREMENTAL: 0 run: | make build-x86_64-unknown-linux-gnu-nextgen - JEMALLOC_SYS_WITH_LG_PAGE=16 make build-aarch64-unknown-linux-gnu-nextgen + # Clean up intermediate files to save disk space + find target/x86_64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true + find target/x86_64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true + + - name: Build aarch64 binary (nextgen) + env: + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 + CARGO_PROFILE_RELEASE_LTO: "thin" + CARGO_BUILD_JOBS: 2 + CARGO_INCREMENTAL: 0 + JEMALLOC_SYS_WITH_LG_PAGE: 16 + run: | + make build-aarch64-unknown-linux-gnu-nextgen + # Clean up intermediate files to save disk space + find target/aarch64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true + find target/aarch64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true - name: Build and push nextgen image env: @@ -106,3 +163,21 @@ jobs: echo "" echo "šŸ” **Git SHA:** ${{ github.sha }}" echo "šŸ“… **Build Date:** ${{ steps.set_tags.outputs.build_date }}" + + - name: Final cleanup and disk space check + if: always() + run: | + echo "Final disk space usage:" + df -h . + echo "" + echo "Target directory size:" + du -sh target 2>/dev/null || echo "target directory not found" + echo "" + echo "Cleaning up remaining build artifacts..." + # Remove all build artifacts except final binaries + find target -type d -name "build" -exec rm -rf {} + 2>/dev/null || true + find target -type f -name "*.o" -delete 2>/dev/null || true + find target -type f -name "*.d" -delete 2>/dev/null || true + find target -type f -name "*.rmeta" -delete 2>/dev/null || true + echo "Cleanup completed" + df -h . diff --git a/.gitignore b/.gitignore index 6893226..9482176 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ .DS_Store .specstory/ test_data/ +coverage/ +*.lcov +tarpaulin-report.html +cobertura.xml \ No newline at end of file diff --git a/Makefile b/Makefile index 961acc4..9e05bc1 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,21 @@ test: @cargo test --workspace --lib @echo "Done testing." +.PHONY: test-coverage +test-coverage: + @echo "Running tests with coverage..." + @cargo install cargo-tarpaulin --quiet 2>/dev/null || true + @cargo tarpaulin --workspace --lib --out Xml --out Html --output-dir coverage --timeout 120 || true + @echo "Coverage report generated in coverage/ directory" + @echo "Done testing with coverage." + +.PHONY: test-coverage-summary +test-coverage-summary: + @echo "Running tests with coverage summary..." + @cargo install cargo-tarpaulin --quiet 2>/dev/null || true + @cargo tarpaulin --workspace --lib --out Stdout --out Xml --output-dir coverage --timeout 120 || true + @echo "Done testing with coverage." + .PHONY: build build: @echo "Building..." From 8228d962f903a5a875577661cd524c18e5e6187b Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Fri, 23 Jan 2026 15:43:59 +0800 Subject: [PATCH 06/11] add 0.49 ci --- .github/workflows/build_image.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 8add643..c5952a3 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -48,6 +48,8 @@ jobs: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable echo "$HOME/.cargo/bin" >> $GITHUB_PATH rustup component add rustfmt clippy + # Install lld linker for faster linking (optional, falls back to default if unavailable) + sudo apt-get update && sudo apt-get install -y lld || echo "lld not available, using default linker" - name: Check disk space run: | @@ -81,26 +83,33 @@ jobs: password: ${{ secrets.DOCKERHUBTOKEN }} - name: Build x86_64 binary (standard) + timeout-minutes: 60 env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 CARGO_PROFILE_RELEASE_LTO: "thin" - CARGO_BUILD_JOBS: 2 + CARGO_BUILD_JOBS: 4 CARGO_INCREMENTAL: 0 run: | + echo "Starting x86_64 build at $(date)" make build-x86_64-unknown-linux-gnu + echo "Finished x86_64 build at $(date)" # Clean up intermediate files to save disk space find target/x86_64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true find target/x86_64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true - name: Build aarch64 binary (standard) + timeout-minutes: 60 env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 CARGO_PROFILE_RELEASE_LTO: "thin" - CARGO_BUILD_JOBS: 2 + CARGO_BUILD_JOBS: 4 CARGO_INCREMENTAL: 0 JEMALLOC_SYS_WITH_LG_PAGE: 16 + RUSTFLAGS: "-C link-arg=-fuse-ld=lld" run: | + echo "Starting aarch64 build at $(date)" make build-aarch64-unknown-linux-gnu + echo "Finished aarch64 build at $(date)" # Clean up intermediate files to save disk space find target/aarch64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true find target/aarch64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true @@ -122,26 +131,33 @@ jobs: echo "Cleaned up intermediate build artifacts" - name: Build x86_64 binary (nextgen) + timeout-minutes: 60 env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 CARGO_PROFILE_RELEASE_LTO: "thin" - CARGO_BUILD_JOBS: 2 + CARGO_BUILD_JOBS: 4 CARGO_INCREMENTAL: 0 run: | + echo "Starting x86_64 nextgen build at $(date)" make build-x86_64-unknown-linux-gnu-nextgen + echo "Finished x86_64 nextgen build at $(date)" # Clean up intermediate files to save disk space find target/x86_64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true find target/x86_64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true - name: Build aarch64 binary (nextgen) + timeout-minutes: 60 env: CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 CARGO_PROFILE_RELEASE_LTO: "thin" - CARGO_BUILD_JOBS: 2 + CARGO_BUILD_JOBS: 4 CARGO_INCREMENTAL: 0 JEMALLOC_SYS_WITH_LG_PAGE: 16 + RUSTFLAGS: "-C link-arg=-fuse-ld=lld" run: | + echo "Starting aarch64 nextgen build at $(date)" make build-aarch64-unknown-linux-gnu-nextgen + echo "Finished aarch64 nextgen build at $(date)" # Clean up intermediate files to save disk space find target/aarch64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true find target/aarch64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true From 86e1d740be1f2a62442c60102ee00712bdd9ff9b Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Mon, 26 Jan 2026 12:02:24 +0800 Subject: [PATCH 07/11] add 0.49 ci --- .github/workflows/build_image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index c5952a3..d5c3100 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -128,7 +128,9 @@ jobs: find target/x86_64-unknown-linux-gnu/release/deps -type f ! -name "*.rlib" -delete 2>/dev/null || true find target/aarch64-unknown-linux-gnu/release/deps -type f ! -name "*.rlib" -delete 2>/dev/null || true # Keep only the final binaries - echo "Cleaned up intermediate build artifacts" + df -h + echo "Available disk space after Cleaned up intermediate build artifacts:" + df -h . | tail -1 - name: Build x86_64 binary (nextgen) timeout-minutes: 60 From 5fec581c8865ce4674b2112e8b12135fd60dcb57 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Mon, 26 Jan 2026 16:05:03 +0800 Subject: [PATCH 08/11] add 0.49 ci --- .github/workflows/build_image.yml | 4 ---- scripts/cross/aarch64-unknown-linux-gnu.dockerfile | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index d5c3100..287040a 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -104,8 +104,6 @@ jobs: CARGO_PROFILE_RELEASE_LTO: "thin" CARGO_BUILD_JOBS: 4 CARGO_INCREMENTAL: 0 - JEMALLOC_SYS_WITH_LG_PAGE: 16 - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" run: | echo "Starting aarch64 build at $(date)" make build-aarch64-unknown-linux-gnu @@ -154,8 +152,6 @@ jobs: CARGO_PROFILE_RELEASE_LTO: "thin" CARGO_BUILD_JOBS: 4 CARGO_INCREMENTAL: 0 - JEMALLOC_SYS_WITH_LG_PAGE: 16 - RUSTFLAGS: "-C link-arg=-fuse-ld=lld" run: | echo "Starting aarch64 nextgen build at $(date)" make build-aarch64-unknown-linux-gnu-nextgen diff --git a/scripts/cross/aarch64-unknown-linux-gnu.dockerfile b/scripts/cross/aarch64-unknown-linux-gnu.dockerfile index 67e41c7..a3243b3 100644 --- a/scripts/cross/aarch64-unknown-linux-gnu.dockerfile +++ b/scripts/cross/aarch64-unknown-linux-gnu.dockerfile @@ -4,3 +4,8 @@ COPY bootstrap-ubuntu.sh . COPY install-protoc.sh . RUN ./bootstrap-ubuntu.sh RUN ./install-protoc.sh + +RUN apt-get update && \ + apt-get remove --assume-yes gcc-9 && \ + apt-get --assume-yes install clang && \ + rm -rf /var/lib/apt/lists/* From d9698bf0331b90fb8362b17d8200eea178feea63 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Wed, 28 Jan 2026 15:51:46 +0800 Subject: [PATCH 09/11] topsql v2 --- .github/workflows/build_image.yml | 26 ++++++++++++++++++++++++++ src/main.rs | 4 ++++ 2 files changed, 30 insertions(+) diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 287040a..56bdfb3 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -160,6 +160,32 @@ jobs: find target/aarch64-unknown-linux-gnu/release/deps -name "*.rlib" -not -name "libvector*.rlib" -delete 2>/dev/null || true find target/aarch64-unknown-linux-gnu/release/build -type f -name "*.o" -delete 2>/dev/null || true + - name: Check nextgen binaries before building image + run: | + echo "Checking nextgen binary files..." + echo "" + echo "x86_64 binary:" + if [ -f target/x86_64-unknown-linux-gnu/release/vector-nextgen ]; then + ls -lh target/x86_64-unknown-linux-gnu/release/vector-nextgen + echo " āœ… EXISTS" + else + echo " āŒ NOT FOUND" + fi + echo "" + echo "aarch64 binary:" + if [ -f target/aarch64-unknown-linux-gnu/release/vector-nextgen ]; then + ls -lh target/aarch64-unknown-linux-gnu/release/vector-nextgen + echo " āœ… EXISTS" + else + echo " āŒ NOT FOUND" + fi + echo "" + if [ -f target/x86_64-unknown-linux-gnu/release/vector-nextgen ] && [ -f target/aarch64-unknown-linux-gnu/release/vector-nextgen ]; then + echo "āœ… Both nextgen binaries exist - Makefile should skip rebuild" + else + echo "āš ļø Some binaries missing - Makefile will trigger rebuild" + fi + - name: Build and push nextgen image env: REPO: ${{ env.DOCKER_REPO }} diff --git a/src/main.rs b/src/main.rs index 438b2f1..e0ab365 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,10 @@ mod utils; #[cfg(unix)] fn main() -> ExitCode { + // Initialize rustls CryptoProvider early, before any threads are spawned + // This prevents panics when rustls is used in worker threads + crate::utils::rustls::init_rustls(); + let exit_code = Application::run(ExtraContext::default()) .code() .unwrap_or(exitcode::UNAVAILABLE) as u8; From 5713af7b3891f609e4c11e5678e3eb4cf85c7313 Mon Sep 17 00:00:00 2001 From: "zhou.cai" Date: Thu, 29 Jan 2026 17:30:55 +0800 Subject: [PATCH 10/11] add 0.49 ci --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 9e05bc1..52e5564 100644 --- a/Makefile +++ b/Makefile @@ -172,6 +172,11 @@ target/%/vector-nextgen: cargo-install-cross --target ${TRIPLE} \ --no-default-features \ --features ${FEATURES} + @# Rename vector to vector-nextgen after build + @if [ -f target/${TRIPLE}/${PROFILE}/vector ] && [ ! -f target/${TRIPLE}/${PROFILE}/vector-nextgen ]; then \ + mv target/${TRIPLE}/${PROFILE}/vector target/${TRIPLE}/${PROFILE}/vector-nextgen; \ + echo "Renamed vector to vector-nextgen"; \ + fi .PHONY: cargo-install-% cargo-install-%: override TOOL = $(@:cargo-install-%=%) From 8cc1cf93226f9caa3c8a044d5efe92642f04c2e3 Mon Sep 17 00:00:00 2001 From: yibin Date: Thu, 29 Jan 2026 22:06:09 +0800 Subject: [PATCH 11/11] Fix topsql plan incorrect escape behavior (#68) (#69) Signed-off-by: yibin87 --- src/common/deltalake_writer/converter.rs | 55 +++++++++++++++++++----- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/src/common/deltalake_writer/converter.rs b/src/common/deltalake_writer/converter.rs index 518b657..8b8f04c 100644 --- a/src/common/deltalake_writer/converter.rs +++ b/src/common/deltalake_writer/converter.rs @@ -18,6 +18,23 @@ impl EventConverter { Self } + /// Convert a Vector `Value` into a plain string for Arrow Utf8 columns. + /// + /// IMPORTANT: + /// - Prefer the raw string via `as_str()` to avoid JSON-style escaping (e.g. newline -> `\n`). + /// - Fall back to `to_string()` for non-string types. + fn log_value_to_plain_string(value: &LogValue) -> Option { + if let Some(s) = value.as_str() { + return Some(s.to_string()); + } + + match value { + LogValue::Null => None, + // For everything else, `to_string()` is fine (numbers/bools/objects, etc.). + _ => Some(value.to_string()), + } + } + /// Convert events to Arrow RecordBatch pub fn events_to_record_batch( schema_manager: &mut SchemaManager, @@ -140,14 +157,14 @@ impl EventConverter { // For data fields, try exact match first, then case-insensitive match let field_name = field.name(); if let Some(value) = log_event.get(field_name.as_str()) { - Some(value.to_string()) + Self::log_value_to_plain_string(value) } else { // Try case-insensitive match for data fields if let Some(iter) = log_event.all_event_fields() { let mut found_value = None; for (key, value) in iter { if key.as_ref().to_lowercase() == field_name.to_lowercase() { - found_value = Some(value.to_string()); + found_value = Self::log_value_to_plain_string(value); break; } } @@ -160,9 +177,7 @@ impl EventConverter { }; if let Some(s) = value_opt { - // Trim quotes from string values to avoid query issues - let trimmed = s.trim_matches('"'); - builder.append_value(trimmed); + builder.append_value(&s); } else { builder.append_null(); } @@ -586,11 +601,9 @@ impl EventConverter { .iter() .map(|event| { if let Event::Log(log_event) = event { - log_event.get(field.name().as_str()).map(|v| { - // Trim quotes from string values to avoid query issues - let s = v.to_string(); - s.trim_matches('"').to_string() - }) + log_event + .get(field.name().as_str()) + .and_then(Self::log_value_to_plain_string) } else { None } @@ -612,6 +625,7 @@ mod tests { use super::*; use std::collections::BTreeMap; use vector_lib::event::LogEvent; + use arrow::array::StringArray; fn create_test_log_event() -> LogEvent { let mut log = LogEvent::from(BTreeMap::new()); @@ -655,4 +669,25 @@ mod tests { let result = EventConverter::build_boolean_column(&field, &events); assert!(result.is_ok()); } + + #[test] + fn test_string_column_preserves_newlines() { + let mut log = create_test_log_event(); + let plan = "root\n└─ child\n".to_string(); + log.insert("normalized_plan", plan.clone()); + let events = vec![Event::Log(log)]; + let field = Field::new("normalized_plan", DataType::Utf8, true); + + let array = EventConverter::build_string_column(&field, &events, None).unwrap(); + let string_array = array + .as_any() + .downcast_ref::() + .expect("should be a StringArray"); + + let got = string_array.value(0); + assert_eq!(got, plan.as_str()); + assert!(got.contains('\n')); + // If we accidentally JSON-escaped the string, we'd store a literal backslash-n. + assert!(!got.contains("\\n")); + } }