diff --git a/.github/workflows/build-node-fibers.yml b/.github/workflows/build-node-fibers.yml index 697163b712a2..0d062421a568 100644 --- a/.github/workflows/build-node-fibers.yml +++ b/.github/workflows/build-node-fibers.yml @@ -1,14 +1,17 @@ -name: Build node-fibers with prebuilt Node +name: Build Custom Fibers on: workflow_dispatch: workflow_run: - workflows: [Build Node] + workflows: ["Build Node (Standard)"] types: - completed + branches: + - v22.21.1 jobs: build-fibers: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} strategy: matrix: include: @@ -21,9 +24,12 @@ jobs: runs-on: ${{ matrix.runs_on }} env: - NODE_VERSION: v20.18.3 + NODE_VERSION: v22.21.1 steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Debug Matrix Values run: | echo "Matrix platform: ${{ matrix.platform }}" @@ -34,39 +40,20 @@ jobs: gh release download node-${{ env.NODE_VERSION }}-release \ --repo asana/node \ --pattern "node-${{ env.NODE_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}-LATEST.tar.xz" + mv node-${{ env.NODE_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}-LATEST.tar.xz node.tar.xz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Node archive - run: | - mkdir -p node-install - tar -C node-install -xJf node-${{ env.NODE_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}-LATEST.tar.xz - echo "$GITHUB_WORKSPACE/node-install/usr/local/bin" >> $GITHUB_PATH - - - name: Verify Node Binary Architecture + - name: Execute the Dockerfile run: | - echo "Node File:" - file $GITHUB_WORKSPACE/node-install/usr/local/bin/node - echo "Runner architecture:" - uname -m - - - name: Checkout node-fibers fork - uses: actions/checkout@v3 - with: - repository: asana/node-fibers - ref: jackstrohm_node20_fibers - path: node-fibers + pwd + docker build -t node22_fibers_build -f Dockerfile.Fibers . - - name: Build node-fibers - working-directory: node-fibers + - name: Extract resources run: | - which node - node -v - node -p "process.arch" - npm install --nodedir="$GITHUB_WORKSPACE/node-install/usr/local" - npm test || true - rm bin/repl - find . + docker create --name temp_node_fibers_extract node22_fibers_build + docker cp temp_node_fibers_extract:/usr/src/node/node-fibers $GITHUB_WORKSPACE/node-fibers + docker rm temp_node_fibers_extract - name: Find and archive fibers.node run: | diff --git a/.github/workflows/build-node-openssl-fips.yml b/.github/workflows/build-node-openssl-fips.yml index 524f20d42cb7..1837722ce720 100644 --- a/.github/workflows/build-node-openssl-fips.yml +++ b/.github/workflows/build-node-openssl-fips.yml @@ -1,37 +1,22 @@ -name: Build Node with options around OpenSSL dynamic linking and FIPS +name: Build Node (FIPS) on: workflow_dispatch: inputs: - enableFips: - description: 'Whether OpenSSL should be FIPS-enabled' - default: true - type: boolean - dynamicLink: - description: 'If OpenSSL should be dynamically linked with node (rather than statically linked)' - default: false - type: boolean - sharedOpenSSLIncludes: - description: 'dir containing header files for OpenSSL' - default: '' - type: string - sharedOpenSSLLibname: - description: 'libname for dynamically linking to OpenSSL' - default: '' - type: string - sharedOpenSSLLibpath: - description: 'dir for searching for shared OpenSSL dlls' - default: '' - type: string BUILD_REF: - description: 'ref to build' + description: 'Git ref to build Node.js from' + required: true + default: 'v22.21.1' + type: string + DOCKER_FILE: + description: 'Dockerfile to use for building Node.js' required: true - default: 'main' + default: 'Dockerfile.Node22fips' type: string jobs: build-node: - name: Build ${{ matrix.platform }}-${{ matrix.arch }} with statically-linked FIPS OpenSSL + name: Build ${{ matrix.platform }}-${{ matrix.arch }} with FIPS OpenSSL strategy: matrix: include: @@ -43,17 +28,13 @@ jobs: runs_on: ubuntu-22.04-arm runs-on: ${{ matrix.runs_on }} - env: - S3_BUCKET: your-bucket-name - AWS_REGION: us-east-1 - steps: - name: Checkout Node fork uses: actions/checkout@v3 with: repository: Asana/node path: node - ref: ${{ BUILD_REF }} + ref: ${{ inputs.BUILD_REF }} token: ${{ secrets.GITHUB_TOKEN }} - name: Extract Node Version @@ -74,56 +55,16 @@ jobs: echo "BUILD_ID=${TIMESTAMP}-${SHORT_SHA}" >> $GITHUB_ENV echo "build_id=${TIMESTAMP}-${SHORT_SHA}" >> $GITHUB_OUTPUT - - name: Install dependencies (Linux) - if: matrix.platform == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y python3 g++ make curl tar xz-utils - - - name: Configure OpenSSL for fips - id: openssl-is-fips - if: inputs.enableFips - run: | - ./configure --openssl-is-fips - - - name: Dynamically link OpenSSL in Node.js - id: openssl-dynamic-link - if: inputs.dynamicLink - run: | - ./configure --shared-openssl - - - name: Define headers for OpenSSL - id: openssl-dynamic-link-headers - if: ${{ !empty(inputs.sharedOpenSSLIncludes) }} - run: | - ./configure --shared-openssl-includes ${{inputs.sharedOpenSSLIncludes}} - - - name: alternative libname for openssl - id: openssl-dynamic-link-libname - if: ${{ !empty(inputs.sharedOpenSSLLibname) }} - run: | - ./configure --shared-openssl-libname ${{inputs.sharedOpenSSLLibname}} - - - name: Define headers for OpenSSL - id: openssl-dynamic-link-libpath - if: ${{ !empty(inputs.sharedOpenSSLLibpath) }} - run: | - ./configure --shared-openssl-includes ${{inputs.sharedOpenSSLLibpath}} - - - - name: Build Node (linux) + - name: Execute the Dockerfile working-directory: node - if: matrix.platform == 'linux' run: | - ./configure --experimental-enable-pointer-compression - make -j4 install DESTDIR=$GITHUB_WORKSPACE/node-install + docker build -t node22_fips_build -f ./${{ inputs.DOCKER_FILE }} . - - name: Build Node (darwin) - working-directory: node - if: matrix.platform == 'darwin' + - name: Extract resources run: | - ./configure --experimental-enable-pointer-compression --without-snapshot - make -j2 install DESTDIR=$GITHUB_WORKSPACE/node-install + docker create --name temp_node_extract node22_fips_build + docker cp temp_node_extract:/usr/src/node/node-install $GITHUB_WORKSPACE/node-install + docker rm temp_node_extract - name: Archive Node run: | @@ -150,8 +91,8 @@ jobs: - name: Upload Node archive to release uses: softprops/action-gh-release@v1 with: - name: node-${{ env.NODE_VERSION }}-fips-static-LATEST - tag_name: node-${{ env.NODE_VERSION }}-fips-static-release + name: node-${{ env.NODE_VERSION }}-fips-LATEST + tag_name: node-${{ env.NODE_VERSION }}-fips-release files: ./artifacts/${{ env.NODE_ARCHIVE_LATEST }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-node-packages.yml b/.github/workflows/build-node-packages.yml index 7e74f98ade56..e32ffb031000 100644 --- a/.github/workflows/build-node-packages.yml +++ b/.github/workflows/build-node-packages.yml @@ -12,63 +12,37 @@ on: jobs: build-packages: if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - permissions: - id-token: write - contents: write strategy: matrix: include: - platform: linux arch: x64 - bazel_arch: amd64 runs_on: ubuntu-22.04 - platform: linux arch: arm64 - bazel_arch: arm64 runs_on: ubuntu-22.04-arm runs-on: ${{ matrix.runs_on }} env: NODE_VERSION: v22.21.1 - PLATFORM: ${{ matrix.platform }} - ARCH: ${{ matrix.arch }} - BAZEL_ARCH: ${{ matrix.bazel_arch }} - REPO: ${{ github.repository }} steps: - # Check out the v22.21.1 branch (not the workflow's default branch) so that - # Dockerfile.Packages and the Node source tree are present. The workflow YAML - # itself runs from whichever ref triggered it (main for workflow_dispatch, or - # v22.21.1 for workflow_run) — that's what the OIDC subject claim binds to, - # and it's how the IAM role's ref_patterns gate works. `ref:` here only - # controls which tree gets checked out into $GITHUB_WORKSPACE. - # - # Security note: v22.21.1 is not a protected branch, so in principle any of - # the repo's ~530 collaborators could push a malicious Dockerfile.Packages - # and have this workflow build+upload the resulting image. That same risk - # already existed for the Node source itself (which also lives on this - # branch), so this change does not expand the attack surface. A follow-up - # PR will propose a structural fix (branch protection, patch series, or - # submodule model) — tracked in our internal project notes. - name: Checkout repository uses: actions/checkout@v3 - with: - ref: ${{ env.NODE_VERSION }} - name: Debug Matrix Values run: | - echo "Matrix platform: $PLATFORM" - echo "Matrix arch: $ARCH" + echo "Matrix platform: ${{ matrix.platform }}" + echo "Matrix arch: ${{ matrix.arch }}" - name: Download Node archive - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - ASSET="node-${NODE_VERSION}-${PLATFORM}-${ARCH}-LATEST.tar.xz" - gh release download "node-${NODE_VERSION}-release" \ + gh release download node-${{ env.NODE_VERSION }}-release \ --repo asana/node \ - --pattern "$ASSET" - mv "$ASSET" node.tar.xz + --pattern "node-${{ env.NODE_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}-LATEST.tar.xz" + mv node-${{ env.NODE_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}-LATEST.tar.xz node.tar.xz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Execute the Dockerfile run: | @@ -91,73 +65,13 @@ jobs: mv node_modules/cld ./cld@2.9.1/node_modules/ mv node_modules/unix-dgram ./unix-dgram@2.0.6/node_modules/ mv "node_modules/@datadog/pprof" "./@datadog+pprof@5.8.0/node_modules/@datadog/" - tar --hard-dereference -cvzf "packages_${ARCH}.tar.gz" bcrypt@5.1.0 cld@2.9.1 unix-dgram@2.0.6 "@datadog+pprof@5.8.0" + tar --hard-dereference -cvzf packages_${{matrix.arch}}.tar.gz bcrypt@5.1.0 cld@2.9.1 unix-dgram@2.0.6 "@datadog+pprof@5.8.0" - name: Upload archive to release - # Use `gh release upload` (first-party GitHub CLI, pre-installed on runners) - # instead of softprops/action-gh-release (one-maintainer third-party action). - # Behavior: --clobber overwrites an existing asset with the same name, matching - # softprops's default. The release must already exist (created by build-node.yml). + uses: softprops/action-gh-release@v1 + with: + name: node-${{ env.NODE_VERSION }}-LATEST + tag_name: node-${{ env.NODE_VERSION }}-release + files: packages_${{matrix.arch}}.tar.gz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload "node-${NODE_VERSION}-release" \ - "packages_${ARCH}.tar.gz" \ - --clobber \ - --repo "$REPO" - - # S3 upload is restricted to the protected main branch only. The IAM role - # (push_node_gyp_packages) trusts only refs/heads/main via OIDC. To upload - # packages to S3 after a Node upgrade, trigger workflow_dispatch from main. - - name: Configure AWS credentials - if: github.ref == 'refs/heads/main' - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::403483446840:role/autogen_github_actions_beta_push_node_gyp_packages - - - name: Upload packages to S3 - if: github.ref == 'refs/heads/main' - run: | - # Upload to s3://asana-oss-cache/node-gyp/... (CloudFront path_patterns entry - # added in codez PR #390222 — that must be merged + applied via Spacelift - # before this workflow can successfully publish fetchable objects). - # - # No --acl public-read: the bucket has BucketOwnerEnforced - # (disable_confusing_acls = true), which disables ACLs entirely. - # BlockPublicAcls + IgnorePublicAcls provide additional coverage. - # Reads come via CloudFront OAC. - NODE_MAJOR=$(echo "$NODE_VERSION" | sed 's/^v//' | cut -d. -f1) - SHA256=$(sha256sum "packages_${ARCH}.tar.gz" | awk '{print $1}') - SHORT_HASH=${SHA256:0:8} - S3_KEY="node-gyp/packages_${BAZEL_ARCH}_node${NODE_MAJOR}-${SHORT_HASH}.tar.gz" - echo "Uploading packages_${ARCH}.tar.gz to s3://asana-oss-cache/$S3_KEY" - aws s3 cp "packages_${ARCH}.tar.gz" "s3://asana-oss-cache/$S3_KEY" - { - echo "S3_KEY=$S3_KEY" - echo "SHA256=$SHA256" - echo "NODE_MAJOR=$NODE_MAJOR" - } >> "$GITHUB_ENV" - - - name: Verify upload is reachable via CloudFront - if: github.ref == 'refs/heads/main' - run: | - # Mac Bazel builds rewrite asana-oss-cache.s3.us-east-1.amazonaws.com/* - # to asana-oss-cache.asana.biz/* (CloudFront). If the S3 key prefix isn't - # allowlisted in CloudFront's path_patterns, Bazel fetches will 403. - # Fail fast here rather than after someone tries to build. - URL="https://asana-oss-cache.asana.biz/$S3_KEY" - echo "Checking $URL" - if ! curl -fsSI "$URL"; then - echo "CloudFront returned an error for $URL. Check path_patterns in system_packages.tf." - exit 1 - fi - - - name: Print tools_repositories.bzl stanza - if: github.ref == 'refs/heads/main' - run: | - echo "" - echo "=== Update tools_repositories.bzl in codez ===" - echo " name = \"node_gyp_packages_${BAZEL_ARCH}_node${NODE_MAJOR}\"," - echo " urls = [\"https://asana-oss-cache.s3.us-east-1.amazonaws.com/$S3_KEY\"]," - echo " sha256 = \"$SHA256\"," diff --git a/.github/workflows/build-node.yml b/.github/workflows/build-node.yml index b74f4af4973d..5d3e80fc1931 100644 --- a/.github/workflows/build-node.yml +++ b/.github/workflows/build-node.yml @@ -1,13 +1,7 @@ -name: Build Node +name: Build Node (Standard) on: workflow_dispatch: - push: - branches: - - v20.18.3 - - workflows-for-v20.18.3 - pull_request: - paths: .github/workflows/build-node.yml jobs: build-node: @@ -54,25 +48,16 @@ jobs: echo "BUILD_ID=${TIMESTAMP}-${SHORT_SHA}" >> $GITHUB_ENV echo "build_id=${TIMESTAMP}-${SHORT_SHA}" >> $GITHUB_OUTPUT - - name: Install dependencies (Linux) - if: matrix.platform == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y python3 g++ make curl tar xz-utils - - - name: Build Node (linux) + - name: Execute the Dockerfile working-directory: node - if: matrix.platform == 'linux' run: | - ./configure --experimental-enable-pointer-compression - make -j4 install DESTDIR=$GITHUB_WORKSPACE/node-install + docker build -t node22_build -f ./Dockerfile.Node22 . - - name: Build Node (darwin) - working-directory: node - if: matrix.platform == 'darwin' + - name: Extract resources run: | - ./configure --experimental-enable-pointer-compression --without-snapshot - make -j2 install DESTDIR=$GITHUB_WORKSPACE/node-install + docker create --name temp_node_extract node22_build + docker cp temp_node_extract:/usr/src/node/node-install $GITHUB_WORKSPACE/node-install + docker rm temp_node_extract - name: Archive Node run: | diff --git a/Dockerfile.Fibers b/Dockerfile.Fibers new file mode 100644 index 000000000000..cc2c9b9b843c --- /dev/null +++ b/Dockerfile.Fibers @@ -0,0 +1,35 @@ +# Stage 1 +FROM ubuntu:20.04 AS base + +# Set non-interactive mode to avoid prompts during installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install necessary dependencies +RUN apt-get update +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN apt-get update +RUN apt-get install -y build-essential python3 python3-distutils g++-10 make curl git pkg-config libssl-dev libffi-dev libgmp-dev libtool autoconf automake cmake wget xz-utils unzip vim + +# Copy local Node.js source into the image +WORKDIR /usr/src/node +COPY node.tar.xz . + +RUN mkdir -p node-install && tar -C node-install -xJf node.tar.xz + +ENV PATH="/usr/src/node/node-install/usr/local/bin:$PATH" + +RUN which node +RUN node -v +RUN node -p "process.arch" +RUN echo ------------------- +RUN git clone --branch jackstrohm_node20_fibers --depth 1 https://github.com/asana/node-fibers.git node-fibers + +WORKDIR /usr/src/node/node-fibers +RUN npm install --nodedir="/usr/src/node/node-install/usr/local" +RUN npm test || true +RUN rm bin/repl +RUN find . + + +CMD ["bash"] diff --git a/Dockerfile.Node20fips b/Dockerfile.Node20fips new file mode 100644 index 000000000000..4c47d8b3315a --- /dev/null +++ b/Dockerfile.Node20fips @@ -0,0 +1,29 @@ +# Stage 1 +FROM ubuntu:22.04 AS base +ARG ENABLE_FIPS +ARG DYNAMIC_LINK +ARG SHARED_OPENSSL_INCLUDES +ARG SHARED_OPENSSL_LIBNAME +ARG SHARED_OPENSSL_LIBPATH + +# Set non-interactive mode to avoid prompts during installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install necessary dependencies +RUN apt-get update +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN apt-get update && apt-get install -y build-essential python3 python3-distutils g++-10 make curl git pkg-config libssl-dev libffi-dev libgmp-dev libtool autoconf automake cmake wget xz-utils unzip vim +RUN rm -rf /var/lib/apt/lists/* + +# Set g++ 10 as the default +RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 + +# Copy local Node.js source into the image +WORKDIR /usr/src/node +COPY . . + +RUN ./configure --shared-openssl --experimental-enable-pointer-compression --openssl-conf-name 'openssl_conf' +RUN make -j4 install DESTDIR=./node-install + +CMD ["bash"] diff --git a/Dockerfile.Node22 b/Dockerfile.Node22 new file mode 100644 index 000000000000..bd59c3ffcc5e --- /dev/null +++ b/Dockerfile.Node22 @@ -0,0 +1,50 @@ +# Stage 1 +FROM ubuntu:20.04 AS base + +# Set non-interactive mode to avoid prompts during installation +ENV DEBIAN_FRONTEND=noninteractive + +# Combine update, PPA setup, and install into one robust retry loop +RUN set -e; \ + for i in 1 2 3 4 5; do \ + echo "Attempt $i: Setting up build environment..." && \ + apt-get update --allow-releaseinfo-change && \ + apt-get install -y --no-install-recommends software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update --allow-releaseinfo-change && \ + apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + python3 \ + python3-distutils \ + g++-10 \ + make \ + curl \ + git \ + pkg-config \ + libssl-dev \ + libffi-dev \ + libgmp-dev \ + libtool \ + autoconf \ + automake \ + cmake \ + wget \ + xz-utils \ + unzip \ + vim \ + && break || \ + (echo "Attempt $i failed! Sleeping 15s..." && sleep 15); \ + done +RUN rm -rf /var/lib/apt/lists/* + +# Set g++ 10 as the default +RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 + +# Copy local Node.js source into the image +WORKDIR /usr/src/node +COPY . . + +RUN ./configure --experimental-enable-pointer-compression +RUN make -j4 install DESTDIR=./node-install + +CMD ["bash"] diff --git a/Dockerfile.Node22fips b/Dockerfile.Node22fips new file mode 100644 index 000000000000..926a914fee09 --- /dev/null +++ b/Dockerfile.Node22fips @@ -0,0 +1,50 @@ +# Stage 1 +FROM ubuntu:22.04 AS base + +# Set non-interactive mode to avoid prompts during installation +ENV DEBIAN_FRONTEND=noninteractive + +# Combine update, PPA setup, and install into one robust retry loop +RUN set -e; \ + for i in 1 2 3 4 5; do \ + echo "Attempt $i: Setting up build environment..." && \ + apt-get update --allow-releaseinfo-change && \ + apt-get install -y --no-install-recommends software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt-get update --allow-releaseinfo-change && \ + apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + python3 \ + python3-distutils \ + g++-10 \ + make \ + curl \ + git \ + pkg-config \ + libssl-dev \ + libffi-dev \ + libgmp-dev \ + libtool \ + autoconf \ + automake \ + cmake \ + wget \ + xz-utils \ + unzip \ + vim \ + && break || \ + (echo "Attempt $i failed! Sleeping 15s..." && sleep 15); \ + done +RUN rm -rf /var/lib/apt/lists/* + +# Set g++ 10 as the default +RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 + +# Copy local Node.js source into the image +WORKDIR /usr/src/node +COPY . . + +RUN ./configure --shared-openssl --experimental-enable-pointer-compression --openssl-conf-name 'openssl_conf' +RUN make -j4 install DESTDIR=./node-install + +CMD ["bash"] diff --git a/Dockerfile.Packages b/Dockerfile.Packages new file mode 100644 index 000000000000..a18babc06af9 --- /dev/null +++ b/Dockerfile.Packages @@ -0,0 +1,56 @@ +# Stage 1 +FROM ubuntu:20.04 AS base + +# Set non-interactive mode to avoid prompts during installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install necessary dependencies +RUN apt-get update +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN apt-get update +RUN apt-get install -y build-essential python3 python3-distutils g++-10 make curl git pkg-config libssl-dev libffi-dev libgmp-dev libtool autoconf automake cmake wget xz-utils unzip vim + +# Copy local Node.js source into the image +WORKDIR /usr/src/node +COPY node.tar.xz . + +RUN mkdir -p node-install && tar -C node-install -xJf node.tar.xz + +ENV PATH="/usr/src/node/node-install/usr/local/bin:$PATH" + +RUN which node +RUN node -v +RUN node -p "process.arch" +RUN echo ------------------- + +RUN npm i --nodedir="/usr/src/node/node-install/usr/local" bcrypt@5.1.0 +RUN npm i --nodedir="/usr/src/node/node-install/usr/local" cld@2.9.1 +RUN npm i --nodedir="/usr/src/node/node-install/usr/local" unix-dgram@2.0.6 + +# Build @datadog/pprof from source against Asana custom Node. +# The npm package ships with "install: exit 0" and no C++ source (only prebuilts), +# so we clone from GitHub to build the native binding against our pointer-compressed V8. +# The standard prebuilts are ABI-incompatible with --experimental-enable-pointer-compression. +RUN npm i @datadog/pprof@5.8.0 --ignore-scripts +RUN git clone --branch v5.8.0 --depth 1 https://github.com/DataDog/pprof-nodejs.git /usr/src/pprof-source +WORKDIR /usr/src/pprof-source +RUN npm install --unsafe-perm --build-from-source +RUN npm install --unsafe-perm node-gyp +RUN node_modules/.bin/node-gyp rebuild --jobs=max --nodedir=/usr/src/node/node-install/usr/local +# Replace the linux prebuilt with our pointer-compression-compatible binary. +# NODE_MODULE_VERSION and arch are read dynamically from the custom Node binary. +# Use find to locate the built .node file since the exact name may vary by version. +WORKDIR /usr/src/node +RUN ARCH=$(node -e "process.stdout.write(process.arch)") && \ + MODULES_VER=$(node -e "process.stdout.write(process.versions.modules)") && \ + PROFILER_NODE=$(find /usr/src/pprof-source/build/Release -name "*.node" | head -1) && \ + echo "Using profiler binary: $PROFILER_NODE" && \ + test -f "$PROFILER_NODE" && \ + cp "$PROFILER_NODE" \ + node_modules/@datadog/pprof/prebuilds/linuxglibc-${ARCH}/node-${MODULES_VER}.node + +RUN ls +RUN ls node_modules + +CMD ["bash"] diff --git a/build-local.sh b/build-local.sh new file mode 100755 index 000000000000..4d2f5f29fa01 --- /dev/null +++ b/build-local.sh @@ -0,0 +1,132 @@ +#!/bin/bash +set -e + +# Build Node.js with fibers and native packages locally +# Combines: Dockerfile.Node22, Dockerfile.Fibers, Dockerfile.Packages + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +OUTPUT_DIR="${SCRIPT_DIR}/local-build-output" +NODE_INSTALL_DIR="${OUTPUT_DIR}/node" +FIBERS_DIR="${OUTPUT_DIR}/node-fibers" +PACKAGES_DIR="${OUTPUT_DIR}/packages" + +# Number of parallel jobs for make +JOBS=${JOBS:-$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)} + +echo "=== Local Node.js + Fibers Build Script ===" +echo "Output directory: ${OUTPUT_DIR}" +echo "Parallel jobs: ${JOBS}" +echo "" + +# Clean previous output +rm -rf "${OUTPUT_DIR}" +mkdir -p "${OUTPUT_DIR}" + +# ============================================ +# Stage 1: Build Node.js (Dockerfile.Node22) +# ============================================ +echo "=== Stage 1: Building Node.js ===" + +cd "${SCRIPT_DIR}" + +# Configure if needed +if [ ! -f config.gypi ] || [ ! -f config.mk ]; then + echo "Configuring Node.js with pointer compression..." + ./configure --experimental-enable-pointer-compression +fi + +# Build Node.js +echo "Building Node.js (this may take a while)..." +make -j${JOBS} + +# Install to local directory +echo "Installing Node.js to ${NODE_INSTALL_DIR}..." +make install DESTDIR="${NODE_INSTALL_DIR}" PREFIX="" + +# Verify installation +NODE_BIN="${NODE_INSTALL_DIR}/bin/node" +NPM_BIN="${NODE_INSTALL_DIR}/bin/npm" + +echo "Verifying Node.js installation..." +"${NODE_BIN}" --version +"${NODE_BIN}" -p "process.arch" +echo "Node module version: $("${NODE_BIN}" -p "process.versions.modules")" + +# IMPORTANT: Put our custom Node first in PATH so npm/node-gyp use it for native builds +export PATH="${NODE_INSTALL_DIR}/bin:${PATH}" +echo "Using node from: $(which node)" + +# ============================================ +# Stage 2: Build node-fibers (Dockerfile.Fibers) +# ============================================ +echo "" +echo "=== Stage 2: Building node-fibers ===" + +mkdir -p "${FIBERS_DIR}" +cd "${FIBERS_DIR}" + +# Clone node-fibers if not already present +if [ ! -d "node-fibers" ]; then + echo "Cloning node-fibers..." + git clone --branch jackstrohm_node20_fibers --depth 1 https://github.com/asana/node-fibers.git node-fibers +fi + +cd node-fibers + +echo "Building node-fibers..." +"${NPM_BIN}" install --nodedir="${NODE_INSTALL_DIR}" + +echo "Running node-fibers tests (failures allowed)..." +"${NPM_BIN}" test || true + +# Remove repl as in Dockerfile +rm -f bin/repl + +echo "node-fibers built successfully" + +# ============================================ +# Stage 3: Build native packages (Dockerfile.Packages) +# ============================================ +echo "" +echo "=== Stage 3: Building native packages ===" + +mkdir -p "${PACKAGES_DIR}" +cd "${PACKAGES_DIR}" + +# Initialize a package.json if needed +if [ ! -f package.json ]; then + echo '{"name": "native-packages", "private": true}' > package.json +fi + +echo "Installing bcrypt@5.1.0..." +"${NPM_BIN}" install --nodedir="${NODE_INSTALL_DIR}" bcrypt@5.1.0 + +echo "Installing cld@2.9.1..." +"${NPM_BIN}" install --nodedir="${NODE_INSTALL_DIR}" cld@2.9.1 + +echo "Installing unix-dgram@2.0.6..." +"${NPM_BIN}" install --nodedir="${NODE_INSTALL_DIR}" unix-dgram@2.0.6 + +# ============================================ +# Summary +# ============================================ +echo "" +echo "=== Build Complete ===" +echo "" +echo "Output directory structure:" +echo " ${OUTPUT_DIR}/" +echo " ├── node/ # Node.js installation" +echo " │ ├── bin/node" +echo " │ ├── bin/npm" +echo " │ └── ..." +echo " ├── node-fibers/ # node-fibers addon" +echo " │ └── node-fibers/" +echo " └── packages/ # Native packages (bcrypt, cld, unix-dgram)" +echo " └── node_modules/" +echo "" +echo "To use this Node.js:" +echo " export PATH=\"${NODE_INSTALL_DIR}/bin:\$PATH\"" +echo " node --version" +echo "" +echo "To test fibers:" +echo " cd ${FIBERS_DIR}/node-fibers && ${NODE_BIN} test.js" diff --git a/deps/v8/src/base/platform/platform.h b/deps/v8/src/base/platform/platform.h index 99be31da7019..9be154693165 100644 --- a/deps/v8/src/base/platform/platform.h +++ b/deps/v8/src/base/platform/platform.h @@ -582,7 +582,13 @@ class V8_BASE_EXPORT Thread { static LocalStorageKey CreateThreadLocalKey(); static void DeleteThreadLocalKey(LocalStorageKey key); static void* GetThreadLocal(LocalStorageKey key); + static int GetThreadLocalInt(LocalStorageKey key) { + return static_cast(reinterpret_cast(GetThreadLocal(key))); + } static void SetThreadLocal(LocalStorageKey key, void* value); + static void SetThreadLocalInt(LocalStorageKey key, int value) { + SetThreadLocal(key, reinterpret_cast(static_cast(value))); + } static bool HasThreadLocal(LocalStorageKey key) { return GetThreadLocal(key) != nullptr; } diff --git a/deps/v8/src/execution/isolate-inl.h b/deps/v8/src/execution/isolate-inl.h index 256db29d1407..e3d2d838118f 100644 --- a/deps/v8/src/execution/isolate-inl.h +++ b/deps/v8/src/execution/isolate-inl.h @@ -24,22 +24,24 @@ namespace v8 { namespace internal { // static -V8_INLINE Isolate::PerIsolateThreadData* -Isolate::CurrentPerIsolateThreadData() { - return g_current_per_isolate_thread_data_; -} +// Commented out for fibers support - these are now non-inline in isolate.h +// using LocalStorageKey instead of thread_local +//V8_INLINE Isolate::PerIsolateThreadData* +//Isolate::CurrentPerIsolateThreadData() { +// return g_current_per_isolate_thread_data_; +//} // static -V8_INLINE Isolate* Isolate::TryGetCurrent() { return g_current_isolate_; } +//V8_INLINE Isolate* Isolate::TryGetCurrent() { return g_current_isolate_; } // static -V8_INLINE Isolate* Isolate::Current() { - Isolate* isolate = TryGetCurrent(); - DCHECK_NOT_NULL(isolate); - return isolate; -} +//V8_INLINE Isolate* Isolate::Current() { +// Isolate* isolate = TryGetCurrent(); +// DCHECK_NOT_NULL(isolate); +// return isolate; +//} -bool Isolate::IsCurrent() const { return this == TryGetCurrent(); } +//bool Isolate::IsCurrent() const { return this == TryGetCurrent(); } void Isolate::set_context(Tagged context) { DCHECK(context.is_null() || IsContext(context)); diff --git a/deps/v8/src/execution/isolate.cc b/deps/v8/src/execution/isolate.cc index c2cc0458bb64..4fa3b0cd496a 100644 --- a/deps/v8/src/execution/isolate.cc +++ b/deps/v8/src/execution/isolate.cc @@ -494,9 +494,14 @@ size_t Isolate::HashIsolateForEmbeddedBlob() { Isolate* Isolate::process_wide_shared_space_isolate_{nullptr}; -thread_local Isolate::PerIsolateThreadData* g_current_per_isolate_thread_data_ - V8_CONSTINIT = nullptr; -thread_local Isolate* g_current_isolate_ V8_CONSTINIT = nullptr; +// LocalStorageKey-based thread locals for fibers support +base::Thread::LocalStorageKey Isolate::isolate_key_; +base::Thread::LocalStorageKey Isolate::per_isolate_thread_data_key_; +std::atomic Isolate::isolate_key_created_{false}; +// Commented out for fibers support - replaced with LocalStorageKey +//thread_local Isolate::PerIsolateThreadData* g_current_per_isolate_thread_data_ +// V8_CONSTINIT = nullptr; +//thread_local Isolate* g_current_isolate_ V8_CONSTINIT = nullptr; namespace { // A global counter for all generated Isolates, might overflow. @@ -551,7 +556,24 @@ Isolate::PerIsolateThreadData* Isolate::FindPerThreadDataForThread( return per_thread; } -void Isolate::InitializeOncePerProcess() { Heap::InitializeOncePerProcess(); } +void Isolate::InitializeOncePerProcess() { + // Initialize LocalStorageKeys for fibers support + isolate_key_ = base::Thread::CreateThreadLocalKey(); + bool expected = false; + CHECK(isolate_key_created_.compare_exchange_strong( + expected, true, std::memory_order_relaxed)); + per_isolate_thread_data_key_ = base::Thread::CreateThreadLocalKey(); + + Heap::InitializeOncePerProcess(); +} + +void Isolate::DisposeOncePerProcess() { + base::Thread::DeleteThreadLocalKey(isolate_key_); + bool expected = true; + CHECK(isolate_key_created_.compare_exchange_strong( + expected, false, std::memory_order_relaxed)); + base::Thread::DeleteThreadLocalKey(per_isolate_thread_data_key_); +} Address Isolate::get_address_from_id(IsolateAddressId id) { return isolate_addresses_[id]; @@ -3547,10 +3569,14 @@ void Isolate::Delete(Isolate* isolate) { // direct pointer. We don't use Enter/Exit here to avoid // initializing the thread data. PerIsolateThreadData* saved_data = isolate->CurrentPerIsolateThreadData(); - Isolate* saved_isolate = isolate->TryGetCurrent(); + // Use LocalStorageKey directly for fibers support + DCHECK_EQ(true, isolate_key_created_.load(std::memory_order_relaxed)); + Isolate* saved_isolate = reinterpret_cast( + base::Thread::GetThreadLocal(isolate->isolate_key_)); SetIsolateThreadLocals(isolate, nullptr); isolate->set_thread_id(ThreadId::Current()); - isolate->heap()->SetStackStart(base::Stack::GetStackStart()); + // Commented out for fibers support - can cause issues with fiber switching + //isolate->heap()->SetStackStart(base::Stack::GetStackStart()); isolate->Deinit(); @@ -4020,8 +4046,11 @@ void Isolate::Deinit() { void Isolate::SetIsolateThreadLocals(Isolate* isolate, PerIsolateThreadData* data) { - g_current_isolate_ = isolate; - g_current_per_isolate_thread_data_ = data; + // Use LocalStorageKey for fibers support + //g_current_isolate_ = isolate; + //g_current_per_isolate_thread_data_ = data; + base::Thread::SetThreadLocal(isolate_key_, isolate); + base::Thread::SetThreadLocal(per_isolate_thread_data_key_, data); #ifdef V8_COMPRESS_POINTERS_IN_ISOLATE_CAGE if (isolate) { @@ -4397,10 +4426,17 @@ void Isolate::AddCrashKeysForIsolateAndHeapPointers() { add_crash_key_callback_(v8::CrashKeyId::kReadonlySpaceFirstPageAddress, ToHexString(ro_space_firstpage_address)); - const uintptr_t old_space_firstpage_address = - heap()->old_space()->FirstPageAddress(); - add_crash_key_callback_(v8::CrashKeyId::kOldSpaceFirstPageAddress, - ToHexString(old_space_firstpage_address)); + // Commented out for pointer compression / fibers support + //const uintptr_t old_space_firstpage_address = + // heap()->old_space()->FirstPageAddress(); + //add_crash_key_callback_(v8::CrashKeyId::kOldSpaceFirstPageAddress, + // ToHexString(old_space_firstpage_address)); + if (heap()->new_space()) { + const uintptr_t map_space_firstpage_address = + heap()->new_space()->FirstPageAddress(); + add_crash_key_callback_(v8::CrashKeyId::kMapSpaceFirstPageAddress, + ToHexString(map_space_firstpage_address)); + } if (heap()->code_range_base()) { const uintptr_t code_range_base_address = heap()->code_range_base(); diff --git a/deps/v8/src/execution/isolate.h b/deps/v8/src/execution/isolate.h index 8f40fe7e1018..f641774d284e 100644 --- a/deps/v8/src/execution/isolate.h +++ b/deps/v8/src/execution/isolate.h @@ -625,6 +625,7 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { }; static void InitializeOncePerProcess(); + static void DisposeOncePerProcess(); // Creates Isolate object. Must be used instead of constructing Isolate with // new operator. @@ -645,15 +646,30 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { // Returns the PerIsolateThreadData for the current thread (or nullptr if one // is not currently set). - V8_INLINE static PerIsolateThreadData* CurrentPerIsolateThreadData(); + // Changed from V8_INLINE with thread_local to use LocalStorageKey for fibers support + static PerIsolateThreadData* CurrentPerIsolateThreadData() { + return reinterpret_cast( + base::Thread::GetThreadLocal(per_isolate_thread_data_key_)); + } // Returns the isolate inside which the current thread is running or nullptr. - V8_INLINE static Isolate* TryGetCurrent(); + // Changed from V8_INLINE with thread_local to use LocalStorageKey for fibers support + V8_INLINE static Isolate* TryGetCurrent() { + DCHECK_EQ(true, isolate_key_created_.load(std::memory_order_relaxed)); + return reinterpret_cast( + base::Thread::GetExistingThreadLocal(isolate_key_)); + } // Returns the isolate inside which the current thread is running. - V8_INLINE static Isolate* Current(); + // Changed from V8_INLINE with thread_local to use LocalStorageKey for fibers support + V8_INLINE static Isolate* Current() { + Isolate* isolate = TryGetCurrent(); + DCHECK_NOT_NULL(isolate); + return isolate; + } - inline bool IsCurrent() const; + // Changed from inline to use TryGetCurrent() with LocalStorageKey for fibers support + bool IsCurrent() const { return this == TryGetCurrent(); } // Usually called by Init(), but can be called early e.g. to allow // testing components that require logging but not the whole @@ -2285,6 +2301,12 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { static Isolate* process_wide_shared_space_isolate_; + // LocalStorageKey-based thread locals for fibers support + // (replaces thread_local variables) + static base::Thread::LocalStorageKey per_isolate_thread_data_key_; + static base::Thread::LocalStorageKey isolate_key_; + static std::atomic isolate_key_created_; + void Deinit(); static void SetIsolateThreadLocals(Isolate* isolate, @@ -2719,9 +2741,11 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { // These are outside the Isolate class with extern storage because in clang-cl, // thread_local is incompatible with dllexport linkage caused by // V8_EXPORT_PRIVATE being applied to Isolate. -extern thread_local Isolate::PerIsolateThreadData* - g_current_per_isolate_thread_data_ V8_CONSTINIT; -extern thread_local Isolate* g_current_isolate_ V8_CONSTINIT; +// +// Commented out for fibers support - replaced with LocalStorageKey in Isolate class +//extern thread_local Isolate::PerIsolateThreadData* +// g_current_per_isolate_thread_data_ V8_CONSTINIT; +//extern thread_local Isolate* g_current_isolate_ V8_CONSTINIT; #undef FIELD_ACCESSOR #undef THREAD_LOCAL_TOP_ACCESSOR diff --git a/deps/v8/src/execution/thread-id.cc b/deps/v8/src/execution/thread-id.cc index ec4e95ad5d83..d20308e7673b 100644 --- a/deps/v8/src/execution/thread-id.cc +++ b/deps/v8/src/execution/thread-id.cc @@ -11,7 +11,9 @@ namespace internal { namespace { -thread_local int thread_id = 0; +// Use LocalStorageKey instead of thread_local for fibers support +DEFINE_LAZY_LEAKY_OBJECT_GETTER(base::Thread::LocalStorageKey, GetThreadIdKey, + base::Thread::CreateThreadLocalKey()) std::atomic next_thread_id{1}; @@ -19,14 +21,18 @@ std::atomic next_thread_id{1}; // static ThreadId ThreadId::TryGetCurrent() { + int thread_id = base::Thread::GetThreadLocalInt(*GetThreadIdKey()); return thread_id == 0 ? Invalid() : ThreadId(thread_id); } // static int ThreadId::GetCurrentThreadId() { + auto key = *GetThreadIdKey(); + int thread_id = base::Thread::GetThreadLocalInt(key); if (thread_id == 0) { thread_id = next_thread_id.fetch_add(1); CHECK_LE(1, thread_id); + base::Thread::SetThreadLocalInt(key, thread_id); } return thread_id; } diff --git a/lib/internal/bootstrap/realm.js b/lib/internal/bootstrap/realm.js index af283265d37d..ea95107b5d0a 100644 --- a/lib/internal/bootstrap/realm.js +++ b/lib/internal/bootstrap/realm.js @@ -156,7 +156,17 @@ const experimentalModuleList = new SafeSet(['sqlite']); return internalBinding(module); } if (legacyWrapperList.has(module)) { - return requireBuiltin('internal/legacy/processbinding')[module](); + const legacy = requireBuiltin('internal/legacy/processbinding')[module](); + if (module === 'util') { + // We need getPromiseDetails and constants.kFulfilled from `internalBinding('util')` + // for fiber elimination utilities + return { + ...legacy, + ...internalBinding('util'), + }; + } else { + return legacy; + } } if (processBindingAllowList.has(module)) { return internalBinding(module); diff --git a/src/env.cc b/src/env.cc index 926645dc647f..a735654b8c22 100644 --- a/src/env.cc +++ b/src/env.cc @@ -586,6 +586,9 @@ IsolateData::IsolateData(Isolate* isolate, // safe guard against this. DCHECK_NE(descriptor.wrappable_type_index, BaseObject::kSlot); } else { + // Commented out for fibers support - CppHeap can cause issues + // with fiber context switching + /* cpp_heap_ = CppHeap::Create( platform, CppHeapCreateParams{ @@ -593,6 +596,7 @@ IsolateData::IsolateData(Isolate* isolate, WrapperDescriptor( BaseObject::kEmbedderType, BaseObject::kSlot, cppgc_id)}); isolate->AttachCppHeap(cpp_heap_.get()); + */ } // We do not care about overflow since we just want this to be different // from the cppgc id. diff --git a/stage_for_s3.bash b/stage_for_s3.bash index 22ee4c9c62f0..794de5173386 100755 --- a/stage_for_s3.bash +++ b/stage_for_s3.bash @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + mkdir stage cd stage || exit @@ -7,43 +9,73 @@ TIMESTAMP=$(date '+%Y%m%d.%H%M') echo "Current timestamp is $TIMESTAMP" -gh release download -p "*.gz" -gh release download -p "*.xz" +# Download packages and move to codez +gh release download -R Asana/node node-v22.21.1-release -p "packages_*.gz" +echo "Moving packages to $CODEZ/node/gyp" +mv packages_arm64.tar.gz $CODEZ/node/gyp/packages_arm64_node22.tar.gz +mv packages_x64.tar.gz $CODEZ/node/gyp/packages_amd64_node22.tar.gz -# Separate packages tarballs — these are uploaded to S3 by the build-node-packages.yml -# workflow (with content-hashed keys like packages_amd64_node22-bb5ac136.tar.gz) and -# consumed by Bazel via http_file in codez. They should NOT be mixed into the fibers archive. -echo "" -echo "=== Native packages (node-gyp) ===" -echo "These are uploaded to s3://asana-oss-cache/node-gyp/ by the build-node-packages.yml workflow" -echo "(triggered via workflow_dispatch from main) with content-hashed S3 keys." -echo "Each build produces an immutable artifact." -for pkg in packages_*.tar.gz; do - if [ -f "$pkg" ]; then - echo " $pkg: sha256=$(sha256sum "$pkg" | awk '{print $1}')" - rm "$pkg" - fi -done -echo "No manual action needed for packages if you've already dispatched build-node-packages.yml from main." -echo "" +# Download fibers binaries and node tarballs (standard build) +gh release download -R Asana/node node-v22.21.1-release -p "linux-*.gz" +gh release download -R Asana/node node-v22.21.1-release -p "*.xz" -curl "https://asana-oss-cache.s3.us-east-1.amazonaws.com/node-fibers/fibers-5.0.4.pc.tgz" --output fibers-5.0.4.tar.gz -tar -xzf fibers-5.0.4.tar.gz +# Download FIPS node tarballs +gh release download -R Asana/node node-v22.21.1-fips-release -p "*.xz" -find . -name "*.gz" | while read -r a -do - tar -xzf "$a" -C package/bin - rm "$a" +# Download fibers base package via CloudFront (public S3 access was disabled) +curl "https://asana-oss-cache.asana.biz/node-fibers/fibers-5.0.4-pc-20250515.1411-960a.tgz" --output fibers-5.0.4.tar.gz +tar -xzf fibers-5.0.4.tar.gz +rm fibers-5.0.4.tar.gz + +# Extract linux fibers binaries into the package +LINUX_GZ_FILES=$(find . -name "linux-*.gz") +if [ -z "$LINUX_GZ_FILES" ]; then + echo "WARNING: No linux-*.gz fibers archives were found to inject (check CI artifact names)" +else + echo "$LINUX_GZ_FILES" | while read -r a; do + echo "Injecting $(basename "$a") into package/bin..." + tar -xzf "$a" -C package/bin + rm "$a" + done +fi +echo "Contents of package/bin after injection:" +ls package/bin/ + +# Repackage fibers with combined binaries +tar -czf temp.tgz package/ +rm -rf package + +# Validate combined archive contains Node 18, 20, and 22 binaries +echo "Validating combined fibers archive..." +echo "fibers.node entries in archive:" +tar -tzf temp.tgz | grep 'fibers\.node' || echo " (none found)" +FOUND_VERSIONS="" +for bin_path in $(tar -tzf temp.tgz | grep 'fibers\.node$'); do + abi_ver=$(echo "$bin_path" | cut -d/ -f3 | cut -d- -f3) + case "$abi_ver" in + 109) FOUND_VERSIONS="$FOUND_VERSIONS Node18(abi=$abi_ver)" ;; + 115) FOUND_VERSIONS="$FOUND_VERSIONS Node20(abi=$abi_ver)" ;; + 127) FOUND_VERSIONS="$FOUND_VERSIONS Node22(abi=$abi_ver)" ;; + esac done +echo "Found binaries:${FOUND_VERSIONS}" +MISSING="" +echo "$FOUND_VERSIONS" | grep -q "Node18" || MISSING="$MISSING Node18(abi=109)" +echo "$FOUND_VERSIONS" | grep -q "Node20" || MISSING="$MISSING Node20(abi=115)" +echo "$FOUND_VERSIONS" | grep -q "Node22" || MISSING="$MISSING Node22(abi=127)" +if [ -n "$MISSING" ]; then + echo "ERROR: Combined fibers archive is missing binaries for:$MISSING" >&2 + exit 1 +fi +echo "Validation passed: archive contains binaries for Node 18, 20, and 22" -tar -czf temp.tgz package/ -rm -fr package SHORT_HASH=$(cat temp.tgz | sha1sum | cut -c1-4) echo "HASH: $SHORT_HASH" UNIQUE="pc-${TIMESTAMP}-${SHORT_HASH}" mv temp.tgz "fibers-5.0.4-${UNIQUE}.tgz" +# Process node tarballs for file in *.tar.xz; do if [[ "$file" == *-LATEST.tar.xz ]]; then base="${file%-LATEST.tar.xz}" @@ -52,23 +84,28 @@ for file in *.tar.xz; do echo "Renaming: $file -> $new_name" mv "$file" "$new_name" - if [[ "$new_name" =~ node-v([0-9.]+)-(darwin|linux)-(arm64|x64)-pc.*\.tar\.xz$ ]]; then + if [[ "$new_name" =~ node-v([0-9.]+)-(fips-)?(darwin|linux)-(arm64|x64).*\.tar\.xz$ ]]; then version="${BASH_REMATCH[1]}" - os="${BASH_REMATCH[2]}" - arch="${BASH_REMATCH[3]}" - target_dir="node-v${version}-${os}-${arch}" + fips="${BASH_REMATCH[2]}" + os="${BASH_REMATCH[3]}" + arch="${BASH_REMATCH[4]}" + target_dir="node-v${version}-${fips}${os}-${arch}" echo "Target Dir: $target_dir" mkdir "$target_dir" - tar -xzf "$new_name" -C "$target_dir" - mv "$target_dir/usr/local/*" "$target_dir" - rm -fr "$target_dir/usr/local" + tar -xJf "$new_name" -C "$target_dir" + + # Flatten directory structure if needed (handle both usr/local and direct layouts) + if [ -d "$target_dir/usr/local" ]; then + mv $target_dir/usr/local/* "$target_dir" + rm -rf "$target_dir/usr" + fi tar -cJf "$new_name" "$target_dir" - rm -fr "$target_dir" + rm -rf "$target_dir" - echo "✅ Done: Archive now contains:" + echo "Done: Archive now contains:" tar -tf "$new_name" | head else @@ -77,11 +114,120 @@ for file in *.tar.xz; do fi done - cd .. mv stage "node-${UNIQUE}" echo "Files are in node-${UNIQUE}, please upload to s3" +echo "Packages were moved to $CODEZ/node/gyp/packages_{arm64,amd64}_node22.tar.gz" + +# Compute SHA256 hashes for node archives +echo "" +echo "Computing SHA256 hashes..." +STAGE_DIR="node-${UNIQUE}" +SHA_LINUX_ARM64=$(shasum -a 256 "${STAGE_DIR}/node-v22.21.1-linux-arm64-${UNIQUE}.tar.xz" | cut -d' ' -f1) +SHA_LINUX_X64=$(shasum -a 256 "${STAGE_DIR}/node-v22.21.1-linux-x64-${UNIQUE}.tar.xz" | cut -d' ' -f1) +SHA_FIPS_LINUX_ARM64=$(shasum -a 256 "${STAGE_DIR}/node-v22.21.1-fips-linux-arm64-${UNIQUE}.tar.xz" | cut -d' ' -f1) +SHA_FIPS_LINUX_X64=$(shasum -a 256 "${STAGE_DIR}/node-v22.21.1-fips-linux-x64-${UNIQUE}.tar.xz" | cut -d' ' -f1) +echo " Standard linux-arm64: $SHA_LINUX_ARM64" +echo " Standard linux-x64: $SHA_LINUX_X64" +echo " FIPS linux-arm64: $SHA_FIPS_LINUX_ARM64" +echo " FIPS linux-x64: $SHA_FIPS_LINUX_X64" + +# Update package.json with new fibers URL +echo "" +echo "Updating $CODEZ/asana2/package.json..." +FIBERS_URL="https://asana-oss-cache.s3.us-east-1.amazonaws.com/node-fibers/fibers-5.0.4-${UNIQUE}.tgz" +sed -i '' "s|\"fibers\": \"https://asana-oss-cache.s3.us-east-1.amazonaws.com/node-fibers/fibers-5.0.4-[^\"]*\"|\"fibers\": \"${FIBERS_URL}\"|" "$CODEZ/asana2/package.json" +echo " Updated fibers URL to: $FIBERS_URL" +# Update node_setup.bzl +echo "" +echo "Updating $CODEZ/asana2/third_party/node/node_setup.bzl..." +NODE_SETUP_FILE="$CODEZ/asana2/third_party/node/node_setup.bzl" + +# Remove existing nodejs22pc and nodejs22fips blocks if present +python3 - "$NODE_SETUP_FILE" <<'PYEOF' +import sys + +def remove_toolchain_block(lines, name): + result = [] + i = 0 + while i < len(lines): + if 'nodejs_register_toolchains(' in lines[i]: + block = [lines[i]] + j = i + 1 + while j < len(lines): + block.append(lines[j]) + if lines[j].strip() == ')': + break + j += 1 + if any(f'name = "{name}"' in l for l in block): + i = j + 1 # skip the whole block + continue + else: + result.extend(block) + i = j + 1 + else: + result.append(lines[i]) + i += 1 + return result + +with open(sys.argv[1], 'r') as f: + lines = f.read().splitlines(keepends=True) + +for name in ['nodejs22pc', 'nodejs22fips']: + lines = remove_toolchain_block(lines, name) + +with open(sys.argv[1], 'w') as f: + f.writelines(lines) +PYEOF + +# Write the new toolchain blocks to a temp file +TEMP_BLOCKS=$(mktemp) +cat > "$TEMP_BLOCKS" << EOF + + nodejs_register_toolchains( + name = "nodejs22pc", + node_version = "22.21.1", + node_repositories = { + "22.21.1-linux_arm64": ("node-v22.21.1-linux-arm64-${UNIQUE}.tar.xz", "node-v22.21.1-linux-arm64", "${SHA_LINUX_ARM64}"), + "22.21.1-linux_amd64": ("node-v22.21.1-linux-x64-${UNIQUE}.tar.xz", "node-v22.21.1-linux-x64", "${SHA_LINUX_X64}"), + }, + node_urls = NODE_FORK_URLS, + ) + + nodejs_register_toolchains( + name = "nodejs22fips", + node_version = "22.21.1", + node_repositories = { + "22.21.1-linux_arm64": ("node-v22.21.1-fips-linux-arm64-${UNIQUE}.tar.xz", "node-v22.21.1-fips-linux-arm64", "${SHA_FIPS_LINUX_ARM64}"), + "22.21.1-linux_amd64": ("node-v22.21.1-fips-linux-x64-${UNIQUE}.tar.xz", "node-v22.21.1-fips-linux-x64", "${SHA_FIPS_LINUX_X64}"), + }, + node_urls = NODE_FIPS_URLS, + ) +EOF + +# Find the line number of the closing paren after nodejs20pc and insert after it +LINE_NUM=$(awk '/name = "nodejs20pc"/{found=1} found && /^[[:space:]]*\)/{print NR; exit}' "$NODE_SETUP_FILE") +if [ -n "$LINE_NUM" ]; then + head -n "$LINE_NUM" "$NODE_SETUP_FILE" > "${NODE_SETUP_FILE}.tmp" + cat "$TEMP_BLOCKS" >> "${NODE_SETUP_FILE}.tmp" + tail -n +$((LINE_NUM + 1)) "$NODE_SETUP_FILE" >> "${NODE_SETUP_FILE}.tmp" + mv "${NODE_SETUP_FILE}.tmp" "$NODE_SETUP_FILE" +fi +rm "$TEMP_BLOCKS" + +echo " Added nodejs22pc and nodejs22fips toolchains" + +echo "" +echo "==========================================" +echo "DONE! Please review the changes before committing:" +echo " - $CODEZ/asana2/package.json" +echo " - $CODEZ/asana2/third_party/node/node_setup.bzl" +echo " - Files in ${STAGE_DIR}/ ready for S3 upload" +echo "" +echo "After uploading files to S3, run:" +echo " cd $CODEZ/asana2 && z npm update-lockfile" +echo "=========================================="