[Performance] Debugging issue with node

handwritten/spanner/.gitignore

@@ -4,11 +4,13 @@
 /coverage
 /.nyc_output
 /docs/
+dist/
 /out/
 /build/
 system-test/secrets.js
 system-test/*key.json
 *.lock
 .DS_Store
 package-lock.json
+*.cpuprofile
 __pycache__

handwritten/spanner/issue/Dockerfile.go

@@ -0,0 +1,11 @@
+FROM golang:1.25-bookworm AS build
+WORKDIR /src
+COPY issue/go/go.mod ./go.mod
+COPY issue/go/go.sum ./go.sum
+RUN go mod download
+COPY issue/go/main.go ./main.go
+RUN CGO_ENABLED=0 GOOS=linux go build -o /out/insert-benchmark ./main.go
+
+FROM gcr.io/distroless/static-debian12:nonroot
+COPY --from=build /out/insert-benchmark /insert-benchmark
+ENTRYPOINT ["/insert-benchmark"]

handwritten/spanner/issue/Dockerfile.go-raw

@@ -0,0 +1,11 @@
+FROM golang:1.25-bookworm AS build
+WORKDIR /src
+COPY issue/go/go.mod ./go.mod
+COPY issue/go/go.sum ./go.sum
+RUN go mod download
+COPY issue/go/raw_grpc_benchmark.go ./raw_grpc_benchmark.go
+RUN CGO_ENABLED=0 GOOS=linux go build -o /out/raw-grpc-benchmark ./raw_grpc_benchmark.go
+
+FROM gcr.io/distroless/static-debian12:nonroot
+COPY --from=build /out/raw-grpc-benchmark /raw-grpc-benchmark
+ENTRYPOINT ["/raw-grpc-benchmark"]

handwritten/spanner/issue/Dockerfile.node

@@ -0,0 +1,10 @@
+FROM node:22-bookworm-slim
+ENV NODE_ENV=production
+WORKDIR /app
+ARG SPANNER_VERSION=8.7.1
+RUN npm init -y \
+ && npm install --omit=dev "@google-cloud/spanner@${SPANNER_VERSION}" @grpc/grpc-js tsx \
+ && npm cache clean --force
+COPY --chown=node:node --chmod=0444 issue/node/benchmark-insert-repl.ts ./benchmark-insert-repl.mts
+USER node
+ENTRYPOINT ["/app/node_modules/.bin/tsx", "/app/benchmark-insert-repl.mts"]

handwritten/spanner/issue/Dockerfile.node-current

@@ -0,0 +1,27 @@
+FROM node:22-bookworm-slim AS local-lib
+WORKDIR /workspace/spanner
+COPY package*.json ./
+RUN if [ -f package-lock.json ]; then \
+      npm ci --ignore-scripts; \
+    else \
+      npm install --ignore-scripts; \
+    fi
+COPY . .
+RUN npm run compile \
+ && mkdir -p /tmp/spanner-pack \
+ && npm pack --pack-destination /tmp/spanner-pack \
+ && cp /tmp/spanner-pack/google-cloud-spanner-*.tgz /tmp/google-cloud-spanner.tgz
+
+FROM node:22-bookworm-slim
+ENV NODE_ENV=production
+WORKDIR /app
+RUN npm init -y >/dev/null 2>&1 || true \
+ && npm install --omit=dev @grpc/grpc-js tsx \
+ && npm cache clean --force
+COPY --from=local-lib /tmp/google-cloud-spanner.tgz /tmp/google-cloud-spanner.tgz
+RUN npm install --omit=dev /tmp/google-cloud-spanner.tgz \
+ && npm cache clean --force \
+ && node -e "console.log(require('@google-cloud/spanner/package.json').version)"
+COPY --chown=node:node --chmod=0444 issue/node/benchmark-insert-repl.ts ./benchmark-insert-repl.mts
+USER node
+ENTRYPOINT ["/app/node_modules/.bin/tsx", "/app/benchmark-insert-repl.mts"]

handwritten/spanner/issue/Dockerfile.node-raw

@@ -0,0 +1,26 @@
+FROM node:22-bookworm-slim AS local-lib
+WORKDIR /workspace/spanner
+COPY package*.json ./
+RUN if [ -f package-lock.json ]; then \
+      npm ci --ignore-scripts; \
+    else \
+      npm install --ignore-scripts; \
+    fi
+COPY . .
+RUN npm run compile \
+ && mkdir -p /tmp/spanner-pack \
+ && npm pack --pack-destination /tmp/spanner-pack \
+ && cp /tmp/spanner-pack/google-cloud-spanner-*.tgz /tmp/google-cloud-spanner.tgz
+
+FROM node:22-bookworm-slim
+ENV NODE_ENV=production
+WORKDIR /app
+RUN npm init -y >/dev/null 2>&1 || true \
+ && npm install --omit=dev @grpc/grpc-js \
+ && npm cache clean --force
+COPY --from=local-lib /tmp/google-cloud-spanner.tgz /tmp/google-cloud-spanner.tgz
+RUN npm install --omit=dev /tmp/google-cloud-spanner.tgz \
+ && npm cache clean --force
+COPY --chown=node:node --chmod=0444 issue/node/raw_grpc_benchmark.js ./raw_grpc_benchmark.js
+USER node
+ENTRYPOINT ["node", "/app/raw_grpc_benchmark.js"]

handwritten/spanner/issue/README.md

@@ -0,0 +1,89 @@
+# Insert benchmark issue repro
+
+Build and push all three images:
+
+```sh
+IMAGE_REPO=us-central1-docker.pkg.dev/span-cloud-testing/gargsurbhi-images \
+SPANNER_VERSION=8.7.1 \
+BUILD_CURRENT=true \
+./issue/build-images.sh
+```
+
+Run jobs:
+
+Connect to Kubernetes cluster
+```sh
+gcloud container clusters get-credentials cluster-1 --region us-central1 --project span-cloud-testing
+```
+
+Release baseline uses `issue-insert-node:latest` from npm `SPANNER_VERSION`. Current branch uses `issue-insert-node:current`. Release cluster uses `issue-insert-node-cluster:release-8.6.0`; current cluster uses `issue-insert-node-cluster:current`.
+
+```sh
+kubectl apply -f issue/k8s/go-insert-benchmark.yaml
+kubectl apply -f issue/k8s/node-insert-benchmark.yaml
+kubectl apply -f issue/k8s/node-current-insert-benchmark.yaml
+kubectl apply -f issue/k8s/node-cluster-insert-benchmark.yaml
+kubectl apply -f issue/k8s/node-current-cluster-insert-benchmark.yaml
+```
+
+Watch logs:
+
+```sh
+kubectl -n spanner-ns logs -f job/issue-insert-go
+kubectl -n spanner-ns logs -f job/issue-insert-node
+kubectl -n spanner-ns logs -f job/issue-insert-node-current
+kubectl -n spanner-ns logs -f job/issue-insert-node-cluster
+kubectl -n spanner-ns logs -f job/issue-insert-node-current-cluster
+```
+
+Defaults match the customer repro shape:
+
+- `INSERT_COUNT=1000`
+- `INSERT_CONCURRENCY=110`
+- `BATCH_COUNT=1`
+- 3 CPU request/limit
+
+Node cluster job uses `CLUSTER_WORKERS=3`. It splits total work across workers:
+
+- total batches stays `1000`
+- total concurrency stays about `110` (`37 + 37 + 36`)
+- each worker has a separate Node event loop and Spanner client
+
+Use `VERBOSE_BATCH_LOGS=false` to remove per-batch logging overhead.
+
+## Reference table setup for Go benchmark
+
+`issue/go/main.go` samples IDs from these tables before inserting into `DeviceRecentActivityLog`:
+
+- `tracking.Devices(deviceRecordId)`
+- `tracking.DeviceDetails(deviceDetailsId)`
+- `tracking.HttpRequestDetails(httpRequestDetailsId)`
+- `tracking.HttpRequestLocations(httpRequestLocationId)`
+
+The insert target table does not require these rows unless you add foreign keys. They are only needed by the Go benchmark script. Minimum to run: 1 row in the first 3 tables; locations can be empty. Recommended for default `SAMPLE_SIZE=10000`: seed 10000 rows in each table.
+
+DDL:
+
+```sh
+gcloud spanner databases ddl update jack_henry_db \
+  --instance=gargsurbhi-testing1 \
+  --project=span-cloud-testing \
+  --ddl-file=issue/create-reference.sql
+```
+
+Seed 10000 rows per reference table:
+
+```sh
+node issue/seed-reference-data.js
+```
+
+Use env overrides if needed:
+
+```sh
+DB_PROJECT_ID=span-cloud-testing \
+DB_INSTANCE=gargsurbhi-testing1 \
+DB_DATABASE=jack_henry_db \
+DB_SCHEMA=tracking \
+SAMPLE_SIZE=10000 \
+node issue/seed-reference-data.js
+```

handwritten/spanner/issue/build-images.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+IMAGE_REPO="${IMAGE_REPO:-us-central1-docker.pkg.dev/span-cloud-testing/gargsurbhi-images}"
+SPANNER_VERSION="${SPANNER_VERSION:-8.17.1}"
+PUSH="${PUSH:-true}"
+PLATFORM="${PLATFORM:-linux/amd64}"
+CONTAINER_TOOL="${CONTAINER_TOOL:-podman}"
+
+# Build Node Raw gRPC image
+$CONTAINER_TOOL build --platform linux/amd64 -f issue/Dockerfile.node-raw -t $IMAGE_REPO/issue-raw-grpc-node:latest .
+# Build Go Raw gRPC image
+$CONTAINER_TOOL build --platform linux/amd64 -f issue/Dockerfile.go-raw -t $IMAGE_REPO/issue-raw-grpc-go:latest .
+
+# Build Go image
+"$CONTAINER_TOOL" build --platform "$PLATFORM" -f "$ROOT/issue/Dockerfile.go" \
+  -t "$IMAGE_REPO/issue-insert-go:latest" \
+  "$ROOT"
+
+# Build Node image
+"$CONTAINER_TOOL" build --platform "$PLATFORM" -f "$ROOT/issue/Dockerfile.node" \
+  --build-arg "SPANNER_VERSION=$SPANNER_VERSION" \
+  -t "$IMAGE_REPO/issue-insert-node:release-${SPANNER_VERSION}" \
+  -t "$IMAGE_REPO/issue-insert-node:latest" \
+  "$ROOT"
+
+# Build Node image with current code changes
+"$CONTAINER_TOOL" build --platform "$PLATFORM" -f "$ROOT/issue/Dockerfile.node-current" \
+  -t "$IMAGE_REPO/issue-insert-node:current" \
+  "$ROOT"
+
+
+if [[ "$PUSH" == "true" ]]; then
+  $CONTAINER_TOOL push $IMAGE_REPO/issue-raw-grpc-node:latest
+  $CONTAINER_TOOL push $IMAGE_REPO/issue-raw-grpc-go:latest
+  "$CONTAINER_TOOL" push "$IMAGE_REPO/issue-insert-go:latest"
+  "$CONTAINER_TOOL" push "$IMAGE_REPO/issue-insert-node:release-${SPANNER_VERSION}"
+  "$CONTAINER_TOOL" push "$IMAGE_REPO/issue-insert-node:latest"
+  "$CONTAINER_TOOL" push "$IMAGE_REPO/issue-insert-node:current"
+fi

handwritten/spanner/issue/create-reference.sql

@@ -0,0 +1,23 @@
+-- Minimal reference tables needed by issue/go/main.go.
+-- DeviceRecentActivityLog insert itself has no foreign keys in issue/create.sql;
+-- these tables only feed benchmark ID sampling.
+
+CREATE TABLE tracking.Devices (
+  deviceRecordId STRING(64) NOT NULL,
+  createdAt TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)
+) PRIMARY KEY (deviceRecordId);
+
+CREATE TABLE tracking.DeviceDetails (
+  deviceDetailsId STRING(64) NOT NULL,
+  createdAt TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)
+) PRIMARY KEY (deviceDetailsId);
+
+CREATE TABLE tracking.HttpRequestDetails (
+  httpRequestDetailsId STRING(64) NOT NULL,
+  createdAt TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)
+) PRIMARY KEY (httpRequestDetailsId);
+
+CREATE TABLE tracking.HttpRequestLocations (
+  httpRequestLocationId STRING(64) NOT NULL,
+  createdAt TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)
+) PRIMARY KEY (httpRequestLocationId);

handwritten/spanner/issue/create.sql

@@ -0,0 +1,20 @@
+CREATE TABLE
+  tracking.DeviceRecentActivityLog ( deviceRecentActivityLogId STRING(64) NOT NULL,
+    deviceRecordId STRING(64) NOT NULL,
+    httpRequestDetailsId STRING(64) NOT NULL,
+    deviceDetailsId STRING(64) NOT NULL,
+    ipAddress BYTES(24) NOT NULL,
+    ipAddressText STRING(MAX) AS (NET.IP_TO_STRING(ipAddress)),
+    xRequestId STRING(64) NOT NULL,
+    institutionId STRING(64),
+    userId STRING(64),
+    username STRING(64),
+    httpRequestLocationId STRING(64),
+    latency INT64,
+    createdAt TIMESTAMP NOT NULL DEFAULT (CURRENT_TIMESTAMP()),
+    sessionId STRING(256),
+    )
+PRIMARY KEY
+  (deviceRecentActivityLogId),
+  ROW DELETION POLICY (OLDER_THAN(createdAt,
+      INTERVAL 20 DAY));

handwritten/spanner/issue/go/go.mod

@@ -0,0 +1,55 @@
+module spanner-insert-benchmark
+
+go 1.25.0
+
+require (
+	cloud.google.com/go/spanner v1.91.0
+	google.golang.org/api v0.278.0
+	google.golang.org/grpc v1.81.0
+)
+
+require (
+	cel.dev/expr v0.25.1 // indirect
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.20.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	cloud.google.com/go/monitoring v1.25.0 // indirect
+	github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
+	github.com/googleapis/gax-go/v2 v2.22.0 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
+	golang.org/x/crypto v0.50.0 // indirect
+	golang.org/x/net v0.53.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
+	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+)