Drop pnpm from CMD so Node is PID 1 and SIGTERM reaches its handler

.github/workflows/manual-deploy.yml

@@ -180,7 +180,7 @@ jobs:
           environment: ${{ inputs.environment }}
           dockerfile: "packages/realm-server/realm-server.Dockerfile"
           build-args: |
-            "realm_server_script=start:${{ inputs.environment }}"
+            "realm_server_script=scripts/start-${{ inputs.environment }}.sh"
 
   build-prerender-manager:
     name: Build prerender manager Docker image
@@ -199,7 +199,7 @@ jobs:
           environment: ${{ inputs.environment }}
           dockerfile: "packages/realm-server/prerender-manager.Dockerfile"
           build-args: |
-            "prerender_manager_script=start:prerender-manager"
+            "prerender_manager_script=scripts/start-prerender-manager.sh"
 
   build-prerender:
     name: Build prerender Docker image
@@ -218,7 +218,7 @@ jobs:
           environment: ${{ inputs.environment }}
           dockerfile: "packages/realm-server/prerender.Dockerfile"
           build-args: |
-            "prerender_script=start:prerender-${{ inputs.environment }}"
+            "prerender_script=scripts/start-prerender-${{ inputs.environment }}.sh"
 
   build-worker:
     name: Build worker Docker image
@@ -237,7 +237,7 @@ jobs:
           environment: ${{ inputs.environment }}
           dockerfile: "packages/realm-server/worker.Dockerfile"
           build-args: |
-            "worker_script=start:worker-${{ inputs.environment }}"
+            "worker_script=scripts/start-worker-${{ inputs.environment }}.sh"
 
   build-pg-migration:
     name: Build pg-migration Docker image

packages/realm-server/main.ts

@@ -1,6 +1,7 @@
 import './instrument';
 import './setup-logger'; // This should be first
 import './lib/wtfnode-on-signal';
+import { writeSync } from 'node:fs';
 import {
   Realm,
   VirtualNetwork,
@@ -43,6 +44,18 @@ import { ModuleCacheCoordinator } from './lib/module-cache-coordination';
 import { resolveFullIndexOnStartup } from './lib/full-index-on-startup';
 import { PUBLISHED_DIRECTORY_NAME } from '@cardstack/runtime-common';
 
+// FD-level synchronous stderr write — `writeSync(2, ...)` calls the
+// write(2) syscall directly, bypassing Node's stream layer.
+// `process.stderr.write` is libuv-async when stderr is a pipe (the
+// Docker / ECS case), so it can be lost if the process exits before
+// libuv flushes. Stamps that fire just before death need to use the
+// FD-level form. Proof the Node process actually started, at what
+// pid/ppid, independent of the logger pipeline.
+writeSync(
+  2,
+  `[realm-server] STARTUP pid=${process.pid} ppid=${process.ppid} argv=${JSON.stringify(process.argv)}\n`,
+);
+
 let log = logger('main');
 const runtimeMetadataFile = process.env.TEST_HARNESS_REALM_SERVER_METADATA_FILE;
 
@@ -596,8 +609,22 @@ const getIndexHTML = async () => {
   // orchestrators trigger graceful cleanup (close httpServer, unsubscribe
   // realm watchers, drain queue + DB) instead of leaking the open
   // handles into a SIGKILL escalation.
-  process.on('SIGTERM', () => stopRealmServer(false));
-  process.on('SIGINT', () => stopRealmServer(false));
+  // `writeSync(2, ...)` (FD-level, syscall-synchronous) for the same
+  // reason as the STARTUP stamp at the top of this file.
+  process.on('SIGTERM', () => {
+    writeSync(
+      2,
+      `[realm-server] SIGTERM received pid=${process.pid} ppid=${process.ppid}\n`,
+    );
+    stopRealmServer(false);
+  });
+  process.on('SIGINT', () => {
+    writeSync(
+      2,
+      `[realm-server] SIGINT received pid=${process.pid} ppid=${process.ppid}\n`,
+    );
+    stopRealmServer(false);
+  });
   process.on('message', (message) => {
     if (message === 'stop') {
       stopRealmServer(true);

packages/realm-server/prerender-manager.Dockerfile

@@ -25,4 +25,4 @@ EXPOSE 4222
 HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
   CMD curl --fail --silent --show-error --max-time 5 --output /dev/null http://localhost:4222/ || exit 1
 
-CMD exec pnpm --filter "./packages/realm-server" $prerender_manager_script
+CMD exec /realm-server/packages/realm-server/$prerender_manager_script

packages/realm-server/prerender.Dockerfile

@@ -79,4 +79,4 @@ EXPOSE 4221
 HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
   CMD curl --fail --silent --show-error --max-time 5 --output /dev/null http://localhost:4221/ || exit 1
 
-CMD exec pnpm --filter "./packages/realm-server" $prerender_script
+CMD exec /realm-server/packages/realm-server/$prerender_script

packages/realm-server/prerender/manager-server.ts

@@ -1,6 +1,7 @@
 import '../instrument';
 import '../setup-logger';
 import '../lib/wtfnode-on-signal';
+import { writeSync } from 'node:fs';
 import { logger } from '@cardstack/runtime-common';
 import type { Server } from 'http';
 import { createServer } from 'http';
@@ -11,6 +12,18 @@ import {
   registerService,
 } from '../lib/dev-service-registry';
 
+// FD-level synchronous stderr write — `writeSync(2, ...)` calls the
+// write(2) syscall directly, bypassing Node's stream layer.
+// `process.stderr.write` is libuv-async when stderr is a pipe (the
+// Docker / ECS case), so it can be lost if the process exits before
+// libuv flushes. Stamps that fire just before death need to use the
+// FD-level form. Proof the Node process actually started, at what
+// pid/ppid, independent of the logger pipeline.
+writeSync(
+  2,
+  `[prerender-manager] STARTUP pid=${process.pid} ppid=${process.ppid} argv=${JSON.stringify(process.argv)}\n`,
+);
+
 let log = logger('prerender-manager');
 
 let { port, exitOnSignal, forceExitTimeoutMs } = yargs(process.argv.slice(2))
@@ -86,5 +99,19 @@ function shutdown(signal: NodeJS.Signals) {
   // process manager to terminate after grace period.
 }
 
-process.on('SIGINT', () => shutdown('SIGINT'));
-process.on('SIGTERM', () => shutdown('SIGTERM'));
+// `writeSync(2, ...)` (FD-level, syscall-synchronous) for the same
+// reason as the STARTUP stamp at the top of this file.
+process.on('SIGINT', () => {
+  writeSync(
+    2,
+    `[prerender-manager] SIGINT received pid=${process.pid} ppid=${process.ppid}\n`,
+  );
+  shutdown('SIGINT');
+});
+process.on('SIGTERM', () => {
+  writeSync(
+    2,
+    `[prerender-manager] SIGTERM received pid=${process.pid} ppid=${process.ppid}\n`,
+  );
+  shutdown('SIGTERM');
+});

packages/realm-server/prerender/prerender-server.ts

@@ -1,11 +1,24 @@
 import '../instrument';
 import '../setup-logger';
 import '../lib/wtfnode-on-signal';
+import { writeSync } from 'node:fs';
 import { logger } from '@cardstack/runtime-common';
 import yargs from 'yargs';
 import type { Server } from 'http';
 import { createPrerenderHttpServer } from './prerender-app';
 
+// FD-level synchronous stderr write — `writeSync(2, ...)` calls the
+// write(2) syscall directly, bypassing Node's stream layer.
+// `process.stderr.write` is libuv-async when stderr is a pipe (the
+// Docker / ECS case), so it can be lost if the process exits before
+// libuv flushes. Stamps that fire just before death need to use the
+// FD-level form. Proof the Node process actually started, at what
+// pid/ppid, independent of the logger pipeline.
+writeSync(
+  2,
+  `[prerender-server] STARTUP pid=${process.pid} ppid=${process.ppid} argv=${JSON.stringify(process.argv)}\n`,
+);
+
 let log = logger('prerender-server');
 
 let { port, count } = yargs(process.argv.slice(2))
@@ -60,5 +73,19 @@ function shutdown() {
   }
 }
 
-process.on('SIGINT', shutdown);
-process.on('SIGTERM', shutdown);
+// `writeSync(2, ...)` (FD-level, syscall-synchronous) for the same
+// reason as the STARTUP stamp at the top of this file.
+process.on('SIGINT', () => {
+  writeSync(
+    2,
+    `[prerender-server] SIGINT received pid=${process.pid} ppid=${process.ppid}\n`,
+  );
+  shutdown();
+});
+process.on('SIGTERM', () => {
+  writeSync(
+    2,
+    `[prerender-server] SIGTERM received pid=${process.pid} ppid=${process.ppid}\n`,
+  );
+  shutdown();
+});

packages/realm-server/realm-server.Dockerfile

@@ -22,4 +22,4 @@ RUN CI=1 pnpm install -r --offline
 
 EXPOSE 3000
 
-CMD exec pnpm --filter "./packages/realm-server" $realm_server_script
+CMD exec /realm-server/packages/realm-server/$realm_server_script

packages/realm-server/scripts/start-prerender-manager.sh

@@ -2,6 +2,17 @@
 
 # Start the prerender manager in staging
 
+# Run from the realm-server package directory so ts-node finds the package's
+# tsconfig. Previously the CMD wrapped pnpm --filter, which set CWD for us;
+# now PID 1 execs into this script directly so we set it ourselves. The PATH
+# prepend gives us the local ts-node binary that `pnpm --filter` used to put
+# on PATH automatically.
+cd "$(cd "$(dirname "$0")" && pwd)/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
+
+echo "[start-prerender-manager] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_ENV=production \
   NODE_NO_WARNINGS=1 \
   exec ts-node \

packages/realm-server/scripts/start-prerender-production.sh

@@ -3,6 +3,17 @@
 # Start the prerender server in production
 # Expects REALM_SECRET_SEED to be set in the environment
 
+# Run from the realm-server package directory so ts-node finds the package's
+# tsconfig. Previously the CMD wrapped pnpm --filter, which set CWD for us;
+# now PID 1 execs into this script directly so we set it ourselves. The PATH
+# prepend gives us the local ts-node binary that `pnpm --filter` used to put
+# on PATH automatically.
+cd "$(cd "$(dirname "$0")" && pwd)/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
+
+echo "[start-prerender-production] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_ENV=production \
   NODE_NO_WARNINGS=1 \
   BOXEL_HOST_URL=https://app.boxel.ai \

packages/realm-server/scripts/start-prerender-staging.sh

@@ -3,6 +3,17 @@
 # Start the prerender server in staging
 # Expects REALM_SECRET_SEED to be set in the environment
 
+# Run from the realm-server package directory so ts-node finds the package's
+# tsconfig. Previously the CMD wrapped pnpm --filter, which set CWD for us;
+# now PID 1 execs into this script directly so we set it ourselves. The PATH
+# prepend gives us the local ts-node binary that `pnpm --filter` used to put
+# on PATH automatically.
+cd "$(cd "$(dirname "$0")" && pwd)/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
+
+echo "[start-prerender-staging] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_ENV=production \
   NODE_NO_WARNINGS=1 \
   BOXEL_HOST_URL=https://realms-staging.stack.cards \

packages/realm-server/scripts/start-production.sh

@@ -1,5 +1,14 @@
 #! /bin/sh
 SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+# Run from the realm-server package directory so `pnpm setup:*` resolves the
+# package's npm scripts (the relative `../base/` paths inside those scripts
+# depend on this CWD) and ts-node can find the package's tsconfig. Previously
+# the CMD wrapped pnpm --filter, which set CWD for us; now PID 1 execs into
+# this script directly so we set it ourselves. The PATH prepend gives us the
+# local ts-node binary that `pnpm --filter` used to put on PATH automatically.
+cd "$SCRIPTS_DIR/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
 pnpm setup:base-in-deployment
 pnpm setup:experiments-in-deployment
 pnpm setup:catalog-in-deployment
@@ -19,6 +28,8 @@ SOFTWARE_FACTORY_REALM_URL="${RESOLVED_SOFTWARE_FACTORY_REALM_URL:-$DEFAULT_SOFT
 DEFAULT_BOXEL_HOMEPAGE_REALM_URL='https://app.boxel.ai/boxel-homepage/'
 BOXEL_HOMEPAGE_REALM_URL="${RESOLVED_BOXEL_HOMEPAGE_REALM_URL:-$DEFAULT_BOXEL_HOMEPAGE_REALM_URL}"
 
+echo "[start-production] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_NO_WARNINGS=1 \
   LOW_CREDIT_THRESHOLD=2000 \
   MATRIX_URL=https://matrix.boxel.ai \

packages/realm-server/scripts/start-staging.sh

@@ -1,5 +1,14 @@
 #! /bin/sh
 SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+# Run from the realm-server package directory so `pnpm setup:*` resolves the
+# package's npm scripts (the relative `../base/` paths inside those scripts
+# depend on this CWD) and ts-node can find the package's tsconfig. Previously
+# the CMD wrapped pnpm --filter, which set CWD for us; now PID 1 execs into
+# this script directly so we set it ourselves. The PATH prepend gives us the
+# local ts-node binary that `pnpm --filter` used to put on PATH automatically.
+cd "$SCRIPTS_DIR/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
 pnpm setup:base-in-deployment
 pnpm setup:experiments-in-deployment
 pnpm setup:catalog-in-deployment
@@ -19,6 +28,8 @@ SOFTWARE_FACTORY_REALM_URL="${RESOLVED_SOFTWARE_FACTORY_REALM_URL:-$DEFAULT_SOFT
 DEFAULT_BOXEL_HOMEPAGE_REALM_URL='https://realms-staging.stack.cards/boxel-homepage/'
 BOXEL_HOMEPAGE_REALM_URL="${RESOLVED_BOXEL_HOMEPAGE_REALM_URL:-$DEFAULT_BOXEL_HOMEPAGE_REALM_URL}"
 
+echo "[start-staging] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_NO_WARNINGS=1 \
   LOW_CREDIT_THRESHOLD=2000 \
   MATRIX_URL=https://matrix-staging.stack.cards \

packages/realm-server/scripts/start-worker-production.sh

@@ -1,10 +1,21 @@
 #! /bin/sh
 
+# Run from the realm-server package directory so ts-node finds the package's
+# tsconfig. Previously the CMD wrapped pnpm --filter, which set CWD for us;
+# now PID 1 execs into this script directly so we set it ourselves. The PATH
+# prepend gives us the local ts-node binary that `pnpm --filter` used to put
+# on PATH automatically.
+cd "$(cd "$(dirname "$0")" && pwd)/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
+
 DEFAULT_CATALOG_REALM_URL='https://app.boxel.ai/catalog/'
 CATALOG_REALM_URL="${RESOLVED_CATALOG_REALM_URL:-$DEFAULT_CATALOG_REALM_URL}"
 DEFAULT_SOFTWARE_FACTORY_REALM_URL='https://app.boxel.ai/software-factory/'
 SOFTWARE_FACTORY_REALM_URL="${RESOLVED_SOFTWARE_FACTORY_REALM_URL:-$DEFAULT_SOFTWARE_FACTORY_REALM_URL}"
 
+echo "[start-worker-production] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_NO_WARNINGS=1 \
   NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \
   REALM_SERVER_MATRIX_USERNAME=realm_server \

packages/realm-server/scripts/start-worker-staging.sh

@@ -1,10 +1,21 @@
 #! /bin/sh
 
+# Run from the realm-server package directory so ts-node finds the package's
+# tsconfig. Previously the CMD wrapped pnpm --filter, which set CWD for us;
+# now PID 1 execs into this script directly so we set it ourselves. The PATH
+# prepend gives us the local ts-node binary that `pnpm --filter` used to put
+# on PATH automatically.
+cd "$(cd "$(dirname "$0")" && pwd)/.."
+PATH="./node_modules/.bin:$PATH"
+export PATH
+
 DEFAULT_CATALOG_REALM_URL='https://realms-staging.stack.cards/catalog/'
 CATALOG_REALM_URL="${RESOLVED_CATALOG_REALM_URL:-$DEFAULT_CATALOG_REALM_URL}"
 DEFAULT_SOFTWARE_FACTORY_REALM_URL='https://realms-staging.stack.cards/software-factory/'
 SOFTWARE_FACTORY_REALM_URL="${RESOLVED_SOFTWARE_FACTORY_REALM_URL:-$DEFAULT_SOFTWARE_FACTORY_REALM_URL}"
 
+echo "[start-worker-staging] pid=$$ ppid=$PPID about to exec ts-node at $(date -Iseconds)" >&2
+
 NODE_NO_WARNINGS=1 \
   NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=4096}" \
   REALM_SERVER_MATRIX_USERNAME=realm_server \