fix(ci): match E2E app tags literally, make them unique per run, and never stop the caller (#8721)

* fix(ci): match E2E app tags literally, make them unique per run, and never stop the caller

- stop-session.sh matches `E2E_APP=<tag>` as a fixed string, so a tag like `s.im` or a
  prefix like `sc` no longer matches another app's `E2E_APP=scim`
- each HTTP end-to-end app's tag is `<app>-<run id>-<attempt>-<shell pid>`, so it is unique
  among the runner user's processes
- the script never signals itself, its ancestors or its own subshells, even when the caller
  exported the same tag; /proc is read without forking so scans stay cheap
- a surviving Next telemetry flush is labelled as expected, so an unexpected escapee stands out

* fix(ci): never spare an app process whose ancestry changes mid-scan, and confirm an empty app on two scans

Author: waleedlatif1

.github/scripts/stop-session.sh

@@ -3,8 +3,11 @@
 #
 # Usage: stop-session.sh <session-leader-pid> <app-tag>
 #
-# Start the app as `E2E_APP=<app-tag> setsid <command> &`. Its processes are the members of the
-# leader's session, plus any process that inherited `E2E_APP=<app-tag>` but left the session.
+# Start the app as `E2E_APP=<app-tag> setsid <command> &`, with a tag no other process of this
+# user carries (CI uses `<app>-<run id>-<attempt>-<shell pid>`). Its processes are the members of
+# the leader's session, plus any process whose environment holds exactly `E2E_APP=<app-tag>` but
+# left the session. This script, its ancestors and its own children are never signalled, even
+# when they carry the tag.
 # `next dev` needs both: its server and workers stay in the session, while its telemetry flush is
 # spawned detached, in a session of its own, and writes .next/dev/trace after `next dev` exits.
 # The next app in the job wipes that directory, so it must not start while any of them runs.
@@ -20,11 +23,55 @@ leader=$1
 tag=$2
 grace_seconds=10
 
+# Sets `state` and `parent` from /proc/<pid>/stat without forking, so a scan stays cheap, and
+# fails once the process is gone. The command name can hold spaces and parentheses, so the
+# fields are read after its last ")".
+read_stat() {
+  local stat rest
+  read -r stat < "/proc/$1/stat" 2>/dev/null || return 1
+  rest=${stat##*") "}
+  state=${rest%% *}
+  rest=${rest#* }
+  parent=${rest%% *}
+}
+
+ancestors=" "
+pid=$$
+while read_stat "$pid" && [ "$parent" -gt 0 ]; do
+  ancestors+="$parent "
+  pid=$parent
+  [ "$pid" -eq 1 ] && break
+done
+
+# This script, an ancestor of it, or one of its own subshells and commands, which inherit the tag
+# when the caller exported it. When a process on the walk exits mid-walk, the candidate has been
+# reparented, so the walk restarts from it; ancestry that still won't resolve counts as not ours,
+# so an app process is never spared by accident.
+is_own() {
+  local candidate=$1 pid attempt
+  [[ "$ancestors" == *" $candidate "* ]] && return 0
+  for attempt in 1 2 3; do
+    pid=$candidate
+    while [ "$pid" -gt 1 ]; do
+      [ "$pid" -eq $$ ] && return 0
+      read_stat "$pid" || continue 2
+      pid=$parent
+    done
+    return 1
+  done
+  return 1
+}
+
 app_pids() {
+  local pid
   {
-    ps -s "$leader" -o pid=,stat= 2>/dev/null | awk '$2 !~ /^Z/ { print $1 }'
-    grep -lzx "E2E_APP=$tag" /proc/[0-9]*/environ 2>/dev/null | cut -d/ -f3
-  } | sort -u
+    ps -s "$leader" -o pid= 2>/dev/null
+    grep -Flzx "E2E_APP=$tag" /proc/[0-9]*/environ 2>/dev/null | cut -d/ -f3
+  } | sort -u | while read -r pid; do
+    read_stat "$pid" || continue
+    [ "$state" = Z ] && continue
+    is_own "$pid" || echo "$pid"
+  done
 }
 
 signal_app() {
@@ -44,7 +91,10 @@ leader_running() {
 list_app() {
   local pids
   pids=$(app_pids | paste -sd, -)
-  [ -z "$pids" ] || ps -p "$pids" -o pid,sid,stat,etimes,args 2>/dev/null | cut -c1-200 || true
+  [ -z "$pids" ] && return
+  ps -p "$pids" -o pid,sid,stat,etimes,args 2>/dev/null |
+    awk '/next\/dist\/telemetry\/detached-flush\.js/ { print "(expected telemetry flush) " $0; next } { print }' |
+    cut -c1-220 || true
 }
 
 # Centiseconds since boot: monotonic, and independent of the locale's decimal separator.
@@ -54,14 +104,20 @@ now_cs() {
   echo "${uptime/./}"
 }
 
-# Re-sends the signal every 0.1s while the check holds, until the shared deadline.
-# Succeeds once the check stops holding.
+# Re-sends the signal every 0.1s while the check holds, until the shared deadline. Succeeds
+# once the check has stopped holding on two scans 0.1s apart, so a process missed by one scan
+# (spawned, or mid-reparenting, while it ran) is still caught.
 signal_while() {
-  local signal=$1
+  local signal=$1 clear=0
   shift
   while ((10#$(now_cs) < deadline)); do
-    "$@" || return 0
-    signal_app "$signal"
+    if "$@"; then
+      clear=0
+      signal_app "$signal"
+    else
+      clear=$((clear + 1))
+      ((clear >= 2)) && return 0
+    fi
     sleep 0.1
   done
   ! "$@"

.github/workflows/test-build.yml

@@ -242,11 +242,12 @@ jobs:
           server_log="$report_dir/scim-next.log"
           mkdir -p "$report_dir"
           rm -rf .next/dev
-          E2E_APP=scim setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 &
+          app_tag="scim-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
+          E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 &
           server_pid=$!
           finish() {
             status=$?
-            bash ../../.github/scripts/stop-session.sh "$server_pid" scim || status=1
+            bash ../../.github/scripts/stop-session.sh "$server_pid" "$app_tag" || status=1
             wait "$server_pid" 2>/dev/null || true
             awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/scim-http-status.log"
             if [ "$status" -ne 0 ]; then
@@ -295,11 +296,12 @@ jobs:
           server_log="$report_dir/cli-next.log"
           mkdir -p "$report_dir"
           rm -rf .next/dev
-          E2E_APP=cli setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 &
+          app_tag="cli-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
+          E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 &
           server_pid=$!
           finish() {
             status=$?
-            bash ../../.github/scripts/stop-session.sh "$server_pid" cli || status=1
+            bash ../../.github/scripts/stop-session.sh "$server_pid" "$app_tag" || status=1
             wait "$server_pid" 2>/dev/null || true
             if [ "$status" -ne 0 ]; then
               tail -n 200 "$server_log"
@@ -346,11 +348,12 @@ jobs:
           server_log="$report_dir/stop-after-next.log"
           mkdir -p "$report_dir"
           rm -rf .next/dev
-          E2E_APP=stop-after setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 &
+          app_tag="stop-after-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
+          E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3018 > "$server_log" 2>&1 &
           server_pid=$!
           finish() {
             status=$?
-            bash ../../.github/scripts/stop-session.sh "$server_pid" stop-after || status=1
+            bash ../../.github/scripts/stop-session.sh "$server_pid" "$app_tag" || status=1
             wait "$server_pid" 2>/dev/null || true
             awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/stop-after-http-status.log"
             if [ "$status" -ne 0 ]; then
@@ -397,11 +400,12 @@ jobs:
           server_log="$report_dir/desktop-inbox-next.log"
           mkdir -p "$report_dir"
           rm -rf .next/dev
-          E2E_APP=desktop-inbox setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3019 > "$server_log" 2>&1 &
+          app_tag="desktop-inbox-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
+          E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3019 > "$server_log" 2>&1 &
           server_pid=$!
           finish() {
             status=$?
-            bash ../../.github/scripts/stop-session.sh "$server_pid" desktop-inbox || status=1
+            bash ../../.github/scripts/stop-session.sh "$server_pid" "$app_tag" || status=1
             wait "$server_pid" 2>/dev/null || true
             awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$report_dir/desktop-inbox-http-status.log"
             if [ "$status" -ne 0 ]; then