diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index c8062c37d..9200cc515 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -167,6 +167,7 @@ modules: - name: tutorials-credstore - name: tutorials-alert-notification - name: tutorials-objectstore # @cap-js/attachments S3 store for durable tutorial images (#1869/#1870) + - name: tutorials-redis-websocket # Socket.IO Redis adapter (#2524) - name: tutorials-srv-qa type: nodejs @@ -210,6 +211,12 @@ modules: # 'true'. Mirrors tutorials-srv; srv-qa runs the same srv/ code and can # exercise resolveGithubToken via the shared fetch/cron libs. USE_GITHUB_APP: 'false' + # #2524 — QA shares the one tutorials-redis-websocket instance with + # tutorials-srv. Override the adapter channel-key prefix so QA and primary + # broadcasts never collide on the shared Redis. package.json sets + # cds.websocket.adapter.options.key="socket.io"; this env var wins at + # runtime (CDS env override) for srv-qa only. + CDS_WEBSOCKET_ADAPTER_OPTIONS_KEY: socket.io-qa provides: - name: srv-qa-api properties: @@ -218,6 +225,7 @@ modules: - name: tutorials-hana-qa - name: tutorials-xsuaa - name: tutorials-credstore + - name: tutorials-redis-websocket # Socket.IO Redis adapter (#2524) — shared instance, distinct channel key - name: tutorials-approuter type: approuter.nodejs @@ -554,3 +562,14 @@ resources: service: objectstore service-plan: s3-standard service-name: tutorials-objectstore + + # Redis (Socket.IO adapter) for cross-instance WebSocket fan-out (#2524). + # Shared by tutorials-srv and tutorials-srv-qa; broadcast isolation between + # them is by distinct adapter channel key (socket.io vs socket.io-qa), not a + # second instance. Plan `standard` in every environment. + - name: tutorials-redis-websocket + type: org.cloudfoundry.managed-service + parameters: + service: redis-cache + service-plan: standard + service-name: tutorials-redis-websocket diff --git a/deploy/prod.mtaext b/deploy/prod.mtaext index 343bff96e..d90d0436d 100644 --- a/deploy/prod.mtaext +++ b/deploy/prod.mtaext @@ -31,12 +31,13 @@ modules: # settled and per-instance CPU proved to sit in single digits (no CPU # bottleneck — the 2GB/instance was an OOM fix, not a capacity fix). # - # Do NOT raise above 2 without addressing the two things the codebase is - # single-instance-designed for, both of which DEGRADE with more instances: - # 1. WebSocket broadcasts (/ws/display, /ws/event-stream) have no sticky - # sessions or Redis adapter — a completion emitted on one instance - # never reaches WS clients parked on the others (silent message loss). - # See scaling-playbook.md row #4 (tracked separately). + # Do NOT raise above 2 without addressing the remaining constraint the + # codebase is single-instance-designed for (the one below still DEGRADES + # with more instances): + # 1. WebSocket fan-out (/ws/display, /ws/event-stream) — RESOLVED by + # #2524: handshakes are now sticky (JSESSIONID Gorouter affinity) and + # broadcasts are Redis-backed (@socket.io/redis-adapter over + # tutorials-redis-websocket). This constraint no longer blocks scaling. # 2. In-process rate limiters (srv/lib/ip-rate-limit.js, # srv/lib/chat-rate-limit.js, srv/developer-service.js) run at N× the # intended aggregate ceiling. See mta-deployment.md § Scaling diff --git a/docs/developers/architecture/scaling-playbook.md b/docs/developers/architecture/scaling-playbook.md index defe81c05..63a45294d 100644 --- a/docs/developers/architecture/scaling-playbook.md +++ b/docs/developers/architecture/scaling-playbook.md @@ -15,7 +15,7 @@ The cheapest path to ~5x current traffic without the bigger Redis/replica work: 1. **Row #1 — AppRouter auto-scaling** (this PR / #749). Pure deploy-config, zero code risk. 2. **Row #3 — Cron separation** (same-binary, env-flag-gated). `JobLocks` already covers correctness at the lock level; pre-flight audit needed for caller-side acquire-before-side-effect ordering. 3. **Row #2 — Replace in-process rate limiters** with HANA-backed `RateLimitBuckets`. Unlocks `tutorials-srv` auto-scaling. -4. **Row #4 — WebSocket sticky sessions** (only if Socket.IO traffic justifies it). +4. **Row #4 — WebSocket sticky sessions** (**resolved in #2524** — Redis adapter + JSESSIONID affinity shipped). 5. Others as traffic forces them. ## The 15 constraints @@ -25,7 +25,7 @@ The cheapest path to ~5x current traffic without the bigger Redis/replica work: | 1 | **AppRouter sessions** | `instances: 1` (deploy default) | **#749 (this PR)** — CF Autoscaler + `instances: 1..4`. XSUAA cookie-based, stateless. | Hours | | 2 | **In-process rate limiters** (feedback, search, chat) | Documented in [mta-deployment.md § Scaling Constraints](../operations/mta-deployment.md#scaling-constraints) | HANA-backed `RateLimitBuckets` entity (the "option 2" already recommended). | 2-3 days | | 3 | **Scheduled cron jobs** | All run inside `tutorials-srv` instance 0; safe-by-lock via `JobLocks` but couples web latency to cron load | **Same-binary, env-flag-gated separation** — new `tutorials-cron` MTA module reuses `path: gen/srv`. One-line guard `if (process.env.ENABLE_CRON === 'true') registerJobs()`. `JobLocks` covers correctness at the LOCK level, but each job must `acquireLock()` BEFORE non-idempotent side effects (email send, GitHub dispatch). **Pre-flight: audit all 16 `srv/jobs/*.js` files for acquire-then-side-effect ordering.** Not a codebase split. | 1-2 days (longer if audit surfaces non-idempotent ordering) | -| 4 | **WebSocket sticky sessions** | Socket.IO transport in `tutorials-srv` requires sticky sessions at N>1 srv | Three options: (a) approuter sticky-session config, (b) Socket.IO Redis adapter, (c) pin srv to 1 and only scale srv if WebSocket is removed (last resort). | 1-3 days | +| 4 | **WebSocket sticky sessions** | Socket.IO transport in `tutorials-srv` requires sticky sessions at N>1 srv | **Resolved in #2524.** Chosen path: **A+B** — sticky sessions (JSESSIONID cookie set on the engine.io handshake) + plugin-native Socket.IO Redis adapter (`@socket.io/redis-adapter` over the shared `tutorials-redis-websocket` standard instance). Fan-out is now Redis-backed; handshakes are sticky. See [PR/issue #2524](https://github.com/sap-tutorials/tutorials-ims/issues/2524). | Shipped | | 5 | **Per-instance in-memory caches** (content BLOBs, advocate photos, alerts, admin docs, generic TTL) | Multiple module-level caches across `srv/lib/` each hold per-instance state: `srv/lib/content-store.js` (50MB BLOB LRU), `srv/lib/advocate-photo-store.js` (bounded photo LRU), `srv/lib/alerts-cache.js` (60s TTL — alerts freshness budget per CLAUDE.md), `srv/lib/admin-docs-index.js` (module-level `_cache`), `srv/lib/ttl-cache.js` + `srv/lib/khoros-cache.js` (generic helpers). At N>1 srv each instance has its own copy → 2× memory, cold cache on each new instance, AND inconsistent TTL behavior (a 60s alerts TTL becomes "up to 60s per instance, observed inconsistently"). | At-scale fix: shared Redis or accept 2-3× memory per added instance. For our traffic profile probably acceptable until 4+ srv instances. Audit each cache for "inconsistency tolerable?" — alerts and admin-docs are tolerable; advocate photos and content BLOBs are bounded memory; nothing in this list breaks correctness. | Decision deferred to post-#2 | | 6 | **Content publish race** | `POST /content/publish` at N>1 srv could race concurrent publishes | Transaction-level advisory lock or `UPDATE ... WHERE version = $expected` optimistic concurrency. | 2-3 days | | 7 | **HANA connection pool** | `@sap/hana-client` defaults; not tuned for high concurrency | Document settings + levers (`max`, `min`, `idleTimeoutMillis`); runtime probe in `/health/db`. | 1-2 days | @@ -72,14 +72,16 @@ The split: a new `tutorials-cron` module in `mta.yaml` reuses `path: gen/srv` (s **Pre-flight audit (mandatory before splitting):** walk `srv/jobs/*.js` (16 files) and confirm every job calls `acquireLock()` BEFORE any non-idempotent side effect. The `instances: 1` pin masks any acquire-after-side-effect ordering today. Once split, two instances of the same code (web + cron) both register schedulers — if `ENABLE_CRON` is mis-set or both happen to be true, the lock catches duplicates, but only if the lock acquisition runs before the side effect. Skip the audit and risk double-sends on email, double-dispatches on GitHub rebuild triggers, etc. -### Row #4 — WebSocket sticky sessions +### Row #4 — WebSocket sticky sessions (**resolved in #2524**) `tutorials-srv` uses Socket.IO via `@cap-js-community/websocket`. The `DisplayService` and `EventStreamService` emit CDS events to clients on the `/ws/display` and `/ws/event-stream` namespaces. At N>1 srv, a client's WebSocket connection lands on one instance and must STAY on that instance — Socket.IO state lives in process memory. -The three fix options: -- **(a) Approuter sticky-session config.** Cheapest. Approuter routes the same client (by cookie or source IP) to the same backend instance. Risk: harder to balance load if a few users have long-lived display sessions. -- **(b) Socket.IO Redis adapter.** Textbook. Bind a BTP managed Redis service; configure `@socket.io/redis-adapter`. State shared across instances. Best architecturally; adds a service dependency for one feature. -- **(c) Pin srv to 1.** Last resort. Acceptable until WebSocket traffic is a meaningful share of overall load. +**Shipped in [#2524](https://github.com/sap-tutorials/tutorials-ims/issues/2524): path A+B.** + +- **(A) Sticky sessions** — a `JSESSIONID` cookie is set on the engine.io handshake (`srv/server.js`), pinning each client's connection to one srv instance via CF Gorouter affinity. Fixes handshake integrity at N>1. +- **(B) Plugin-native Socket.IO Redis adapter** — `@socket.io/redis-adapter` over the `tutorials-redis-websocket` managed `redis-cache` standard instance. Bridges cross-instance broadcasts so a completion emitted on any instance reaches every subscribed client. Fixes the core fan-out bug. + +Fan-out is now Redis-backed and handshakes are sticky. The WebSocket constraint no longer blocks raising `instances` above 2; the remaining gate is row #2 (rate limiters). ### Row #5 — Per-instance in-memory caches @@ -147,4 +149,4 @@ Every srv instance decrypts secrets on boot via the `tutorials-credstore` bindin --- -**Last updated:** 2026-06-29 (#749 PR). When you crack off a row, update its status here and link to the implementing PR. +**Last updated:** 2026-09-29 (#2524 PR — row #4 resolved). When you crack off a row, update its status here and link to the implementing PR. diff --git a/docs/superpowers/plans/2026-09-29-websocket-scaling-2524.md b/docs/superpowers/plans/2026-09-29-websocket-scaling-2524.md new file mode 100644 index 000000000..6f840a39d --- /dev/null +++ b/docs/superpowers/plans/2026-09-29-websocket-scaling-2524.md @@ -0,0 +1,417 @@ +# WebSocket scaling at N>1 srv — sticky sessions + Redis adapter (#2524) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make Socket.IO broadcasts (`tutorialCompleted`) survive `tutorials-srv` running at more than one instance, by activating the plugin-native Socket.IO Redis adapter (fan-out) and setting a Gorouter sticky-session cookie on the engine.io handshake (handshake integrity). + +**Architecture:** `@cap-js-community/websocket` (`kind: "socket.io"`) already reads `cds.env.websocket.adapter`; when `adapter.impl === "@socket.io/redis-adapter"` and a Redis binding resolves, it mounts the real `@socket.io/redis-adapter` and bridges all broadcasts through Redis. We add that config + the one missing npm dep, provision one `redis-cache` (plan `standard`) instance in the MTA and bind it to both srv modules (sharing one instance, isolated by distinct adapter channel keys), and add a `ws:ready` hook in `srv/server.js` that stamps a `JSESSIONID` cookie on the handshake. On CF (`USER=vcap`) the adapter auto-activates; off-CF it stays in-memory, so local dev and unit tests are untouched. + +**Tech Stack:** SAP CAP (`@sap/cds`), `@cap-js-community/websocket` 1.11.1, `socket.io` 4.x, `@socket.io/redis-adapter` 8.x (new dep), `redis`/`@cap-js-community/common` (already transitive), MTA (`org.cloudfoundry.managed-service`), CF Gorouter session affinity. + +**Spec:** `docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md` + +## Global Constraints + +- **PR targets `DEV`.** `main` is protected; no main-hotfix path. Never deploy a feature branch — deploy from a fresh `origin/DEV`. +- **Never store credentials in source** — Redis is resolved via the CF service binding only (`cds.env.requires["redis-websocket"]`), never hardcoded. +- **Never write raw SQL / never touch emit sites** — this PR changes config, one server hook, MTA, and docs only. `srv/developer-service.js` emit sites (822-838) are out of scope. +- **`@socket.io/redis-adapter` must be v8.x** — 8.x is the current major and the correct pairing for `socket.io` 4.x (the adapter major runs ahead of socket.io's; they are not meant to match). Pin the exact 8.x resolved at install. +- **No `"active": true` in config** — rely on the CF `USER=vcap` auto-activation guard so local/tests stay in-memory with zero extra config. +- **Adapter channel key differs per module** — `tutorials-srv` uses `"socket.io"`, `tutorials-srv-qa` uses `"socket.io-qa"`, so both can share one Redis instance without colliding broadcasts. Both modules load the **same** `package.json`, so the QA key MUST be an env-var override (`CDS_WEBSOCKET_ADAPTER_OPTIONS_KEY`) set on the `tutorials-srv-qa` module, NOT a static value in `package.json`. +- **Editing paths:** write only under `D:\projects\tutorials-poc\.claude\worktrees\ws-scaling-2524\...` (absolute `D:\projects\tutorials-poc\...` paths write to the primary tree even in a worktree). +- **Noisy commands** (`npm test`, `mbt build`, `cf deploy`) run through `scripts/quiet-run.sh`. +- **The actual thing before done (Tom's #1 rule):** the DEV fan-out proof at 2 instances gates the PR — it is not optional. + +--- + +### Task 1: Activate the Redis adapter in CAP config + add the dependency + +**Files:** +- Modify: `package.json` (the `cds` block — `cds.websocket` is a top-level sibling of `cds.requires`, both already present) +- Modify: `package.json` (`dependencies` — add `@socket.io/redis-adapter`) + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `cds.env.websocket.adapter.impl === "@socket.io/redis-adapter"`, `cds.env.websocket.adapter.options.key === "socket.io"`, and `cds.env.requires["redis-websocket"]` — Task 2 reads `cds.env.websocket.adapter.options.key`; Task 3's MTA binding satisfies `redis-websocket`. + +- [ ] **Step 1: Add the dependency, pinned to the resolved 8.x** + +Run (in the worktree root): + +```bash +npm install --save --save-exact @socket.io/redis-adapter@^8 +``` + +Then read back the exact version npm resolved and confirm it is 8.x: + +```bash +jq -r '.dependencies["@socket.io/redis-adapter"]' package.json +``` + +Expected: a concrete `8.x.y` string (e.g. `8.3.0`). If npm resolves anything below 8, stop — that is a mismatch for `socket.io` 4.x. + +- [ ] **Step 2: Add the adapter block to `cds.websocket` and the `redis-websocket` requires entry** + +In `package.json`, the `cds.websocket` object today is: + +```jsonc +"websocket": { "kind": "socket.io" } +``` + +Change it to: + +```jsonc +"websocket": { + "kind": "socket.io", + "adapter": { + "impl": "@socket.io/redis-adapter", + "options": { "key": "socket.io" } + } +} +``` + +And add a `redis-websocket` entry to the existing `cds.requires` object (alongside AICore, attachments, db, audit-log, caching, etc.) — do NOT nest it under `websocket`: + +```jsonc +"redis-websocket": { + "vcap": { "label": "redis-cache" } +} +``` + +- [ ] **Step 3: Verify config parses and the adapter is NOT active off-CF** + +Run: + +```bash +node -e "const cds=require('@sap/cds'); const w=cds.env.websocket; const r=cds.env.requires['redis-websocket']; console.log(JSON.stringify({adapter:w.adapter, redis:r})); if(w.adapter.impl!=='@socket.io/redis-adapter') throw new Error('adapter impl not set'); if(w.adapter.options.key!=='socket.io') throw new Error('key not set'); if(!r||r.vcap.label!=='redis-cache') throw new Error('redis-websocket requires missing'); console.log('OK');" +``` + +Expected: prints the resolved objects then `OK`. (This runs off-CF, so `USER!==vcap` — the adapter parses but stays dormant; that is the desired local behavior, no Redis needed.) + +- [ ] **Step 4: Run the unit suite to prove no regression with the adapter dormant** + +Run: + +```bash +scripts/quiet-run.sh npm test +``` + +Expected: green — the in-memory websocket path is unchanged locally because `connectionCheck` returns false off-CF. + +- [ ] **Step 5: Commit** + +```bash +git add package.json package-lock.json +git commit -m "feat(ws): activate plugin-native Socket.IO Redis adapter config (#2524)" +``` + +--- + +### Task 2: Sticky-session handshake cookie in `srv/server.js` + +**Files:** +- Modify: `srv/server.js` (append a `cds.on('ws:ready', …)` hook; file currently ends at line 2256, no existing `ws:ready`/`cds.io` usage) +- Test: `test/smoke/websocket-handshake.test.js` (existing route-existence smoke; extend to assert the handshake sets a `JSESSIONID` cookie against a deployed srv) + +**Interfaces:** +- Consumes: `cds.io` (exposed by the plugin's `ws:ready` event) and its `.engine` (engine.io server). +- Produces: every engine.io handshake response carries `Set-Cookie: JSESSIONID=; Path=/; HttpOnly; SameSite=Lax`. No later task consumes this programmatically; it is a platform-facing behavior verified in Task 5. + +- [ ] **Step 1: Write the failing smoke assertion** + +Add to `test/smoke/websocket-handshake.test.js` a test that opens the Socket.IO polling handshake against the deployed srv and asserts a `JSESSIONID` cookie comes back. Append inside the existing `describe('WebSocket endpoints', …)`: + +```js +it('engine.io handshake sets a JSESSIONID affinity cookie', async () => { + // Socket.IO v4 polling handshake — the first GET returns the session + // open packet AND (after this change) a Set-Cookie for Gorouter affinity. + const res = await fetchWithRetry( + `${SRV_URL}/socket.io/?EIO=4&transport=polling&namespace=/ws/event-stream`, + ); + expect(res.status).toBe(200); + const setCookie = res.headers.get('set-cookie') || ''; + expect(setCookie).toMatch(/JSESSIONID=/); +}); +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run (against a deployed srv — this test self-requires `SMOKE_SRV_URL`; if unset the smoke config skips, so set it to the DEV srv URL to see a real red): + +```bash +SMOKE_SRV_URL="https://tutorial-system-dev-tutorials-srv.cfapps.eu10-005.hana.ondemand.com" \ + scripts/quiet-run.sh npx vitest run test/smoke/websocket-handshake.test.js +``` + +Expected: the new test FAILS — no `JSESSIONID` in the handshake response (the hook isn't deployed yet). The two existing route tests still pass. (This red is confirmed after deploy in Task 5; locally the smoke suite has no server to hit.) + +- [ ] **Step 3: Add the `ws:ready` hook at the tail of `srv/server.js`** + +Append (after the final existing `cds.on(...)` block, at file end): + +```js +// #2524 — Gorouter session affinity for Socket.IO. The engine.io handshake is +// multi-request (HTTP polling → WS upgrade); if those requests land on +// different tutorials-srv instances the handshake flaps. Stamping a JSESSIONID +// cookie on the handshake response pins the client to one instance for the +// connection's lifetime. This is complementary to the Redis adapter (Task 1): +// affinity fixes handshake integrity; the adapter fixes cross-instance fan-out. +// Scoped to the engine.io handshake only — does not touch application cookies. +cds.on('ws:ready', () => { + const { randomUUID } = require('node:crypto'); + cds.io?.engine?.on('initial_headers', (headers /*, req */) => { + headers['set-cookie'] = [ + `JSESSIONID=${randomUUID()}; Path=/; HttpOnly; SameSite=Lax`, + ]; + }); +}); +``` + +- [ ] **Step 4: Verify it loads without error locally** + +Run: + +```bash +node -e "process.env.CDS_ENV='development'; require('@sap/cds'); const fs=require('node:fs'); const s=fs.readFileSync('srv/server.js','utf8'); if(!/ws:ready/.test(s)) throw new Error('hook not present'); if(!/initial_headers/.test(s)) throw new Error('initial_headers not wired'); console.log('hook present OK');" \ + && scripts/quiet-run.sh npm test +``` + +Expected: `hook present OK` then the unit suite stays green (the hook only fires when the plugin emits `ws:ready`, which is harmless locally). Do NOT attempt to assert the cookie locally — the plugin's Redis path and the deployed multi-instance context are what this proves; the cookie assertion is a deployed-only smoke (Task 5). + +- [ ] **Step 5: Commit** + +```bash +git add srv/server.js test/smoke/websocket-handshake.test.js +git commit -m "feat(ws): set JSESSIONID affinity cookie on engine.io handshake (#2524)" +``` + +--- + +### Task 3: MTA — provision `redis-cache` (standard) and bind to both srv modules + +**Files:** +- Modify: `.deploy/mta.yaml` — add the `tutorials-redis-websocket` resource (after the last resource, `tutorials-objectstore`, at line 556); add a `requires` entry to `tutorials-srv` (after line 169) and to `tutorials-srv-qa` (after line 220); add the QA channel-key env-var property to the `tutorials-srv-qa` `properties` block (after line 212) + +**Interfaces:** +- Consumes: `cds.env.requires["redis-websocket"]` from Task 1 (resolved by the binding this task creates); `cds.env.websocket.adapter.options.key` from Task 1 (overridden per-module here). +- Produces: a bound `redis-cache` instance named `tutorials-redis-websocket`; `tutorials-srv` broadcasts on channel key `socket.io`, `tutorials-srv-qa` on `socket.io-qa`. + +- [ ] **Step 1: Add the managed-service resource** + +Append to `.deploy/mta.yaml` immediately after the `tutorials-objectstore` resource (after line 556, at the end of the `resources:` list), matching the existing resource indentation: + +```yaml + # Redis (Socket.IO adapter) for cross-instance WebSocket fan-out (#2524). + # Shared by tutorials-srv and tutorials-srv-qa; broadcast isolation between + # them is by distinct adapter channel key (socket.io vs socket.io-qa), not a + # second instance. Plan `standard` in every environment. + - name: tutorials-redis-websocket + type: org.cloudfoundry.managed-service + parameters: + service: redis-cache + service-plan: standard + service-name: tutorials-redis-websocket +``` + +- [ ] **Step 2: Bind Redis to `tutorials-srv`** + +In the `tutorials-srv` module's `requires:` block, add after the `tutorials-objectstore` line (line 169): + +```yaml + - name: tutorials-redis-websocket # Socket.IO Redis adapter (#2524) +``` + +- [ ] **Step 3: Bind Redis to `tutorials-srv-qa` and set its distinct channel key** + +In the `tutorials-srv-qa` module's `requires:` block, add after the `tutorials-credstore` line (line 220): + +```yaml + - name: tutorials-redis-websocket # Socket.IO Redis adapter (#2524) — shared instance, distinct channel key +``` + +In the same module's `properties:` block, add after the `USE_GITHUB_APP` property (after line 212): + +```yaml + # #2524 — QA shares the one tutorials-redis-websocket instance with + # tutorials-srv. Override the adapter channel-key prefix so QA and primary + # broadcasts never collide on the shared Redis. package.json sets + # cds.websocket.adapter.options.key="socket.io"; this env var wins at + # runtime (CDS env override) for srv-qa only. + CDS_WEBSOCKET_ADAPTER_OPTIONS_KEY: socket.io-qa +``` + +- [ ] **Step 4: Validate the MTA descriptor parses and the wiring is correct** + +Run: + +```bash +yq '.resources[] | select(.name=="tutorials-redis-websocket")' .deploy/mta.yaml \ + && echo "--- srv requires ---" \ + && yq '.modules[] | select(.name=="tutorials-srv").requires[].name' .deploy/mta.yaml \ + && echo "--- srv-qa requires ---" \ + && yq '.modules[] | select(.name=="tutorials-srv-qa").requires[].name' .deploy/mta.yaml \ + && echo "--- srv-qa qa key ---" \ + && yq '.modules[] | select(.name=="tutorials-srv-qa").properties.CDS_WEBSOCKET_ADAPTER_OPTIONS_KEY' .deploy/mta.yaml +``` + +Expected: the resource block prints with `service: redis-cache` / `service-plan: standard`; both `requires` lists include `tutorials-redis-websocket`; the srv-qa key prints `socket.io-qa`. (No mtaext edits are needed — none of `dev/qa/prod.mtaext` overrides these resources or the srv-qa module, and `standard` is used everywhere so there is no per-env plan override.) + +- [ ] **Step 5: Commit** + +```bash +git add .deploy/mta.yaml +git commit -m "feat(mta): provision + bind redis-cache for Socket.IO adapter (#2524)" +``` + +--- + +### Task 4: Docs + stale-comment updates + +**Files:** +- Modify: `docs/developers/architecture/scaling-playbook.md` (row #4 — mark resolved) +- Modify: `srv/lib/content-cache-coherence.js:20` (comment now conditionally false) +- Modify: `deploy/prod.mtaext:34-39` (the "no sticky sessions or Redis adapter" warning under item #1 is now addressed by this PR; item #2 rate limiters still stand) +- Modify: `docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md` (correct the two "instances: 4" references — prod is actually `instances: 2`) + +**Interfaces:** +- Consumes: the behavior delivered by Tasks 1-3. +- Produces: nothing consumed by later tasks (doc-only). + +- [ ] **Step 1: Mark scaling-playbook row #4 resolved** + +Open `docs/developers/architecture/scaling-playbook.md`, find row #4 (WebSocket sticky sessions, options a/b/c). Replace the open "three options" text with a resolved note: chosen path is **A+B** — sticky sessions (JSESSIONID on the engine.io handshake) **plus** the plugin-native Socket.IO Redis adapter (`@socket.io/redis-adapter` over a shared `redis-cache` standard instance) — shipped in #2524; link the PR/issue. Keep the row; do not delete it. + +- [ ] **Step 2: Update the content-cache-coherence comment** + +In `srv/lib/content-cache-coherence.js`, the comment at line 20 claims "no pub/sub bus exists; websocket has no Redis adapter." Reword to: a Redis pub/sub bus now exists **for the websocket Socket.IO adapter** (#2524), but cache-coherence deliberately does **not** use it — wiring cache invalidation onto Redis is a separate concern (scaling-playbook row #5, out of scope here). Do NOT change any code in this file. + +- [ ] **Step 3: Update the prod.mtaext scaling warning** + +In `deploy/prod.mtaext`, the comment block at lines 34-43 warns against raising `instances` above 2 for two reasons. Reason #1 (WebSocket has no sticky sessions or Redis adapter) is resolved by #2524 — update it to say fan-out is now Redis-backed and handshakes are sticky, so the WebSocket constraint no longer blocks scaling. Leave reason #2 (in-process rate limiters) intact — it still gates raising instances. Do NOT change the `instances: 2` value itself. + +- [ ] **Step 4: Correct the spec's instance count** + +In `docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md`, two lines say prod wants `instances: 4` (the Problem section around line 20-21, and the Rejected-approaches "(c)" note around line 92). The actual `deploy/prod.mtaext` value is `instances: 2`. Change both "4" references to "2" and adjust the surrounding wording so the argument (pinning to 1 caps throughput; prod already runs multi-instance) still reads correctly at 2. + +- [ ] **Step 5: Verify no stale claim remains, then commit** + +Run: + +```bash +grep -rniE "no redis adapter|no sticky session|instances: 4|instances 4" \ + docs/developers/architecture/scaling-playbook.md \ + srv/lib/content-cache-coherence.js \ + deploy/prod.mtaext \ + docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md +``` + +Expected: no matches (every stale claim updated). Then: + +```bash +git add docs/developers/architecture/scaling-playbook.md srv/lib/content-cache-coherence.js deploy/prod.mtaext docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md +git commit -m "docs(ws): mark scaling row #4 resolved; refresh stale WS-adapter comments (#2524)" +``` + +--- + +### Task 5: DEV fan-out verification (the actual thing — gates the PR) + +**Files:** none modified — this task is the deployed proof that Tasks 1-4 work at N=2. + +**Interfaces:** +- Consumes: everything from Tasks 1-4, deployed to DEV from a fresh `origin/DEV`-based build. +- Produces: recorded evidence that a completion emitted on one instance reaches a WS client on another, and that handshake success stays green at 2 instances. This is the gate; the PR opens only after it passes. + +- [ ] **Step 1: Confirm CF target and fetch before deploy** + +Run: + +```bash +cf target +git fetch origin +``` + +Expected: `cf target` shows the **DEV** space (`tutorial-system` / dev, `eu10-005`) — NOT prod. If it shows prod, stop and re-target. Pause the sap-devs scheduler for the duration of the deploy so the target can't drift. + +- [ ] **Step 2: Verify Redis entitlement/quota before provisioning** + +Run: + +```bash +cf marketplace -e redis-cache +``` + +Expected: the `standard` plan is listed and available in the DEV space (entitlement propagated). If `standard` is absent, stop — the binding will fail at deploy. + +- [ ] **Step 3: Build + deploy A+B to DEV** + +Build the full site first (Hugo before mbt), then deploy. Point `CAP_BASE_URL` at the deployed DEV backend as the repo's deploy convention requires: + +```bash +export CAP_BASE_URL="https://tutorial-system-dev-tutorials-srv.cfapps.eu10-005.hana.ondemand.com" +QR="$(pwd)/scripts/quiet-run.sh" +"$QR" npm run build:all +cd .deploy && "$QR" mbt build && "$QR" cf deploy mta_archives/*.mtar -e ../deploy/dev.mtaext -f ; cd - +``` + +Expected: deploy succeeds; the `tutorials-redis-websocket` service instance is created and bound. Confirm the binding resolved (this is spec open-item #1 — `vcap.label` resolution): + +```bash +cf services | grep tutorials-redis-websocket +cf env tutorials-srv | grep -iA3 redis +``` + +Expected: the service shows bound to `tutorials-srv`; `VCAP_SERVICES` contains a `redis-cache` entry. If the plugin can't resolve it by label, check the srv logs for the `websocket/redis` connection check and fall back to matching by instance name (`cds.env.requires.redis-websocket.name`). + +- [ ] **Step 4: Confirm the adapter actually mounted (not silently in-memory)** + +Run: + +```bash +cf logs tutorials-srv --recent | grep -iE "websocket|redis|adapter|ws:ready" +``` + +Expected: log lines showing the Redis connection check passed and the Socket.IO adapter was mounted (`USER=vcap` on CF activates it). If it says the adapter is disabled / in-memory, the binding didn't resolve — fix before proceeding. This is the single most important check: an in-memory adapter at N>1 is exactly the bug, and it fails silently. + +- [ ] **Step 5: Scale srv to 2 and prove cross-instance fan-out** + +```bash +cf scale tutorials-srv -i 2 +``` + +Then, with two instances live: +1. Open a WS client to `/ws/display` for a chosen `legacyId` (via the approuter URL so affinity applies). Record which instance holds it — the `JSESSIONID` cookie in the handshake response, cross-referenced with `cf logs` / `__VCAP_ID__`. +2. Drive a completion for that same `legacyId` (through the normal completion path) — repeat until the completion transaction demonstrably runs on the **other** instance (check logs for which instance processed it). +3. Confirm the WS client receives `tutorialCompleted` even when emitter ≠ client's instance. + +Expected: the client receives the event across instances. **If it does not, the fix is not working — do not open the PR.** Record the instance IDs (emitter vs holder) as the evidence. + +- [ ] **Step 6: Handshake load test at 2 instances (proves sticky sessions)** + +Run: + +```bash +scripts/quiet-run.sh npm run loadtest:ws +``` + +Expected: `ws_session_errors` stays within threshold and the 101 handshake check passes — sticky sessions hold the multi-request handshake together at N=2. Also run the deployed smoke assertion from Task 2: + +```bash +SMOKE_SRV_URL="https://tutorial-system-dev-tutorials-srv.cfapps.eu10-005.hana.ondemand.com" \ + scripts/quiet-run.sh npx vitest run test/smoke/websocket-handshake.test.js +``` + +Expected: all three smoke tests pass, including the new `JSESSIONID` assertion (now green post-deploy). + +- [ ] **Step 7: Open the PR to DEV** + +Only after Steps 5-6 pass. Push the branch and open the PR with `gh pr create --base DEV`, citing #2524, the spec path, and the recorded fan-out evidence (emitter/holder instance IDs, loadtest summary). Never direct-merge; PR review is required. + +--- + +## Notes for the executor + +- **Do not** wire Redis into `content-cache-coherence.js` or the rate limiters — explicitly out of scope (spec §Scope boundaries). +- **Do not** change emit sites (`srv/developer-service.js:822-838`) or the WS service `.cds` definitions. +- **Windows/worktree line endings:** after any subagent edit, verify files did not flip LF→CRLF (`file `) and normalize before commit. +- **srv-qa cp-list audit** is NOT triggered here — no `srv/lib/` runtime file changes (the only `srv/lib` touch is a comment in `content-cache-coherence.js`, which is already shipped to srv-qa if it was a dep; confirm it's unchanged in behavior). diff --git a/docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md b/docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md new file mode 100644 index 000000000..b6517a0bf --- /dev/null +++ b/docs/superpowers/specs/2026-09-29-websocket-scaling-2524-design.md @@ -0,0 +1,286 @@ +# WebSocket scaling at N>1 srv — sticky sessions + Redis adapter (#2524) + +**Status:** Design — awaiting review +**Date:** 2026-09-29 +**Issue:** [sap-tutorials/tutorials-ims#2524](https://github.com/sap-tutorials/tutorials-ims/issues/2524) +**Target branch:** `DEV` + +## Problem + +`tutorials-srv` runs Socket.IO through the `@cap-js-community/websocket` plugin +(`cds.websocket.kind: "socket.io"`) on two namespaces: + +- `/ws/display` — `DisplayService` (`srv/display-service.cds`), `@requires: 'DisplayApp'` +- `/ws/event-stream` — `EventStreamService` (`srv/event-stream-service.cds`), `@requires: 'any'` + +On tutorial completion, `srv/developer-service.js:780-790` emits `tutorialCompleted` +to both services via `service.tx({ user: privileged }, tx => tx.emit(...))`, scoped +by `contexts: [String(event.legacyId)]`. + +At `instances: 1` this works. `deploy/prod.mtaext` already sets `tutorials-srv +instances: 2`. **At N>1 srv there are two distinct, independent failure modes:** + +### Failure mode 1 — cross-instance fan-out is lossy (the core bug) + +Socket.IO connection state lives in the process memory of whichever srv instance +accepted the WebSocket. A `tx.emit('tutorialCompleted')` executing on instance A +only reaches sockets held by instance A. A display client parked on instance B +never sees it. There is no shared message bus between instances, so broadcasts are +**silently partial** — no error, just missed events. This is the bug the issue +reports. + +### Failure mode 2 — handshake integrity across a multi-instance backend + +The Socket.IO/engine.io handshake is multi-request (HTTP polling upgrade → WS). If +the polling requests in a single handshake are load-balanced across different srv +instances, the handshake fails or flaps, because engine.io session state for an +in-flight handshake is also per-instance. This needs request affinity for the +duration of a client's connection. + +**Sticky sessions alone do NOT fix mode 1.** Affinity keeps a *client* pinned to one +instance, but the *emitter* (the instance running the completion transaction) can be +a different instance. Only a shared adapter fixes fan-out. Both fixes are required +and they address different layers — this is why the chosen scope is A+B, not either +alone. + +## Chosen approach: A + B together + +- **(A) Sticky sessions** — pin each client's connection to one srv instance for + handshake integrity (mode 2). +- **(B) Shared Redis adapter** — bridge broadcasts across all srv instances so a + completion on any instance reaches every subscribed client (mode 1). + +Decided with Tom: +- Scope: **both (a)+(b)**. +- Sequencing: **block on the Redis entitlement first** — do A+B together against a + real bound service. No dormant/unbound code. (Entitlement has since been added; + `redis-cache` is in the marketplace with plans development/standard/premium.) +- Redis plan: **standard, in every environment** (DEV/QA/PROD). +- Verification: **full DEV fan-out test** — bind Redis on DEV, deploy A+B, and prove + a completion emitted on one instance reaches a WS client on another, before opening + the PR. + +### How the plugin actually implements Redis (verified against installed 1.11.1) + +Reading the installed plugin source rather than assuming: + +- For `kind: "socket.io"`, `src/socket/socket.io.js#applyAdapter()` reads + `cds.env.websocket.adapter`. When `adapter.impl === "@socket.io/redis-adapter"` it + runs a Redis connection check, creates a primary + secondary client, and calls the + **real** `@socket.io/redis-adapter`'s `createAdapter(pub, sub, options)`, then + `io.adapter(...)`. So (B) is the genuine Socket.IO Redis adapter, not a bespoke + bridge. `options.key` (default `"websocket"`) is the adapter's channel prefix. +- The Redis clients come from `@cap-js-community/common`'s `RedisClient` + (`src/redis-client/RedisClient.js`). It resolves credentials from + **`cds.env.requires["redis-websocket"]`** first (pattern `redis-${env}`, where the + plugin instantiates `RedisClient.create("websocket")`), falling back to + `cds.env.requires["redis"]`. That is what binds the adapter to a specific BTP + service. +- Activation guard (`src/redis/index.js#connectionCheck`): active when + `process.env.USER === "vcap"` (i.e. on CF) **or** `adapter.active === true` + explicitly **or** `adapter.local === true`. Off-CF without an explicit flag it + stays disabled and the server runs in-memory — so local `cds watch` and unit tests + are unaffected with zero extra config. + +**Dependency reality:** `@cap-js-community/common@0.5.0` and `redis@6.1.0` are +already installed (transitive via the websocket plugin). The **only** missing package +is `@socket.io/redis-adapter`, which the plugin `require`s by name at runtime. + +## Rejected approaches + +- **(c) Pin srv to 1** — the issue's last-resort option. Rejected: `prod.mtaext` + already wants 2 instances; pinning to 1 caps the whole backend's throughput to + protect one feature. Kicks the can. +- **Sticky sessions only (A alone)** — does not fix the core fan-out bug (mode 1). + Rejected on correctness. +- **Redis adapter only (B alone)** — fixes fan-out but leaves the multi-request + handshake vulnerable to cross-instance load-balancing (mode 2). The adapter shares + *broadcasts*, not *handshake session state*. Rejected as incomplete. +- **A custom HANA-backed pub/sub bridge** (mirroring the rate-limiter "option 2" + pattern) — reinvents what the plugin already does natively over Redis; adds DB write + load on the hot completion path. Rejected. + +## Precondition (satisfied) + +Redis entitlement in the target subaccount(s). Was the hard blocker for (B); Tom has +added it. `redis-cache` now appears in the marketplace (plans development / standard / +premium). Plan chosen: **standard everywhere**. + +## Components to change + +### 1. CAP config — activate the Redis adapter (`package.json`) + +Today `package.json` has (sibling of `cds.requires`, not under it): + +```jsonc +"cds": { + "websocket": { "kind": "socket.io" } +} +``` + +Change to add the adapter block, and add a `redis-websocket` requires entry so the +plugin's `RedisClient` finds the CF binding: + +```jsonc +"cds": { + "websocket": { + "kind": "socket.io", + "adapter": { + "impl": "@socket.io/redis-adapter", + "options": { "key": "socket.io" } + } + }, + "requires": { + "redis-websocket": { + "vcap": { "label": "redis-cache" } + } + } +} +``` + +- No `"active": true` in config: on CF `USER=vcap` auto-activates; off-CF it stays + in-memory (desired — local/tests untouched). +- `vcap.label: "redis-cache"` binds by service label; the MTA gives the instance the + name `tutorials-redis-websocket` (see §3). If label matching proves ambiguous at + bind time we fall back to matching by instance name — resolved during the DEV + deploy, noted as an open item. +- `options.key` is the **prefix for the Redis pub/sub channel names** the adapter + broadcasts on (plugin default `"websocket"`). `tutorials-srv` uses `"socket.io"`; + `tutorials-srv-qa` uses `"socket.io-qa"` so QA and primary can share one Redis + instance without their broadcasts colliding (see §3). +- Add `@socket.io/redis-adapter` to `dependencies` (`npm install --save`). Use the + current major, **v8.x** — that is the release line that pairs with `socket.io 4.x` + (the adapter's major runs ahead of socket.io's; they are not meant to match + numbers). Confirm the exact 8.x at install. + +### 2. Sticky sessions — `srv/server.js` handshake cookie hook + +CF's Gorouter provides session affinity when the app sets a `JSESSIONID` cookie +(alongside CF's own `__VCAP_ID__`). This is a **platform/Gorouter** mechanism, not an +`xs-app.json`/approuter config — `approuter/xs-app.json` already has +`websockets.enabled: true` and routes `/socket.io/*` and `/ws/*` through, which is all +it needs to do. + +Set the cookie on the engine.io handshake only, via the plugin's `ws:ready` lifecycle +event (which exposes `cds.io`): + +```js +cds.on('ws:ready', () => { + cds.io?.engine?.on('initial_headers', (headers /*, req */) => { + // Gorouter affinity: pin this client to this srv instance for the + // lifetime of the Socket.IO connection. Scoped to the handshake; does + // not touch application cookies. + headers['set-cookie'] = [ + `JSESSIONID=${require('node:crypto').randomUUID()}; Path=/; HttpOnly; SameSite=Lax`, + ]; + }); +}); +``` + +- Placed alongside the existing `cds.on('bootstrap')` (:228) / `cds.on('served')` + (:1570) hooks. +- Affinity + adapter are complementary: affinity keeps a client's handshake on one + instance; the adapter makes cross-instance emits arrive regardless. + +### 3. MTA — provision + bind Redis (`.deploy/mta.yaml` + 3 mtaexts) + +New managed-service resource, following the existing `tutorials-objectstore` pattern +(`.deploy/mta.yaml:497+`): + +```yaml +- name: tutorials-redis-websocket + type: org.cloudfoundry.managed-service + parameters: + service: redis-cache + service-plan: standard + service-name: tutorials-redis-websocket +``` + +Bind it to `tutorials-srv` (add to the module's `requires:` block, `.deploy/mta.yaml` +:157-163). `standard` plan in `.deploy/mta.yaml` base; confirm the resource is present +for each of `deploy/{dev,qa,prod}.mtaext` (standard everywhere → no per-env plan +override needed). + +**`tutorials-srv-qa` — resolved: share one instance, distinct channel key.** The QA +module (`.deploy/mta.yaml:164`) also serves WebSocket and has the same fan-out bug at +N>1. QA binds to the **same** `tutorials-redis-websocket` instance rather than a +dedicated one — QA is low-traffic and a second managed Redis is unwarranted. Broadcast +isolation between QA and the primary is achieved by giving each module a distinct +adapter channel-key (see §1): primary uses `key: "socket.io"`, QA uses +`key: "socket.io-qa"`. Same Redis instance, non-colliding pub/sub channels. Hard +instance-level isolation is rejected — QA volume cannot swamp the shared instance. + +### 4. Docs + stale comment + +- `docs/developers/architecture/scaling-playbook.md` row #4 — mark resolved, record + chosen path (A+B via plugin-native Redis adapter), link this PR. +- `srv/lib/content-cache-coherence.js:20` — comment claims "no pub/sub bus exists; + websocket has no Redis adapter." After this change that is conditionally false (a + Redis bus now exists for the websocket adapter). Update the comment; **do not** wire + cache-coherence into Redis here — that is explicitly out of scope (see Scope + boundaries). + +## Data flow (after change, N=2 srv) + +1. Client opens `/ws/display`; handshake polling requests all pinned to srv-A by + `JSESSIONID` (A). +2. Client's socket lives on srv-A; subscribes to context `legacyId=123`. +3. A completion for tutorial 123 is processed on **srv-B**; + `developer-service.js` emits `tutorialCompleted`. +4. Plugin's Socket.IO Redis adapter on srv-B publishes the broadcast to Redis + (channel prefixed by `key: "socket.io"`). +5. srv-A's adapter, subscribed to the same channel, receives it and delivers to the + local socket. Client sees the event. **Fan-out no longer lossy.** + +## Error handling + +- **No Redis binding present** (e.g. entitlement not yet propagated, local dev): + `connectionCheck` returns false, adapter not mounted, server runs in-memory. No + crash — same as today. This is why the local/test path needs no change. +- **Redis unreachable at runtime:** the `redis` client's reconnect handling applies; + the plugin logs via `cds.log("websocket/redis")`. Worst case degrades to + per-instance in-memory behavior (today's baseline) rather than failing requests. +- The completion emit path already wraps in try/catch (`developer-service.js:792`, + `cds.log('ws').warn`), so an adapter hiccup never breaks the completion transaction. + +## Testing plan + +**Local (no Redis):** +- `npm test` — unit suite stays in-memory; assert no regression, adapter dormant. +- `test/smoke/websocket-handshake.test.js` — handshake still succeeds. + +**DEV fan-out (gates the PR — the actual thing):** +1. `git fetch origin`; deploy A+B from fresh `origin/DEV` to DEV (Redis bound). +2. Scale `tutorials-srv` to 2 instances on DEV. +3. Connect a WS client to `/ws/display` for a given `legacyId`; confirm via + `__VCAP_ID__`/logs which instance holds it. +4. Drive a completion for that `legacyId` and confirm (retry until observed on the + *other* instance is exercised) the client receives `tutorialCompleted`. +5. `npm run loadtest:ws` (`test/load/scenarios/05-websocket-handshake.js`) at 2 + instances — handshake success rate stays green (proves sticky sessions). +6. Only after fan-out is demonstrably non-lossy: open the PR against DEV. + +## Scope boundaries + +**In scope:** activate the plugin's Redis adapter for the websocket namespaces; sticky +sessions; provision/bind `redis-cache` standard; docs/comment updates; DEV fan-out +verification. + +**Out of scope (do NOT touch in this PR):** +- Wiring Redis into `content-cache-coherence.js` or any `srv/lib/*cache*` module + (scaling-playbook row #5) — different problem, different PR. +- Replacing the in-process rate limiters with Redis (row #2) — even though `common`'s + RedisClient could back them; not this change. +- Approuter auto-scaling (row #1) / cron separation (row #3). +- Changing emit sites or WS service definitions. + +## Open items (resolve at implementation / DEV deploy) + +1. **`vcap` binding resolution** — confirm `vcap.label: "redis-cache"` resolves the + instance at deploy; fall back to instance-name match if needed. Verify during the + DEV deploy. +2. **`@socket.io/redis-adapter` exact 8.x** — confirm the specific 8.x release at + `npm install` (v8.x is the correct line for `socket.io 4.x`). + +Resolved during review: QA shares one Redis instance with a distinct channel key +(`socket.io-qa`); `options.key` collision is thereby avoided (§3). diff --git a/package-lock.json b/package-lock.json index 739651116..146233b58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "@sap/cds": "^10.1.0", "@sap/xsenv": "^6.2.1", "@sap/xssec": "^4.13.1", + "@socket.io/redis-adapter": "8.3.0", "ajv": "8.20.0", "archiver": "8.0.0", "cds-caching": "2.1.0", @@ -7335,6 +7336,40 @@ "version": "3.1.2", "license": "MIT" }, + "node_modules/@socket.io/redis-adapter": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@socket.io/redis-adapter/-/redis-adapter-8.3.0.tgz", + "integrity": "sha512-ly0cra+48hDmChxmIpnESKrc94LjRL80TEmZVscuQ/WWkRP81nNj8W8cCGMqbI4L6NCuAaPRSzZF1a9GlAxxnA==", + "license": "MIT", + "dependencies": { + "debug": "~4.3.1", + "notepack.io": "~3.0.1", + "uid2": "1.0.0" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "socket.io-adapter": "^2.5.4" + } + }, + "node_modules/@socket.io/redis-adapter/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -18440,6 +18475,12 @@ "node": ">=0.10.0" } }, + "node_modules/notepack.io": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/notepack.io/-/notepack.io-3.0.1.tgz", + "integrity": "sha512-TKC/8zH5pXIAMVQio2TvVDTtPRX+DJPHDqjRbxogtFiByHyzKmy96RA0JtCQJ+WouyyL4A10xomQzgbUT+1jCg==", + "license": "MIT" + }, "node_modules/nth-check": { "version": "2.1.1", "license": "BSD-2-Clause", @@ -21237,6 +21278,15 @@ "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", "license": "MIT" }, + "node_modules/uid2": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/uid2/-/uid2-1.0.0.tgz", + "integrity": "sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==", + "license": "MIT", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/undici": { "version": "8.10.2", "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", diff --git a/package.json b/package.json index 518619244..c1dff5896 100644 --- a/package.json +++ b/package.json @@ -139,7 +139,7 @@ "validate-api-docs-yaml": "node scripts/validate-api-docs-yaml.cjs", "build:sdl": "tsx scripts/emit-graphql-sdl.ts", "check:graphql-breaking": "tsx scripts/check-graphql-breaking.ts", - "install-notify-workflows": "tsx scripts/install-notify-workflows.ts", + "install-notify-workflows": "tsx scripts/install-notify-workflows.ts", "lint:boundaries": "node scripts/lint-package-boundaries.cjs" }, "devDependencies": { @@ -208,6 +208,7 @@ "@sap/cds": "^10.1.0", "@sap/xsenv": "^6.2.1", "@sap/xssec": "^4.13.1", + "@socket.io/redis-adapter": "8.3.0", "ajv": "8.20.0", "archiver": "8.0.0", "cds-caching": "2.1.0", @@ -440,13 +441,20 @@ "AlertingTest" ], "dedupWindowMs": 300000 + }, + "redis-websocket": { + "vcap": { "label": "redis-cache" } } }, "hana": { "fuzzy": false }, "websocket": { - "kind": "socket.io" + "kind": "socket.io", + "adapter": { + "impl": "@socket.io/redis-adapter", + "options": { "key": "socket.io" } + } }, "ord": { "namespace": "sap.tutorials", diff --git a/packages/content/content-cache-coherence.js b/packages/content/content-cache-coherence.js index 820c7d0a9..538b180b4 100644 --- a/packages/content/content-cache-coherence.js +++ b/packages/content/content-cache-coherence.js @@ -17,7 +17,10 @@ * serving old content until LRU eviction or `cf restart` (#1621), and an * Admin-UI reorder "didn't propagate" for the same reason (#1592). * - * THE FIX (no pub/sub bus exists; websocket has no Redis adapter) + * THE FIX (a Redis pub/sub bus now exists for the Socket.IO websocket adapter, + * but cache-coherence deliberately does NOT use it — wiring cache invalidation + * onto Redis is a separate concern tracked in scaling-playbook.md row #5, out + * of scope for #2524) * -------------------------------------------------------------- * Keep the fast local caches for read latency, but publish a shared * *generation token* into the already-wired `caching` service. In hybrid/prod diff --git a/srv/server.js b/srv/server.js index b5d3db69f..cb20e76ec 100644 --- a/srv/server.js +++ b/srv/server.js @@ -1,4 +1,5 @@ import cds from '@sap/cds'; +import { randomUUID } from 'node:crypto'; import express from 'express'; import { AsyncLocalStorage } from 'node:async_hooks'; import { readFileSync } from 'node:fs'; @@ -2253,3 +2254,18 @@ cds.on('served', () => { cds.log('metrics-db-wrap').info('METRICS_DB_WRAP enabled: cds.db.run / cds.db.tx wrapped'); } }); + +// #2524 — Gorouter session affinity for Socket.IO. The engine.io handshake is +// multi-request (HTTP polling → WS upgrade); if those requests land on +// different tutorials-srv instances the handshake flaps. Stamping a JSESSIONID +// cookie on the handshake response pins the client to one instance for the +// connection's lifetime. This is complementary to the Redis adapter (Task 1): +// affinity fixes handshake integrity; the adapter fixes cross-instance fan-out. +// Scoped to the engine.io handshake only — does not touch application cookies. +cds.on('ws:ready', () => { + cds.io?.engine?.on('initial_headers', (headers /*, req */) => { + headers['set-cookie'] = [ + `JSESSIONID=${randomUUID()}; Path=/; HttpOnly; SameSite=Lax`, + ]; + }); +}); diff --git a/test/smoke/websocket-handshake.test.js b/test/smoke/websocket-handshake.test.js index eaf6bb5a5..f89a1f182 100644 --- a/test/smoke/websocket-handshake.test.js +++ b/test/smoke/websocket-handshake.test.js @@ -18,4 +18,15 @@ describe('WebSocket endpoints', () => { expect([200, 404]).toContain(res.status); expect(res.headers.get('content-type')).toMatch(/application\/json/); }); + + it('engine.io handshake sets a JSESSIONID affinity cookie', async () => { + // Socket.IO v4 polling handshake — the first GET returns the session + // open packet AND (after this change) a Set-Cookie for Gorouter affinity. + const res = await fetchWithRetry( + `${SRV_URL}/socket.io/?EIO=4&transport=polling&namespace=/ws/event-stream`, + ); + expect(res.status).toBe(200); + const setCookie = res.headers.get('set-cookie') || ''; + expect(setCookie).toMatch(/JSESSIONID=/); + }); });