From 28f8c8d9b9e6a59be202abb1d9c6ae97335ee1ec Mon Sep 17 00:00:00 2001 From: Roomote Date: Thu, 9 Jul 2026 06:33:50 +0000 Subject: [PATCH] feat(api): add rate-limited /health alias reporting DB and Redis connectivity with app version --- .agent-guidance/api/api-app.md | 37 ++-- .agent-guidance/operations/monitoring.md | 14 +- .env.production.example | 3 + .../__tests__/live-server.integration.test.ts | 33 ++++ .../src/handlers/health/__tests__/api.test.ts | 49 ++++- apps/api/src/handlers/health/api.ts | 102 ++++------- apps/api/src/handlers/health/version.ts | 17 ++ .../healthRateLimitMiddleware.test.ts | 173 ++++++++++++++++++ .../requestObservabilityMiddleware.test.ts | 1 + .../middleware/healthRateLimitMiddleware.ts | 123 +++++++++++++ apps/api/src/middleware/index.ts | 1 + .../requestObservabilityMiddleware.ts | 1 + apps/api/src/server.ts | 8 + packages/env/src/__tests__/index.test.ts | 2 + packages/env/src/index.ts | 7 + 15 files changed, 479 insertions(+), 92 deletions(-) create mode 100644 apps/api/src/handlers/health/version.ts create mode 100644 apps/api/src/middleware/__tests__/healthRateLimitMiddleware.test.ts create mode 100644 apps/api/src/middleware/healthRateLimitMiddleware.ts diff --git a/.agent-guidance/api/api-app.md b/.agent-guidance/api/api-app.md index 7679f1eb..d82fe2cc 100644 --- a/.agent-guidance/api/api-app.md +++ b/.agent-guidance/api/api-app.md @@ -1,7 +1,7 @@ --- title: API App status: active -last_reviewed: 2026-07-04 +last_reviewed: 2026-07-09 owner: engineering summary: Technical documentation of the Hono API app covering route mounting, middleware, and the handler families hosted in apps/api. --- @@ -23,22 +23,22 @@ summary: Technical documentation of the Hono API app covering route mounting, mi [`createApiApp()` in `apps/api/src/server.ts`](../../apps/api/src/server.ts) mounts the current route families in one place: -| Public path prefix | Mounted handler | Main responsibility | -| ------------------------------------------------------------ | ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/`, `/health/api`, `/health/liveness`, `/health/controller` | `apiHealth`, `apiLiveness`, `controllerHealth` | API-process and controller heartbeat health reporting; `/health/liveness` is the dependency-free process check, and authenticated callers receive the fuller diagnostic payloads | -| `/api/webhooks/github` | `github` | GitHub App webhook intake | -| `/api/webhooks/slack` | `slack` | Slack Events API and interactive payload intake | -| `/api/webhooks/teams` | `teams` | Microsoft Teams Bot Framework activity intake | -| `/api/webhooks/telegram` | `telegram` | Telegram Bot API update intake | -| `/api/webhooks/linear` | `linear` | Linear Agent Session webhook intake | -| `/api/mcp` | `mcp` | Worker-facing MCP routes, including integration proxies, native MCP handlers such as Snowflake, and task and environment sub-routes | -| `/api/mcp/tasks` | `mcp -> tasksRouter` | Task search, summaries, transcript history, follow-up messages, steering, stop/cancel, compute logs, launch, and task-suggestion APIs | -| `/api/mcp-routing` | `mcpRouting` | Router-facing allowlisted MCP access during task routing | -| `/api/cloud-jobs` | `cloudJobsRouter` | Cloud job logs and worker-facing runtime streams | -| `/api/artifacts` | `artifactsRouter` | Artifact upload, completion, metadata, and download URL APIs | -| `/api/tasks` | `taskArtifactsRouter` | Task-scoped artifact listing and artifact metadata lookup by task-relative artifact path | -| `/.well-known/openid-configuration`, `/api/oidc/jwks` | `oidcRouter` | Public sandbox OIDC discovery metadata and JWKS for machine trust configuration | -| `/trpc` | `trpc` | Backend-to-backend SDK tRPC router | +| Public path prefix | Mounted handler | Main responsibility | +| ----------------------------------------------------------------------- | ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/`, `/health`, `/health/api`, `/health/liveness`, `/health/controller` | `apiHealth`, `apiLiveness`, `controllerHealth` | API-process and controller heartbeat health reporting; `/health` is a rate-limited alias of the same DB+Redis readiness handler as `/` and `/health/api`, `/health/liveness` is the dependency-free process check, and authenticated callers receive the fuller diagnostic payloads | +| `/api/webhooks/github` | `github` | GitHub App webhook intake | +| `/api/webhooks/slack` | `slack` | Slack Events API and interactive payload intake | +| `/api/webhooks/teams` | `teams` | Microsoft Teams Bot Framework activity intake | +| `/api/webhooks/telegram` | `telegram` | Telegram Bot API update intake | +| `/api/webhooks/linear` | `linear` | Linear Agent Session webhook intake | +| `/api/mcp` | `mcp` | Worker-facing MCP routes, including integration proxies, native MCP handlers such as Snowflake, and task and environment sub-routes | +| `/api/mcp/tasks` | `mcp -> tasksRouter` | Task search, summaries, transcript history, follow-up messages, steering, stop/cancel, compute logs, launch, and task-suggestion APIs | +| `/api/mcp-routing` | `mcpRouting` | Router-facing allowlisted MCP access during task routing | +| `/api/cloud-jobs` | `cloudJobsRouter` | Cloud job logs and worker-facing runtime streams | +| `/api/artifacts` | `artifactsRouter` | Artifact upload, completion, metadata, and download URL APIs | +| `/api/tasks` | `taskArtifactsRouter` | Task-scoped artifact listing and artifact metadata lookup by task-relative artifact path | +| `/.well-known/openid-configuration`, `/api/oidc/jwks` | `oidcRouter` | Public sandbox OIDC discovery metadata and JWKS for machine trust configuration | +| `/trpc` | `trpc` | Backend-to-backend SDK tRPC router | The production self-host proxy exposes this Hono app under `ROOMOTE_APP_DOMAIN/_roomote-api/*` and strips `/_roomote-api` before proxying @@ -62,7 +62,7 @@ routes. | `apps/api/src/handlers/cloud-jobs/` | api | documented | [API App](./api-app.md#task-and-session-handlers) | Cloud job log and stream endpoints used by running task UIs and workers. | | `apps/api/src/handlers/oidc/` | api | documented | [API App](./api-app.md#public-oidc-handlers) | Public sandbox OIDC discovery and JWKS routes mounted at the root app level. | | `apps/api/src/handlers/artifacts/` | api | documented | [API App](./api-app.md#artifact-handlers) | Artifact creation, upload completion, metadata, per-task listing, and signed download URLs. | -| `apps/api/src/middleware/` | architecture | documented | [API App](./api-app.md#runtime-and-middleware) | Request observability, bearer-token auth middleware. | +| `apps/api/src/middleware/` | architecture | documented | [API App](./api-app.md#runtime-and-middleware) | Request observability, bearer-token auth, and `/health` rate-limit middleware. | | `apps/api/src/monitoring/` | operations | documented | [API App](./api-app.md#runtime-and-middleware) | API Sentry capture and request-level observability support. | ## Runtime And Middleware @@ -70,6 +70,7 @@ routes. The API app uses one shared Hono process with global middleware applied before route mounting: - `requestObservabilityMiddleware` runs on every request and supplies the request metadata used by the API's slow-request and trace logging. +- `healthRateLimitMiddleware()` applies a basic in-memory fixed-window per-client-IP budget (default 60/min, `API_HEALTH_RATE_LIMIT_PER_MINUTE`, `0` disables) to the bare `/health` alias only; over-limit requests get `429` with `Retry-After`, and the `/`, `/health/api`, and `/health/liveness` mounts stay unlimited for external monitors and deployment healthchecks. - CORS is configured separately for `/api/*` and `/trpc/*`, with credentials enabled for both surfaces. - `tokenAuthMiddleware()` resolves bearer tokens for worker, MCP, backend, and health-check callers. - `/admin` gets HTTP basic auth outside development. diff --git a/.agent-guidance/operations/monitoring.md b/.agent-guidance/operations/monitoring.md index 99507fcc..df9257d0 100644 --- a/.agent-guidance/operations/monitoring.md +++ b/.agent-guidance/operations/monitoring.md @@ -1,7 +1,7 @@ --- title: Monitoring & Health Checks status: active -last_reviewed: 2026-07-08 +last_reviewed: 2026-07-09 owner: engineering summary: Technical documentation of local health monitoring covering API endpoints, request observability, optional Sentry capture, controller heartbeat, orphan detection, and common debugging patterns. --- @@ -202,7 +202,7 @@ This is intended to answer "which endpoints are getting slow?" without enabling The API process also keeps a per-instance, since-start tally of inbound requests grouped by method and endpoint. - Normal REST routes use Hono's matched route templates so dynamic segments collapse into stable keys like `/api/mcp/tasks/:taskId/messages`. -- `GET /health/api`, `GET /health/liveness`, `GET /health/controller`, and `GET /` are excluded so the monitoring endpoint does not measure itself or count platform health probes. +- `GET /health`, `GET /health/api`, `GET /health/liveness`, `GET /health/controller`, and `GET /` are excluded so the monitoring endpoint does not measure itself or count platform health probes. - `/trpc/*` keeps its raw pathname so procedure names remain distinct. - The tracked table is capped at `256` unique endpoints. If the process sees additional unseen endpoints after that, they do not evict existing rows; instead they increment `overflowedUniqueEndpointCount` and `overflowedRequestCount`. @@ -214,7 +214,7 @@ Self-host Caddy exposes API health paths under `ROOMOTE_APP_DOMAIN/_roomote-api` and strips that prefix before proxying to Hono, so the health routes intentionally split their response shape: -- unauthenticated requests to `GET /_roomote-api/`, +- unauthenticated requests to `GET /_roomote-api/`, `GET /_roomote-api/health`, `GET /_roomote-api/health/api`, `GET /_roomote-api/health/liveness`, and `GET /_roomote-api/health/controller` return only `server`, `ok`, and `timestamp` @@ -250,6 +250,7 @@ Explicit outbound timeouts now live only at the call sites that intentionally wa These settings are defined in `packages/env/src/index.ts` and default safely when unset: - `API_SLOW_REQUEST_THRESHOLD_MS` — inbound request warning threshold, default `5000` +- `API_HEALTH_RATE_LIMIT_PER_MINUTE` — per-client-IP request budget for the `/health` alias, default `60`, `0` disables the limiter - `API_SLOW_EXTERNAL_REQUEST_THRESHOLD_MS` — outbound warning threshold, default `2000` - `SLACK_API_TIMEOUT_MS` — explicit timeout used by the Slack WebClient wrapper, default `10000` - `API_EXTERNAL_REQUEST_TIMEOUT_MS` — legacy compatibility fallback for `SLACK_API_TIMEOUT_MS`; the global observed `fetch` wrapper no longer applies this timeout automatically @@ -296,7 +297,7 @@ environment metadata used for deeper operator diagnostics. ### GET /health/api -Basic health check for the API server. Validates connectivity to PostgreSQL and Redis. Monitored externally by Better Stack (the "Roomote / API" monitor checks `/`, which serves the same handler). +Basic health check for the API server. Validates connectivity to PostgreSQL and Redis, always running both checks in parallel so a database outage still reports Redis state. Monitored externally by Better Stack (the "Roomote / API" monitor checks `/`, which serves the same handler). `GET /health` serves the same handler as a convenience alias, with one difference: the `/health` path is rate limited per client IP by `healthRateLimitMiddleware` (in-memory fixed window, default 60 requests/minute via `API_HEALTH_RATE_LIMIT_PER_MINUTE`, `429` + `Retry-After` when exceeded, `0` disables). The `/` and `/health/api` mounts stay unlimited for external monitors, and deployment healthchecks use the untouched `/health/liveness`. **Implementation:** `apps/api/src/handlers/health/api.ts` @@ -315,6 +316,7 @@ Basic health check for the API server. Validates connectivity to PostgreSQL and ```json { "server": "api", + "version": "v1.2.3", "environment": { "NODE_ENV": "production", "APP_ENV": "production" @@ -394,6 +396,8 @@ Basic health check for the API server. Validates connectivity to PostgreSQL and - PostgreSQL: `SELECT 1` - Redis: `PING` +- Both checks always run (in parallel); either failure yields `503` +- Authenticated payloads include `version`: the baked `RELEASE_VERSION`, falling back to `VERCEL_GIT_COMMIT_SHA` / `GITHUB_SHA`, then `development` - Long-lived proxy stream snapshot: - `activeCount` is the number of currently streaming MCP proxy responses on this instance - `oldestAgeMs` and `countsByAge` show whether stream lifetimes are clustering in the 1m/5m/15m buckets (cumulative — a 6m stream counts in both `atLeast1m` and `atLeast5m`) @@ -403,7 +407,7 @@ Basic health check for the API server. Validates connectivity to PostgreSQL and - `sinceStartedAt` marks when this process started tracking requests - `endpoints` is sorted by request count, then average duration - dynamic REST routes are normalized to matched route templates, while `/trpc/*` keeps raw pathnames - - `/`, `/health/api`, `/health/liveness`, and `/health/controller` are excluded from the tally + - `/`, `/health`, `/health/api`, `/health/liveness`, and `/health/controller` are excluded from the tally - overflow counters indicate that more unique endpoints were seen than the in-memory table can retain ### GET /health/controller diff --git a/.env.production.example b/.env.production.example index faca1182..6772388c 100644 --- a/.env.production.example +++ b/.env.production.example @@ -52,6 +52,9 @@ DASHBOARD_PASSWORD= # one-command installer generates it automatically. SETUP_TOKEN= +# Per-client-IP request budget for the API /health endpoint (requests per minute; 0 disables). +# API_HEALTH_RATE_LIMIT_PER_MINUTE=60 + # Optional env-level model provider overrides. Use models.dev-style provider/model # ids. Leave unset to manage task models through Roomote runtime settings. # ROOMOTE_MODEL=openrouter/anthropic/claude-sonnet-4 diff --git a/apps/api/src/__tests__/live-server.integration.test.ts b/apps/api/src/__tests__/live-server.integration.test.ts index 3d69cce5..80d99276 100644 --- a/apps/api/src/__tests__/live-server.integration.test.ts +++ b/apps/api/src/__tests__/live-server.integration.test.ts @@ -6,6 +6,7 @@ vi.mock('@roomote/env', async (importOriginal) => { Env: { ...actual.Env, API_SLOW_REQUEST_THRESHOLD_MS: 0, + API_HEALTH_RATE_LIMIT_PER_MINUTE: 2, }, }; }); @@ -48,6 +49,38 @@ describe('api live server integration', () => { vi.restoreAllMocks(); }); + it('serves the /health alias with a public payload and rate limits only that path', async () => { + const baseUrl = requireApi().baseUrl; + + for (let i = 0; i < 2; i += 1) { + const response = await fetch(`${baseUrl}/health`); + + expect([200, 503]).toContain(response.status); + + const payload = (await response.json()) as Record; + + // Unauthenticated callers get only the redacted public fields. + expect(Object.keys(payload).sort()).toEqual([ + 'ok', + 'server', + 'timestamp', + ]); + expect(payload.server).toBe('api'); + } + + const limited = await fetch(`${baseUrl}/health`); + + expect(limited.status).toBe(429); + expect(limited.headers.get('retry-after')).toMatch(/^\d+$/); + await expect(limited.json()).resolves.toEqual({ + error: 'Too many requests', + }); + + const unlimitedSibling = await fetch(`${baseUrl}/health/liveness`); + + expect(unlimitedSibling.status).toBe(200); + }); + it('rejects unauthenticated cloud job log requests over HTTP', async () => { const response = await fetch( `${requireApi().baseUrl}/api/cloud-jobs/123/logs`, diff --git a/apps/api/src/handlers/health/__tests__/api.test.ts b/apps/api/src/handlers/health/__tests__/api.test.ts index 54836fe0..a200b994 100644 --- a/apps/api/src/handlers/health/__tests__/api.test.ts +++ b/apps/api/src/handlers/health/__tests__/api.test.ts @@ -9,6 +9,7 @@ const envMock = vi.hoisted(() => ({ | 'production' | undefined, API_SLOW_REQUEST_THRESHOLD_MS: 3_000, + RELEASE_VERSION: 'test-release' as string | undefined, })); const dbExecuteMock = vi.hoisted(() => vi.fn()); @@ -73,6 +74,7 @@ describe('apiHealth', () => { vi.clearAllMocks(); resetLongLivedProxyStreamRegistryForTests(); resetRequestEndpointMetricsForTests(); + envMock.RELEASE_VERSION = 'test-release'; dbExecuteMock.mockResolvedValue(undefined); redisPingMock.mockResolvedValue('PONG'); consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -110,6 +112,7 @@ describe('apiHealth', () => { expect(response.status).toBe(200); await expect(response.json()).resolves.toEqual({ server: 'api', + version: 'test-release', environment: { NODE_ENV: 'test', APP_ENV: 'development', @@ -182,6 +185,7 @@ describe('apiHealth', () => { expect(response.status).toBe(200); await expect(response.json()).resolves.toEqual({ server: 'api', + version: 'test-release', environment: { NODE_ENV: 'test', APP_ENV: 'development', @@ -263,6 +267,47 @@ describe('apiHealth', () => { }); }); + it('returns a redacted healthy response without a version to unauthenticated callers', async () => { + const response = await createApp().request('/health/api'); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ + server: 'api', + ok: true, + timestamp: '2026-03-21T06:00:00.000Z', + }); + }); + + it('reports an unhealthy response when only Redis is down', async () => { + redisPingMock.mockRejectedValue(new Error('redis unavailable')); + + const response = await createApp(authContext).request('/health/api'); + + expect(response.status).toBe(503); + await expect(response.json()).resolves.toMatchObject({ + ok: false, + error: 'redis unavailable', + }); + expect(dbExecuteMock).toHaveBeenCalledTimes(1); + }); + + it('falls back to a development version when no release metadata is set', async () => { + envMock.RELEASE_VERSION = undefined; + vi.stubEnv('VERCEL_GIT_COMMIT_SHA', ''); + vi.stubEnv('GITHUB_SHA', ''); + + try { + const response = await createApp(authContext).request('/health/api'); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + version: 'development', + }); + } finally { + vi.unstubAllEnvs(); + } + }); + it('returns detailed diagnostics to authenticated callers', async () => { registerLongLivedProxyStream({ route: 'mcp:Docs', @@ -320,7 +365,7 @@ describe('apiHealth', () => { error: 'database unavailable', }); expect(consoleErrorSpy).toHaveBeenCalledTimes(1); - expect(redisPingMock).not.toHaveBeenCalled(); + expect(redisPingMock).toHaveBeenCalledTimes(1); const [message] = consoleErrorSpy.mock.calls[0] ?? []; expect(message).toContain('[Health Check]'); @@ -328,7 +373,7 @@ describe('apiHealth', () => { expect(message).toContain('"status":"unhealthy"'); expect(message).toContain('"name":"db"'); expect(message).toContain('Error: database unavailable'); - expect(message).not.toContain('"name":"redis"'); + expect(message).toContain('"name":"redis"'); expect(consoleWarnSpy).not.toHaveBeenCalled(); }); }); diff --git a/apps/api/src/handlers/health/api.ts b/apps/api/src/handlers/health/api.ts index e024cb52..5477c411 100644 --- a/apps/api/src/handlers/health/api.ts +++ b/apps/api/src/handlers/health/api.ts @@ -15,6 +15,7 @@ import { toHealthCheckLogEntry, } from './diagnostics'; import { buildHealthResponse } from './response'; +import { resolveAppVersion } from './version'; export const apiHealth = new Hono<{ Variables: Variables }>(); @@ -22,74 +23,40 @@ apiHealth.get('/', async (c) => { const requestStartedAt = Date.now(); const slowThresholdMs = Env.API_SLOW_REQUEST_THRESHOLD_MS; - const dbCheck = await runTimedHealthCheck('db', async () => { - try { - await db.execute('SELECT 1'); - return { - ok: true, - error: undefined as string | undefined, - responseError: undefined as string | undefined, - }; - } catch (error) { - return { - ok: false, - error: getHealthCheckErrorMessage(error), - responseError: getHealthCheckResponseErrorMessage(error), - }; - } - }); - - if (!dbCheck.value.ok) { - logHealthCheckDiagnostics({ - route: c.req.path, - totalDurationMs: Date.now() - requestStartedAt, - slowThresholdMs, - checks: [ - toHealthCheckLogEntry({ - check: dbCheck, - ok: dbCheck.value.ok, - error: dbCheck.value.error, - }), - ], - }); - - return c.json( - buildHealthResponse( - c, - { - server: 'api', + const [dbCheck, redisCheck] = await Promise.all([ + runTimedHealthCheck('db', async () => { + try { + await db.execute('SELECT 1'); + return { + ok: true, + error: undefined as string | undefined, + responseError: undefined as string | undefined, + }; + } catch (error) { + return { ok: false, - timestamp: new Date().toISOString(), - }, - { - environment: { NODE_ENV: Env.NODE_ENV, APP_ENV: Env.APP_ENV }, - monitoring: { - longLivedProxyStreams: getLongLivedProxyStreamHealthSnapshot(), - requestEndpointMetrics: getRequestEndpointMetricsSnapshot(), - }, - error: dbCheck.value.responseError, - }, - ), - { status: 503 }, - ); - } - - const redisCheck = await runTimedHealthCheck('redis', async () => { - try { - await getRedis().ping(); - return { - ok: true, - error: undefined as string | undefined, - responseError: undefined as string | undefined, - }; - } catch (error) { - return { - ok: false, - error: getHealthCheckErrorMessage(error), - responseError: getHealthCheckResponseErrorMessage(error), - }; - } - }); + error: getHealthCheckErrorMessage(error), + responseError: getHealthCheckResponseErrorMessage(error), + }; + } + }), + runTimedHealthCheck('redis', async () => { + try { + await getRedis().ping(); + return { + ok: true, + error: undefined as string | undefined, + responseError: undefined as string | undefined, + }; + } catch (error) { + return { + ok: false, + error: getHealthCheckErrorMessage(error), + responseError: getHealthCheckResponseErrorMessage(error), + }; + } + }), + ]); const ok = dbCheck.value.ok && redisCheck.value.ok; const error = dbCheck.value.responseError ?? redisCheck.value.responseError; @@ -121,6 +88,7 @@ apiHealth.get('/', async (c) => { timestamp: new Date().toISOString(), }, { + version: resolveAppVersion(), environment: { NODE_ENV: Env.NODE_ENV, APP_ENV: Env.APP_ENV }, monitoring: { longLivedProxyStreams: getLongLivedProxyStreamHealthSnapshot(), diff --git a/apps/api/src/handlers/health/version.ts b/apps/api/src/handlers/health/version.ts new file mode 100644 index 00000000..6cb6dfd3 --- /dev/null +++ b/apps/api/src/handlers/health/version.ts @@ -0,0 +1,17 @@ +import { Env } from '@roomote/env'; + +/** + * Resolve the running app version for health diagnostics. + * + * Prefers the release tag baked into published app images, then falls back to + * the commit SHA sources used by the API's Sentry release resolution + * (monitoring/sentry.ts), and finally to 'development' for local builds. + */ +export function resolveAppVersion(): string { + return ( + Env.RELEASE_VERSION?.trim() || + process.env.VERCEL_GIT_COMMIT_SHA?.trim() || + process.env.GITHUB_SHA?.trim() || + 'development' + ); +} diff --git a/apps/api/src/middleware/__tests__/healthRateLimitMiddleware.test.ts b/apps/api/src/middleware/__tests__/healthRateLimitMiddleware.test.ts new file mode 100644 index 00000000..3105c9d5 --- /dev/null +++ b/apps/api/src/middleware/__tests__/healthRateLimitMiddleware.test.ts @@ -0,0 +1,173 @@ +import { Hono } from 'hono'; + +const envMock = vi.hoisted(() => ({ + API_HEALTH_RATE_LIMIT_PER_MINUTE: 3, +})); + +vi.mock('@roomote/env', async (importOriginal) => { + const actual = await importOriginal(); + + return { + ...actual, + Env: envMock, + }; +}); + +import { healthRateLimitMiddleware } from '../healthRateLimitMiddleware'; + +function createApp(options?: { maxTrackedClientWindows?: number }) { + const app = new Hono(); + + const healthStub = new Hono(); + healthStub.get('/', (c) => c.json({ server: 'api', ok: true })); + + // Mirrors the production mounting in server.ts: the limiter is registered + // on the bare `/health` path while sibling health routes stay unlimited. + app.use('/health', healthRateLimitMiddleware(options)); + app.route('/health', healthStub); + app.route('/health/api', healthStub); + app.route('/health/liveness', healthStub); + + return app; +} + +function requestHealth(app: Hono, ip?: string) { + return app.request( + '/health', + ip ? { headers: { 'x-forwarded-for': ip } } : undefined, + ); +} + +describe('healthRateLimitMiddleware', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-03-21T06:00:00.000Z')); + envMock.API_HEALTH_RATE_LIMIT_PER_MINUTE = 3; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('allows requests up to the limit and rejects the next with 429 and Retry-After', async () => { + const app = createApp(); + + for (let i = 0; i < 3; i += 1) { + const response = await requestHealth(app, '203.0.113.7'); + expect(response.status).toBe(200); + } + + vi.advanceTimersByTime(15_000); + + const limited = await requestHealth(app, '203.0.113.7'); + + expect(limited.status).toBe(429); + expect(limited.headers.get('retry-after')).toBe('45'); + await expect(limited.json()).resolves.toEqual({ + error: 'Too many requests', + }); + }); + + it('resets the budget when the window rolls over', async () => { + const app = createApp(); + + for (let i = 0; i < 4; i += 1) { + await requestHealth(app, '203.0.113.7'); + } + + vi.advanceTimersByTime(60_000); + + const response = await requestHealth(app, '203.0.113.7'); + + expect(response.status).toBe(200); + }); + + it('tracks budgets per client IP', async () => { + const app = createApp(); + + for (let i = 0; i < 3; i += 1) { + await requestHealth(app, '203.0.113.7'); + } + + const limited = await requestHealth(app, '203.0.113.7'); + const otherClient = await requestHealth(app, '198.51.100.9'); + + expect(limited.status).toBe(429); + expect(otherClient.status).toBe(200); + }); + + it('uses the first hop of a multi-entry x-forwarded-for header', async () => { + const app = createApp(); + + for (let i = 0; i < 3; i += 1) { + await requestHealth(app, '203.0.113.7, 10.0.0.1'); + } + + const limited = await requestHealth(app, '203.0.113.7, 10.0.0.2'); + + expect(limited.status).toBe(429); + }); + + it('falls back to a shared key when no client address is available', async () => { + const app = createApp(); + + for (let i = 0; i < 3; i += 1) { + const response = await requestHealth(app); + expect(response.status).toBe(200); + } + + const limited = await requestHealth(app); + + expect(limited.status).toBe(429); + }); + + it('disables limiting when the configured limit is 0', async () => { + envMock.API_HEALTH_RATE_LIMIT_PER_MINUTE = 0; + const app = createApp(); + + for (let i = 0; i < 10; i += 1) { + const response = await requestHealth(app, '203.0.113.7'); + expect(response.status).toBe(200); + } + }); + + it('evicts the earliest-tracked client when the live-window cap is reached', async () => { + envMock.API_HEALTH_RATE_LIMIT_PER_MINUTE = 1; + const app = createApp({ maxTrackedClientWindows: 3 }); + + // Fill the cap with three live windows, one request each. + expect((await requestHealth(app, '203.0.113.1')).status).toBe(200); + expect((await requestHealth(app, '203.0.113.2')).status).toBe(200); + expect((await requestHealth(app, '203.0.113.3')).status).toBe(200); + + // A fourth unique client evicts the earliest-tracked one (.1). + expect((await requestHealth(app, '203.0.113.4')).status).toBe(200); + + // The evicted client restarts a fresh window instead of hitting its + // exhausted budget (a second request on a live window would be 429). + expect((await requestHealth(app, '203.0.113.1')).status).toBe(200); + + // A client whose window survived the evictions still has its count. + expect((await requestHealth(app, '203.0.113.3')).status).toBe(429); + }); + + it('never limits the sibling health routes', async () => { + const app = createApp(); + + for (let i = 0; i < 4; i += 1) { + await requestHealth(app, '203.0.113.7'); + } + + const alias = await requestHealth(app, '203.0.113.7'); + const apiHealth = await app.request('/health/api', { + headers: { 'x-forwarded-for': '203.0.113.7' }, + }); + const liveness = await app.request('/health/liveness', { + headers: { 'x-forwarded-for': '203.0.113.7' }, + }); + + expect(alias.status).toBe(429); + expect(apiHealth.status).toBe(200); + expect(liveness.status).toBe(200); + }); +}); diff --git a/apps/api/src/middleware/__tests__/requestObservabilityMiddleware.test.ts b/apps/api/src/middleware/__tests__/requestObservabilityMiddleware.test.ts index a41a07af..107621bd 100644 --- a/apps/api/src/middleware/__tests__/requestObservabilityMiddleware.test.ts +++ b/apps/api/src/middleware/__tests__/requestObservabilityMiddleware.test.ts @@ -126,6 +126,7 @@ describe('requestObservabilityMiddleware', () => { const app = createApp(); await app.request('http://localhost/'); + await app.request('http://localhost/health'); await app.request('http://localhost/health/api'); await app.request('http://localhost/health/liveness'); await app.request('http://localhost/health/controller'); diff --git a/apps/api/src/middleware/healthRateLimitMiddleware.ts b/apps/api/src/middleware/healthRateLimitMiddleware.ts new file mode 100644 index 00000000..308f0154 --- /dev/null +++ b/apps/api/src/middleware/healthRateLimitMiddleware.ts @@ -0,0 +1,123 @@ +import type { MiddlewareHandler } from 'hono'; +import { getConnInfo } from '@hono/node-server/conninfo'; + +import { Env } from '@roomote/env'; + +const RATE_LIMIT_WINDOW_MS = 60_000; + +// Cap on tracked client windows so spoofed x-forwarded-for values cannot grow +// the map without bound; when the cap is hit, expired windows are swept and, +// if every window is still live, the earliest-tracked client is evicted. +const MAX_TRACKED_CLIENT_WINDOWS = 10_000; + +const UNKNOWN_CLIENT_KEY = 'unknown'; + +type RateLimitWindow = { + windowStartMs: number; + count: number; +}; + +function resolveClientKey(c: Parameters[0]): string { + // First hop of x-forwarded-for is the original client behind the reverse + // proxy (self-host Caddy, hosted load balancers). The header is spoofable; + // this limiter is abuse dampening, not a security control. + const forwardedClientIp = c.req + .header('x-forwarded-for') + ?.split(',')[0] + ?.trim(); + + if (forwardedClientIp) { + return forwardedClientIp; + } + + try { + const remoteAddress = getConnInfo(c).remote.address?.trim(); + + if (remoteAddress) { + return remoteAddress; + } + } catch { + // getConnInfo throws outside the node-server adaptor (for example in + // app.request-based tests); fall through to the shared key. + } + + return UNKNOWN_CLIENT_KEY; +} + +/** + * Basic fixed-window per-client-IP rate limiter for the `/health` alias. + * + * Deliberately in-memory and per-process: a health check that verifies Redis + * connectivity must not itself depend on Redis, and each replica keeping its + * own budget is acceptable for basic abuse dampening. + */ +export function healthRateLimitMiddleware(options?: { + maxTrackedClientWindows?: number; +}): MiddlewareHandler { + const maxTrackedClientWindows = + options?.maxTrackedClientWindows ?? MAX_TRACKED_CLIENT_WINDOWS; + const windows = new Map(); + + const sweepExpiredWindows = (now: number): void => { + for (const [key, window] of windows) { + if (now - window.windowStartMs >= RATE_LIMIT_WINDOW_MS) { + windows.delete(key); + } + } + }; + + const evictForNewClient = (now: number): void => { + sweepExpiredWindows(now); + + if (windows.size < maxTrackedClientWindows) { + return; + } + + // Every tracked window is still live: evict the earliest-tracked client + // so the map stays bounded even under a flood of unique spoofed + // addresses. The evicted client simply restarts a fresh window later. + const earliestTrackedKey = windows.keys().next().value; + + if (earliestTrackedKey !== undefined) { + windows.delete(earliestTrackedKey); + } + }; + + return async (c, next) => { + const limitPerMinute = Env.API_HEALTH_RATE_LIMIT_PER_MINUTE; + + if (limitPerMinute <= 0) { + await next(); + return; + } + + const now = Date.now(); + const clientKey = resolveClientKey(c); + const window = windows.get(clientKey); + + if (!window || now - window.windowStartMs >= RATE_LIMIT_WINDOW_MS) { + if (!window && windows.size >= maxTrackedClientWindows) { + evictForNewClient(now); + } + + windows.set(clientKey, { windowStartMs: now, count: 1 }); + await next(); + return; + } + + window.count += 1; + + if (window.count > limitPerMinute) { + const retryAfterSeconds = Math.max( + 1, + Math.ceil((window.windowStartMs + RATE_LIMIT_WINDOW_MS - now) / 1000), + ); + + return c.json({ error: 'Too many requests' }, 429, { + 'Retry-After': String(retryAfterSeconds), + }); + } + + await next(); + }; +} diff --git a/apps/api/src/middleware/index.ts b/apps/api/src/middleware/index.ts index 90cf51b1..05aeb999 100644 --- a/apps/api/src/middleware/index.ts +++ b/apps/api/src/middleware/index.ts @@ -1,2 +1,3 @@ export { tokenAuthMiddleware } from './tokenAuthMiddleware'; export { requestObservabilityMiddleware } from './requestObservabilityMiddleware'; +export { healthRateLimitMiddleware } from './healthRateLimitMiddleware'; diff --git a/apps/api/src/middleware/requestObservabilityMiddleware.ts b/apps/api/src/middleware/requestObservabilityMiddleware.ts index f8c42b5a..2407f436 100644 --- a/apps/api/src/middleware/requestObservabilityMiddleware.ts +++ b/apps/api/src/middleware/requestObservabilityMiddleware.ts @@ -7,6 +7,7 @@ import { recordRequestEndpointMetric } from '../monitoring/request-endpoint-metr const EXCLUDED_REQUEST_METRICS_PATHS = new Set([ '/', + '/health', '/health/api', '/health/liveness', '/health/controller', diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index e86c2a75..0fb4902f 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -27,6 +27,7 @@ import { resolveApiCorsOrigin } from './cors'; import { createSingleLineWarnLogger } from './logging'; import { captureApiException } from './monitoring/sentry'; import { + healthRateLimitMiddleware, requestObservabilityMiddleware, tokenAuthMiddleware, } from './middleware'; @@ -139,6 +140,12 @@ export function createApiApp(): ApiApp { app.use('*', requestObservabilityMiddleware); + // Basic per-client-IP budget for the `/health` alias only. Runs before the + // auth stack so limited requests skip token validation. The `/`, + // `/health/api`, and `/health/liveness` mounts stay unlimited for external + // monitors and deployment healthchecks. + app.use('/health', healthRateLimitMiddleware()); + const corsOptions = { origin: resolveApiCorsOrigin, credentials: true, @@ -168,6 +175,7 @@ export function createApiApp(): ApiApp { */ app.route('/', apiHealth); + app.route('/health', apiHealth); app.route('/health/api', apiHealth); app.route('/health/liveness', apiLiveness); app.route('/health/controller', controllerHealth); diff --git a/packages/env/src/__tests__/index.test.ts b/packages/env/src/__tests__/index.test.ts index 201853b1..ba37899b 100644 --- a/packages/env/src/__tests__/index.test.ts +++ b/packages/env/src/__tests__/index.test.ts @@ -79,6 +79,7 @@ describe('Env', () => { delete runtimeEnv.ARTIFACT_SIGNING_KEY_PREVIOUS; delete runtimeEnv.SANDBOX_OIDC_PUBLIC_KEY_SECONDARY; delete runtimeEnv.PREVIEW_TOKEN_TTL_SECONDS; + delete runtimeEnv.API_HEALTH_RATE_LIMIT_PER_MINUTE; delete runtimeEnv.SKIP_ENV_VALIDATION; try { @@ -114,6 +115,7 @@ describe('Env', () => { expect(env.S3_SECRET_ACCESS_KEY).toBe('roomote-local-artifacts-password'); expect(env.S3_BUCKET_ARTIFACTS).toBe('roomote-artifacts'); expect(env.PREVIEW_TOKEN_TTL_SECONDS).toBe(3600); + expect(env.API_HEALTH_RATE_LIMIT_PER_MINUTE).toBe(60); } finally { if (previousSkipEnvValidation === undefined) { delete process.env.SKIP_ENV_VALIDATION; diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index e4655003..b680b007 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -207,6 +207,13 @@ const serverSchema = { .int() .positive() .default(2_000), + // Per-client-IP request budget for the API's `/health` alias (requests per + // minute, in-memory per process). 0 disables the limiter. + API_HEALTH_RATE_LIMIT_PER_MINUTE: z.coerce + .number() + .int() + .nonnegative() + .default(60), }; const clientSchema = {