From 356b3fb547d4eda170744a28fc3cee6efb5a7db5 Mon Sep 17 00:00:00 2001 From: Thomas Jung Date: Sat, 12 Sep 2026 23:03:49 -0400 Subject: [PATCH] =?UTF-8?q?feat(security):=20Origin=20load-shedding=20guar?= =?UTF-8?q?d=20=E2=80=94=20PR4=20(#2271)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - srv/lib/load-shed.js: concurrent in-flight guard with fail-open. - srv/lib/runtime-config/load-shed-settings.js: DB-backed (ImsConfig) numeric config. - srv/lib/content-store.js: wire guard into serveStoredSlug after cache-miss. - srv/lib/feature-flags/registry.js: LOADSHED_ENABLED flag entry. - .deploy/mta.yaml: srv-qa cp-list updated for transitive imports. - test/unit/load-shed.test.js: guard admits/sheds/releases/fails-open, config defaults. - test/lib/content-store.test.js: integration tests shed→503+Retry-After, release on success. Unit tests: 8 + 2 integration, all pass. Registry + cp-list drift tests pass. Default OFF, enable on DEV for observation before PROD rollout. --- .deploy/mta.yaml | 2 +- srv/lib/content-store.js | 155 +++++++++++-------- srv/lib/feature-flags/registry.js | 7 + srv/lib/load-shed.js | 91 +++++++++++ srv/lib/runtime-config/load-shed-settings.js | 92 +++++++++++ test/lib/content-store.test.js | 93 +++++++++++ test/unit/load-shed.test.js | 137 ++++++++++++++++ 7 files changed, 509 insertions(+), 68 deletions(-) create mode 100644 srv/lib/load-shed.js create mode 100644 srv/lib/runtime-config/load-shed-settings.js create mode 100644 test/unit/load-shed.test.js diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index 69ed910d1..4f4b7b6bb 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -178,7 +178,7 @@ modules: - cp -r ../../hugo/assets ./hugo/assets - cp -r ../../hugo/data ./hugo/data - cp -r ../../hugo/i18n ./hugo/i18n - - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/assert-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js ../../srv/lib/fast-purge.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" + - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js ../../srv/lib/runtime-config/load-shed-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/assert-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js ../../srv/lib/fast-purge.js ../../srv/lib/load-shed.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" - bash -c "node -e \"const p=require('./package.json'); p.dependencies=Object.assign(p.dependencies||{},{cheerio:'^1.2.0','@sap-ai-sdk/foundation-models':'^2.10.0'}); require('fs').writeFileSync('./package.json', JSON.stringify(p,null,2));\"" properties: EXPOSE_CAP_UI: false diff --git a/srv/lib/content-store.js b/srv/lib/content-store.js index ac5373e43..2237fa5ab 100644 --- a/srv/lib/content-store.js +++ b/srv/lib/content-store.js @@ -25,6 +25,7 @@ import { stampSubmissionId } from './task-record-submission-id.js'; import { isDeltaWrite, isDeltaRead, isDeltaSkipCarryForward } from './content-delta-flags.js'; import { normalizeTutorialMarkdown, prefersMarkdown } from './tutorial-markdown.js'; import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { acquireServeSlot } from './load-shed.js'; import { loadProvenanceInputs } from './provenance-data.js'; import { deriveConfidence } from './provenance-freshness.js'; @@ -1048,77 +1049,97 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap } metrics.counter('content.cache.miss'); // #805 - const { ContentFiles, ContentCurrent } = cds.entities(namespace); - const db = await cds.connect.to('db'); - const isHana = db.options?.kind === 'hana' || db.constructor?.name === 'HANAService'; - - // Option B read cutover (slug-targeted-delta-rebuild), flag-gated + fail-safe. - // When CONTENT_DELTA_READ_ENABLED=true, serve from the mutable ContentCurrent - // (WHERE slug=?, no version join). Fall back to the legacy version-pinned - // ContentFiles snapshot when the flag is off OR the slug isn't in - // ContentCurrent yet (mid-migration, before the full seed) — so a partially- - // populated ContentCurrent never 404s a slug that still lives in ContentFiles. - const READ_DELTA = isDeltaRead(); - let meta; - let source = 'files'; - if (READ_DELTA && ContentCurrent) { - const [cur] = await SELECT.from(ContentCurrent) - // slug-canonical: caller-canonicalizes - .where({ slug }) - .columns('contentHash', 'mimeType'); - if (cur) { meta = cur; source = 'current'; } - } - if (!meta) { - const activeVersion = await getActiveVersion(); - if (activeVersion === null) return 'no-version'; - // Serve only from the active version — legacy full-snapshot path. - const [legacy] = await SELECT.from(ContentFiles) - // slug-canonical: callers canonicalize before calling. - .where({ slug, version: activeVersion }) - .columns('contentHash', 'mimeType', 'version'); - if (!legacy) return 'not-found'; - meta = legacy; - source = 'files'; + // #2271 load-shedding: bound concurrent DB BLOB reads on the anonymous + // content-serve path. Cache hits above never reach here, so a warm cache is + // untouched. Flag-gated (LOADSHED_ENABLED, default OFF) + fail-open — a + // guard fault admits. When in-flight reads exceed the configured ceiling, + // shed with 503 + Retry-After rather than piling up toward OOM. + const slot = await acquireServeSlot(); + if (!slot.admitted) { + metrics.counter('loadshed.triggered'); + res.setHeader('Retry-After', String(slot.retryAfter)); + res.setHeader('Cache-Control', 'no-store'); // don't let the edge cache a shed response + res.status(503).json({ error: 'Service temporarily unavailable' }); + return 'served'; // response handled; callers treat 'served' as done } - const ifNoneMatch = req.headers['if-none-match']; - if (ifNoneMatch && ifNoneMatch === `"${meta.contentHash}"`) { - res.status(304).end(); - return 'served'; - } + // Hold the in-flight slot for the whole DB read + gunzip + send; release on + // EVERY exit path (304 / no-version / not-found / served / throw). + try { + const { ContentFiles, ContentCurrent } = cds.entities(namespace); + const db = await cds.connect.to('db'); + const isHana = db.options?.kind === 'hana' || db.constructor?.name === 'HANAService'; - // Read BLOB separately — CDS QL returns HANA BLOBs as streams with locators - // that expire before consumption. Raw SQL returns a Buffer directly. - // For SQLite (tests), CDS QL works fine since there's no LOB streaming. - let contentBuf; - if (isHana) { - const [blobRow] = source === 'current' - ? await db.run( - `SELECT TOP 1 "CONTENT" FROM "${hanaCurrentTableName()}" WHERE "SLUG" = ?`, - [slug] - ) - : await db.run( - `SELECT TOP 1 "CONTENT" FROM "${hanaTableName()}" WHERE "SLUG" = ? AND "VERSION" = ?`, - [slug, meta.version] - ); - contentBuf = blobRow.CONTENT; - } else { - const blobRow = source === 'current' - ? await SELECT.one.from(ContentCurrent).where({ slug }).columns('content') // slug-canonical: caller-canonicalizes - : await SELECT.one.from(ContentFiles).where({ slug, version: meta.version }).columns('content'); // slug-canonical: caller-canonicalizes - contentBuf = await toBuffer(blobRow.content); + // Option B read cutover (slug-targeted-delta-rebuild), flag-gated + fail-safe. + // When CONTENT_DELTA_READ_ENABLED=true, serve from the mutable ContentCurrent + // (WHERE slug=?, no version join). Fall back to the legacy version-pinned + // ContentFiles snapshot when the flag is off OR the slug isn't in + // ContentCurrent yet (mid-migration, before the full seed) — so a partially- + // populated ContentCurrent never 404s a slug that still lives in ContentFiles. + const READ_DELTA = isDeltaRead(); + let meta; + let source = 'files'; + if (READ_DELTA && ContentCurrent) { + const [cur] = await SELECT.from(ContentCurrent) + // slug-canonical: caller-canonicalizes + .where({ slug }) + .columns('contentHash', 'mimeType'); + if (cur) { meta = cur; source = 'current'; } + } + if (!meta) { + const activeVersion = await getActiveVersion(); + if (activeVersion === null) return 'no-version'; + // Serve only from the active version — legacy full-snapshot path. + const [legacy] = await SELECT.from(ContentFiles) + // slug-canonical: callers canonicalize before calling. + .where({ slug, version: activeVersion }) + .columns('contentHash', 'mimeType', 'version'); + if (!legacy) return 'not-found'; + meta = legacy; + source = 'files'; + } + + const ifNoneMatch = req.headers['if-none-match']; + if (ifNoneMatch && ifNoneMatch === `"${meta.contentHash}"`) { + res.status(304).end(); + return 'served'; + } + + // Read BLOB separately — CDS QL returns HANA BLOBs as streams with locators + // that expire before consumption. Raw SQL returns a Buffer directly. + // For SQLite (tests), CDS QL works fine since there's no LOB streaming. + let contentBuf; + if (isHana) { + const [blobRow] = source === 'current' + ? await db.run( + `SELECT TOP 1 "CONTENT" FROM "${hanaCurrentTableName()}" WHERE "SLUG" = ?`, + [slug] + ) + : await db.run( + `SELECT TOP 1 "CONTENT" FROM "${hanaTableName()}" WHERE "SLUG" = ? AND "VERSION" = ?`, + [slug, meta.version] + ); + contentBuf = blobRow.CONTENT; + } else { + const blobRow = source === 'current' + ? await SELECT.one.from(ContentCurrent).where({ slug }).columns('content') // slug-canonical: caller-canonicalizes + : await SELECT.one.from(ContentFiles).where({ slug, version: meta.version }).columns('content'); // slug-canonical: caller-canonicalizes + contentBuf = await toBuffer(blobRow.content); + } + const decompressed = gunzipSync(contentBuf); + const advisory = await computeAdvisory(slug); + cache.set(slug, decompressed, meta.contentHash, advisory); + + res.setHeader('Content-Type', `${mimeType || meta.mimeType}; charset=utf-8`); + res.setHeader('ETag', `"${meta.contentHash}"`); + setContentCacheHeaders(res, { slug: tagSlug }); + res.setHeader('X-Content-Source', source === 'current' ? 'db-current' : 'db'); + setAdvisoryHeaders(res, advisory); + res.send(decompressed); + return 'served'; + } finally { + slot.release(); } - const decompressed = gunzipSync(contentBuf); - const advisory = await computeAdvisory(slug); - cache.set(slug, decompressed, meta.contentHash, advisory); - - res.setHeader('Content-Type', `${mimeType || meta.mimeType}; charset=utf-8`); - res.setHeader('ETag', `"${meta.contentHash}"`); - setContentCacheHeaders(res, { slug: tagSlug }); - res.setHeader('X-Content-Source', source === 'current' ? 'db-current' : 'db'); - setAdvisoryHeaders(res, advisory); - res.send(decompressed); - return 'served'; } async function serveHandler(req, res) { diff --git a/srv/lib/feature-flags/registry.js b/srv/lib/feature-flags/registry.js index 495247a54..03ef0bb38 100644 --- a/srv/lib/feature-flags/registry.js +++ b/srv/lib/feature-flags/registry.js @@ -104,6 +104,13 @@ export const FEATURE_FLAGS = [ description: 'WAF-equivalent input validation on the anon/agentic write surface (PR3). Body-size caps + JSON depth/shape limits on anon POST bodies before handlers run, and a depth + complexity limit (plus prod-only introspection block) on GraphQL /graphql/public. DB-driven config (ImsConfig key flag.inputvalidation); thresholds in ImsConfig inputvalidation.* via input-validation-settings.js. No env var. Default OFF, fail-open.', howToChange: featureFlagUpsert('INPUT_VALIDATION_ENABLED', 'flag.inputvalidation'), }, + { + key: 'LOADSHED_ENABLED', label: 'Origin load-shedding', category: 'Security', + kind: 'db', imsConfigKey: 'flag.loadshed', + valueType: 'boolean', default: false, issue: 'origin-abuse-protection', status: 'dev-only', + description: 'In-flight concurrency guard on the anonymous HANA content-serve path (content-store.js serveStoredSlug — tutorial HTML + content pages + author/advocate pages). When concurrent per-request gzip-BLOB reads exceed loadshed.maxConcurrent, excess requests are shed with 503 + Retry-After instead of piling up toward OOM under a scraper flood that gets past the edge cache (the serve path has no static fallback). Cache hits are never counted. Per-instance ceiling (in-memory, not cross-instance). DB-driven config (ImsConfig key flag.loadshed); ceiling + Retry-After in ImsConfig loadshed.maxConcurrent / loadshed.retryAfterSeconds via load-shed-settings.js. No env var. Default OFF, fail-open (a guard fault admits).', + howToChange: featureFlagUpsert('LOADSHED_ENABLED', 'flag.loadshed'), + }, // ---- Navigator ---- { key: 'NAV_INCLUDE_NESTED_GROUPS', label: 'Navigator nested-group cards', diff --git a/srv/lib/load-shed.js b/srv/lib/load-shed.js new file mode 100644 index 000000000..affb3ffdf --- /dev/null +++ b/srv/lib/load-shed.js @@ -0,0 +1,91 @@ +// srv/lib/load-shed.js +// Origin-side load-shedding guard (PR4, #2271) for the anonymous HANA +// content-serve path. Bounds the number of *concurrent* per-request gzip-BLOB +// reads: the serve path reads a compressed HTML BLOB from HANA and gunzips it +// per request with no static fallback, so a scraper flood that gets past the +// edge cache can saturate the DB pool and OOM the instance. Above a configured +// ceiling, excess requests are shed with 503 + Retry-After rather than piling +// up. +// +// Contract: +// acquireServeSlot() -> Promise<{ admitted, retryAfter, release() }> +// admitted:true → a slot was taken; the caller MUST call release() exactly +// once (on success AND on every error path — use finally). +// admitted:false → shed; retryAfter is the Retry-After hint (seconds). +// release() is a no-op (nothing was taken). +// +// Design notes: +// - Flag-gated (LOADSHED_ENABLED, ImsConfig flag.loadshed). OFF → every call +// is admitted as a no-op and NOTHING is counted, so the guard is inert and +// a warm cache path is untouched. +// - Fail-open: any error resolving the flag/config admits the request. The +// guard must never block content on its own failure. +// - The in-flight counter is pinned on globalThis so every ESM instance of +// this module (CAP loads service modules via dynamic file:// imports whose +// URL can differ from a static import's on Windows) shares one count — +// same rationale as the feature-flag cache in db-flags.js. +// - Single-process, per-instance ceiling (NOT cross-instance): the metric to +// bound here is per-instance memory/pool pressure, so a plain in-memory +// counter is exactly right (and needs no shared store). With N app +// instances the effective origin ceiling is N × maxConcurrent. +// - The check + increment run with NO await between them, so they are atomic +// under Node's single-threaded event loop — no check-then-act race. + +import cds from '@sap/cds'; +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { resolveLoadShedConfig } from './runtime-config/load-shed-settings.js'; + +const LOG = cds.log('load-shed'); + +const STATE_KEY = Symbol.for('com.sap.developers.ims:load-shed-inflight'); +const _state = (globalThis[STATE_KEY] ??= { inFlight: 0 }); + +// Shared no-op admission (flag off / fail-open): admitted, no slot taken. +const ADMIT_NOOP = Object.freeze({ admitted: true, retryAfter: 0, release() {} }); + +/** + * Try to take an in-flight serve slot. See the contract in the file header. + * @returns {Promise<{admitted:boolean, retryAfter:number, release:()=>void}>} + */ +export async function acquireServeSlot() { + try { + // Flag off → guard inert; do not count, always admit. + if (!isFlagEnabled('LOADSHED_ENABLED')) return ADMIT_NOOP; + + const { maxConcurrent, retryAfterSeconds } = await resolveLoadShedConfig(); + + // NB: no `await` between this read of _state.inFlight and the increment + // below — the pair is atomic on the single-threaded event loop. + if (_state.inFlight >= maxConcurrent) { + return { admitted: false, retryAfter: retryAfterSeconds, release() {} }; + } + + _state.inFlight += 1; + let released = false; + return { + admitted: true, + retryAfter: 0, + release() { + // Idempotent: a double-release must not under-count (which would let the + // ceiling drift upward and defeat the guard). + if (released) return; + released = true; + _state.inFlight -= 1; + }, + }; + } catch (err) { + // Fail-open: never block content on the guard's own failure. + LOG.warn(`load-shed guard error; admitting request: ${err.message}`); + return ADMIT_NOOP; + } +} + +/** Test-only: current in-flight count. */ +export function _inFlightForTest() { + return _state.inFlight; +} + +/** Test-only: reset the in-flight counter between cases. */ +export function _resetForTest() { + _state.inFlight = 0; +} diff --git a/srv/lib/runtime-config/load-shed-settings.js b/srv/lib/runtime-config/load-shed-settings.js new file mode 100644 index 000000000..cd2b77840 --- /dev/null +++ b/srv/lib/runtime-config/load-shed-settings.js @@ -0,0 +1,92 @@ +// srv/lib/runtime-config/load-shed-settings.js +// Numeric threshold config for the origin-side load-shedding guard (PR4, +// #2271). Mirrors rate-limit-settings.js: DB-backed (ImsConfig key/value), +// ~5s cache, globalThis-pinned, never throws, NO env-var fallback (project +// rule: tunable behavior lives in the DB, toggled live in the admin UI — env +// vars are dropped by the blue-green swap). +// +// The master on/off is a separate feature flag (LOADSHED_ENABLED, ImsConfig +// flag.loadshed) resolved by srv/lib/feature-flags/db-flags.js. THIS module +// only supplies the numbers once the guard is on. Any unset / malformed key +// falls back to its hardcoded default, so a partial config never disables +// protection. + +import cds from '@sap/cds'; + +const LOG = cds.log('load-shed-settings'); +const NS = 'com.sap.developers.ims'; +const TTL_MS = 5_000; + +const STATE_KEY = Symbol.for('com.sap.developers.ims:load-shed-settings-resolver'); +const _state = (globalThis[STATE_KEY] ??= { cached: null, cachedAt: 0 }); + +// maxConcurrent — the ceiling on simultaneous in-flight DB content reads on the +// anonymous serve path before excess requests are shed. Chosen well above +// normal browsing concurrency (a warm cache never reaches the guard) but low +// enough to bound the blast radius of a scraper flood that gets past the edge +// cache — the per-request gzip-BLOB read + gunzip is the known OOM-sensitive +// hot path with no static fallback. retryAfterSeconds — the Retry-After hint on +// a 503 shed. Both overridable per env via ImsConfig. +export const DEFAULT_CONFIG = Object.freeze({ + maxConcurrent: 64, + retryAfterSeconds: 2, +}); + +// ImsConfig key -> path into the config object. +const KEY_MAP = { + 'loadshed.maxConcurrent': ['maxConcurrent'], + 'loadshed.retryAfterSeconds': ['retryAfterSeconds'], +}; +const ALL_KEYS = Object.keys(KEY_MAP); + +function freshClone() { + return { ...DEFAULT_CONFIG }; +} + +function applyRow(cfg, key, value) { + const path = KEY_MAP[key]; + if (!path) return; + const n = Number(value); + if (!Number.isFinite(n) || n <= 0) return; // reject unset/garbage/zero → keep default + cfg[path[0]] = n; +} + +/** + * Resolve the effective numeric config. NEVER throws; on any read failure the + * hardcoded defaults are returned. Cached ~5s. + * @returns {Promise<{maxConcurrent:number, retryAfterSeconds:number}>} + */ +export async function resolveLoadShedConfig() { + const now = Date.now(); + if (_state.cached && now - _state.cachedAt < TTL_MS) return _state.cached; + + const cfg = freshClone(); + try { + if (typeof cds.entities === 'function') { + const db = await cds.connect.to('db'); + const { ImsConfig } = cds.entities(NS); + const rows = await db.run( + SELECT.from(ImsConfig).columns('key', 'value').where({ key: { in: ALL_KEYS } }) + ); + for (const r of rows || []) applyRow(cfg, r.key, r.value); + } + } catch (err) { + LOG.warn(`config read failed; using defaults: ${err.message}`); + } + + _state.cached = cfg; + _state.cachedAt = now; + return cfg; +} + +/** Test-only: clear the memoised config between cases. */ +export function _resetForTest() { + _state.cached = null; + _state.cachedAt = 0; +} + +/** Test-only: prime the cache with a known config (skips the DB read). */ +export function _primeForTest(cfg) { + _state.cached = { ...DEFAULT_CONFIG, ...cfg }; + _state.cachedAt = Date.now(); +} diff --git a/test/lib/content-store.test.js b/test/lib/content-store.test.js index 19593e4ff..4c354f280 100644 --- a/test/lib/content-store.test.js +++ b/test/lib/content-store.test.js @@ -8,6 +8,16 @@ import * as catalogRenderer from '../../srv/lib/catalog-renderer.js'; import { refreshContentDeltaFlags, bustContentDeltaFlagsCache, DELTA_WRITE_KEY, } from '../../srv/lib/content-delta-flags.js'; +import { + acquireServeSlot, + _inFlightForTest, + _resetForTest as _resetLoadShed, +} from '../../srv/lib/load-shed.js'; +import { + _primeForTest as _primeLoadShedCfg, + _resetForTest as _resetLoadShedCfg, +} from '../../srv/lib/runtime-config/load-shed-settings.js'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; const project = cds.test('serve', '--project', '.', '--in-memory'); @@ -917,3 +927,86 @@ describe('content-store catalog render failure → nice error page (#1938)', () expect(res._body.toString('utf-8')).toContain('Tutorial not found'); }); }); + +describe('content-store load-shedding guard on the serve path (#2271)', () => { + const API_KEY = 'test-content-key-12345'; + const html = '

Shed Tutorial

'; + + beforeAll(() => { + process.env.CONTENT_API_KEY = API_KEY; + }); + + beforeEach(async () => { + // Fresh publish → the slug is a cache MISS, so the serve reaches the guard + // (the guard sits after the cache-miss counter, so warm hits never count). + await project.axios.post('/content/publish', { + trigger: 'loadshed-seed', + files: makePayload({ 'shed-tut': html }), + }, { headers: { Authorization: `Bearer ${API_KEY}` } }); + _resetLoadShed(); + _resetLoadShedCfg(); + __resetFlagsForTest(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + _resetLoadShed(); + _resetLoadShedCfg(); + __resetFlagsForTest(); + }); + + function makeServeRes() { + return { + _status: null, + _headers: {}, + _body: null, + status(code) { this._status = code; return this; }, + setHeader(k, v) { this._headers[k] = v; }, + json(b) { this._body = b; return this; }, + send(b) { this._body = b; return this; }, + end() { return this; }, + }; + } + + it('sheds with 503 + Retry-After once the in-flight ceiling is reached', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + _primeLoadShedCfg({ maxConcurrent: 1, retryAfterSeconds: 3 }); + + // Occupy the single slot so the incoming request must be shed. State is + // globalThis-pinned, so this held slot is visible to the served handler. + const held = await acquireServeSlot(); + expect(held.admitted).toBe(true); + + const { serveHandler } = createContentHandlers(); + const res = makeServeRes(); + await serveHandler( + { params: { slug: 'shed-tut' }, url: '/content/tutorials/shed-tut', headers: {} }, + res, + ); + + expect(res._status).toBe(503); + expect(String(res._headers['Retry-After'])).toBe('3'); + expect(res._headers['Cache-Control']).toBe('no-store'); + expect(res._body).toEqual({ error: 'Service temporarily unavailable' }); + + held.release(); + }); + + it('admits and serves normally below the ceiling, releasing the slot afterward', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + _primeLoadShedCfg({ maxConcurrent: 4, retryAfterSeconds: 3 }); + + const { serveHandler } = createContentHandlers(); + const res = makeServeRes(); + await serveHandler( + { params: { slug: 'shed-tut' }, url: '/content/tutorials/shed-tut', headers: {} }, + res, + ); + + expect(res._body.toString('utf-8')).toBe(html); + expect(res._headers['X-Content-Source']).toBe('db'); + expect(res._status).not.toBe(503); + // finally{} must release the slot on the success path. + expect(_inFlightForTest()).toBe(0); + }); +}); diff --git a/test/unit/load-shed.test.js b/test/unit/load-shed.test.js new file mode 100644 index 000000000..7ba63348f --- /dev/null +++ b/test/unit/load-shed.test.js @@ -0,0 +1,137 @@ +// test/unit/load-shed.test.js +// Unit coverage for the origin-side load-shedding guard (PR4, #2271): the +// in-flight concurrency guard (acquireServeSlot) and its numeric config +// resolver. Pure logic — no cds runtime. The config resolver is primed +// deterministically via _primeForTest so the guard tests control the ceiling +// without a DB, and the flag is toggled via db-flags' test seams. + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import cds from '@sap/cds'; + +// Wrap the settings resolver in a vi.fn that delegates to the real logic, so a +// single test can force it to reject (proving the guard's own fail-open) +// without disturbing the config-defaulting tests. vi.mock is hoisted and +// applies to load-shed.js's own static import too. +vi.mock('../../srv/lib/runtime-config/load-shed-settings.js', async (importActual) => { + const actual = await importActual(); + return { ...actual, resolveLoadShedConfig: vi.fn(actual.resolveLoadShedConfig) }; +}); + +import { + acquireServeSlot, + _inFlightForTest, + _resetForTest as _resetGuard, +} from '../../srv/lib/load-shed.js'; +import { + resolveLoadShedConfig, + DEFAULT_CONFIG, + _resetForTest as _resetCfg, + _primeForTest as _primeCfg, +} from '../../srv/lib/runtime-config/load-shed-settings.js'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; + +describe('load-shed: acquireServeSlot', () => { + beforeEach(() => { + _resetGuard(); + _resetCfg(); + __resetFlagsForTest(); + }); + afterEach(() => { + vi.restoreAllMocks(); + _resetGuard(); + _resetCfg(); + __resetFlagsForTest(); + }); + + it('is inert when the flag is off: admits without counting', async () => { + __setFlagForTest('LOADSHED_ENABLED', false); + _primeCfg({ maxConcurrent: 1 }); // even a ceiling of 1 must not bite + const a = await acquireServeSlot(); + const b = await acquireServeSlot(); + expect(a.admitted).toBe(true); + expect(b.admitted).toBe(true); + expect(_inFlightForTest()).toBe(0); // nothing counted while off + }); + + it('admits below the ceiling and increments the in-flight count', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + _primeCfg({ maxConcurrent: 2 }); + const a = await acquireServeSlot(); + expect(a.admitted).toBe(true); + expect(_inFlightForTest()).toBe(1); + const b = await acquireServeSlot(); + expect(b.admitted).toBe(true); + expect(_inFlightForTest()).toBe(2); + }); + + it('sheds at the ceiling with a Retry-After hint and takes no slot', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + _primeCfg({ maxConcurrent: 2, retryAfterSeconds: 7 }); + await acquireServeSlot(); + await acquireServeSlot(); // at ceiling now + const shed = await acquireServeSlot(); + expect(shed.admitted).toBe(false); + expect(shed.retryAfter).toBe(7); + expect(_inFlightForTest()).toBe(2); // shed request did NOT take a slot + }); + + it('release() decrements the count and frees a slot for the next request', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + _primeCfg({ maxConcurrent: 1 }); + const a = await acquireServeSlot(); + expect((await acquireServeSlot()).admitted).toBe(false); // full + a.release(); + expect(_inFlightForTest()).toBe(0); + expect((await acquireServeSlot()).admitted).toBe(true); // slot freed + }); + + it('release() is idempotent — a double call cannot under-count', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + _primeCfg({ maxConcurrent: 4 }); + const a = await acquireServeSlot(); + const b = await acquireServeSlot(); + expect(_inFlightForTest()).toBe(2); + a.release(); + a.release(); // second call is a no-op + expect(_inFlightForTest()).toBe(1); // only b remains, not -1 or 0 + b.release(); + expect(_inFlightForTest()).toBe(0); + }); + + it('fails open when config resolution throws: admits as a no-op', async () => { + __setFlagForTest('LOADSHED_ENABLED', true); + // resolveLoadShedConfig never throws on its own (it catches read errors), + // so force it to reject once to exercise the guard's OWN fail-open catch. + resolveLoadShedConfig.mockRejectedValueOnce(new Error('cfg down')); + const r = await acquireServeSlot(); + expect(r.admitted).toBe(true); + expect(_inFlightForTest()).toBe(0); // no slot taken on the fail-open path + r.release(); // must be a safe no-op + expect(_inFlightForTest()).toBe(0); + }); +}); + +describe('load-shed-settings: resolveLoadShedConfig', () => { + beforeEach(() => { + _resetCfg(); + vi.spyOn(cds.connect, 'to').mockRejectedValue(new Error('no db in unit')); + }); + afterEach(() => { + vi.restoreAllMocks(); + _resetCfg(); + }); + + it('falls back to hardcoded defaults on read failure', async () => { + const cfg = await resolveLoadShedConfig(); + expect(cfg.maxConcurrent).toBe(DEFAULT_CONFIG.maxConcurrent); + expect(cfg.retryAfterSeconds).toBe(DEFAULT_CONFIG.retryAfterSeconds); + }); + + it('returns a mutable clone, not the frozen default', async () => { + const cfg = await resolveLoadShedConfig(); + expect(() => { + cfg.maxConcurrent = 999; + }).not.toThrow(); + expect(DEFAULT_CONFIG.maxConcurrent).not.toBe(999); + }); +});