diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index 69ed910d1..4f4b7b6bb 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -178,7 +178,7 @@ modules: - cp -r ../../hugo/assets ./hugo/assets - cp -r ../../hugo/data ./hugo/data - cp -r ../../hugo/i18n ./hugo/i18n - - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/assert-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js ../../srv/lib/fast-purge.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" + - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js ../../srv/lib/runtime-config/load-shed-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/assert-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js ../../srv/lib/fast-purge.js ../../srv/lib/load-shed.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" - bash -c "node -e \"const p=require('./package.json'); p.dependencies=Object.assign(p.dependencies||{},{cheerio:'^1.2.0','@sap-ai-sdk/foundation-models':'^2.10.0'}); require('fs').writeFileSync('./package.json', JSON.stringify(p,null,2));\"" properties: EXPOSE_CAP_UI: false diff --git a/srv/lib/content-store.js b/srv/lib/content-store.js index ac5373e43..2237fa5ab 100644 --- a/srv/lib/content-store.js +++ b/srv/lib/content-store.js @@ -25,6 +25,7 @@ import { stampSubmissionId } from './task-record-submission-id.js'; import { isDeltaWrite, isDeltaRead, isDeltaSkipCarryForward } from './content-delta-flags.js'; import { normalizeTutorialMarkdown, prefersMarkdown } from './tutorial-markdown.js'; import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { acquireServeSlot } from './load-shed.js'; import { loadProvenanceInputs } from './provenance-data.js'; import { deriveConfidence } from './provenance-freshness.js'; @@ -1048,77 +1049,97 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap } metrics.counter('content.cache.miss'); // #805 - const { ContentFiles, ContentCurrent } = cds.entities(namespace); - const db = await cds.connect.to('db'); - const isHana = db.options?.kind === 'hana' || db.constructor?.name === 'HANAService'; - - // Option B read cutover (slug-targeted-delta-rebuild), flag-gated + fail-safe. - // When CONTENT_DELTA_READ_ENABLED=true, serve from the mutable ContentCurrent - // (WHERE slug=?, no version join). Fall back to the legacy version-pinned - // ContentFiles snapshot when the flag is off OR the slug isn't in - // ContentCurrent yet (mid-migration, before the full seed) — so a partially- - // populated ContentCurrent never 404s a slug that still lives in ContentFiles. - const READ_DELTA = isDeltaRead(); - let meta; - let source = 'files'; - if (READ_DELTA && ContentCurrent) { - const [cur] = await SELECT.from(ContentCurrent) - // slug-canonical: caller-canonicalizes - .where({ slug }) - .columns('contentHash', 'mimeType'); - if (cur) { meta = cur; source = 'current'; } - } - if (!meta) { - const activeVersion = await getActiveVersion(); - if (activeVersion === null) return 'no-version'; - // Serve only from the active version — legacy full-snapshot path. - const [legacy] = await SELECT.from(ContentFiles) - // slug-canonical: callers canonicalize before calling. - .where({ slug, version: activeVersion }) - .columns('contentHash', 'mimeType', 'version'); - if (!legacy) return 'not-found'; - meta = legacy; - source = 'files'; + // #2271 load-shedding: bound concurrent DB BLOB reads on the anonymous + // content-serve path. Cache hits above never reach here, so a warm cache is + // untouched. Flag-gated (LOADSHED_ENABLED, default OFF) + fail-open — a + // guard fault admits. When in-flight reads exceed the configured ceiling, + // shed with 503 + Retry-After rather than piling up toward OOM. + const slot = await acquireServeSlot(); + if (!slot.admitted) { + metrics.counter('loadshed.triggered'); + res.setHeader('Retry-After', String(slot.retryAfter)); + res.setHeader('Cache-Control', 'no-store'); // don't let the edge cache a shed response + res.status(503).json({ error: 'Service temporarily unavailable' }); + return 'served'; // response handled; callers treat 'served' as done } - const ifNoneMatch = req.headers['if-none-match']; - if (ifNoneMatch && ifNoneMatch === `"${meta.contentHash}"`) { - res.status(304).end(); - return 'served'; - } + // Hold the in-flight slot for the whole DB read + gunzip + send; release on + // EVERY exit path (304 / no-version / not-found / served / throw). + try { + const { ContentFiles, ContentCurrent } = cds.entities(namespace); + const db = await cds.connect.to('db'); + const isHana = db.options?.kind === 'hana' || db.constructor?.name === 'HANAService'; - // Read BLOB separately — CDS QL returns HANA BLOBs as streams with locators - // that expire before consumption. Raw SQL returns a Buffer directly. - // For SQLite (tests), CDS QL works fine since there's no LOB streaming. - let contentBuf; - if (isHana) { - const [blobRow] = source === 'current' - ? await db.run( - `SELECT TOP 1 "CONTENT" FROM "${hanaCurrentTableName()}" WHERE "SLUG" = ?`, - [slug] - ) - : await db.run( - `SELECT TOP 1 "CONTENT" FROM "${hanaTableName()}" WHERE "SLUG" = ? AND "VERSION" = ?`, - [slug, meta.version] - ); - contentBuf = blobRow.CONTENT; - } else { - const blobRow = source === 'current' - ? await SELECT.one.from(ContentCurrent).where({ slug }).columns('content') // slug-canonical: caller-canonicalizes - : await SELECT.one.from(ContentFiles).where({ slug, version: meta.version }).columns('content'); // slug-canonical: caller-canonicalizes - contentBuf = await toBuffer(blobRow.content); + // Option B read cutover (slug-targeted-delta-rebuild), flag-gated + fail-safe. + // When CONTENT_DELTA_READ_ENABLED=true, serve from the mutable ContentCurrent + // (WHERE slug=?, no version join). Fall back to the legacy version-pinned + // ContentFiles snapshot when the flag is off OR the slug isn't in + // ContentCurrent yet (mid-migration, before the full seed) — so a partially- + // populated ContentCurrent never 404s a slug that still lives in ContentFiles. + const READ_DELTA = isDeltaRead(); + let meta; + let source = 'files'; + if (READ_DELTA && ContentCurrent) { + const [cur] = await SELECT.from(ContentCurrent) + // slug-canonical: caller-canonicalizes + .where({ slug }) + .columns('contentHash', 'mimeType'); + if (cur) { meta = cur; source = 'current'; } + } + if (!meta) { + const activeVersion = await getActiveVersion(); + if (activeVersion === null) return 'no-version'; + // Serve only from the active version — legacy full-snapshot path. + const [legacy] = await SELECT.from(ContentFiles) + // slug-canonical: callers canonicalize before calling. + .where({ slug, version: activeVersion }) + .columns('contentHash', 'mimeType', 'version'); + if (!legacy) return 'not-found'; + meta = legacy; + source = 'files'; + } + + const ifNoneMatch = req.headers['if-none-match']; + if (ifNoneMatch && ifNoneMatch === `"${meta.contentHash}"`) { + res.status(304).end(); + return 'served'; + } + + // Read BLOB separately — CDS QL returns HANA BLOBs as streams with locators + // that expire before consumption. Raw SQL returns a Buffer directly. + // For SQLite (tests), CDS QL works fine since there's no LOB streaming. + let contentBuf; + if (isHana) { + const [blobRow] = source === 'current' + ? await db.run( + `SELECT TOP 1 "CONTENT" FROM "${hanaCurrentTableName()}" WHERE "SLUG" = ?`, + [slug] + ) + : await db.run( + `SELECT TOP 1 "CONTENT" FROM "${hanaTableName()}" WHERE "SLUG" = ? AND "VERSION" = ?`, + [slug, meta.version] + ); + contentBuf = blobRow.CONTENT; + } else { + const blobRow = source === 'current' + ? await SELECT.one.from(ContentCurrent).where({ slug }).columns('content') // slug-canonical: caller-canonicalizes + : await SELECT.one.from(ContentFiles).where({ slug, version: meta.version }).columns('content'); // slug-canonical: caller-canonicalizes + contentBuf = await toBuffer(blobRow.content); + } + const decompressed = gunzipSync(contentBuf); + const advisory = await computeAdvisory(slug); + cache.set(slug, decompressed, meta.contentHash, advisory); + + res.setHeader('Content-Type', `${mimeType || meta.mimeType}; charset=utf-8`); + res.setHeader('ETag', `"${meta.contentHash}"`); + setContentCacheHeaders(res, { slug: tagSlug }); + res.setHeader('X-Content-Source', source === 'current' ? 'db-current' : 'db'); + setAdvisoryHeaders(res, advisory); + res.send(decompressed); + return 'served'; + } finally { + slot.release(); } - const decompressed = gunzipSync(contentBuf); - const advisory = await computeAdvisory(slug); - cache.set(slug, decompressed, meta.contentHash, advisory); - - res.setHeader('Content-Type', `${mimeType || meta.mimeType}; charset=utf-8`); - res.setHeader('ETag', `"${meta.contentHash}"`); - setContentCacheHeaders(res, { slug: tagSlug }); - res.setHeader('X-Content-Source', source === 'current' ? 'db-current' : 'db'); - setAdvisoryHeaders(res, advisory); - res.send(decompressed); - return 'served'; } async function serveHandler(req, res) { diff --git a/srv/lib/feature-flags/registry.js b/srv/lib/feature-flags/registry.js index 495247a54..03ef0bb38 100644 --- a/srv/lib/feature-flags/registry.js +++ b/srv/lib/feature-flags/registry.js @@ -104,6 +104,13 @@ export const FEATURE_FLAGS = [ description: 'WAF-equivalent input validation on the anon/agentic write surface (PR3). Body-size caps + JSON depth/shape limits on anon POST bodies before handlers run, and a depth + complexity limit (plus prod-only introspection block) on GraphQL /graphql/public. DB-driven config (ImsConfig key flag.inputvalidation); thresholds in ImsConfig inputvalidation.* via input-validation-settings.js. No env var. Default OFF, fail-open.', howToChange: featureFlagUpsert('INPUT_VALIDATION_ENABLED', 'flag.inputvalidation'), }, + { + key: 'LOADSHED_ENABLED', label: 'Origin load-shedding', category: 'Security', + kind: 'db', imsConfigKey: 'flag.loadshed', + valueType: 'boolean', default: false, issue: 'origin-abuse-protection', status: 'dev-only', + description: 'In-flight concurrency guard on the anonymous HANA content-serve path (content-store.js serveStoredSlug — tutorial HTML + content pages + author/advocate pages). When concurrent per-request gzip-BLOB reads exceed loadshed.maxConcurrent, excess requests are shed with 503 + Retry-After instead of piling up toward OOM under a scraper flood that gets past the edge cache (the serve path has no static fallback). Cache hits are never counted. Per-instance ceiling (in-memory, not cross-instance). DB-driven config (ImsConfig key flag.loadshed); ceiling + Retry-After in ImsConfig loadshed.maxConcurrent / loadshed.retryAfterSeconds via load-shed-settings.js. No env var. Default OFF, fail-open (a guard fault admits).', + howToChange: featureFlagUpsert('LOADSHED_ENABLED', 'flag.loadshed'), + }, // ---- Navigator ---- { key: 'NAV_INCLUDE_NESTED_GROUPS', label: 'Navigator nested-group cards', diff --git a/srv/lib/load-shed.js b/srv/lib/load-shed.js new file mode 100644 index 000000000..affb3ffdf --- /dev/null +++ b/srv/lib/load-shed.js @@ -0,0 +1,91 @@ +// srv/lib/load-shed.js +// Origin-side load-shedding guard (PR4, #2271) for the anonymous HANA +// content-serve path. Bounds the number of *concurrent* per-request gzip-BLOB +// reads: the serve path reads a compressed HTML BLOB from HANA and gunzips it +// per request with no static fallback, so a scraper flood that gets past the +// edge cache can saturate the DB pool and OOM the instance. Above a configured +// ceiling, excess requests are shed with 503 + Retry-After rather than piling +// up. +// +// Contract: +// acquireServeSlot() -> Promise<{ admitted, retryAfter, release() }> +// admitted:true → a slot was taken; the caller MUST call release() exactly +// once (on success AND on every error path — use finally). +// admitted:false → shed; retryAfter is the Retry-After hint (seconds). +// release() is a no-op (nothing was taken). +// +// Design notes: +// - Flag-gated (LOADSHED_ENABLED, ImsConfig flag.loadshed). OFF → every call +// is admitted as a no-op and NOTHING is counted, so the guard is inert and +// a warm cache path is untouched. +// - Fail-open: any error resolving the flag/config admits the request. The +// guard must never block content on its own failure. +// - The in-flight counter is pinned on globalThis so every ESM instance of +// this module (CAP loads service modules via dynamic file:// imports whose +// URL can differ from a static import's on Windows) shares one count — +// same rationale as the feature-flag cache in db-flags.js. +// - Single-process, per-instance ceiling (NOT cross-instance): the metric to +// bound here is per-instance memory/pool pressure, so a plain in-memory +// counter is exactly right (and needs no shared store). With N app +// instances the effective origin ceiling is N × maxConcurrent. +// - The check + increment run with NO await between them, so they are atomic +// under Node's single-threaded event loop — no check-then-act race. + +import cds from '@sap/cds'; +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { resolveLoadShedConfig } from './runtime-config/load-shed-settings.js'; + +const LOG = cds.log('load-shed'); + +const STATE_KEY = Symbol.for('com.sap.developers.ims:load-shed-inflight'); +const _state = (globalThis[STATE_KEY] ??= { inFlight: 0 }); + +// Shared no-op admission (flag off / fail-open): admitted, no slot taken. +const ADMIT_NOOP = Object.freeze({ admitted: true, retryAfter: 0, release() {} }); + +/** + * Try to take an in-flight serve slot. See the contract in the file header. + * @returns {Promise<{admitted:boolean, retryAfter:number, release:()=>void}>} + */ +export async function acquireServeSlot() { + try { + // Flag off → guard inert; do not count, always admit. + if (!isFlagEnabled('LOADSHED_ENABLED')) return ADMIT_NOOP; + + const { maxConcurrent, retryAfterSeconds } = await resolveLoadShedConfig(); + + // NB: no `await` between this read of _state.inFlight and the increment + // below — the pair is atomic on the single-threaded event loop. + if (_state.inFlight >= maxConcurrent) { + return { admitted: false, retryAfter: retryAfterSeconds, release() {} }; + } + + _state.inFlight += 1; + let released = false; + return { + admitted: true, + retryAfter: 0, + release() { + // Idempotent: a double-release must not under-count (which would let the + // ceiling drift upward and defeat the guard). + if (released) return; + released = true; + _state.inFlight -= 1; + }, + }; + } catch (err) { + // Fail-open: never block content on the guard's own failure. + LOG.warn(`load-shed guard error; admitting request: ${err.message}`); + return ADMIT_NOOP; + } +} + +/** Test-only: current in-flight count. */ +export function _inFlightForTest() { + return _state.inFlight; +} + +/** Test-only: reset the in-flight counter between cases. */ +export function _resetForTest() { + _state.inFlight = 0; +} diff --git a/srv/lib/runtime-config/load-shed-settings.js b/srv/lib/runtime-config/load-shed-settings.js new file mode 100644 index 000000000..cd2b77840 --- /dev/null +++ b/srv/lib/runtime-config/load-shed-settings.js @@ -0,0 +1,92 @@ +// srv/lib/runtime-config/load-shed-settings.js +// Numeric threshold config for the origin-side load-shedding guard (PR4, +// #2271). Mirrors rate-limit-settings.js: DB-backed (ImsConfig key/value), +// ~5s cache, globalThis-pinned, never throws, NO env-var fallback (project +// rule: tunable behavior lives in the DB, toggled live in the admin UI — env +// vars are dropped by the blue-green swap). +// +// The master on/off is a separate feature flag (LOADSHED_ENABLED, ImsConfig +// flag.loadshed) resolved by srv/lib/feature-flags/db-flags.js. THIS module +// only supplies the numbers once the guard is on. Any unset / malformed key +// falls back to its hardcoded default, so a partial config never disables +// protection. + +import cds from '@sap/cds'; + +const LOG = cds.log('load-shed-settings'); +const NS = 'com.sap.developers.ims'; +const TTL_MS = 5_000; + +const STATE_KEY = Symbol.for('com.sap.developers.ims:load-shed-settings-resolver'); +const _state = (globalThis[STATE_KEY] ??= { cached: null, cachedAt: 0 }); + +// maxConcurrent — the ceiling on simultaneous in-flight DB content reads on the +// anonymous serve path before excess requests are shed. Chosen well above +// normal browsing concurrency (a warm cache never reaches the guard) but low +// enough to bound the blast radius of a scraper flood that gets past the edge +// cache — the per-request gzip-BLOB read + gunzip is the known OOM-sensitive +// hot path with no static fallback. retryAfterSeconds — the Retry-After hint on +// a 503 shed. Both overridable per env via ImsConfig. +export const DEFAULT_CONFIG = Object.freeze({ + maxConcurrent: 64, + retryAfterSeconds: 2, +}); + +// ImsConfig key -> path into the config object. +const KEY_MAP = { + 'loadshed.maxConcurrent': ['maxConcurrent'], + 'loadshed.retryAfterSeconds': ['retryAfterSeconds'], +}; +const ALL_KEYS = Object.keys(KEY_MAP); + +function freshClone() { + return { ...DEFAULT_CONFIG }; +} + +function applyRow(cfg, key, value) { + const path = KEY_MAP[key]; + if (!path) return; + const n = Number(value); + if (!Number.isFinite(n) || n <= 0) return; // reject unset/garbage/zero → keep default + cfg[path[0]] = n; +} + +/** + * Resolve the effective numeric config. NEVER throws; on any read failure the + * hardcoded defaults are returned. Cached ~5s. + * @returns {Promise<{maxConcurrent:number, retryAfterSeconds:number}>} + */ +export async function resolveLoadShedConfig() { + const now = Date.now(); + if (_state.cached && now - _state.cachedAt < TTL_MS) return _state.cached; + + const cfg = freshClone(); + try { + if (typeof cds.entities === 'function') { + const db = await cds.connect.to('db'); + const { ImsConfig } = cds.entities(NS); + const rows = await db.run( + SELECT.from(ImsConfig).columns('key', 'value').where({ key: { in: ALL_KEYS } }) + ); + for (const r of rows || []) applyRow(cfg, r.key, r.value); + } + } catch (err) { + LOG.warn(`config read failed; using defaults: ${err.message}`); + } + + _state.cached = cfg; + _state.cachedAt = now; + return cfg; +} + +/** Test-only: clear the memoised config between cases. */ +export function _resetForTest() { + _state.cached = null; + _state.cachedAt = 0; +} + +/** Test-only: prime the cache with a known config (skips the DB read). */ +export function _primeForTest(cfg) { + _state.cached = { ...DEFAULT_CONFIG, ...cfg }; + _state.cachedAt = Date.now(); +} diff --git a/test/lib/content-store.test.js b/test/lib/content-store.test.js index 19593e4ff..4c354f280 100644 --- a/test/lib/content-store.test.js +++ b/test/lib/content-store.test.js @@ -8,6 +8,16 @@ import * as catalogRenderer from '../../srv/lib/catalog-renderer.js'; import { refreshContentDeltaFlags, bustContentDeltaFlagsCache, DELTA_WRITE_KEY, } from '../../srv/lib/content-delta-flags.js'; +import { + acquireServeSlot, + _inFlightForTest, + _resetForTest as _resetLoadShed, +} from '../../srv/lib/load-shed.js'; +import { + _primeForTest as _primeLoadShedCfg, + _resetForTest as _resetLoadShedCfg, +} from '../../srv/lib/runtime-config/load-shed-settings.js'; +import { __setFlagForTest, __resetFlagsForTest } from '../../srv/lib/feature-flags/db-flags.js'; const project = cds.test('serve', '--project', '.', '--in-memory'); @@ -917,3 +927,86 @@ describe('content-store catalog render failure → nice error page (#1938)', () expect(res._body.toString('utf-8')).toContain('Tutorial not found'); }); }); + +describe('content-store load-shedding guard on the serve path (#2271)', () => { + const API_KEY = 'test-content-key-12345'; + const html = '