diff --git a/.deploy/mta.yaml b/.deploy/mta.yaml index cd35788eb..69ed910d1 100644 --- a/.deploy/mta.yaml +++ b/.deploy/mta.yaml @@ -178,7 +178,7 @@ modules: - cp -r ../../hugo/assets ./hugo/assets - cp -r ../../hugo/data ./hugo/data - cp -r ../../hugo/i18n ./hugo/i18n - - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/assert-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" + - bash -c "mkdir -p srv/jobs && mkdir -p srv/handlers && mkdir -p srv/lib/branch && mkdir -p srv/lib/runtime-config && mkdir -p srv/lib/prompts && mkdir -p srv/lib/kg && mkdir -p srv/mcp/prompts && cp ../../srv/lib/branch/condition.js ../../srv/lib/branch/engine.js ../../srv/lib/branch/ranker.js ../../srv/lib/branch/user-state.js ../../srv/lib/branch/loaders.js ../../srv/lib/branch/mission-detail.js ../../srv/lib/branch/slug-key.js ../../srv/lib/branch/decide.js ../../srv/lib/branch/joule-tool.js ../../srv/lib/branch/branch-telemetry.js ../../srv/lib/branch/group-by-alt.js ../../srv/lib/branch/profile-fields.js ../../srv/lib/branch/profile-override.js srv/lib/branch/ && cp ../../srv/lib/runtime-config/kg-settings.js ../../srv/lib/runtime-config/ui-events-settings.js ../../srv/lib/runtime-config/search-settings.js ../../srv/lib/runtime-config/navigator-settings.js ../../srv/lib/runtime-config/display-settings.js ../../srv/lib/runtime-config/tenant-settings.js ../../srv/lib/runtime-config/alert-settings.js srv/lib/runtime-config/ && cp ../../srv/lib/kg/on-demand-enqueue.js ../../srv/lib/kg/on-demand-cosine-rank.js srv/lib/kg/ && cp ../../srv/lib/credstore.js ../../srv/lib/secret-resolver.js ../../srv/lib/content-store.js ../../srv/lib/tutorial-markdown.js ../../srv/lib/content-delta-flags.js ../../srv/lib/content-cache-coherence.js ../../srv/lib/edge-cache-headers.js ../../srv/lib/content-publish-session.js ../../srv/lib/resolve-tutorial-author.js ../../srv/lib/_tutorials-table.js ../../srv/lib/catalog-renderer.js ../../srv/lib/catalog-data.js ../../srv/lib/catalog-mission-hierarchy.js ../../srv/lib/chrome-shell.js ../../srv/lib/pipeline-log.js ../../srv/lib/legacy-id.js ../../srv/lib/embedding-pipeline.js ../../srv/lib/step-text-extractor.js ../../srv/lib/embedding-client.js ../../srv/lib/step-vectors.js ../../srv/lib/user-progress.js ../../srv/lib/co-completion.js ../../srv/lib/tutorial-centroid.js ../../srv/lib/tag-label-map.js ../../srv/lib/code-check-tool.js ../../srv/lib/code-check-prompt.js ../../srv/lib/code-check-handler.js ../../srv/lib/code-check-llm.js ../../srv/lib/code-check-step-loader.js ../../srv/lib/code-check-spec-publish.js ../../srv/lib/assert-spec-publish.js ../../srv/lib/validate-answer-spec-publish.js ../../srv/lib/category-classifier.js ../../srv/lib/category-classifier-llm.js ../../srv/lib/category-seed-embeddings.js ../../srv/lib/build-catalog-categories.js ../../srv/lib/chat-settings-resolver.js ../../srv/lib/kg-extract.js ../../srv/lib/kg-queries.js ../../srv/lib/kg-projection.js ../../srv/lib/kg-similarity.js ../../srv/lib/kg-cycles.js ../../srv/lib/kg-graph-rebuild.js ../../srv/lib/kg-sparql-client.js ../../srv/lib/kg-merge-pair.js ../../srv/lib/kg-concept-loader.js ../../srv/lib/kg-neighborhood-cache.js ../../srv/lib/kg-neighborhood-merge.js ../../srv/lib/kg-neighborhood-full-helpers.js ../../srv/lib/kg-other-resources-loader.js ../../srv/lib/kg-stamp-meta-text.js ../../srv/lib/kg-tutorial-teaches-map.js ../../srv/lib/kg-resource-type-config.js ../../srv/lib/kg-meta-formatters.js ../../srv/lib/discovery-mission-categories.js ../../srv/lib/external-content-ttl.js ../../srv/lib/recompute-tutorial-progress-bulk-sql.js ../../srv/lib/youtube-fetcher.js ../../srv/lib/homepage-events-merger.js ../../srv/lib/homepage-rss-fetcher.js ../../srv/lib/rss-parse.js ../../srv/lib/community-blogs-fetcher.js ../../srv/lib/community-blog-source-defaults.js ../../srv/lib/community-blogs-classifier.js ../../srv/lib/safe-fetch.js ../../srv/lib/curl-transport.js ../../srv/lib/khoros-transport.js ../../srv/lib/explainer-generator.js ../../srv/lib/_token-cost.js ../../srv/lib/metrics.js ../../srv/lib/alerting.js ../../srv/lib/relevance-classifier.js ../../srv/lib/relevance-seed-embeddings.js ../../srv/lib/relevance-keyword-rules.js ../../srv/lib/canonicalize-link.js ../../srv/lib/detect-language-en.js ../../srv/lib/kg-community-coverage.js ../../srv/lib/page-key-map.js ../../srv/lib/page-fallback.js ../../srv/lib/task-record-submission-id.js ../../srv/lib/image-store.cjs ../../srv/lib/image-ingest.cjs ../../srv/lib/image-source-handler.js ../../srv/lib/img-cdn-fetch.cjs ../../srv/lib/img-cdn-retry.cjs ../../srv/lib/image-warm-utils.js ../../srv/lib/attachment-store.cjs ../../srv/lib/attachment-ingest.cjs ../../srv/lib/attachment-mime.cjs ../../srv/lib/attachment-warm-utils.js ../../srv/lib/attachment-source-handler.js ../../srv/lib/attachment-ingest-handler.js ../../srv/lib/contributors-publish.js ../../srv/lib/validation-rules-publish.js ../../srv/lib/topics-query.js ../../srv/lib/topic-slug.js ../../srv/lib/tag-md-format.js ../../srv/lib/semaphore-tags.js ../../srv/lib/publish-channels.js ../../srv/lib/media-diet-picks.js ../../srv/lib/media-diet-export.js ../../srv/lib/build-channel-detail.js ../../srv/lib/channel-detail-render.js ../../srv/lib/build-channel-atlas.js ../../srv/lib/island-manifest.json ../../srv/lib/provenance-handlers.js ../../srv/lib/provenance-envelope.js ../../srv/lib/provenance-data.js ../../srv/lib/provenance-keys.js ../../srv/lib/provenance-freshness.js ../../srv/lib/fast-purge.js srv/lib/ && mkdir -p srv/lib/channels && cp ../../srv/lib/channels/normalize.cjs srv/lib/channels/ && mkdir -p srv/lib/feature-flags && cp ../../srv/lib/feature-flags/db-flags.js ../../srv/lib/feature-flags/registry.js srv/lib/feature-flags/ && cp ../../srv/handlers/categories-after-hooks.js ../../srv/handlers/completion-path-items-altgroup.js srv/handlers/ && mkdir -p srv && cp ../../srv/content-moderation-service.js srv/ && cp ../../srv/jobs/consolidate-concepts-job.js ../../srv/jobs/extract-concepts-job.js ../../srv/jobs/job-lock.js ../../srv/jobs/secret-expiry-check.js ../../srv/jobs/homepage-link-health.js ../../srv/jobs/kg-ondemand-job.js ../../srv/jobs/community-blogs-fetch-job.js ../../srv/jobs/community-blogs-classify-job.js ../../srv/jobs/fetch-news-job.js srv/jobs/ && cp ../../srv/lib/prompts/explainer-verb.md ../../srv/lib/prompts/explainer-shelf.md ../../srv/lib/prompts/explainer-shelf-entry.md ../../srv/lib/prompts/community-blogs-classifier.md srv/lib/prompts/ && cp ../../srv/mcp/prompts/summarize_mission_for_beginner.md ../../srv/mcp/prompts/generate_lab_exercise.md ../../srv/mcp/prompts/explain_concept.md ../../srv/mcp/prompts/suggest_learning_path.md srv/mcp/prompts/" - bash -c "node -e \"const p=require('./package.json'); p.dependencies=Object.assign(p.dependencies||{},{cheerio:'^1.2.0','@sap-ai-sdk/foundation-models':'^2.10.0'}); require('fs').writeFileSync('./package.json', JSON.stringify(p,null,2));\"" properties: EXPOSE_CAP_UI: false diff --git a/srv/lib/content-publish-session.js b/srv/lib/content-publish-session.js index 2454523be..91d5cac16 100644 --- a/srv/lib/content-publish-session.js +++ b/srv/lib/content-publish-session.js @@ -438,6 +438,7 @@ export function createSessionHelpers({ namespace }) { fileCount: existing.fileCount, totalSizeBytes: existing.totalSizeBytes, durationMs: existing.publishDurationMs || 0, + freshSlugs: [], // already active → nothing freshly published to purge alreadyActive: true }; } @@ -655,6 +656,10 @@ export function createSessionHelpers({ namespace }) { totalSizeBytes: freshSize + carriedSize, durationMs, carriedForward, + // freshSlugs — the slugs freshly written by this publish (post revert + // filtering), surfaced so commitHandler can issue a targeted edge + // Fast-Purge by per-item tag (srv/lib/fast-purge.js). + freshSlugs, // #672 — empty array (not omitted) so clients can rely on the field // being present in every commit response. `allowedReverts` lists slugs // that WERE detected as reverts but committed anyway per an explicit diff --git a/srv/lib/content-store.js b/srv/lib/content-store.js index b88a9a6dc..ac5373e43 100644 --- a/srv/lib/content-store.js +++ b/srv/lib/content-store.js @@ -18,6 +18,7 @@ import * as metrics from './metrics.js'; import * as alerting from './alerting.js'; import { resolveSecret } from './secret-resolver.js'; import { setContentCacheHeaders } from './edge-cache-headers.js'; +import { purgePublishedSlugs, purgeAllContent } from './fast-purge.js'; import { pageKeyForPath, mimeTypeForPageKey } from './page-key-map.js'; import { loadPageFallback } from './page-fallback.js'; import { stampSubmissionId } from './task-record-submission-id.js'; @@ -1874,6 +1875,11 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap cache.invalidate(); await bumpCacheGeneration(); // #1592/#1621: propagate wipe to peer instances + // Rollback knows the manifest version, not a per-slug list — purge the + // whole content corpus at the edge (coarse `content` tag). Fire-and-forget, + // fail-open, no-op unless EDGE_PURGE_ENABLED. See fast-purge.js. + purgeAllContent(); + res.json({ rolledBackTo: target.version, status: 'ACTIVE' }); } catch (err) { console.error('[content/rollback]', err instanceof Error ? err.message : String(err)); @@ -2032,6 +2038,9 @@ export function createContentHandlers({ namespace = 'com.sap.developers.ims', ap const result = await sessionHelpers.commitSession({ sessionId, allowRevertSlugs }); cache.invalidate(); await bumpCacheGeneration(); // #1592/#1621: propagate wipe to peer instances + // Fire-and-forget edge Fast-Purge of the freshly published slugs — void, + // never awaited, fail-open (no-op unless EDGE_PURGE_ENABLED). See fast-purge.js. + purgePublishedSlugs(result.freshSlugs); LOG.info(`[content/publish/commit] sessionId=${sessionId} version=${result.version} duration=${result.durationMs}ms alreadyActive=${result.alreadyActive}`); res.status(200).json(result); } catch (err) { diff --git a/srv/lib/edge-cache-headers.js b/srv/lib/edge-cache-headers.js index 1e6b1d660..c47767abe 100644 --- a/srv/lib/edge-cache-headers.js +++ b/srv/lib/edge-cache-headers.js @@ -1,4 +1,5 @@ import cds from '@sap/cds'; +import { isFlagEnabled } from './feature-flags/db-flags.js'; // Shared Cache-Control / edge-cache-tag helper for the public content-serving // paths (tutorials, group/mission catalog pages, concepts index). A CDN @@ -6,27 +7,48 @@ import cds from '@sap/cds'; // // The value splits the *browser* TTL from the *shared-edge* TTL: // - max-age (browser): short (60s), so a hard refresh picks up a new publish fast. -// - s-maxage (edge): modest (600s / 10 min). There is NO active purge-by-tag -// signal wired today — the Akamai Fast-Purge hook is still -// proposed (see docs/developers/architecture/cdn-caching.md, -// "publish-time purge hook remains proposed"). With no -// purge, the edge TTL is the ONLY thing bounding how long a -// content update stays stale at the CDN, so we keep it short -// (worst-case ~10 min stale). When purge-by-tag lands, this -// can safely be raised back toward a day. +// - s-maxage (edge): the shared-edge TTL is COUPLED to whether an active +// purge-by-tag signal exists (EDGE_PURGE_ENABLED / the +// Akamai Fast-Purge hook in srv/lib/fast-purge.js): +// * flag OFF → 600s (10 min). With no purge, the edge +// TTL is the ONLY thing bounding post-publish +// staleness, so it stays short (worst-case ~10 min). +// * flag ON → 86400s (1 day). A publish now issues a +// targeted purge-by-tag within seconds, so the long +// TTL is safe and offloads scraper/bot floods to the +// edge instead of HANA. +// Coupling them means we can never raise the staleness +// ceiling without a purge mechanism in place to bound it. // - stale-while-revalidate: serve the stale copy instantly while the edge // revalidates in the background. // -// The Edge-Cache-Tag header below is still emitted so a future purge-by-tag hook -// works with no further change here. +// The Edge-Cache-Tag header below drives that purge-by-tag hook (see +// fast-purge.js — it maps freshly published slugs to their item- tags). // // Only call setContentCacheHeaders on 200 content responses — never on // redirects, 404s, or the no-cache delta/drift probes. + +// Base (flag-off) value — kept as the exported constant for tests/importers. const CONTENT_CACHE_CONTROL = 'public, max-age=60, s-maxage=600, stale-while-revalidate=600'; +// Long shared-edge TTL applied only when EDGE_PURGE_ENABLED is on. +const CONTENT_CACHE_CONTROL_PURGE = + 'public, max-age=60, s-maxage=86400, stale-while-revalidate=600'; + const LOG = cds.log('edge-cache'); +// Effective Cache-Control for a cacheable content response. Fail-open: any +// fault reading the flag falls back to the short-TTL base value (never the long +// TTL — a stale-ceiling regression must never be the failure mode). +function effectiveCacheControl() { + try { + return isFlagEnabled('EDGE_PURGE_ENABLED') ? CONTENT_CACHE_CONTROL_PURGE : CONTENT_CACHE_CONTROL; + } catch { + return CONTENT_CACHE_CONTROL; + } +} + // Sanitize a slug into a valid Cache-Tag token (Akamai tags are alnum + [-_], // capped in length). Never let an unbounded/exotic slug produce a malformed // header value. @@ -65,7 +87,7 @@ function cacheTagsFor(slug) { // break content serving. function setContentCacheHeaders(res, { slug } = {}) { try { - res.setHeader('Cache-Control', CONTENT_CACHE_CONTROL); + res.setHeader('Cache-Control', effectiveCacheControl()); res.setHeader('Vary', 'Accept-Encoding'); res.setHeader('Edge-Cache-Tag', cacheTagsFor(slug).join(', ')); } catch (err) { diff --git a/srv/lib/fast-purge.js b/srv/lib/fast-purge.js new file mode 100644 index 000000000..915ca224a --- /dev/null +++ b/srv/lib/fast-purge.js @@ -0,0 +1,264 @@ +// srv/lib/fast-purge.js +// +// Akamai Fast-Purge (CCU v3) purge-by-tag hook for the content-serving edge +// (PR2 of the origin-abuse-protection sequence; see the plan + the +// docs/developers/architecture/cdn-caching.md §"Invalidation on publish"). +// +// Why this exists: developers.sap.com fronts with Akamai. Content is served +// per-request from HANA gzip BLOBs with a split browser/edge Cache-Control +// (srv/lib/edge-cache-headers.js). Without an active purge signal the ONLY +// thing bounding edge staleness after a publish is `s-maxage`, so that TTL has +// to stay short (10 min). Wiring a purge-by-tag call on the publish path lets +// the edge hold content much longer (offloading scraper/bot floods away from +// HANA — the biggest availability win) while a publish invalidates the exact +// changed items within seconds. +// +// Design constraints honored here: +// - Master flag EDGE_PURGE_ENABLED (ImsConfig flag.edgepurge, default OFF). +// Off → every entry point is a no-op: no secret read, no network. This is +// also what makes dev a natural mock (no Akamai credential bound → no-op). +// - FAIL-OPEN + FIRE-AND-FORGET everywhere. A purge fault (missing/garbled +// credential, network blip, Akamai 5xx) must NEVER fail a content commit or +// rollback. Callers use purgePublishedSlugs()/purgeAllContent() which return +// void and swallow all errors; the awaitable purgeTags() (used by tests) +// also never throws. +// - No new dependency: the Akamai EG1-HMAC-SHA256 request signing is +// implemented with node:crypto (the algorithm is a documented HMAC-SHA256 +// construction). Native fetch performs the call (AbortSignal timeout). +// - Env-free: the credential bundle is one JSON secret in Credential Store +// (alias AKAMAI_FASTPURGE_EDGERC), resolved via secret-resolver.js and +// rotatable through /admin-ui/#secrets. Numeric/string tunables come from +// ImsConfig (edgepurge.*), never process.env (blue-green drops set-env). + +import cds from '@sap/cds'; +import { createHmac, createHash, randomUUID } from 'node:crypto'; +import { resolveSecret } from './secret-resolver.js'; +import { cacheTagsFor } from './edge-cache-headers.js'; +import { isFlagEnabled } from './feature-flags/db-flags.js'; +import { counter } from './metrics.js'; + +const LOG = cds.log('fast-purge'); +const NS = 'com.sap.developers.ims'; + +// Credential Store alias holding the EdgeGrid credentials as a JSON string: +// {"host":"akab-….purge.akamaiapis.net","client_token":"akab-…", +// "client_secret":"…","access_token":"akab-…"} +const SECRET_ALIAS = 'AKAMAI_FASTPURGE_EDGERC'; + +// Coarse tags that must NOT drive a targeted per-publish purge: purging any of +// these would invalidate the whole corpus / whole category on a single-slug +// publish. cacheTagsFor() always includes 'content'; category pages also carry +// 'group'/'mission'/'page'/'concepts'. We keep only the per-item tags. +const COARSE_TAGS = new Set(['content', 'group', 'mission', 'page', 'concepts']); + +// Akamai caps objects per Fast-Purge request; batch conservatively. +const MAX_TAGS_PER_CALL = 100; + +// ---- config (ImsConfig-backed, ~5s cache, never throws) -------------------- +// Mirrors srv/lib/runtime-config/rate-limit-settings.js but kept inline so this +// hook adds exactly one file to the srv-qa cp list. +const TTL_MS = 5_000; +const CFG_KEY = Symbol.for('com.sap.developers.ims:fast-purge-settings'); +const _cfgState = (globalThis[CFG_KEY] ??= { cached: null, cachedAt: 0 }); + +const DEFAULT_CFG = Object.freeze({ + network: 'production', // Akamai purge network: 'production' | 'staging' + timeoutMs: 5_000, +}); + +async function resolveConfig() { + const now = Date.now(); + if (_cfgState.cached && now - _cfgState.cachedAt < TTL_MS) return _cfgState.cached; + const cfg = { ...DEFAULT_CFG }; + try { + if (typeof cds.entities === 'function') { + const db = await cds.connect.to('db'); + const { ImsConfig } = cds.entities(NS); + const rows = await db.run( + SELECT.from(ImsConfig) + .columns('key', 'value') + .where({ key: { in: ['edgepurge.network', 'edgepurge.timeoutMs'] } }) + ); + for (const r of rows || []) { + if (r.key === 'edgepurge.network' && (r.value === 'production' || r.value === 'staging')) { + cfg.network = r.value; + } else if (r.key === 'edgepurge.timeoutMs') { + const n = Number(r.value); + if (Number.isFinite(n) && n >= 500 && n <= 60_000) cfg.timeoutMs = n; + } + } + } + } catch (err) { + LOG.warn(`config read failed; using defaults: ${err.message}`); + } + _cfgState.cached = cfg; + _cfgState.cachedAt = now; + return cfg; +} + +// ---- EdgeGrid EG1-HMAC-SHA256 signing -------------------------------------- + +function base64HmacSha256(data, key) { + return createHmac('sha256', key).update(data, 'utf8').digest('base64'); +} + +function base64Sha256(data) { + return createHash('sha256').update(data, 'utf8').digest('base64'); +} + +// Akamai timestamp: yyyyMMdd'T'HH:mm:ss+0000 in UTC. +function edgeGridTimestamp(d = new Date()) { + const p = (n) => String(n).padStart(2, '0'); + return ( + `${d.getUTCFullYear()}${p(d.getUTCMonth() + 1)}${p(d.getUTCDate())}` + + `T${p(d.getUTCHours())}:${p(d.getUTCMinutes())}:${p(d.getUTCSeconds())}+0000` + ); +} + +// Build the Authorization header per the EdgeGrid spec. `creds` = {clientToken, +// accessToken, clientSecret}; `req` = {method, host, path, body}. timestamp + +// nonce are injectable for deterministic tests. +function makeAuthHeader(creds, req, timestamp = edgeGridTimestamp(), nonce = randomUUID()) { + let authHeader = + `EG1-HMAC-SHA256 client_token=${creds.clientToken};` + + `access_token=${creds.accessToken};timestamp=${timestamp};nonce=${nonce};`; + const signingKey = base64HmacSha256(timestamp, creds.clientSecret); + const contentHash = req.method === 'POST' && req.body ? base64Sha256(req.body) : ''; + const dataToSign = [ + req.method.toUpperCase(), + 'https', + req.host.toLowerCase(), + req.path, + '', // canonicalized request headers — none signed + contentHash, + authHeader, + ].join('\t'); + const signature = base64HmacSha256(dataToSign, signingKey); + return `${authHeader}signature=${signature}`; +} + +// Parse + validate the JSON credential bundle from Credential Store. Returns +// null (never throws) when absent/malformed — the caller fails open. +function parseCreds(raw) { + if (!raw) return null; + let obj; + try { + obj = JSON.parse(raw); + } catch { + return null; + } + const host = obj.host; + const clientToken = obj.client_token; + const clientSecret = obj.client_secret; + const accessToken = obj.access_token; + if (!host || !clientToken || !clientSecret || !accessToken) return null; + return { host, clientToken, clientSecret, accessToken }; +} + +// Test seam: allow a mock fetch/credential without touching credstore or the +// network. Pinned to globalThis for the CDS+Vitest module-multiplicity reason. +const TEST_KEY = Symbol.for('com.sap.developers.ims:fast-purge-testhooks'); +const _test = (globalThis[TEST_KEY] ??= { fetchImpl: null }); + +/** + * Purge a set of Edge-Cache-Tag values via Akamai Fast-Purge (CCU v3). + * Awaitable core used by tests and by the fire-and-forget wrappers. NEVER + * throws. Returns a small result object describing what happened. + * + * @param {string[]} tags + * @returns {Promise<{ok:boolean, skipped?:string, status?:number, purged?:number}>} + */ +export async function purgeTags(tags) { + try { + if (!isFlagEnabled('EDGE_PURGE_ENABLED')) return { ok: true, skipped: 'flag-off' }; + const unique = [...new Set((tags || []).filter((t) => typeof t === 'string' && t))]; + if (!unique.length) return { ok: true, skipped: 'no-tags' }; + + const raw = await resolveSecret(SECRET_ALIAS, { logTag: '[fast-purge]' }); + const creds = parseCreds(raw); + if (!creds) { + LOG.warn('EDGE_PURGE_ENABLED is on but no valid Akamai credential is configured — skipping purge (fail-open)'); + return { ok: false, skipped: 'no-credential' }; + } + + const cfg = await resolveConfig(); + const fetchImpl = _test.fetchImpl || fetch; + let purged = 0; + let lastStatus = 0; + for (let i = 0; i < unique.length; i += MAX_TAGS_PER_CALL) { + const batch = unique.slice(i, i + MAX_TAGS_PER_CALL); + const path = `/ccu/v3/invalidate/tag/${cfg.network}`; + const body = JSON.stringify({ objects: batch }); + const auth = makeAuthHeader( + creds, + { method: 'POST', host: creds.host, path, body } + ); + const res = await fetchImpl(`https://${creds.host}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: auth }, + body, + signal: AbortSignal.timeout(cfg.timeoutMs), + }); + lastStatus = res.status; + if (!res.ok) { + const detail = await res.text().catch(() => ''); + LOG.warn(`Fast-Purge returned ${res.status} for ${batch.length} tag(s): ${detail.slice(0, 200)}`); + counter('edgepurge.failed'); + return { ok: false, status: res.status, purged }; + } + purged += batch.length; + } + counter('edgepurge.purged'); + LOG.info(`Fast-Purge invalidated ${purged} tag(s) on ${cfg.network}`); + return { ok: true, status: lastStatus, purged }; + } catch (err) { + LOG.warn(`Fast-Purge failed (allowing content change to proceed): ${err.message}`); + counter('edgepurge.failed'); + return { ok: false, skipped: 'error' }; + } +} + +// Map a set of freshly published slugs to their per-item Edge-Cache-Tag set, +// dropping the coarse corpus/category tags so a targeted publish purges only +// the changed items (not the whole edge cache). +function itemTagsForSlugs(slugs) { + const tags = new Set(); + for (const slug of slugs || []) { + for (const t of cacheTagsFor(slug)) { + if (!COARSE_TAGS.has(t)) tags.add(t); + } + } + return [...tags]; +} + +/** + * Fire-and-forget: purge the edge for the freshly published slugs. Void — + * returns immediately, never awaited from the commit path. Call after the + * in-process cache invalidate in commitHandler. + */ +export function purgePublishedSlugs(slugs) { + const tags = itemTagsForSlugs(slugs); + if (!tags.length) return; + Promise.resolve(purgeTags(tags)).catch(() => {}); +} + +/** + * Fire-and-forget: purge the entire content corpus (coarse `content` tag). Used + * on rollback, where only a manifest version — not a per-slug list — is known. + */ +export function purgeAllContent() { + Promise.resolve(purgeTags(['content'])).catch(() => {}); +} + +// ---- test-only hooks ------------------------------------------------------- +export const _makeAuthHeaderForTest = makeAuthHeader; +export const _edgeGridTimestampForTest = edgeGridTimestamp; +export const _itemTagsForSlugsForTest = itemTagsForSlugs; +export function _setFetchForTest(fn) { + _test.fetchImpl = fn; +} +export function _resetForTest() { + _test.fetchImpl = null; + _cfgState.cached = null; + _cfgState.cachedAt = 0; +} diff --git a/srv/lib/feature-flags/registry.js b/srv/lib/feature-flags/registry.js index 3982310d2..bf6856932 100644 --- a/srv/lib/feature-flags/registry.js +++ b/srv/lib/feature-flags/registry.js @@ -363,4 +363,12 @@ export const FEATURE_FLAGS = [ description: 'Enables the /explore/ktt/ acronym trainer and its /ktt CAP endpoints. Off → completeLesson/syncProgress reject 503.', howToChange: featureFlagUpsert('KTT_ENABLED', 'flag.ktt.enabled'), }, + // ---- Edge cache ---- + { + key: 'EDGE_PURGE_ENABLED', label: 'Akamai edge Fast-Purge on publish', category: 'Security', + kind: 'db', imsConfigKey: 'flag.edgepurge', + valueType: 'boolean', default: false, status: 'dev-only', + description: 'When true, a successful content publish/rollback fires a fire-and-forget Akamai Fast-Purge (CCU v3) purge-by-tag for the changed slugs, AND the served content Cache-Control s-maxage is raised from 600s to 86400s (safe only because the purge now bounds staleness). Requires the AKAMAI_FASTPURGE_EDGERC JSON credential in Credential Store; inert (no-op, short TTL) without it. Fail-open. Numeric tunables: ImsConfig edgepurge.network (production|staging), edgepurge.timeoutMs. DB-driven config (ImsConfig key flag.edgepurge); no env var. Default OFF.', + howToChange: featureFlagUpsert('EDGE_PURGE_ENABLED', 'flag.edgepurge'), + }, ]; diff --git a/test/unit/fast-purge.test.js b/test/unit/fast-purge.test.js new file mode 100644 index 000000000..1b9e42266 --- /dev/null +++ b/test/unit/fast-purge.test.js @@ -0,0 +1,176 @@ +// test/unit/fast-purge.test.js +// Unit coverage for the Akamai Fast-Purge hook (PR2): EdgeGrid EG1-HMAC-SHA256 +// request signing, the flag/credential gating, the purge-by-tag HTTP call +// (mocked), fail-open behavior, and the slug→item-tag mapping. No network and +// no cds runtime are required — fetch and the credential are injected. + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + +import { + purgeTags, + purgePublishedSlugs, + _makeAuthHeaderForTest, + _edgeGridTimestampForTest, + _itemTagsForSlugsForTest, + _setFetchForTest, + _resetForTest, +} from '../../srv/lib/fast-purge.js'; +import { + __setFlagForTest, + __resetFlagsForTest, +} from '../../srv/lib/feature-flags/db-flags.js'; +import { + _primeForTests, + _resetForTests as _resetSecrets, +} from '../../srv/lib/secret-resolver.js'; + +const CREDS_JSON = JSON.stringify({ + host: 'akab-example.purge.akamaiapis.net', + client_token: 'akab-client-token', + client_secret: 'client-secret-value', + access_token: 'akab-access-token', +}); + +describe('fast-purge: EdgeGrid signing', () => { + it('formats the timestamp as yyyyMMddTHH:mm:ss+0000 in UTC', () => { + const ts = _edgeGridTimestampForTest(new Date(Date.UTC(2026, 8, 12, 7, 5, 3))); + expect(ts).toBe('20260912T07:05:03+0000'); + }); + + const creds = { clientToken: 'ct', accessToken: 'at', clientSecret: 'cs' }; + const ts = '20260912T07:05:03+0000'; + const nonce = 'fixed-nonce'; + + it('builds a well-formed EG1-HMAC-SHA256 Authorization header', () => { + const h = _makeAuthHeaderForTest( + creds, + { method: 'POST', host: 'h.example.net', path: '/ccu/v3/invalidate/tag/production', body: '{"objects":["x"]}' }, + ts, + nonce + ); + expect(h).toMatch(/^EG1-HMAC-SHA256 /); + expect(h).toContain('client_token=ct;'); + expect(h).toContain('access_token=at;'); + expect(h).toContain(`timestamp=${ts};`); + expect(h).toContain(`nonce=${nonce};`); + expect(h).toMatch(/signature=[A-Za-z0-9+/=]+$/); + }); + + it('is deterministic for fixed timestamp + nonce', () => { + const args = [creds, { method: 'POST', host: 'h.example.net', path: '/p', body: 'b' }, ts, nonce]; + expect(_makeAuthHeaderForTest(...args)).toBe(_makeAuthHeaderForTest(...args)); + }); + + it('signature changes when the body changes (content hash is signed)', () => { + const base = { method: 'POST', host: 'h.example.net', path: '/p' }; + const a = _makeAuthHeaderForTest(creds, { ...base, body: '{"objects":["a"]}' }, ts, nonce); + const b = _makeAuthHeaderForTest(creds, { ...base, body: '{"objects":["b"]}' }, ts, nonce); + expect(a).not.toBe(b); + }); + + it('signature changes when the path changes', () => { + const a = _makeAuthHeaderForTest(creds, { method: 'POST', host: 'h', path: '/one', body: 'x' }, ts, nonce); + const b = _makeAuthHeaderForTest(creds, { method: 'POST', host: 'h', path: '/two', body: 'x' }, ts, nonce); + expect(a).not.toBe(b); + }); +}); + +describe('fast-purge: slug → item-tag mapping', () => { + it('keeps per-item tags and drops the coarse corpus/category tags', () => { + const tags = _itemTagsForSlugsForTest(['abap-basics']); + expect(tags).toContain('item-abap-basics'); + expect(tags).not.toContain('content'); + }); + + it('drops the coarse `group` tag but keeps the group item tag', () => { + const tags = _itemTagsForSlugsForTest(['group-getting-started']); + expect(tags).toContain('item-group-getting-started'); + expect(tags).not.toContain('group'); + expect(tags).not.toContain('content'); + }); + + it('dedupes across multiple slugs', () => { + const tags = _itemTagsForSlugsForTest(['abap-basics', 'abap-basics', 'cap-intro']); + expect(tags.filter((t) => t === 'item-abap-basics')).toHaveLength(1); + expect(tags).toContain('item-cap-intro'); + }); +}); + +describe('fast-purge: purgeTags gating + HTTP', () => { + beforeEach(() => { + _resetForTest(); + _resetSecrets(); + __resetFlagsForTest(); + }); + afterEach(() => { + _resetForTest(); + _resetSecrets(); + __resetFlagsForTest(); + }); + + it('is a no-op when EDGE_PURGE_ENABLED is off — no fetch', async () => { + let called = false; + _setFetchForTest(() => { called = true; }); + const r = await purgeTags(['item-x']); + expect(r).toEqual({ ok: true, skipped: 'flag-off' }); + expect(called).toBe(false); + }); + + it('skips when the flag is on but no credential is configured', async () => { + __setFlagForTest('EDGE_PURGE_ENABLED', true); + _primeForTests('AKAMAI_FASTPURGE_EDGERC', null); // explicitly absent + delete process.env.AKAMAI_FASTPURGE_EDGERC; + let called = false; + _setFetchForTest(() => { called = true; }); + const r = await purgeTags(['item-x']); + expect(r.ok).toBe(false); + expect(r.skipped).toBe('no-credential'); + expect(called).toBe(false); + }); + + it('POSTs the objects to the CCU v3 tag endpoint with an Authorization header', async () => { + __setFlagForTest('EDGE_PURGE_ENABLED', true); + _primeForTests('AKAMAI_FASTPURGE_EDGERC', CREDS_JSON); + const seen = {}; + _setFetchForTest(async (url, init) => { + seen.url = url; + seen.init = init; + return { ok: true, status: 201, text: async () => '' }; + }); + const r = await purgeTags(['item-a', 'item-b']); + expect(r.ok).toBe(true); + expect(r.purged).toBe(2); + expect(seen.url).toBe('https://akab-example.purge.akamaiapis.net/ccu/v3/invalidate/tag/production'); + expect(seen.init.method).toBe('POST'); + expect(seen.init.headers.Authorization).toMatch(/^EG1-HMAC-SHA256 /); + expect(JSON.parse(seen.init.body)).toEqual({ objects: ['item-a', 'item-b'] }); + }); + + it('fails open (never throws) when fetch rejects', async () => { + __setFlagForTest('EDGE_PURGE_ENABLED', true); + _primeForTests('AKAMAI_FASTPURGE_EDGERC', CREDS_JSON); + _setFetchForTest(async () => { throw new Error('network down'); }); + const r = await purgeTags(['item-a']); + expect(r.ok).toBe(false); + expect(r.skipped).toBe('error'); + }); + + it('reports the status on an Akamai error response', async () => { + __setFlagForTest('EDGE_PURGE_ENABLED', true); + _primeForTests('AKAMAI_FASTPURGE_EDGERC', CREDS_JSON); + _setFetchForTest(async () => ({ ok: false, status: 403, text: async () => 'forbidden' })); + const r = await purgeTags(['item-a']); + expect(r.ok).toBe(false); + expect(r.status).toBe(403); + }); + + it('purgePublishedSlugs with no slugs never calls fetch', async () => { + __setFlagForTest('EDGE_PURGE_ENABLED', true); + _primeForTests('AKAMAI_FASTPURGE_EDGERC', CREDS_JSON); + let called = false; + _setFetchForTest(async () => { called = true; return { ok: true, status: 201, text: async () => '' }; }); + purgePublishedSlugs([]); + await new Promise((r) => setTimeout(r, 10)); + expect(called).toBe(false); + }); +});