From 6260d06d3604f67d2c634cbbea105f97ffacaa75 Mon Sep 17 00:00:00 2001 From: Dave Rolsky Date: Wed, 2 Sep 2026 15:31:47 -0500 Subject: [PATCH] feat(mongodb-runner): default slsImageTag to pinned_sls_commit from the build manifest --- .prettierignore | 1 + packages/mongodb-runner/.prettierignore | 1 + .../docs/disaggregated-storage.md | 24 +++++----- packages/mongodb-runner/src/cli.ts | 5 +- packages/mongodb-runner/src/index.ts | 1 + packages/mongodb-runner/src/sls.spec.ts | 47 ++++++++++++++++++ packages/mongodb-runner/src/sls.ts | 48 +++++++++++++++++-- .../test/fixtures/sls/complete/manifest.json | 4 ++ .../test/fixtures/sls/malformed/manifest.json | 1 + .../test/fixtures/sls/no-key/manifest.json | 3 ++ 10 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 packages/mongodb-runner/src/sls.spec.ts create mode 100644 packages/mongodb-runner/test/fixtures/sls/complete/manifest.json create mode 100644 packages/mongodb-runner/test/fixtures/sls/malformed/manifest.json create mode 100644 packages/mongodb-runner/test/fixtures/sls/no-key/manifest.json diff --git a/.prettierignore b/.prettierignore index fe7b3b3b6..707d7e2c7 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,3 @@ resources/cidrs.json packages/mongodb-schema/examples/fanclub.json +packages/mongodb-runner/test/fixtures/sls/malformed/manifest.json diff --git a/packages/mongodb-runner/.prettierignore b/packages/mongodb-runner/.prettierignore index 4d28df660..7ce446d29 100644 --- a/packages/mongodb-runner/.prettierignore +++ b/packages/mongodb-runner/.prettierignore @@ -1,3 +1,4 @@ .nyc_output dist coverage +test/fixtures/sls/malformed diff --git a/packages/mongodb-runner/docs/disaggregated-storage.md b/packages/mongodb-runner/docs/disaggregated-storage.md index 31e7c4599..d1da307f1 100644 --- a/packages/mongodb-runner/docs/disaggregated-storage.md +++ b/packages/mongodb-runner/docs/disaggregated-storage.md @@ -70,24 +70,23 @@ override. server checkout. Files it references (`slsbackup.proto`, `flags-state.json`) are resolved relative to it, and the services/ports are parsed from it, so any version of the file works as-is. -- The SLS image tag to use, typically the `pinned_sls_commit` from - `buildscripts/modules/atlas/manifest.json` in the mongodb server repository. +- The image tag is read automatically from the `pinned_sls_commit` in + `manifest.json` sitting next to the compose file (the server repository's + `buildscripts/modules/atlas/manifest.json`). Pass `--slsImageTag` to + override it. ## Quick start -Given the compose file and image tag, everything else (compose environment +Given the compose file, everything else (image tag, compose environment variables, readiness polling, per-shard log creation, and the `disaggregatedStorageConfig` server parameter) is generated automatically. Full sequence, assuming a mongodb server checkout at `$MONGO_REPO`: ```bash -# 1. Look up the pinned SLS image tag -SLS_IMAGE_TAG=$(python3 -c "import json; print(json.load(open('$MONGO_REPO/buildscripts/modules/atlas/manifest.json'))['pinned_sls_commit'])") - -# 2. Start a 2-node replica set backed by SLS (logs in to ECR automatically) +# The image tag is read from the manifest.json next to the compose file; pass +# --slsImageTag to override it. Logs in to ECR automatically. @mongodb-js/mongodb-runner start -t replset \ --slsCompose=$MONGO_REPO/buildscripts/modules/atlas/sls-multicell-docker-compose.yml \ - --slsImageTag=$SLS_IMAGE_TAG \ --binDir=/path/to/dsc-mongod/bin \ --debug # or, instead of --binDir: @@ -227,7 +226,8 @@ from the server codebase and returns: | `ports` | The allocated host port per service, e.g. `ports['crs-cell1-0']` | | `services` | Host `addr`/`uri` per service, e.g. `services['cms-cell1-0'].uri` | -Required options: `composeFile`, `imageTag`. Optional: `imageRepo`, +Required options: `composeFile`. Optional: `imageTag` (defaults to the +`pinned_sls_commit` from the manifest next to the compose file), `imageRepo`, `thirdPartyImageRepo`, `testDataId` (container label for test attribution), `hostInternalIP`. The service list is parsed from the compose file's `ports:` mappings (`parseSLSComposeServices`), so it adapts to whatever @@ -241,13 +241,13 @@ server parameter for one shard; options: `logId`, `cellMetadataService`, ## CLI use -For an SLS project, `--slsCompose` + `--slsImageTag` handle everything (see -Quick start): +For an SLS project, `--slsCompose` handles everything (see Quick start); +`--slsImageTag` is optional and overrides the tag read from the manifest: ```bash @mongodb-js/mongodb-runner start -t replset \ --slsCompose=/path/to/sls-multicell-docker-compose.yml \ - --slsImageTag= --binDir=... + --binDir=... ``` Custom (non-SLS) storage backends are only supported through the programmatic diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index a11d9dc80..ce9f73738 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -87,12 +87,11 @@ import type { MongoClientOptions } from 'mongodb'; type: 'string', describe: 'Path to an SLS multi-cell docker-compose.yml; launches the SLS DSC project and configures mongod to use it (requires a DSC-capable mongod via --binDir or --downloadUrl)', - implies: 'slsImageTag', }) .option('slsImageTag', { type: 'string', describe: - 'SLS docker image tag to use with --slsCompose (e.g. the pinned_sls_commit from the server repo manifest)', + 'SLS docker image tag to use with --slsCompose (defaults to the pinned_sls_commit from the manifest.json next to the compose file)', }) .option('slsSkipEcrLogin', { type: 'boolean', @@ -139,7 +138,7 @@ import type { MongoClientOptions } from 'mongodb'; const disaggregatedStorage = argv.slsCompose ? await utilities.createSLSDisaggregatedStorageOptions({ composeFile: argv.slsCompose, - imageTag: argv.slsImageTag!, + imageTag: argv.slsImageTag, ecrLogin: !argv.slsSkipEcrLogin, }) : undefined; diff --git a/packages/mongodb-runner/src/index.ts b/packages/mongodb-runner/src/index.ts index 8c17a5b7e..3b06664b7 100644 --- a/packages/mongodb-runner/src/index.ts +++ b/packages/mongodb-runner/src/index.ts @@ -22,6 +22,7 @@ export { type SLSDisaggregatedStorageConfigOptions, type SLSDisaggregatedStorageSetupOptions, parseSLSComposeServices, + readPinnedSlsCommit, SLS_HOSTNAME, SLS_CELL1, SLS_CELL2, diff --git a/packages/mongodb-runner/src/sls.spec.ts b/packages/mongodb-runner/src/sls.spec.ts new file mode 100644 index 000000000..540b0aa0a --- /dev/null +++ b/packages/mongodb-runner/src/sls.spec.ts @@ -0,0 +1,47 @@ +import { expect } from 'chai'; +import path from 'path'; +import { readPinnedSlsCommit } from './sls'; + +const FIXTURES = path.resolve(__dirname, '..', 'test', 'fixtures', 'sls'); + +describe('readPinnedSlsCommit', function () { + it('reads pinned_sls_commit from a manifest', async function () { + expect( + await readPinnedSlsCommit(path.join(FIXTURES, 'complete')), + 'should return the pinned commit verbatim', + ).to.equal('abc123def456'); + }); + + it('names the path it looked at when the manifest is absent', async function () { + const missing = path.join(FIXTURES, 'does-not-exist'); + const err = await readPinnedSlsCommit(missing).catch((e: Error) => e); + expect( + (err as Error).message, + 'error should name the manifest path that was checked', + ).to.include(path.join(missing, 'manifest.json')); + expect( + (err as Error).message, + 'error should mention the override flag', + ).to.include('--slsImageTag'); + }); + + it('reports a manifest that is missing the key', async function () { + const err = await readPinnedSlsCommit(path.join(FIXTURES, 'no-key')).catch( + (e: Error) => e, + ); + expect( + (err as Error).message, + 'error should name the missing key', + ).to.include('pinned_sls_commit'); + }); + + it('reports a malformed manifest', async function () { + const err = await readPinnedSlsCommit( + path.join(FIXTURES, 'malformed'), + ).catch((e: Error) => e); + expect( + (err as Error).message, + 'error should say the manifest could not be parsed', + ).to.match(/parse/i); + }); +}); diff --git a/packages/mongodb-runner/src/sls.ts b/packages/mongodb-runner/src/sls.ts index 1a9b9e889..7b1b7fe79 100644 --- a/packages/mongodb-runner/src/sls.ts +++ b/packages/mongodb-runner/src/sls.ts @@ -26,6 +26,44 @@ export const SLS_HOSTNAME = 'local.sls.mmscloudteam.com'; const DEFAULT_SLS_IMAGE_REPO = '664315256653.dkr.ecr.us-east-1.amazonaws.com/disagg-storage/'; +/** Name of the build manifest inside a Server build's atlas module directory. */ +export const SLS_MANIFEST_FILE = 'manifest.json'; + +/** + * Read `pinned_sls_commit` from a Server build's + * `buildscripts/modules/atlas/manifest.json`. This is the SLS image tag + * matching the commit the binaries were built from. + */ +export async function readPinnedSlsCommit(atlasDir: string): Promise { + const manifestPath = path.join(atlasDir, SLS_MANIFEST_FILE); + let contents: string; + try { + contents = await fs.readFile(manifestPath, 'utf8'); + } catch (err) { + throw new Error( + `Could not read the SLS build manifest at ${manifestPath}: ` + + `${(err as Error).message}. Pass --slsImageTag explicitly to skip this lookup.`, + ); + } + let manifest: unknown; + try { + manifest = JSON.parse(contents); + } catch (err) { + throw new Error( + `Could not parse the SLS build manifest at ${manifestPath}: ` + + `${(err as Error).message}. Pass --slsImageTag explicitly to skip this lookup.`, + ); + } + const pinned = (manifest as Record)?.pinned_sls_commit; + if (typeof pinned !== 'string' || !pinned) { + throw new Error( + `The SLS build manifest at ${manifestPath} has no pinned_sls_commit key. ` + + `Pass --slsImageTag explicitly to skip this lookup.`, + ); + } + return pinned; +} + export interface SLSServiceInfo { /** Environment variable through which the compose file receives the host port. */ portVar: string; @@ -96,10 +134,10 @@ export interface SLSMultiCellEnvironmentOptions { */ composeFile: string; /** - * Image tag for the SLS images (typically the `pinned_sls_commit` from the - * server repository's buildscripts/modules/atlas/manifest.json). + * Image tag for the SLS images. Defaults to the `pinned_sls_commit` from + * the `manifest.json` sitting next to the compose file. */ - imageTag: string; + imageTag?: string; /** Docker image repository for SLS images. */ imageRepo?: string; /** Docker image repository for third-party images (default: imageRepo with 'disagg-storage' replaced by 'thirdparty'). */ @@ -130,7 +168,9 @@ export interface SLSMultiCellEnvironment { export async function createSLSMultiCellEnvironment( options: SLSMultiCellEnvironmentOptions, ): Promise { - const { composeFile, imageTag } = options; + const { composeFile } = options; + const imageTag = + options.imageTag ?? (await readPinnedSlsCommit(path.dirname(composeFile))); const serviceInfo = parseSLSComposeServices( await fs.readFile(composeFile, 'utf8'), diff --git a/packages/mongodb-runner/test/fixtures/sls/complete/manifest.json b/packages/mongodb-runner/test/fixtures/sls/complete/manifest.json new file mode 100644 index 000000000..bfc2ac600 --- /dev/null +++ b/packages/mongodb-runner/test/fixtures/sls/complete/manifest.json @@ -0,0 +1,4 @@ +{ + "pinned_sls_commit": "abc123def456", + "other_key": "ignored" +} diff --git a/packages/mongodb-runner/test/fixtures/sls/malformed/manifest.json b/packages/mongodb-runner/test/fixtures/sls/malformed/manifest.json new file mode 100644 index 000000000..dcf314808 --- /dev/null +++ b/packages/mongodb-runner/test/fixtures/sls/malformed/manifest.json @@ -0,0 +1 @@ +{ this is not valid json diff --git a/packages/mongodb-runner/test/fixtures/sls/no-key/manifest.json b/packages/mongodb-runner/test/fixtures/sls/no-key/manifest.json new file mode 100644 index 000000000..9aa8eadbf --- /dev/null +++ b/packages/mongodb-runner/test/fixtures/sls/no-key/manifest.json @@ -0,0 +1,3 @@ +{ + "other_key": "present but not the one we need" +}