From 4561526103541cb1453dc9fb24892bb207d52ee8 Mon Sep 17 00:00:00 2001 From: notgitika Date: Wed, 25 Mar 2026 04:14:22 -0400 Subject: [PATCH 1/5] feat: add CDK synthesis validation tests (19 cases) Synth realistic agentcore.json specs through AgentCoreStack and assert valid CloudFormation output. Covers agents, memories, credentials, evaluators, online eval configs, policy engines, and edge cases. --- .../__tests__/cdk-synth-validation.test.ts | 480 ++++++++++++++++++ vitest.config.ts | 1 + 2 files changed, 481 insertions(+) create mode 100644 src/assets/__tests__/cdk-synth-validation.test.ts diff --git a/src/assets/__tests__/cdk-synth-validation.test.ts b/src/assets/__tests__/cdk-synth-validation.test.ts new file mode 100644 index 000000000..82a93b79e --- /dev/null +++ b/src/assets/__tests__/cdk-synth-validation.test.ts @@ -0,0 +1,480 @@ +/** + * CDK Synthesis Validation Tests + * + * Validates that realistic agentcore.json configurations can be synthesized + * into valid CloudFormation templates by the vended CDK stack. + * + * These tests catch schema mismatches and construct bugs before deploy time. + */ +import { AgentCoreStack } from '../cdk/lib/cdk-stack'; +import { setSessionProjectRoot } from '@aws/agentcore-cdk'; +import type { AgentCoreProjectSpec } from '@aws/agentcore-cdk'; +import * as cdk from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +// ─── CFN Resource Types ────────────────────────────────────────────────────── + +const CFN_RUNTIME = 'AWS::BedrockAgentCore::Runtime'; +const CFN_MEMORY = 'AWS::BedrockAgentCore::Memory'; +const CFN_EVALUATOR = 'AWS::BedrockAgentCore::Evaluator'; +const CFN_POLICY_ENGINE = 'AWS::BedrockAgentCore::PolicyEngine'; +const CFN_POLICY = 'AWS::BedrockAgentCore::Policy'; +const CFN_ECR_REPO = 'AWS::ECR::Repository'; +const CFN_CODEBUILD = 'AWS::CodeBuild::Project'; +const CFN_IAM_ROLE = 'AWS::IAM::Role'; + +// ─── Test project directory ────────────────────────────────────────────────── +// AgentCoreApplication calls findConfigRoot() which walks up from cwd looking +// for agentcore/agentcore.json. We use setSessionProjectRoot() to point it at +// our temp directory. + +let tmpDir: string; +let originalCwd: string; + +beforeAll(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'agentcore-cdk-synth-test-')); + const agentcoreDir = join(tmpDir, 'agentcore'); + mkdirSync(agentcoreDir, { recursive: true }); + // Create minimal agentcore.json so findConfigRoot() succeeds + writeFileSync(join(agentcoreDir, 'agentcore.json'), '{}'); + // Create agent code directories that constructs may reference + const agentNames = [ + 'myagent', + 'agent1', + 'agent2', + 'primaryagent', + 'secondaryagent', + 'containeragent', + 'mcpagent', + 'a2aagent', + 'a'.repeat(48), + ]; + const minimalPyproject = '[project]\nname = "agent"\nversion = "0.1.0"\n'; + for (const dir of agentNames) { + mkdirSync(join(tmpDir, 'agents', dir), { recursive: true }); + writeFileSync(join(tmpDir, 'agents', dir, 'main.py'), '# placeholder'); + writeFileSync(join(tmpDir, 'agents', dir, 'pyproject.toml'), minimalPyproject); + writeFileSync(join(tmpDir, 'agents', dir, 'Dockerfile'), 'FROM python:3.12-slim\n'); + } + // Tell the CDK L3 construct where the project root is so findConfigRoot() succeeds + setSessionProjectRoot(tmpDir); + originalCwd = process.cwd(); + process.chdir(tmpDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + rmSync(tmpDir, { recursive: true, force: true }); +}); + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function synthStack( + spec: AgentCoreProjectSpec, + mcpSpec?: unknown, + credentials?: Record +): Template { + const app = new cdk.App(); + const stack = new AgentCoreStack(app, `TestStack${Date.now()}`, { + spec, + mcpSpec: mcpSpec as never, + credentials, + env: { account: '123456789012', region: 'us-east-1' }, + }); + return Template.fromStack(stack); +} + +function baseSpec(overrides: Partial = {}): AgentCoreProjectSpec { + return { + name: 'testproject', + version: 1, + agents: [], + memories: [], + credentials: [], + evaluators: [], + onlineEvalConfigs: [], + policyEngines: [], + ...overrides, + } as AgentCoreProjectSpec; +} + +function makeAgent(name: string, overrides: Record = {}) { + return { + type: 'AgentEnvironment', + name, + build: 'CodeZip', + entrypoint: 'main.py', + codeLocation: `agents/${name}`, + runtimeVersion: 'PYTHON_3_12', + ...overrides, + }; +} + +function makeMemory(name: string, strategies: unknown[] = []) { + return { + type: 'AgentCoreMemory', + name, + eventExpiryDuration: 30, + strategies, + }; +} + +function makeEvaluator(name: string) { + return { + type: 'CustomEvaluator', + name, + level: 'SESSION', + config: { + type: 'LlmAsAJudge', + llmAsAJudge: { + model: 'anthropic.claude-3-haiku-20240307-v1:0', + instructions: 'Rate the response quality based on helpfulness and accuracy.', + ratingScale: { + numerical: [ + { value: 1, label: 'Poor', definition: 'Unhelpful or incorrect' }, + { value: 3, label: 'Good', definition: 'Mostly helpful and accurate' }, + { value: 5, label: 'Excellent', definition: 'Very helpful and fully accurate' }, + ], + }, + }, + }, + }; +} + +function makeOnlineEvalConfig(name: string, agent: string, evaluators: string[]) { + return { + type: 'OnlineEvaluationConfig', + name, + agent, + evaluators, + samplingRate: 50, + }; +} + +function makeCredential( + name: string, + type: 'ApiKeyCredentialProvider' | 'OAuthCredentialProvider' = 'ApiKeyCredentialProvider' +) { + if (type === 'OAuthCredentialProvider') { + return { + type, + name, + discoveryUrl: 'https://example.com/.well-known/openid-configuration', + scopes: ['openid'], + }; + } + return { type, name }; +} + +function makePolicyEngine(name: string) { + return { + type: 'PolicyEngine', + name, + policies: [ + { + type: 'Policy', + name: `${name}Policy`, + statement: 'permit(principal, action, resource);', + }, + ], + }; +} + +// ─── Tests ──────────────────────────────────────────────────────────────────── + +describe('CDK Synthesis Validation', () => { + // ─── Empty and minimal specs ────────────────────────────────────────────── + + it('synthesizes empty spec with only StackNameOutput', () => { + const template = synthStack(baseSpec()); + template.hasOutput('StackNameOutput', { + Description: 'Name of the CloudFormation Stack', + }); + }); + + // ─── Agent specs ────────────────────────────────────────────────────────── + + it('synthesizes a single CodeZip agent', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + }) + ); + + // Should create an AgentCore Runtime resource + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('myagent'), + }); + + // Should create an IAM role for the agent + template.hasResourceProperties(CFN_IAM_ROLE, { + AssumeRolePolicyDocument: Match.objectLike({ + Statement: Match.arrayWith([ + Match.objectLike({ + Effect: 'Allow', + Principal: Match.objectLike({ + Service: Match.anyValue(), + }), + }), + ]), + }), + }); + }); + + it('synthesizes a Container agent with ECR and CodeBuild', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('containeragent', { build: 'Container' })] as never, + }) + ); + + // Should create an ECR repository + template.hasResourceProperties(CFN_ECR_REPO, Match.anyValue()); + + // Should create a CodeBuild project for building the container + template.hasResourceProperties(CFN_CODEBUILD, Match.anyValue()); + }); + + it('synthesizes multiple agents', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('agent1'), makeAgent('agent2')] as never, + }) + ); + + // Should create 2 runtimes + template.resourceCountIs(CFN_RUNTIME, 2); + }); + + // ─── Memory specs ───────────────────────────────────────────────────────── + + it('synthesizes agent with short-term memory (no strategies)', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + memories: [makeMemory('ShortTermMem')] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + template.hasResourceProperties(CFN_MEMORY, { + Name: Match.stringLikeRegexp('ShortTermMem'), + }); + }); + + it('synthesizes agent with long-term memory strategies', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + memories: [ + makeMemory('LongTermMem', [{ type: 'SEMANTIC' }, { type: 'SUMMARIZATION' }, { type: 'USER_PREFERENCE' }]), + ] as never, + }) + ); + + template.hasResourceProperties(CFN_MEMORY, { + Name: Match.stringLikeRegexp('LongTermMem'), + }); + }); + + // ─── Credential specs ───────────────────────────────────────────────────── + + it('synthesizes agent with API key credential', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + credentials: [makeCredential('MyApiKey')] as never, + }) + ); + + // Agent runtime should exist — credential wiring happens at deploy time + template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + }); + + it('synthesizes agent with OAuth credential', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + credentials: [makeCredential('MyOAuth', 'OAuthCredentialProvider')] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + }); + + // ─── Evaluator specs ────────────────────────────────────────────────────── + + it('synthesizes custom evaluator', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + evaluators: [makeEvaluator('QualityCheck')] as never, + }) + ); + + template.hasResourceProperties(CFN_EVALUATOR, { + EvaluatorName: Match.stringLikeRegexp('QualityCheck'), + }); + }); + + // ─── Online eval config specs ───────────────────────────────────────────── + + it('synthesizes online eval config referencing project agent', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + evaluators: [makeEvaluator('QualityCheck')] as never, + onlineEvalConfigs: [makeOnlineEvalConfig('MonitorQuality', 'myagent', ['QualityCheck'])] as never, + }) + ); + + template.hasResourceProperties(CFN_EVALUATOR, Match.anyValue()); + }); + + it('synthesizes online eval config with builtin evaluator', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + onlineEvalConfigs: [makeOnlineEvalConfig('BuiltinMonitor', 'myagent', ['Builtin.GoalSuccessRate'])] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + }); + + // ─── Policy engine specs ────────────────────────────────────────────────── + + it('synthesizes policy engine', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + policyEngines: [makePolicyEngine('SafetyGuard')] as never, + }) + ); + + template.hasResourceProperties(CFN_POLICY_ENGINE, Match.anyValue()); + template.hasResourceProperties(CFN_POLICY, { + Definition: Match.objectLike({ + Cedar: Match.objectLike({ + Statement: Match.anyValue(), + }), + }), + }); + }); + + // ─── Full project specs ─────────────────────────────────────────────────── + + it('synthesizes a complete project with all resource types', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('primaryagent'), makeAgent('secondaryagent')] as never, + memories: [makeMemory('ProjectMemory', [{ type: 'SEMANTIC' }])] as never, + credentials: [ + makeCredential('ProdApiKey'), + makeCredential('OAuthProvider', 'OAuthCredentialProvider'), + ] as never, + evaluators: [makeEvaluator('ResponseQuality')] as never, + onlineEvalConfigs: [ + makeOnlineEvalConfig('LiveMonitor', 'primaryagent', ['Builtin.GoalSuccessRate', 'ResponseQuality']), + ] as never, + policyEngines: [makePolicyEngine('ContentFilter')] as never, + }) + ); + + // Verify resource counts + template.resourceCountIs(CFN_RUNTIME, 2); + template.hasResourceProperties(CFN_MEMORY, Match.anyValue()); + template.hasResourceProperties(CFN_EVALUATOR, Match.anyValue()); + template.hasResourceProperties(CFN_POLICY_ENGINE, Match.anyValue()); + }); + + // ─── Agent configuration variants ───────────────────────────────────────── + + it('synthesizes agent with custom environment variables', () => { + const template = synthStack( + baseSpec({ + agents: [ + makeAgent('myagent', { + envVars: [ + { name: 'MODEL_ID', value: 'anthropic.claude-3-haiku-20240307-v1:0' }, + { name: 'TEMPERATURE', value: '0.7' }, + ], + }), + ] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('myagent'), + }); + }); + + it('synthesizes agent with MCP protocol', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('mcpagent', { protocol: 'MCP' })] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('mcpagent'), + }); + }); + + it('synthesizes agent with A2A protocol', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('a2aagent', { protocol: 'A2A' })] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('a2aagent'), + }); + }); + + // ─── Edge cases ─────────────────────────────────────────────────────────── + + it('synthesizes with memories but no agents', () => { + // Valid scenario: user may add memory before adding agents + const template = synthStack( + baseSpec({ + memories: [makeMemory('StandaloneMemory')] as never, + }) + ); + + template.hasResourceProperties(CFN_MEMORY, Match.anyValue()); + template.resourceCountIs(CFN_RUNTIME, 0); + }); + + it('synthesizes with evaluators but no online eval configs', () => { + const template = synthStack( + baseSpec({ + agents: [makeAgent('myagent')] as never, + evaluators: [makeEvaluator('UnusedEval')] as never, + }) + ); + + template.hasResourceProperties(CFN_EVALUATOR, Match.anyValue()); + }); + + it('synthesizes spec with maximum name lengths', () => { + // Agent name max is 48 chars, memory name max is 48 chars + const longAgentName = 'a'.repeat(48); + const longMemoryName = 'M'.repeat(48); + + const template = synthStack( + baseSpec({ + agents: [makeAgent(longAgentName)] as never, + memories: [makeMemory(longMemoryName)] as never, + }) + ); + + template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + template.hasResourceProperties(CFN_MEMORY, Match.anyValue()); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index fec90f1ce..7cbf21e3d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -29,6 +29,7 @@ export default defineConfig({ resolve: { alias: { '@': path.resolve(__dirname, './src'), + '@aws/agentcore-cdk': path.resolve(__dirname, './node_modules/@aws/agentcore-cdk/dist/index.js'), }, }, plugins: [textLoaderPlugin], From b529bf788af87865432cb49466686257248eb44e Mon Sep 17 00:00:00 2001 From: notgitika Date: Wed, 25 Mar 2026 17:34:49 -0400 Subject: [PATCH 2/5] fix: exclude CDK synth tests from CI unit suite @aws/agentcore-cdk is npm-linked locally, not available in CI. Exclude the test from the unit project, matching the existing exclusion pattern for src/assets/cdk/test/*.test.ts. --- vitest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vitest.config.ts b/vitest.config.ts index 7cbf21e3d..963f60fc1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -40,7 +40,7 @@ export default defineConfig({ test: { name: 'unit', include: ['src/**/*.test.ts', 'src/**/*.test.tsx'], - exclude: ['src/assets/cdk/test/*.test.ts'], + exclude: ['src/assets/cdk/test/*.test.ts', 'src/assets/__tests__/cdk-synth-validation.test.ts'], }, }, { From 57382e0317f890b4a275cbfaf30d16662cfe170f Mon Sep 17 00:00:00 2001 From: notgitika Date: Wed, 25 Mar 2026 17:36:31 -0400 Subject: [PATCH 3/5] fix: remove unused expect import --- src/assets/__tests__/cdk-synth-validation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assets/__tests__/cdk-synth-validation.test.ts b/src/assets/__tests__/cdk-synth-validation.test.ts index 82a93b79e..58c410b39 100644 --- a/src/assets/__tests__/cdk-synth-validation.test.ts +++ b/src/assets/__tests__/cdk-synth-validation.test.ts @@ -14,7 +14,7 @@ import { Match, Template } from 'aws-cdk-lib/assertions'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, it } from 'vitest'; // ─── CFN Resource Types ────────────────────────────────────────────────────── From 23fa32dbe5db228d4b43bb3a8bd59934e9fa3959 Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 10 Apr 2026 23:35:08 -0400 Subject: [PATCH 4/5] fix: use correct schema types and remove 'as never' casts in CDK synth tests --- .../__tests__/cdk-synth-validation.test.ts | 203 ++++++++++-------- 1 file changed, 112 insertions(+), 91 deletions(-) diff --git a/src/assets/__tests__/cdk-synth-validation.test.ts b/src/assets/__tests__/cdk-synth-validation.test.ts index 58c410b39..090fae938 100644 --- a/src/assets/__tests__/cdk-synth-validation.test.ts +++ b/src/assets/__tests__/cdk-synth-validation.test.ts @@ -30,10 +30,9 @@ const CFN_IAM_ROLE = 'AWS::IAM::Role'; // ─── Test project directory ────────────────────────────────────────────────── // AgentCoreApplication calls findConfigRoot() which walks up from cwd looking // for agentcore/agentcore.json. We use setSessionProjectRoot() to point it at -// our temp directory. +// our temp directory instead of mutating process.cwd(). let tmpDir: string; -let originalCwd: string; beforeAll(() => { tmpDir = mkdtempSync(join(tmpdir(), 'agentcore-cdk-synth-test-')); @@ -60,14 +59,11 @@ beforeAll(() => { writeFileSync(join(tmpDir, 'agents', dir, 'pyproject.toml'), minimalPyproject); writeFileSync(join(tmpDir, 'agents', dir, 'Dockerfile'), 'FROM python:3.12-slim\n'); } - // Tell the CDK L3 construct where the project root is so findConfigRoot() succeeds + // Tell the CDK L3 construct where the project root is setSessionProjectRoot(tmpDir); - originalCwd = process.cwd(); - process.chdir(tmpDir); }); afterAll(() => { - process.chdir(originalCwd); rmSync(tmpDir, { recursive: true, force: true }); }); @@ -75,13 +71,13 @@ afterAll(() => { function synthStack( spec: AgentCoreProjectSpec, - mcpSpec?: unknown, + mcpSpec?: AgentCoreProjectSpec['agentCoreGateways'], credentials?: Record ): Template { const app = new cdk.App(); const stack = new AgentCoreStack(app, `TestStack${Date.now()}`, { spec, - mcpSpec: mcpSpec as never, + mcpSpec: mcpSpec ? { agentCoreGateways: mcpSpec } : undefined, credentials, env: { account: '123456789012', region: 'us-east-1' }, }); @@ -92,19 +88,23 @@ function baseSpec(overrides: Partial = {}): AgentCoreProje return { name: 'testproject', version: 1, - agents: [], + managedBy: 'CDK', + runtimes: [], memories: [], credentials: [], evaluators: [], onlineEvalConfigs: [], policyEngines: [], + agentCoreGateways: [], ...overrides, - } as AgentCoreProjectSpec; + }; } -function makeAgent(name: string, overrides: Record = {}) { +function makeRuntime( + name: string, + overrides: Partial = {} +): AgentCoreProjectSpec['runtimes'][number] { return { - type: 'AgentEnvironment', name, build: 'CodeZip', entrypoint: 'main.py', @@ -114,22 +114,22 @@ function makeAgent(name: string, overrides: Record = {}) { }; } -function makeMemory(name: string, strategies: unknown[] = []) { +function makeMemory( + name: string, + strategies: AgentCoreProjectSpec['memories'][number]['strategies'] = [] +): AgentCoreProjectSpec['memories'][number] { return { - type: 'AgentCoreMemory', name, eventExpiryDuration: 30, strategies, }; } -function makeEvaluator(name: string) { +function makeEvaluator(name: string): AgentCoreProjectSpec['evaluators'][number] { return { - type: 'CustomEvaluator', name, level: 'SESSION', config: { - type: 'LlmAsAJudge', llmAsAJudge: { model: 'anthropic.claude-3-haiku-20240307-v1:0', instructions: 'Rate the response quality based on helpfulness and accuracy.', @@ -145,9 +145,12 @@ function makeEvaluator(name: string) { }; } -function makeOnlineEvalConfig(name: string, agent: string, evaluators: string[]) { +function makeOnlineEvalConfig( + name: string, + agent: string, + evaluators: string[] +): AgentCoreProjectSpec['onlineEvalConfigs'][number] { return { - type: 'OnlineEvaluationConfig', name, agent, evaluators, @@ -157,26 +160,24 @@ function makeOnlineEvalConfig(name: string, agent: string, evaluators: string[]) function makeCredential( name: string, - type: 'ApiKeyCredentialProvider' | 'OAuthCredentialProvider' = 'ApiKeyCredentialProvider' -) { - if (type === 'OAuthCredentialProvider') { + authorizerType: 'ApiKeyCredentialProvider' | 'OAuthCredentialProvider' = 'ApiKeyCredentialProvider' +): AgentCoreProjectSpec['credentials'][number] { + if (authorizerType === 'OAuthCredentialProvider') { return { - type, + authorizerType, name, discoveryUrl: 'https://example.com/.well-known/openid-configuration', scopes: ['openid'], }; } - return { type, name }; + return { authorizerType, name }; } -function makePolicyEngine(name: string) { +function makePolicyEngine(name: string): AgentCoreProjectSpec['policyEngines'][number] { return { - type: 'PolicyEngine', name, policies: [ { - type: 'Policy', name: `${name}Policy`, statement: 'permit(principal, action, resource);', }, @@ -201,16 +202,14 @@ describe('CDK Synthesis Validation', () => { it('synthesizes a single CodeZip agent', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, + runtimes: [makeRuntime('myagent')], }) ); - // Should create an AgentCore Runtime resource template.hasResourceProperties(CFN_RUNTIME, { AgentRuntimeName: Match.stringLikeRegexp('myagent'), }); - // Should create an IAM role for the agent template.hasResourceProperties(CFN_IAM_ROLE, { AssumeRolePolicyDocument: Match.objectLike({ Statement: Match.arrayWith([ @@ -228,25 +227,28 @@ describe('CDK Synthesis Validation', () => { it('synthesizes a Container agent with ECR and CodeBuild', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('containeragent', { build: 'Container' })] as never, + runtimes: [makeRuntime('containeragent', { build: 'Container' })], }) ); - // Should create an ECR repository - template.hasResourceProperties(CFN_ECR_REPO, Match.anyValue()); + template.hasResourceProperties(CFN_ECR_REPO, { + RepositoryName: Match.stringLikeRegexp('containeragent'), + }); - // Should create a CodeBuild project for building the container - template.hasResourceProperties(CFN_CODEBUILD, Match.anyValue()); + template.hasResourceProperties(CFN_CODEBUILD, { + Source: Match.objectLike({ + Type: Match.anyValue(), + }), + }); }); it('synthesizes multiple agents', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('agent1'), makeAgent('agent2')] as never, + runtimes: [makeRuntime('agent1'), makeRuntime('agent2')], }) ); - // Should create 2 runtimes template.resourceCountIs(CFN_RUNTIME, 2); }); @@ -255,12 +257,14 @@ describe('CDK Synthesis Validation', () => { it('synthesizes agent with short-term memory (no strategies)', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - memories: [makeMemory('ShortTermMem')] as never, + runtimes: [makeRuntime('myagent')], + memories: [makeMemory('ShortTermMem')], }) ); - template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('myagent'), + }); template.hasResourceProperties(CFN_MEMORY, { Name: Match.stringLikeRegexp('ShortTermMem'), }); @@ -269,10 +273,10 @@ describe('CDK Synthesis Validation', () => { it('synthesizes agent with long-term memory strategies', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, + runtimes: [makeRuntime('myagent')], memories: [ makeMemory('LongTermMem', [{ type: 'SEMANTIC' }, { type: 'SUMMARIZATION' }, { type: 'USER_PREFERENCE' }]), - ] as never, + ], }) ); @@ -286,24 +290,27 @@ describe('CDK Synthesis Validation', () => { it('synthesizes agent with API key credential', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - credentials: [makeCredential('MyApiKey')] as never, + runtimes: [makeRuntime('myagent')], + credentials: [makeCredential('MyApiKey')], }) ); - // Agent runtime should exist — credential wiring happens at deploy time - template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('myagent'), + }); }); it('synthesizes agent with OAuth credential', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - credentials: [makeCredential('MyOAuth', 'OAuthCredentialProvider')] as never, + runtimes: [makeRuntime('myagent')], + credentials: [makeCredential('MyOAuth', 'OAuthCredentialProvider')], }) ); - template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('myagent'), + }); }); // ─── Evaluator specs ────────────────────────────────────────────────────── @@ -311,8 +318,8 @@ describe('CDK Synthesis Validation', () => { it('synthesizes custom evaluator', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - evaluators: [makeEvaluator('QualityCheck')] as never, + runtimes: [makeRuntime('myagent')], + evaluators: [makeEvaluator('QualityCheck')], }) ); @@ -326,24 +333,28 @@ describe('CDK Synthesis Validation', () => { it('synthesizes online eval config referencing project agent', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - evaluators: [makeEvaluator('QualityCheck')] as never, - onlineEvalConfigs: [makeOnlineEvalConfig('MonitorQuality', 'myagent', ['QualityCheck'])] as never, + runtimes: [makeRuntime('myagent')], + evaluators: [makeEvaluator('QualityCheck')], + onlineEvalConfigs: [makeOnlineEvalConfig('MonitorQuality', 'myagent', ['QualityCheck'])], }) ); - template.hasResourceProperties(CFN_EVALUATOR, Match.anyValue()); + template.hasResourceProperties(CFN_EVALUATOR, { + EvaluatorName: Match.stringLikeRegexp('QualityCheck'), + }); }); it('synthesizes online eval config with builtin evaluator', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - onlineEvalConfigs: [makeOnlineEvalConfig('BuiltinMonitor', 'myagent', ['Builtin.GoalSuccessRate'])] as never, + runtimes: [makeRuntime('myagent')], + onlineEvalConfigs: [makeOnlineEvalConfig('BuiltinMonitor', 'myagent', ['Builtin.GoalSuccessRate'])], }) ); - template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp('myagent'), + }); }); // ─── Policy engine specs ────────────────────────────────────────────────── @@ -351,16 +362,18 @@ describe('CDK Synthesis Validation', () => { it('synthesizes policy engine', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - policyEngines: [makePolicyEngine('SafetyGuard')] as never, + runtimes: [makeRuntime('myagent')], + policyEngines: [makePolicyEngine('SafetyGuard')], }) ); - template.hasResourceProperties(CFN_POLICY_ENGINE, Match.anyValue()); + template.hasResourceProperties(CFN_POLICY_ENGINE, { + Name: Match.stringLikeRegexp('SafetyGuard'), + }); template.hasResourceProperties(CFN_POLICY, { Definition: Match.objectLike({ Cedar: Match.objectLike({ - Statement: Match.anyValue(), + Statement: Match.stringLikeRegexp('permit'), }), }), }); @@ -371,25 +384,27 @@ describe('CDK Synthesis Validation', () => { it('synthesizes a complete project with all resource types', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('primaryagent'), makeAgent('secondaryagent')] as never, - memories: [makeMemory('ProjectMemory', [{ type: 'SEMANTIC' }])] as never, - credentials: [ - makeCredential('ProdApiKey'), - makeCredential('OAuthProvider', 'OAuthCredentialProvider'), - ] as never, - evaluators: [makeEvaluator('ResponseQuality')] as never, + runtimes: [makeRuntime('primaryagent'), makeRuntime('secondaryagent')], + memories: [makeMemory('ProjectMemory', [{ type: 'SEMANTIC' }])], + credentials: [makeCredential('ProdApiKey'), makeCredential('OAuthProvider', 'OAuthCredentialProvider')], + evaluators: [makeEvaluator('ResponseQuality')], onlineEvalConfigs: [ makeOnlineEvalConfig('LiveMonitor', 'primaryagent', ['Builtin.GoalSuccessRate', 'ResponseQuality']), - ] as never, - policyEngines: [makePolicyEngine('ContentFilter')] as never, + ], + policyEngines: [makePolicyEngine('ContentFilter')], }) ); - // Verify resource counts template.resourceCountIs(CFN_RUNTIME, 2); - template.hasResourceProperties(CFN_MEMORY, Match.anyValue()); - template.hasResourceProperties(CFN_EVALUATOR, Match.anyValue()); - template.hasResourceProperties(CFN_POLICY_ENGINE, Match.anyValue()); + template.hasResourceProperties(CFN_MEMORY, { + Name: Match.stringLikeRegexp('ProjectMemory'), + }); + template.hasResourceProperties(CFN_EVALUATOR, { + EvaluatorName: Match.stringLikeRegexp('ResponseQuality'), + }); + template.hasResourceProperties(CFN_POLICY_ENGINE, { + Name: Match.stringLikeRegexp('ContentFilter'), + }); }); // ─── Agent configuration variants ───────────────────────────────────────── @@ -397,14 +412,14 @@ describe('CDK Synthesis Validation', () => { it('synthesizes agent with custom environment variables', () => { const template = synthStack( baseSpec({ - agents: [ - makeAgent('myagent', { + runtimes: [ + makeRuntime('myagent', { envVars: [ { name: 'MODEL_ID', value: 'anthropic.claude-3-haiku-20240307-v1:0' }, { name: 'TEMPERATURE', value: '0.7' }, ], }), - ] as never, + ], }) ); @@ -416,7 +431,7 @@ describe('CDK Synthesis Validation', () => { it('synthesizes agent with MCP protocol', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('mcpagent', { protocol: 'MCP' })] as never, + runtimes: [makeRuntime('mcpagent', { protocol: 'MCP' })], }) ); @@ -428,7 +443,7 @@ describe('CDK Synthesis Validation', () => { it('synthesizes agent with A2A protocol', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('a2aagent', { protocol: 'A2A' })] as never, + runtimes: [makeRuntime('a2aagent', { protocol: 'A2A' })], }) ); @@ -440,41 +455,47 @@ describe('CDK Synthesis Validation', () => { // ─── Edge cases ─────────────────────────────────────────────────────────── it('synthesizes with memories but no agents', () => { - // Valid scenario: user may add memory before adding agents const template = synthStack( baseSpec({ - memories: [makeMemory('StandaloneMemory')] as never, + memories: [makeMemory('StandaloneMemory')], }) ); - template.hasResourceProperties(CFN_MEMORY, Match.anyValue()); + template.hasResourceProperties(CFN_MEMORY, { + Name: Match.stringLikeRegexp('StandaloneMemory'), + }); template.resourceCountIs(CFN_RUNTIME, 0); }); it('synthesizes with evaluators but no online eval configs', () => { const template = synthStack( baseSpec({ - agents: [makeAgent('myagent')] as never, - evaluators: [makeEvaluator('UnusedEval')] as never, + runtimes: [makeRuntime('myagent')], + evaluators: [makeEvaluator('UnusedEval')], }) ); - template.hasResourceProperties(CFN_EVALUATOR, Match.anyValue()); + template.hasResourceProperties(CFN_EVALUATOR, { + EvaluatorName: Match.stringLikeRegexp('UnusedEval'), + }); }); it('synthesizes spec with maximum name lengths', () => { - // Agent name max is 48 chars, memory name max is 48 chars const longAgentName = 'a'.repeat(48); const longMemoryName = 'M'.repeat(48); const template = synthStack( baseSpec({ - agents: [makeAgent(longAgentName)] as never, - memories: [makeMemory(longMemoryName)] as never, + runtimes: [makeRuntime(longAgentName)], + memories: [makeMemory(longMemoryName)], }) ); - template.hasResourceProperties(CFN_RUNTIME, Match.anyValue()); - template.hasResourceProperties(CFN_MEMORY, Match.anyValue()); + template.hasResourceProperties(CFN_RUNTIME, { + AgentRuntimeName: Match.stringLikeRegexp(longAgentName), + }); + template.hasResourceProperties(CFN_MEMORY, { + Name: Match.stringLikeRegexp(longMemoryName), + }); }); }); From 4fa85ddb20ab1c430a0c41629276cef4efa85e6c Mon Sep 17 00:00:00 2001 From: notgitika Date: Fri, 10 Apr 2026 23:37:03 -0400 Subject: [PATCH 5/5] fix: include CDK synth tests in CI unit suite --- vitest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vitest.config.ts b/vitest.config.ts index 963f60fc1..7cbf21e3d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -40,7 +40,7 @@ export default defineConfig({ test: { name: 'unit', include: ['src/**/*.test.ts', 'src/**/*.test.tsx'], - exclude: ['src/assets/cdk/test/*.test.ts', 'src/assets/__tests__/cdk-synth-validation.test.ts'], + exclude: ['src/assets/cdk/test/*.test.ts'], }, }, {