From 6fba5c8aefb230a2fd6638d8decfb117b3aac848 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Tue, 16 Jun 2026 14:12:59 -0700 Subject: [PATCH 1/4] Add deepeval CLI integration: flags, templates, warnings, memory, unit tests --- .../execution-role-policy.json | 15 ++ .../deepeval-lambda/lambda_function.py | 4 + .../evaluators/deepeval-lambda/pyproject.toml | 16 ++ src/cli/primitives/EvaluatorPrimitive.ts | 153 +++++++++++- .../__tests__/EvaluatorPrimitive.test.ts | 231 +++++++++++++++++- src/cli/templates/EvaluatorRenderer.ts | 11 + src/schema/llm-compacted/agentcore.ts | 1 + src/schema/schemas/primitives/evaluator.ts | 1 + 8 files changed, 425 insertions(+), 7 deletions(-) create mode 100644 src/assets/evaluators/deepeval-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/deepeval-lambda/lambda_function.py create mode 100644 src/assets/evaluators/deepeval-lambda/pyproject.toml diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py new file mode 100644 index 000000000..8738f0ea7 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -0,0 +1,4 @@ +from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalHandler +from deepeval.metrics import {{ MetricClass }} + +handler = DeepEvalHandler(metric={{ MetricClass }}({{{ MetricParams }}})) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml new file mode 100644 index 000000000..ac974774c --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (DeepEval)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[deepeval]>=1.6.0", + "deepeval>=2.0.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index 51f82f512..bf5b056e6 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -6,7 +6,7 @@ import { getErrorMessage } from '../errors'; import type { RemovalPreview, SchemaChange } from '../operations/remove/types'; import { runCliCommand } from '../telemetry/cli-command-run.js'; import { EvaluatorLevel, EvaluatorType, standardize } from '../telemetry/schemas/common-shapes.js'; -import { renderCodeBasedEvaluatorTemplate } from '../templates/EvaluatorRenderer'; +import { renderCodeBasedEvaluatorTemplate, renderDeepEvalEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; import { LEVEL_PLACEHOLDERS, @@ -21,18 +21,58 @@ import { existsSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { dirname, join } from 'node:path'; +export interface ThirdPartyLibraryOptions { + library: 'deepeval'; + metricClass: string; + metricParams?: string; +} + export interface AddEvaluatorOptions { name: string; level: EvaluationLevel; description?: string; config: EvaluatorConfig; kmsKeyArn?: string; + thirdParty?: ThirdPartyLibraryOptions; } export type RemovableEvaluator = RemovableResource; const DEFAULT_CODE_ENTRYPOINT = 'lambda_function.handler'; const DEFAULT_CODE_TIMEOUT = 60; +const DEEPEVAL_DEFAULT_TIMEOUT = 300; +const DEEPEVAL_DEFAULT_MEMORY_MB = 1024; + +const METRICS_REQUIRING_RETRIEVAL_CONTEXT = new Set([ + 'FaithfulnessMetric', + 'HallucinationMetric', + 'ContextualRelevancyMetric', + 'ContextualPrecisionMetric', + 'ContextualRecallMetric', +]); + +const METRICS_REQUIRING_EXPECTED_OUTPUT = new Set(['ContextualPrecisionMetric', 'ContextualRecallMetric']); + +export function jsonToPythonValue(value: unknown): string { + if (value === null) return 'None'; + if (value === true) return 'True'; + if (value === false) return 'False'; + if (typeof value === 'number') return String(value); + if (typeof value === 'string') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(jsonToPythonValue).join(', ')}]`; + if (typeof value === 'object') { + const entries = Object.entries(value as Record); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}: ${jsonToPythonValue(v)}`).join(', ')}}`; + } + return String(value); +} + +export function jsonToKwargs(json: string): string { + const obj = JSON.parse(json) as Record; + return Object.entries(obj) + .map(([key, value]) => `${key}=${jsonToPythonValue(value)}`) + .join(', '); +} /** * EvaluatorPrimitive handles all evaluator add/remove operations. @@ -53,7 +93,19 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') + .option( + '--from-3p-library ', + 'Third-party evaluation library to use (currently: deepeval)' + ) + .option('--metric ', '[3P library] Metric class name (e.g. AnswerRelevancyMetric)') + .option('--parameters ', '[3P library] JSON string of metric constructor kwargs') + .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240 (default: 1024 for deepeval)') .option( '--config ', 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]' @@ -200,6 +259,10 @@ export class EvaluatorPrimitive extends BasePrimitive 10240) { + fail('--memory must be an integer between 128 and 10240'); + } + } + + // Default --type to code-based when --from-3p-library is set + const evalType = cliOptions.type ?? (from3pLibrary ? 'code-based' : 'llm-as-a-judge'); if (evalType !== 'llm-as-a-judge' && evalType !== 'code-based') { fail(`Invalid --type "${evalType}". Must be one of: llm-as-a-judge, code-based`); } @@ -234,15 +327,31 @@ export class EvaluatorPrimitive extends BasePrimitive { const project = await this.readProjectSpec(); diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index a521e6a78..e5c50f236 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -1,5 +1,5 @@ import type { EvaluatorConfig } from '../../../schema'; -import { EvaluatorPrimitive } from '../EvaluatorPrimitive.js'; +import { EvaluatorPrimitive, jsonToKwargs, jsonToPythonValue } from '../EvaluatorPrimitive.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mockReadProjectSpec = vi.fn(); @@ -27,6 +27,14 @@ vi.mock('../../../lib/index.js', () => ({ }, })); +const mockRenderCodeBased = vi.fn().mockResolvedValue(undefined); +const mockRenderDeepEval = vi.fn().mockResolvedValue(undefined); + +vi.mock('../../templates/EvaluatorRenderer', () => ({ + renderCodeBasedEvaluatorTemplate: (...args: unknown[]) => mockRenderCodeBased(...args), + renderDeepEvalEvaluatorTemplate: (...args: unknown[]) => mockRenderDeepEval(...args), +})); + const validConfig: EvaluatorConfig = { llmAsAJudge: { model: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', @@ -256,4 +264,225 @@ describe('EvaluatorPrimitive', () => { expect(await primitive.getAllNames()).toEqual([]); }); }); + + describe('buildDeepEvalConfig', () => { + it('returns managed code-based config with deepeval defaults', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval'); + + expect(config).toEqual({ + codeBased: { + managed: { + codeLocation: 'app/my_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 300, + memorySizeMb: 1024, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }); + }); + + it('respects custom timeout', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval', '120'); + + expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); + }); + + it('respects custom memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval', undefined, '2048'); + + expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); + }); + + it('respects both custom timeout and memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('my_eval', '60', '512'); + + expect(config.codeBased!.managed!.timeoutSeconds).toBe(60); + expect(config.codeBased!.managed!.memorySizeMb).toBe(512); + }); + + it('sets codeLocation based on evaluator name', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildDeepEvalConfig']('relevancy_check'); + + expect(config.codeBased!.managed!.codeLocation).toBe('app/relevancy_check/'); + }); + }); + + describe('add with thirdParty (deepeval)', () => { + const deepEvalConfig: EvaluatorConfig = { + codeBased: { + managed: { + codeLocation: 'app/deep_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 300, + memorySizeMb: 1024, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }; + + it('calls renderDeepEvalEvaluatorTemplate when thirdParty.library is deepeval', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const result = await primitive.add({ + name: 'deep_eval', + level: 'SESSION', + config: deepEvalConfig, + thirdParty: { + library: 'deepeval', + metricClass: 'AnswerRelevancyMetric', + metricParams: 'threshold=0.7', + }, + }); + + expect(result.success).toBe(true); + expect(result).toHaveProperty('codePath', 'app/deep_eval/'); + expect(mockRenderDeepEval).toHaveBeenCalledOnce(); + expect(mockRenderDeepEval).toHaveBeenCalledWith( + { Name: 'deep_eval', MetricClass: 'AnswerRelevancyMetric', MetricParams: 'threshold=0.7' }, + expect.stringContaining('app/deep_eval') + ); + expect(mockRenderCodeBased).not.toHaveBeenCalled(); + }); + + it('passes empty string for MetricParams when not provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'deep_eval', + level: 'SESSION', + config: deepEvalConfig, + thirdParty: { + library: 'deepeval', + metricClass: 'HallucinationMetric', + }, + }); + + expect(mockRenderDeepEval).toHaveBeenCalledWith( + { Name: 'deep_eval', MetricClass: 'HallucinationMetric', MetricParams: '' }, + expect.any(String) + ); + }); + + it('calls renderCodeBasedEvaluatorTemplate when thirdParty is not set', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const managedConfig: EvaluatorConfig = { + codeBased: { + managed: { + codeLocation: 'app/plain_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 60, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }; + + await primitive.add({ + name: 'plain_eval', + level: 'SESSION', + config: managedConfig, + }); + + expect(mockRenderCodeBased).toHaveBeenCalledOnce(); + expect(mockRenderCodeBased).toHaveBeenCalledWith('plain_eval', expect.stringContaining('app/plain_eval')); + expect(mockRenderDeepEval).not.toHaveBeenCalled(); + }); + }); +}); + +describe('jsonToPythonValue', () => { + it('converts null to None', () => { + expect(jsonToPythonValue(null)).toBe('None'); + }); + + it('converts true to True', () => { + expect(jsonToPythonValue(true)).toBe('True'); + }); + + it('converts false to False', () => { + expect(jsonToPythonValue(false)).toBe('False'); + }); + + it('converts integers', () => { + expect(jsonToPythonValue(42)).toBe('42'); + }); + + it('converts floats', () => { + expect(jsonToPythonValue(0.7)).toBe('0.7'); + }); + + it('converts strings with quotes', () => { + expect(jsonToPythonValue('gpt-4')).toBe('"gpt-4"'); + }); + + it('converts arrays', () => { + expect(jsonToPythonValue([1, 'two', true])).toBe('[1, "two", True]'); + }); + + it('converts nested objects to Python dicts', () => { + expect(jsonToPythonValue({ key: 'value', n: 3 })).toBe('{"key": "value", "n": 3}'); + }); + + it('handles empty arrays', () => { + expect(jsonToPythonValue([])).toBe('[]'); + }); + + it('handles empty objects', () => { + expect(jsonToPythonValue({})).toBe('{}'); + }); +}); + +describe('jsonToKwargs', () => { + it('converts simple number parameter', () => { + expect(jsonToKwargs('{"threshold": 0.7}')).toBe('threshold=0.7'); + }); + + it('converts simple string parameter', () => { + expect(jsonToKwargs('{"model": "gpt-4"}')).toBe('model="gpt-4"'); + }); + + it('converts multiple parameters', () => { + const result = jsonToKwargs('{"threshold": 0.7, "model": "gpt-4"}'); + expect(result).toBe('threshold=0.7, model="gpt-4"'); + }); + + it('converts boolean parameters', () => { + expect(jsonToKwargs('{"verbose": true, "strict": false}')).toBe('verbose=True, strict=False'); + }); + + it('converts null parameters', () => { + expect(jsonToKwargs('{"callback": null}')).toBe('callback=None'); + }); + + it('converts array parameters', () => { + expect(jsonToKwargs('{"tools": ["search", "calculate"]}')).toBe('tools=["search", "calculate"]'); + }); + + it('converts nested object parameters', () => { + const result = jsonToKwargs('{"config": {"temperature": 0.5}}'); + expect(result).toBe('config={"temperature": 0.5}'); + }); + + it('handles mixed types', () => { + const input = '{"threshold": 0.7, "model": "gpt-4", "verbose": true, "tags": ["eval"], "fallback": null}'; + const result = jsonToKwargs(input); + expect(result).toBe('threshold=0.7, model="gpt-4", verbose=True, tags=["eval"], fallback=None'); + }); + + it('throws on invalid JSON', () => { + expect(() => jsonToKwargs('not json')).toThrow(); + }); + + it('returns empty string for empty object', () => { + expect(jsonToKwargs('{}')).toBe(''); + }); }); diff --git a/src/cli/templates/EvaluatorRenderer.ts b/src/cli/templates/EvaluatorRenderer.ts index 6b2f22c24..458106ddf 100644 --- a/src/cli/templates/EvaluatorRenderer.ts +++ b/src/cli/templates/EvaluatorRenderer.ts @@ -10,3 +10,14 @@ export async function renderCodeBasedEvaluatorTemplate(evaluatorName: string, ou const templateDir = getTemplatePath('evaluators', 'python-lambda'); await copyAndRenderDir(templateDir, outputDir, { Name: evaluatorName }); } + +export interface DeepEvalTemplateData { + Name: string; + MetricClass: string; + MetricParams: string; +} + +export async function renderDeepEvalEvaluatorTemplate(data: DeepEvalTemplateData, outputDir: string): Promise { + const templateDir = getTemplatePath('evaluators', 'deepeval-lambda'); + await copyAndRenderDir(templateDir, outputDir, data); +} diff --git a/src/schema/llm-compacted/agentcore.ts b/src/schema/llm-compacted/agentcore.ts index 2ab5290cf..2ec7ebc06 100644 --- a/src/schema/llm-compacted/agentcore.ts +++ b/src/schema/llm-compacted/agentcore.ts @@ -233,6 +233,7 @@ interface ManagedCodeBasedConfig { codeLocation: string; entrypoint: string; // default 'lambda_function.handler' timeoutSeconds: number; // @min 1 @max 300 (default 60) + memorySizeMb?: number; // @min 128 @max 10240 additionalPolicies?: string[]; } diff --git a/src/schema/schemas/primitives/evaluator.ts b/src/schema/schemas/primitives/evaluator.ts index 97d772d52..7bb2b6a23 100644 --- a/src/schema/schemas/primitives/evaluator.ts +++ b/src/schema/schemas/primitives/evaluator.ts @@ -81,6 +81,7 @@ export const ManagedCodeBasedConfigSchema = z.object({ codeLocation: z.string().min(1), entrypoint: z.string().min(1).default('lambda_function.handler'), timeoutSeconds: z.number().int().min(1).max(300).default(60), + memorySizeMb: z.number().int().min(128).max(10240).optional(), additionalPolicies: z.array(z.string().min(1)).optional(), }); From d44430a244918cbb04b0ba153cb533c0c57c55a5 Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 25 Jun 2026 14:35:32 -0700 Subject: [PATCH 2/4] Refactor CLI to registry-driven third-party evaluator framework (DeepEval + Autoevals) --- .../execution-role-policy.json | 15 ++ .../autoevals-lambda/lambda_function.py | 4 + .../autoevals-lambda/pyproject.toml | 16 ++ .../deepeval-lambda/lambda_function.py | 6 +- src/cli/primitives/EvaluatorPrimitive.ts | 177 ++++++++++---- .../__tests__/EvaluatorPrimitive.test.ts | 215 ++++++++++++++---- src/cli/templates/EvaluatorRenderer.ts | 14 +- 7 files changed, 347 insertions(+), 100 deletions(-) create mode 100644 src/assets/evaluators/autoevals-lambda/execution-role-policy.json create mode 100644 src/assets/evaluators/autoevals-lambda/lambda_function.py create mode 100644 src/assets/evaluators/autoevals-lambda/pyproject.toml diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py new file mode 100644 index 000000000..7c6a256b9 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -0,0 +1,4 @@ +from bedrock_agentcore.evaluation.integrations.autoevals import AutoevalsAdapter +from autoevals import {{ EvaluatorClass }} + +handler = AutoevalsAdapter(scorer={{ EvaluatorClass }}({{{ EvaluatorParams }}})) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml new file mode 100644 index 000000000..b2e920a3f --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (Autoevals)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[autoevals]>=1.6.0", + "autoevals>=0.0.80", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 8738f0ea7..3b1033daf 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -1,4 +1,4 @@ -from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalHandler -from deepeval.metrics import {{ MetricClass }} +from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalAdapter +from deepeval.metrics import {{ EvaluatorClass }} -handler = DeepEvalHandler(metric={{ MetricClass }}({{{ MetricParams }}})) +handler = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index bf5b056e6..fe9fadb13 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -6,7 +6,7 @@ import { getErrorMessage } from '../errors'; import type { RemovalPreview, SchemaChange } from '../operations/remove/types'; import { runCliCommand } from '../telemetry/cli-command-run.js'; import { EvaluatorLevel, EvaluatorType, standardize } from '../telemetry/schemas/common-shapes.js'; -import { renderCodeBasedEvaluatorTemplate, renderDeepEvalEvaluatorTemplate } from '../templates/EvaluatorRenderer'; +import { renderCodeBasedEvaluatorTemplate, renderThirdPartyEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; import { LEVEL_PLACEHOLDERS, @@ -21,8 +21,83 @@ import { existsSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { dirname, join } from 'node:path'; +// ============================================================================ +// Third-Party Library Registry +// ============================================================================ + +interface MetricWarning { + metrics: Set; + message: string; +} + +export interface ThirdPartyLibraryConfig { + templateDir: string; + defaultTimeoutSeconds: number; + defaultMemorySizeMb: number; + warnings: MetricWarning[]; +} + +export const THIRD_PARTY_EVALUATOR_LIBRARIES = { + deepeval: { + templateDir: 'deepeval-lambda', + defaultTimeoutSeconds: 300, + defaultMemorySizeMb: 1024, + warnings: [ + { + metrics: new Set([ + 'FaithfulnessMetric', + 'HallucinationMetric', + 'ContextualRelevancyMetric', + 'ContextualPrecisionMetric', + 'ContextualRecallMetric', + ]), + message: + 'requires retrieval_context from tool-role messages. ' + + 'If your agent has no tool calls, the evaluator will return MISSING_REQUIRED_FIELD at runtime.', + }, + { + metrics: new Set(['ContextualPrecisionMetric', 'ContextualRecallMetric']), + message: + 'requires expected_output via evaluationReferenceInputs. ' + + 'Caller must provide referenceInputs when invoking the Evaluate API.', + }, + ], + }, + autoevals: { + templateDir: 'autoevals-lambda', + defaultTimeoutSeconds: 60, + defaultMemorySizeMb: 512, + warnings: [ + { + metrics: new Set(['Factuality', 'ClosedQA']), + message: + 'requires expected_output via evaluationReferenceInputs. ' + + 'Caller must provide referenceInputs when invoking the Evaluate API.', + }, + { + metrics: new Set(['SQL']), + message: + 'requires expected_output (reference SQL) via evaluationReferenceInputs. ' + + 'Caller must provide referenceInputs when invoking the Evaluate API.', + }, + ], + }, +} satisfies Record; + +export type ThirdPartyLibrary = keyof typeof THIRD_PARTY_EVALUATOR_LIBRARIES; + +const SUPPORTED_LIBRARIES: ThirdPartyLibrary[] = Object.keys(THIRD_PARTY_EVALUATOR_LIBRARIES) as ThirdPartyLibrary[]; + +function isSupportedLibrary(value: string): value is ThirdPartyLibrary { + return value in THIRD_PARTY_EVALUATOR_LIBRARIES; +} + +// ============================================================================ +// Types +// ============================================================================ + export interface ThirdPartyLibraryOptions { - library: 'deepeval'; + library: ThirdPartyLibrary; metricClass: string; metricParams?: string; } @@ -38,20 +113,12 @@ export interface AddEvaluatorOptions { export type RemovableEvaluator = RemovableResource; +// ============================================================================ +// Constants & Utilities +// ============================================================================ + const DEFAULT_CODE_ENTRYPOINT = 'lambda_function.handler'; const DEFAULT_CODE_TIMEOUT = 60; -const DEEPEVAL_DEFAULT_TIMEOUT = 300; -const DEEPEVAL_DEFAULT_MEMORY_MB = 1024; - -const METRICS_REQUIRING_RETRIEVAL_CONTEXT = new Set([ - 'FaithfulnessMetric', - 'HallucinationMetric', - 'ContextualRelevancyMetric', - 'ContextualPrecisionMetric', - 'ContextualRecallMetric', -]); - -const METRICS_REQUIRING_EXPECTED_OUTPUT = new Set(['ContextualPrecisionMetric', 'ContextualRecallMetric']); export function jsonToPythonValue(value: unknown): string { if (value === null) return 'None'; @@ -64,7 +131,7 @@ export function jsonToPythonValue(value: unknown): string { const entries = Object.entries(value as Record); return `{${entries.map(([k, v]) => `${JSON.stringify(k)}: ${jsonToPythonValue(v)}`).join(', ')}}`; } - return String(value); + return String(value as string | number | boolean); } export function jsonToKwargs(json: string): string { @@ -74,6 +141,20 @@ export function jsonToKwargs(json: string): string { .join(', '); } +function getWarningsForMetric(libraryConfig: ThirdPartyLibraryConfig, metricClass: string): string[] { + const messages: string[] = []; + for (const warning of libraryConfig.warnings) { + if (warning.metrics.has(metricClass)) { + messages.push(`⚠️ ${metricClass} ${warning.message}`); + } + } + return messages; +} + +// ============================================================================ +// EvaluatorPrimitive +// ============================================================================ + /** * EvaluatorPrimitive handles all evaluator add/remove operations. */ @@ -94,12 +175,14 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') - .option( - '--from-3p-library ', - 'Third-party evaluation library to use (currently: deepeval)' - ) - .option('--metric ', '[3P library] Metric class name (e.g. AnswerRelevancyMetric)') + .option('--from-3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) + .option('--metric ', '[3P library] Metric/evaluator class name (e.g. AnswerRelevancyMetric)') .option('--parameters ', '[3P library] JSON string of metric constructor kwargs') - .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240 (default: 1024 for deepeval)') + .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( '--config ', 'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]' @@ -288,10 +368,11 @@ export class EvaluatorPrimitive extends BasePrimitive ({ })); const mockRenderCodeBased = vi.fn().mockResolvedValue(undefined); -const mockRenderDeepEval = vi.fn().mockResolvedValue(undefined); +const mockRenderThirdParty = vi.fn().mockResolvedValue(undefined); vi.mock('../../templates/EvaluatorRenderer', () => ({ renderCodeBasedEvaluatorTemplate: (...args: unknown[]) => mockRenderCodeBased(...args), - renderDeepEvalEvaluatorTemplate: (...args: unknown[]) => mockRenderDeepEval(...args), + renderThirdPartyEvaluatorTemplate: (...args: unknown[]) => mockRenderThirdParty(...args), })); const validConfig: EvaluatorConfig = { @@ -265,55 +270,79 @@ describe('EvaluatorPrimitive', () => { }); }); - describe('buildDeepEvalConfig', () => { - it('returns managed code-based config with deepeval defaults', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval'); - - expect(config).toEqual({ - codeBased: { - managed: { - codeLocation: 'app/my_eval/', - entrypoint: 'lambda_function.handler', - timeoutSeconds: 300, - memorySizeMb: 1024, - additionalPolicies: ['execution-role-policy.json'], + describe('buildThirdPartyConfig', () => { + describe('deepeval', () => { + const deepevalConfig = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + + it('returns config with deepeval defaults (300s, 1024MB)', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig); + + expect(config).toEqual({ + codeBased: { + managed: { + codeLocation: 'app/my_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 300, + memorySizeMb: 1024, + additionalPolicies: ['execution-role-policy.json'], + }, }, - }, + }); }); - }); - it('respects custom timeout', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval', '120'); + it('respects custom timeout', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig, '120'); - expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); - }); + expect(config.codeBased!.managed!.timeoutSeconds).toBe(120); + }); - it('respects custom memory', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval', undefined, '2048'); + it('respects custom memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('my_eval', deepevalConfig, undefined, '2048'); - expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); + expect(config.codeBased!.managed!.memorySizeMb).toBe(2048); + }); }); - it('respects both custom timeout and memory', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('my_eval', '60', '512'); + describe('autoevals', () => { + const autoevalsConfig = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + + it('returns config with autoevals defaults (60s, 512MB)', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig); + + expect(config).toEqual({ + codeBased: { + managed: { + codeLocation: 'app/fact_check/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 60, + memorySizeMb: 512, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }); + }); - expect(config.codeBased!.managed!.timeoutSeconds).toBe(60); - expect(config.codeBased!.managed!.memorySizeMb).toBe(512); - }); + it('respects custom timeout', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig, '180'); - it('sets codeLocation based on evaluator name', () => { - // eslint-disable-next-line @typescript-eslint/dot-notation - const config = primitive['buildDeepEvalConfig']('relevancy_check'); + expect(config.codeBased!.managed!.timeoutSeconds).toBe(180); + }); - expect(config.codeBased!.managed!.codeLocation).toBe('app/relevancy_check/'); + it('respects custom memory', () => { + // eslint-disable-next-line @typescript-eslint/dot-notation + const config = primitive['buildThirdPartyConfig']('fact_check', autoevalsConfig, undefined, '1024'); + + expect(config.codeBased!.managed!.memorySizeMb).toBe(1024); + }); }); }); - describe('add with thirdParty (deepeval)', () => { + describe('add with thirdParty', () => { const deepEvalConfig: EvaluatorConfig = { codeBased: { managed: { @@ -326,7 +355,19 @@ describe('EvaluatorPrimitive', () => { }, }; - it('calls renderDeepEvalEvaluatorTemplate when thirdParty.library is deepeval', async () => { + const autoevalsEvalConfig: EvaluatorConfig = { + codeBased: { + managed: { + codeLocation: 'app/auto_eval/', + entrypoint: 'lambda_function.handler', + timeoutSeconds: 60, + memorySizeMb: 512, + additionalPolicies: ['execution-role-policy.json'], + }, + }, + }; + + it('calls renderThirdPartyEvaluatorTemplate with deepeval templateDir', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); @@ -343,15 +384,42 @@ describe('EvaluatorPrimitive', () => { expect(result.success).toBe(true); expect(result).toHaveProperty('codePath', 'app/deep_eval/'); - expect(mockRenderDeepEval).toHaveBeenCalledOnce(); - expect(mockRenderDeepEval).toHaveBeenCalledWith( - { Name: 'deep_eval', MetricClass: 'AnswerRelevancyMetric', MetricParams: 'threshold=0.7' }, + expect(mockRenderThirdParty).toHaveBeenCalledOnce(); + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'deepeval-lambda', + { Name: 'deep_eval', EvaluatorClass: 'AnswerRelevancyMetric', EvaluatorParams: 'threshold=0.7' }, expect.stringContaining('app/deep_eval') ); expect(mockRenderCodeBased).not.toHaveBeenCalled(); }); - it('passes empty string for MetricParams when not provided', async () => { + it('calls renderThirdPartyEvaluatorTemplate with autoevals templateDir', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const result = await primitive.add({ + name: 'auto_eval', + level: 'SESSION', + config: autoevalsEvalConfig, + thirdParty: { + library: 'autoevals', + metricClass: 'Factuality', + metricParams: '', + }, + }); + + expect(result.success).toBe(true); + expect(result).toHaveProperty('codePath', 'app/auto_eval/'); + expect(mockRenderThirdParty).toHaveBeenCalledOnce(); + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'autoevals-lambda', + { Name: 'auto_eval', EvaluatorClass: 'Factuality', EvaluatorParams: '' }, + expect.stringContaining('app/auto_eval') + ); + expect(mockRenderCodeBased).not.toHaveBeenCalled(); + }); + + it('passes empty string for EvaluatorParams when not provided', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); @@ -365,8 +433,9 @@ describe('EvaluatorPrimitive', () => { }, }); - expect(mockRenderDeepEval).toHaveBeenCalledWith( - { Name: 'deep_eval', MetricClass: 'HallucinationMetric', MetricParams: '' }, + expect(mockRenderThirdParty).toHaveBeenCalledWith( + 'deepeval-lambda', + { Name: 'deep_eval', EvaluatorClass: 'HallucinationMetric', EvaluatorParams: '' }, expect.any(String) ); }); @@ -394,11 +463,65 @@ describe('EvaluatorPrimitive', () => { expect(mockRenderCodeBased).toHaveBeenCalledOnce(); expect(mockRenderCodeBased).toHaveBeenCalledWith('plain_eval', expect.stringContaining('app/plain_eval')); - expect(mockRenderDeepEval).not.toHaveBeenCalled(); + expect(mockRenderThirdParty).not.toHaveBeenCalled(); }); }); }); +describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { + it('contains deepeval with expected defaults', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + expect(config).toBeDefined(); + expect(config.templateDir).toBe('deepeval-lambda'); + expect(config.defaultTimeoutSeconds).toBe(300); + expect(config.defaultMemorySizeMb).toBe(1024); + }); + + it('contains autoevals with expected defaults', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + expect(config).toBeDefined(); + expect(config.templateDir).toBe('autoevals-lambda'); + expect(config.defaultTimeoutSeconds).toBe(60); + expect(config.defaultMemorySizeMb).toBe(512); + }); + + it('deepeval has warnings for retrieval_context metrics', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + const retrievalWarning = config.warnings.find(w => w.metrics.has('FaithfulnessMetric')); + expect(retrievalWarning).toBeDefined(); + expect(retrievalWarning!.message).toContain('retrieval_context'); + expect(retrievalWarning!.metrics.has('HallucinationMetric')).toBe(true); + expect(retrievalWarning!.metrics.has('ContextualRelevancyMetric')).toBe(true); + }); + + it('deepeval has warnings for expected_output metrics', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; + const expectedWarning = config.warnings.find(w => w.metrics.has('ContextualPrecisionMetric')); + expect(expectedWarning).toBeDefined(); + expect(expectedWarning!.message).toContain('expected_output'); + }); + + it('autoevals has warnings for reference-input metrics', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + const factWarning = config.warnings.find(w => w.metrics.has('Factuality')); + expect(factWarning).toBeDefined(); + expect(factWarning!.message).toContain('expected_output'); + expect(factWarning!.metrics.has('ClosedQA')).toBe(true); + }); + + it('autoevals has warnings for SQL metric', () => { + const config = THIRD_PARTY_EVALUATOR_LIBRARIES.autoevals; + const sqlWarning = config.warnings.find(w => w.metrics.has('SQL')); + expect(sqlWarning).toBeDefined(); + expect(sqlWarning!.message).toContain('reference SQL'); + }); + + it('does not contain unsupported libraries', () => { + expect((THIRD_PARTY_EVALUATOR_LIBRARIES as Record).ragas).toBeUndefined(); + expect((THIRD_PARTY_EVALUATOR_LIBRARIES as Record).langsmith).toBeUndefined(); + }); +}); + describe('jsonToPythonValue', () => { it('converts null to None', () => { expect(jsonToPythonValue(null)).toBe('None'); diff --git a/src/cli/templates/EvaluatorRenderer.ts b/src/cli/templates/EvaluatorRenderer.ts index 458106ddf..6b50242e5 100644 --- a/src/cli/templates/EvaluatorRenderer.ts +++ b/src/cli/templates/EvaluatorRenderer.ts @@ -11,13 +11,17 @@ export async function renderCodeBasedEvaluatorTemplate(evaluatorName: string, ou await copyAndRenderDir(templateDir, outputDir, { Name: evaluatorName }); } -export interface DeepEvalTemplateData { +export interface ThirdPartyEvaluatorTemplateData { Name: string; - MetricClass: string; - MetricParams: string; + EvaluatorClass: string; + EvaluatorParams: string; } -export async function renderDeepEvalEvaluatorTemplate(data: DeepEvalTemplateData, outputDir: string): Promise { - const templateDir = getTemplatePath('evaluators', 'deepeval-lambda'); +export async function renderThirdPartyEvaluatorTemplate( + templateDirName: string, + data: ThirdPartyEvaluatorTemplateData, + outputDir: string +): Promise { + const templateDir = getTemplatePath('evaluators', templateDirName); await copyAndRenderDir(templateDir, outputDir, data); } From 485000e5f3e04041371e81f08bb62d46d3319bcd Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 16 Jul 2026 09:42:34 -0700 Subject: [PATCH 3/4] fix: Update 3P evaluator templates and add --param/--parameters-file support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DeepEval template: add /tmp workaround + DEEPEVAL_TELEMETRY_OPT_OUT for Lambda - Autoevals template: fix scorer= → metric= to match SDK API - Rename --from-3p-library → --3p-library - Add --param key=value (repeatable) for metric constructor kwargs - Add --parameters-file for JSON file of kwargs - Add parseParamFlags() utility + unit tests - --param and --parameters-file are mutually exclusive --- .gitignore | 3 + .../assets.snapshot.test.ts.snap | 6 + .../autoevals-lambda/lambda_function.py | 15 ++- .../autoevals-lambda/pyproject.toml | 2 +- .../deepeval-lambda/lambda_function.py | 21 +++- .../evaluators/deepeval-lambda/pyproject.toml | 2 +- src/cli/primitives/EvaluatorPrimitive.ts | 110 ++++++++++++------ .../__tests__/EvaluatorPrimitive.test.ts | 43 ++++++- 8 files changed, 161 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 6613a8f02..e544ccddc 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,6 @@ ProtocolTesting/ browser-tests/.browser-test-env browser-tests/test-results/ browser-tests/playwright-report/ + +# E2E test output +test-e2e-output/ diff --git a/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap b/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap index b935e4f72..6bbaad067 100644 --- a/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap +++ b/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap @@ -742,6 +742,12 @@ exports[`Assets Directory Snapshots > File listing > should match the expected f "container/typescript/dockerignore.template", "datasets/predefined-v1.jsonl", "datasets/simulated-v1.jsonl", + "evaluators/autoevals-lambda/execution-role-policy.json", + "evaluators/autoevals-lambda/lambda_function.py", + "evaluators/autoevals-lambda/pyproject.toml", + "evaluators/deepeval-lambda/execution-role-policy.json", + "evaluators/deepeval-lambda/lambda_function.py", + "evaluators/deepeval-lambda/pyproject.toml", "evaluators/python-lambda/execution-role-policy.json", "evaluators/python-lambda/lambda_function.py", "evaluators/python-lambda/pyproject.toml", diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 7c6a256b9..3fa199389 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -1,4 +1,15 @@ -from bedrock_agentcore.evaluation.integrations.autoevals import AutoevalsAdapter from autoevals import {{ EvaluatorClass }} -handler = AutoevalsAdapter(scorer={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter + +adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml index b2e920a3f..bd1fa659f 100644 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "AgentCore Code-Based Evaluator (Autoevals)" requires-python = ">=3.10" dependencies = [ - "bedrock-agentcore[autoevals]>=1.6.0", + "bedrock-agentcore[autoevals]", "autoevals>=0.0.80", ] diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py index 3b1033daf..970dfe465 100644 --- a/src/assets/evaluators/deepeval-lambda/lambda_function.py +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -1,4 +1,21 @@ -from bedrock_agentcore.evaluation.integrations.deepeval import DeepEvalAdapter +import os + +os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") +os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") +os.chdir("/tmp") + from deepeval.metrics import {{ EvaluatorClass }} -handler = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter + +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml index ac974774c..231940432 100644 --- a/src/assets/evaluators/deepeval-lambda/pyproject.toml +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.0" description = "AgentCore Code-Based Evaluator (DeepEval)" requires-python = ">=3.10" dependencies = [ - "bedrock-agentcore[deepeval]>=1.6.0", + "bedrock-agentcore[deepeval]", "deepeval>=2.0.0", ] diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index fe9fadb13..f2d34f4da 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -17,7 +17,7 @@ import { import { BasePrimitive } from './BasePrimitive'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; import type { Command } from '@commander-js/extra-typings'; -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { dirname, join } from 'node:path'; @@ -141,6 +141,26 @@ export function jsonToKwargs(json: string): string { .join(', '); } +export function parseParamFlags(params: string[]): string { + return params + .map(param => { + const eqIndex = param.indexOf('='); + if (eqIndex === -1) { + throw new Error(`"${param}" is not in key=value format`); + } + const key = param.slice(0, eqIndex); + const rawValue = param.slice(eqIndex + 1); + let value: unknown; + try { + value = JSON.parse(rawValue); + } catch { + value = rawValue; + } + return `${key}=${jsonToPythonValue(value)}`; + }) + .join(', '); +} + function getWarningsForMetric(libraryConfig: ThirdPartyLibraryConfig, metricClass: string): string[] { const messages: string[] = []; for (const warning of libraryConfig.warnings) { @@ -319,9 +339,15 @@ export class EvaluatorPrimitive extends BasePrimitive', `[LLM] Rating scale preset: ${presetIds.join(', ')} (default: 1-5-quality)`) .option('--lambda-arn ', '[Code-based] Existing Lambda function ARN (external)') .option('--timeout ', '[Code-based] Lambda timeout in seconds, 1-300 (default: 60)') - .option('--from-3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) + .option('--3p-library ', `Third-party evaluation library (${SUPPORTED_LIBRARIES.join(', ')})`) .option('--metric ', '[3P library] Metric/evaluator class name (e.g. AnswerRelevancyMetric)') - .option('--parameters ', '[3P library] JSON string of metric constructor kwargs') + .option( + '--param ', + '[3P library] Metric parameter as key=value (repeatable)', + (val: string, prev: string[]) => [...prev, val], + [] as string[] + ) + .option('--parameters-file ', '[3P library] JSON file of metric constructor kwargs') .option('--memory ', '[3P library] Lambda memory size in MB, 128-10240') .option( '--config ', @@ -339,9 +365,10 @@ export class EvaluatorPrimitive extends BasePrimitive 0 && !threePLibrary) { + fail('--param requires --3p-library'); } - if (cliOptions.metric && !from3pLibrary) { - fail('--metric requires --from-3p-library'); + if (cliOptions.parametersFile && !threePLibrary) { + fail('--parameters-file requires --3p-library'); } - if (cliOptions.parameters && !from3pLibrary) { - fail('--parameters requires --from-3p-library'); + if (cliOptions.param.length > 0 && cliOptions.parametersFile) { + fail('--param and --parameters-file cannot be used together'); } - if (cliOptions.memory && !from3pLibrary) { - fail('--memory requires --from-3p-library'); + if (cliOptions.memory && !threePLibrary) { + fail('--memory requires --3p-library'); } if (cliOptions.memory) { const memVal = parseInt(cliOptions.memory, 10); @@ -397,8 +430,8 @@ export class EvaluatorPrimitive extends BasePrimitive 0) { + try { + kwargs = parseParamFlags(cliOptions.param); + } catch (e) { + fail(`Invalid --param value: ${getErrorMessage(e)}`); + } + } else if (cliOptions.parametersFile) { + if (!existsSync(cliOptions.parametersFile)) { + fail(`--parameters-file not found: ${cliOptions.parametersFile}`); + } try { - kwargs = jsonToKwargs(cliOptions.parameters); - } catch { - fail('--parameters must be a valid JSON object (e.g. \'{"threshold": 0.7}\')'); + const fileContent = readFileSync(cliOptions.parametersFile, 'utf-8'); + kwargs = jsonToKwargs(fileContent); + } catch (e) { + fail(`Invalid --parameters-file: ${getErrorMessage(e)}`); } } thirdParty = { - library: from3pLibrary, + library: threePLibrary, metricClass: cliOptions.metric!, metricParams: kwargs, }; } else if (cliOptions.config) { - const { readFileSync } = await import('fs'); configJson = JSON.parse(readFileSync(cliOptions.config, 'utf-8')) as EvaluatorConfig; } else if (evalType === 'code-based') { configJson = this.buildCodeBasedConfig(cliOptions.name!, cliOptions.lambdaArn, cliOptions.timeout); diff --git a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts index 33d1d19ba..cc2808ac8 100644 --- a/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts +++ b/src/cli/primitives/__tests__/EvaluatorPrimitive.test.ts @@ -4,6 +4,7 @@ import { THIRD_PARTY_EVALUATOR_LIBRARIES, jsonToKwargs, jsonToPythonValue, + parseParamFlags, } from '../EvaluatorPrimitive.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -496,7 +497,9 @@ describe('THIRD_PARTY_EVALUATOR_LIBRARIES registry', () => { it('deepeval has warnings for expected_output metrics', () => { const config = THIRD_PARTY_EVALUATOR_LIBRARIES.deepeval; - const expectedWarning = config.warnings.find(w => w.metrics.has('ContextualPrecisionMetric')); + const expectedWarning = config.warnings.find( + w => w.metrics.has('ContextualPrecisionMetric') && w.message.includes('expected_output') + ); expect(expectedWarning).toBeDefined(); expect(expectedWarning!.message).toContain('expected_output'); }); @@ -609,3 +612,41 @@ describe('jsonToKwargs', () => { expect(jsonToKwargs('{}')).toBe(''); }); }); + +describe('parseParamFlags', () => { + it('parses number value', () => { + expect(parseParamFlags(['threshold=0.7'])).toBe('threshold=0.7'); + }); + + it('parses string value (JSON-quoted)', () => { + expect(parseParamFlags(['model="gpt-4"'])).toBe('model="gpt-4"'); + }); + + it('parses boolean value', () => { + expect(parseParamFlags(['verbose=true'])).toBe('verbose=True'); + }); + + it('parses array value', () => { + expect(parseParamFlags(['items=[1,2,3]'])).toBe('items=[1, 2, 3]'); + }); + + it('treats unquoted non-JSON string as string', () => { + expect(parseParamFlags(['name=hello world'])).toBe('name="hello world"'); + }); + + it('parses multiple params', () => { + expect(parseParamFlags(['threshold=0.7', 'verbose=true'])).toBe('threshold=0.7, verbose=True'); + }); + + it('throws on missing equals sign', () => { + expect(() => parseParamFlags(['noequalssign'])).toThrow('not in key=value format'); + }); + + it('handles value containing equals sign', () => { + expect(parseParamFlags(['formula=a=b'])).toBe('formula="a=b"'); + }); + + it('parses null value', () => { + expect(parseParamFlags(['callback=null'])).toBe('callback=None'); + }); +}); From 0bff7a8f7b35006b2982ead764f84aedc6c4084f Mon Sep 17 00:00:00 2001 From: Haomiao Shi Date: Thu, 16 Jul 2026 15:55:25 -0700 Subject: [PATCH 4/4] feat: Add 3P evaluator E2E test + fix autoevals template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add e2e-tests/third-party-eval-lifecycle.test.ts: full lifecycle test for DeepEval + Autoevals 3P evaluators (create → add → deploy → invoke → run eval) - Fix autoevals template: params go to AutoevalsAdapter (not metric constructor) - Add openai>=1.0.0 to autoevals pyproject.toml (required by LLM-based scorers) --- e2e-tests/third-party-eval-lifecycle.test.ts | 203 ++++++++++++++++++ .../autoevals-lambda/lambda_function.py | 2 +- .../autoevals-lambda/pyproject.toml | 1 + 3 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 e2e-tests/third-party-eval-lifecycle.test.ts diff --git a/e2e-tests/third-party-eval-lifecycle.test.ts b/e2e-tests/third-party-eval-lifecycle.test.ts new file mode 100644 index 000000000..354dd4fca --- /dev/null +++ b/e2e-tests/third-party-eval-lifecycle.test.ts @@ -0,0 +1,203 @@ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: third-party evaluator lifecycle (DeepEval + Autoevals)', () => { + let testDir: string; + let projectPath: string; + const agentName = `E2e3pEval${String(Date.now()).slice(-8)}`; + const deepevalEvalName = 'answer_relevancy'; + const autoevalsEvalName = 'exact_match'; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-3p-eval-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + it.skipIf(!canRun)( + 'adds a DeepEval 3P evaluator with --3p-library and --param', + async () => { + const result = await run([ + 'add', + 'evaluator', + '--name', + deepevalEvalName, + '--level', + 'TRACE', + '--3p-library', + 'deepeval', + '--metric', + 'AnswerRelevancyMetric', + '--param', + 'threshold=0.5', + '--json', + ]); + expect(result.exitCode, `Add DeepEval evaluator failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; evaluatorName: string; codePath: string }; + expect(json.success).toBe(true); + expect(json.evaluatorName).toBe(deepevalEvalName); + expect(json.codePath).toContain(deepevalEvalName); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds an Autoevals 3P evaluator with --3p-library', + async () => { + const result = await run([ + 'add', + 'evaluator', + '--name', + autoevalsEvalName, + '--level', + 'TRACE', + '--3p-library', + 'autoevals', + '--metric', + 'ExactMatch', + '--json', + ]); + expect(result.exitCode, `Add Autoevals evaluator failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; evaluatorName: string; codePath: string }; + expect(json.success).toBe(true); + expect(json.evaluatorName).toBe(autoevalsEvalName); + expect(json.codePath).toContain(autoevalsEvalName); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'deploys agent with 3P evaluators', + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + expect(result.exitCode, 'Deploy failed').toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed agent to generate traces', + async () => { + await retry( + async () => { + const result = await run(['invoke', '--prompt', 'What is 2+2?', '--runtime', agentName, '--json']); + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'runs on-demand evaluation with DeepEval 3P evaluator', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'eval', + '--runtime', + agentName, + '--evaluator', + deepevalEvalName, + '--days', + '1', + '--json', + ]); + expect(result.exitCode, `Run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('run'); + }, + 18, + 10000 + ); + }, + 300000 + ); + + it.skipIf(!canRun)( + 'runs on-demand evaluation with Autoevals 3P evaluator', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'eval', + '--runtime', + agentName, + '--evaluator', + autoevalsEvalName, + '--days', + '1', + '--json', + ]); + expect(result.exitCode, `Run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('run'); + }, + 18, + 10000 + ); + }, + 300000 + ); +}); diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py index 3fa199389..215302cbd 100644 --- a/src/assets/evaluators/autoevals-lambda/lambda_function.py +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -7,7 +7,7 @@ ) from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoevalsAdapter -adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +adapter = AutoevalsAdapter(metric={{ EvaluatorClass }}(), {{{ EvaluatorParams }}}) @custom_code_based_evaluator() diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml index bd1fa659f..d117c3786 100644 --- a/src/assets/evaluators/autoevals-lambda/pyproject.toml +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.10" dependencies = [ "bedrock-agentcore[autoevals]", "autoevals>=0.0.80", + "openai>=1.0.0", ] [tool.hatch.build.targets.wheel]