diff --git a/src/assets/evaluators/autoevals-lambda/execution-role-policy.json b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/autoevals-lambda/lambda_function.py b/src/assets/evaluators/autoevals-lambda/lambda_function.py new file mode 100644 index 000000000..4161dbbf7 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/lambda_function.py @@ -0,0 +1,37 @@ +{{#if ModelProviderBedrock}} +import os + +# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION. +os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2")) + +from autoevals import {{ EvaluatorClass }}, init +from autoevals.litellm import LiteLLMClient + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +client = LiteLLMClient() +init(client=client, default_model="bedrock/{{ Model }}") + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{else}} +from autoevals import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter + +adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/autoevals-lambda/pyproject.toml b/src/assets/evaluators/autoevals-lambda/pyproject.toml new file mode 100644 index 000000000..b37442fd1 --- /dev/null +++ b/src/assets/evaluators/autoevals-lambda/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (Autoevals)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[autoevals]", + "autoevals>=0.0.80,<1.0.0", +{{#if ModelProviderBedrock}} + # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge + "litellm>=1.60,<1.85", +{{else}} + "openai>=1.0.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/deepeval-lambda/execution-role-policy.json b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json new file mode 100644 index 000000000..6b47af830 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/execution-role-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + }, + { + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": "*" + } + ] +} diff --git a/src/assets/evaluators/deepeval-lambda/lambda_function.py b/src/assets/evaluators/deepeval-lambda/lambda_function.py new file mode 100644 index 000000000..a89b22051 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/lambda_function.py @@ -0,0 +1,29 @@ +import os + +os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval") +os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES") +os.chdir("/tmp") + +{{#if ModelProviderBedrock}} +from deepeval.models import AmazonBedrockModel +{{/if}} +from deepeval.metrics import {{ EvaluatorClass }} + +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + EvaluatorInput, + EvaluatorOutput, + custom_code_based_evaluator, +) +from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter + +{{#if ModelProviderBedrock}} +model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2")) +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})) +{{else}} +adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}})) +{{/if}} + + +@custom_code_based_evaluator() +def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput: + return adapter(evaluator_input, context) diff --git a/src/assets/evaluators/deepeval-lambda/pyproject.toml b/src/assets/evaluators/deepeval-lambda/pyproject.toml new file mode 100644 index 000000000..7385ccfc2 --- /dev/null +++ b/src/assets/evaluators/deepeval-lambda/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator (DeepEval)" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore[deepeval]", + "deepeval>=2.0.0,<3.0.0", +{{#if ModelProviderBedrock}} + "aiobotocore>=2.13.0", +{{/if}} +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/assets/evaluators/python-lambda/execution-role-policy.json b/src/assets/evaluators/python-lambda/execution-role-policy.json new file mode 100644 index 000000000..b3b98be42 --- /dev/null +++ b/src/assets/evaluators/python-lambda/execution-role-policy.json @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*" + } + ] +} diff --git a/src/assets/evaluators/python-lambda/lambda_function.py b/src/assets/evaluators/python-lambda/lambda_function.py new file mode 100644 index 000000000..0f8bd5c6b --- /dev/null +++ b/src/assets/evaluators/python-lambda/lambda_function.py @@ -0,0 +1,19 @@ +from bedrock_agentcore.evaluation.custom_code_based_evaluators import ( + custom_code_based_evaluator, + EvaluatorInput, + EvaluatorOutput, +) + + +@custom_code_based_evaluator() +def handler(input: EvaluatorInput, context) -> EvaluatorOutput: + """Evaluate agent behavior with custom logic. + + Args: + input: Contains evaluation_level, session_spans, target_trace_id, target_span_id + + Returns: + EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure. + """ + # TODO: Replace with your evaluation logic + return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed") diff --git a/src/assets/evaluators/python-lambda/pyproject.toml b/src/assets/evaluators/python-lambda/pyproject.toml new file mode 100644 index 000000000..69ad99b43 --- /dev/null +++ b/src/assets/evaluators/python-lambda/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "{{ Name }}" +version = "0.1.0" +description = "AgentCore Code-Based Evaluator" +requires-python = ">=3.10" +dependencies = [ + "bedrock-agentcore>=1.6.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] diff --git a/src/core/project/fsUtils.ts b/src/core/project/fsUtils.ts index a9e6a4a15..714205349 100644 --- a/src/core/project/fsUtils.ts +++ b/src/core/project/fsUtils.ts @@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined { } } } + +export function toPythonPackageName(name: string): string { + return name + .replace(/[^a-zA-Z0-9._-]/g, "-") + .replace(/^[^a-zA-Z0-9]+/, "") + .replace(/[^a-zA-Z0-9]+$/, ""); +} diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 45b99d4d6..f32b75c31 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -42,6 +42,7 @@ import { } from "./templates/export"; import { HarnessSpecSchema } from "../../projectSchemas/harness"; import { FsTreeNode } from "./templates/fsTree"; +import { getEvaluatorTemplateResolver } from "./templates/evaluator"; import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project"; import { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { @@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager { break; } case "evaluator": { - projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); + if (input.scaffold) { + yield { message: "Scaffolding evaluator in project" }; + const outputPath = join(project.rootPath, "app", input.scaffold.name); + if (existsSync(outputPath)) + throw new InputValidationError( + `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`, + ); + scaffoldedPaths.push(outputPath); + const result = await getEvaluatorTemplateResolver({ + assetSource: this.assetSource, + templateRenderer: this.templateRenderer, + }).resolve(input.scaffold); + await result.tree.write(dirname(outputPath)); + projectSpec.evaluators.push(...(result.spec.evaluators ?? [])); + } else { + projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig)); + } break; } case "gateway": diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts new file mode 100644 index 000000000..0940a2d64 --- /dev/null +++ b/src/core/project/templates/evaluator.ts @@ -0,0 +1,89 @@ +import { FsTreeNode } from "./fsTree"; +import type { AssetSource } from "../source"; +import type { Evaluator, EvaluationLevel } from "../../../projectSchemas/evaluator"; +import type { TemplateRenderer, TemplateResolver } from "./types"; +import { toPythonPackageName } from "../fsUtils"; + +const DEFAULT_TIMEOUT = 60; + +export const EVALUATOR_LIBRARIES = { + deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 }, + autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT }, +} as const; + +export type EvaluatorLibrary = keyof typeof EVALUATOR_LIBRARIES; + +const EMPTY_ASSET_DIR = "evaluators/python-lambda"; + +export type ManagedEvaluatorScaffoldInput = { + name: string; + level: EvaluationLevel; + description?: string; + kmsKeyArn?: string; + tags?: Record; + metric?: { library: EvaluatorLibrary; metricClass: string }; + model?: string; + timeoutSeconds?: number; +}; + +function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator { + const timeoutSeconds = + input.timeoutSeconds ?? + (input.metric + ? EVALUATOR_LIBRARIES[input.metric.library].defaultTimeoutSeconds + : DEFAULT_TIMEOUT); + return { + name: input.name, + level: input.level, + ...(input.description && { description: input.description }), + config: { + codeBased: { + managed: { + codeLocation: `app/${input.name}`, + entrypoint: "lambda_function.handler", + timeoutSeconds, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }), + ...(input.tags && { tags: input.tags }), + }; +} + +function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record { + const context: Record = { Name: toPythonPackageName(input.name) }; + if (input.metric) { + context["EvaluatorClass"] = input.metric.metricClass; + context["Model"] = input.model ?? ""; + context["ModelProviderBedrock"] = input.model !== undefined; + context["EvaluatorParams"] = ""; + } + return context; +} + +type GetEvaluatorTemplateResolverConfig = { + assetSource: AssetSource; + templateRenderer: TemplateRenderer; +}; + +export function getEvaluatorTemplateResolver( + config: GetEvaluatorTemplateResolverConfig, +): TemplateResolver { + return { + async resolve(input) { + const assetDir = input.metric + ? EVALUATOR_LIBRARIES[input.metric.library].assetDir + : EMPTY_ASSET_DIR; + const tree = await FsTreeNode.fromAssetSource( + { assetSource: config.assetSource }, + { assetDir }, + { + rootDirName: input.name, + transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)), + }, + ); + return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } }; + }, + }; +} diff --git a/src/core/project/templates/export.ts b/src/core/project/templates/export.ts index 5cc289010..f83d89302 100644 --- a/src/core/project/templates/export.ts +++ b/src/core/project/templates/export.ts @@ -16,7 +16,7 @@ import { credentialEnvVarName, type Credential } from "../../../projectSchemas/c import type { Memory } from "../../../projectSchemas/memory"; import type { EnvLocalEntry } from "../../../handlers/project/types"; import { InputValidationError } from "../../../errors/errors"; -import { toPythonPackageName } from "./runtime"; +import { toPythonPackageName } from "../fsUtils"; type ProjectSpec = z.infer; diff --git a/src/core/project/templates/runtime.ts b/src/core/project/templates/runtime.ts index 10ab9cd7d..add21b561 100644 --- a/src/core/project/templates/runtime.ts +++ b/src/core/project/templates/runtime.ts @@ -5,6 +5,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime"; import type { TemplateRenderer, TemplateResolver } from "./types"; import type { ScaffoldRuntimeInput } from "../../../handlers/project/types"; import { InputValidationError } from "../../../errors"; +import { toPythonPackageName } from "../fsUtils"; function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { const { scaffoldRuntimeInput, name, ...infra } = input; @@ -38,18 +39,6 @@ function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { }; } -/** - * Normalize a name for use as a Python package name per PEP 508. - * Valid names consist only of ASCII letters, numbers, period, underscore, and - * hyphen, and must start and end with a letter or number. - */ -export function toPythonPackageName(name: string): string { - return name - .replace(/[^a-zA-Z0-9._-]/g, "-") - .replace(/^[^a-zA-Z0-9]+/, "") - .replace(/[^a-zA-Z0-9]+$/, ""); -} - /** * Normalize a name for use as an npm package name. * diff --git a/src/core/project/templates/types.ts b/src/core/project/templates/types.ts index f472be503..798e4ba4b 100644 --- a/src/core/project/templates/types.ts +++ b/src/core/project/templates/types.ts @@ -3,6 +3,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime"; import type { MemorySchema } from "../../../projectSchemas/memory"; import type { CredentialSchema } from "../../../projectSchemas/credential"; import type { HarnessRegistryEntry } from "../../../projectSchemas/harness"; +import type { Evaluator } from "../../../projectSchemas/evaluator"; import type z from "zod"; /** AgentCore Project Spec Entries that rendered as part of a {@link Template} **/ @@ -11,6 +12,7 @@ export type SpecEntries = { credentials?: z.infer[]; memories?: z.infer[]; harnesses?: HarnessRegistryEntry[]; + evaluators?: Evaluator[]; }; /** A group of files and resources that can be rendered into a project **/ diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts new file mode 100644 index 000000000..08faac121 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -0,0 +1,349 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createRootHandler } from "../../../../index"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../../testing"; +import { InputValidationError } from "../../../../../errors"; + +const originalCwd = process.cwd(); +const tempDirectories: string[] = []; + +async function inTempDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "agentcore-code-eval-")); + tempDirectories.push(directory); + process.chdir(directory); + return process.cwd(); +} + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all( + tempDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +async function run(args: string[]) { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + globalConfigAccessor: new TestGlobalConfigAccessor(), + logger: createSilentLogger(), + }); + await root.route(["node", "agentcore", "project", ...args]); + return { io }; +} + +async function inProject(name = "TestProject"): Promise { + const directory = await inTempDirectory(); + await run(["create", "--name", name, "--skip-install", "--skip-git"]); + const projectRoot = join(directory, name); + process.chdir(projectRoot); + return projectRoot; +} + +const spec = (projectRoot: string) => + Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); +const evaluator = async (projectRoot: string, name: string) => + ((await spec(projectRoot)).evaluators ?? []).find((e: { name: string }) => e.name === name); + +describe("project add evaluator code-based", () => { + test("3P metric → managed config + scaffolded, rendered Lambda source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "answer_faithfulness", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + ]); + + expect(await evaluator(projectRoot, "answer_faithfulness")).toMatchObject({ + name: "answer_faithfulness", + level: "SESSION", + config: { + codeBased: { + managed: { + codeLocation: "app/answer_faithfulness", + entrypoint: "lambda_function.handler", + timeoutSeconds: 300, + additionalPolicies: ["execution-role-policy.json"], + }, + }, + }, + }); + + const appDir = join(projectRoot, "app", "answer_faithfulness"); + const handler = await Bun.file(join(appDir, "lambda_function.py")).text(); + expect(handler).toContain("FaithfulnessMetric"); + expect(handler).toContain("AmazonBedrockModel"); + expect(handler).toContain("anthropic.claude-3-5-sonnet-20240620-v1:0"); + expect(await Bun.file(join(appDir, "execution-role-policy.json")).exists()).toBe(true); + expect(handler).not.toContain("{{"); + }); + + test("autoevals metric with default (non-bedrock) model", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "factuality", + "--level", + "TRACE", + "--metric", + "autoevals.Factuality", + ]); + + expect( + (await evaluator(projectRoot, "factuality")).config.codeBased.managed.timeoutSeconds, + ).toBe(60); + const handler = await Bun.file( + join(projectRoot, "app", "factuality", "lambda_function.py"), + ).text(); + expect(handler).toContain("Factuality"); + expect(handler).not.toContain("{{"); + }); + + test("no metric, no lambda → empty managed stub", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "custom_eval", "--level", "TOOL_CALL"]); + + expect((await evaluator(projectRoot, "custom_eval")).config.codeBased.managed).toMatchObject({ + codeLocation: "app/custom_eval", + timeoutSeconds: 60, + }); + const handler = await Bun.file( + join(projectRoot, "app", "custom_eval", "lambda_function.py"), + ).text(); + expect(handler).toContain("TODO"); + expect(handler).toContain("custom_code_based_evaluator"); + }); + + test("--lambda-arn → external config, no scaffold", async () => { + const projectRoot = await inProject(); + const arn = "arn:aws:lambda:us-west-2:123456789012:function:refund-policy"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "refund_policy", + "--level", + "SESSION", + "--lambda-arn", + arn, + ]); + + expect((await evaluator(projectRoot, "refund_policy")).config).toEqual({ + codeBased: { external: { lambdaArn: arn } }, + }); + expect( + await Bun.file(join(projectRoot, "app", "refund_policy", "lambda_function.py")).exists(), + ).toBe(false); + }); + + test("persists description, kms key, and tags", async () => { + const projectRoot = await inProject(); + const kms = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"; + await run([ + "add", + "evaluator", + "code-based", + "--name", + "full", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--description", + "external scorer", + "--kms-key-arn", + kms, + "--tags", + '{"team":"ml"}', + ]); + + expect(await evaluator(projectRoot, "full")).toMatchObject({ + description: "external scorer", + kmsKeyArn: kms, + tags: { team: "ml" }, + }); + }); + + test.each<[string, string[]]>([ + ["missing --name", ["--level", "SESSION"]], + ["missing --level", ["--name", "x"]], + [ + "--metric and --lambda-arn together", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ], + ], + [ + "unknown metric library", + ["--name", "x", "--level", "SESSION", "--metric", "ragas.Faithfulness"], + ], + ["metric without a class", ["--name", "x", "--level", "SESSION", "--metric", "deepeval"]], + [ + "namespaced (multi-dot) metric class", + ["--name", "x", "--level", "SESSION", "--metric", "deepeval.metrics.Faithfulness"], + ], + [ + "non-Bedrock --model", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "gpt-4o", + ], + ], + [ + "--model bedrock with no slash/id", + [ + "--name", + "x", + "--level", + "SESSION", + "--metric", + "autoevals.Factuality", + "--model", + "bedrock", + ], + ], + ["--model without --metric", ["--name", "x", "--level", "SESSION", "--model", "bedrock/foo"]], + [ + "managed flag with --lambda-arn", + [ + "--name", + "x", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + "--timeout-seconds", + "30", + ], + ], + ["invalid --level", ["--name", "x", "--level", "NOPE"]], + ["invalid --lambda-arn", ["--name", "x", "--level", "SESSION", "--lambda-arn", "not-an-arn"]], + ])("%s", async (_label, flags) => { + await inProject(); + await expect(run(["add", "evaluator", "code-based", ...flags])).rejects.toBeInstanceOf( + InputValidationError, + ); + }); + + test("accepts a bare Bedrock inference-profile model id and renders it into the source", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "code-based", + "--name", + "prof", + "--level", + "SESSION", + "--metric", + "deepeval.FaithfulnessMetric", + "--model", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + ]); + expect(await evaluator(projectRoot, "prof")).toBeDefined(); + const src = await Bun.file(join(projectRoot, "app", "prof", "lambda_function.py")).text(); + expect(src).toContain("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + }); + + test("rejects a duplicate evaluator name", async () => { + await inProject(); + const flags = [ + "add", + "evaluator", + "code-based", + "--name", + "dup", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]; + await run(flags); + await expect(run(flags)).rejects.toBeInstanceOf(InputValidationError); + }); + + test("errors before writing when app/ already exists (cross-resource collision)", async () => { + const projectRoot = await inProject(); + const appDir = join(projectRoot, "app", "collide"); + await mkdir(appDir, { recursive: true }); + await Bun.write(join(appDir, "pyproject.toml"), "# pre-existing\n"); + + await expect( + run(["add", "evaluator", "code-based", "--name", "collide", "--level", "SESSION"]), + ).rejects.toBeInstanceOf(InputValidationError); + + expect(await evaluator(projectRoot, "collide")).toBeUndefined(); + expect(await Bun.file(join(appDir, "pyproject.toml")).text()).toBe("# pre-existing\n"); + expect(await Bun.file(join(appDir, "lambda_function.py")).exists()).toBe(false); + }); + + test("empty stub warns it returns Pass until implemented", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "stub", + "--level", + "SESSION", + ]); + expect(io.stderr()).toContain("returns Pass for every session"); + }); + + test("external mode prints no stub note", async () => { + await inProject(); + const { io } = await run([ + "add", + "evaluator", + "code-based", + "--name", + "ext", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:f", + ]); + expect(io.stderr()).not.toContain("returns Pass for every session"); + }); + + test("remove evaluator drops it from the spec", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "gone", "--level", "SESSION"]); + expect(await evaluator(projectRoot, "gone")).toBeDefined(); + await run(["remove", "evaluator", "--name", "gone"]); + expect(await evaluator(projectRoot, "gone")).toBeUndefined(); + }); +}); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts new file mode 100644 index 000000000..58e02b7f3 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -0,0 +1,144 @@ +import z from "zod"; +import { createHandler, flag, ProjectKey } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { + EvaluatorSchema, + EvaluationLevelSchema, + isValidBedrockModelId, +} from "../../../../../projectSchemas/evaluator"; +import { TagsSchema } from "../../../../../projectSchemas/tags"; +import { + EVALUATOR_LIBRARIES, + type EvaluatorLibrary, + type ManagedEvaluatorScaffoldInput, +} from "../../../../../core/project/templates/evaluator"; +import { parseJsonFlagWithSchema } from "../../../../utils"; +import type { AddProjectResourceConfig } from "../../types"; + +export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => + createHandler({ + name: "code-based", + description: + "add a code-based evaluator — a Lambda that scores a session. Pass a 3P metric, an existing Lambda, or neither to scaffold an empty evaluator you fill in", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().optional()), + flag( + "metric", + "3P metric to scaffold as , e.g. deepeval.FaithfulnessMetric or autoevals.Factuality", + z.string().optional(), + ), + flag( + "model", + "judge model for the 3P metric, e.g. bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + z.string().optional(), + ), + flag("lambda-arn", "ARN of an existing Lambda that scores a session", z.string().optional()), + flag( + "timeout-seconds", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), + flag("description", "a description of what this evaluator measures", z.string().optional()), + flag( + "kms-key-arn", + "customer-managed KMS key ARN to encrypt the evaluator", + z.string().optional(), + ), + flag("tags", "tags to apply (JSON object of key/value strings)", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + const levelParsed = EvaluationLevelSchema.safeParse(flags["level"]); + if (!levelParsed.success) throw new InputValidationError(z.prettifyError(levelParsed.error)); + const level = levelParsed.data; + + const hasMetric = flags["metric"] !== undefined; + const hasLambda = flags["lambda-arn"] !== undefined; + if (hasMetric && hasLambda) + throw new InputValidationError( + "provide either --metric (managed) or --lambda-arn (external), not both", + ); + + const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); + const base = { + name: flags["name"], + level, + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags, + }; + const project = ctx.require(ProjectKey); + + if (hasLambda) { + if (flags["metric"] || flags["model"] || flags["timeout-seconds"] !== undefined) + throw new InputValidationError( + "--metric, --model, and --timeout-seconds are managed-only and not valid with --lambda-arn", + ); + const parsed = EvaluatorSchema.safeParse({ + ...base, + config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, + }); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: parsed.data, + })) { + config.io.stderr.write(`${event.message}\n`); + } + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + return; + } + + if (flags["model"] && !hasMetric) throw new InputValidationError("--model requires --metric"); + + const scaffold: ManagedEvaluatorScaffoldInput = { + ...base, + ...(hasMetric && { metric: parseMetric(flags["metric"]!) }), + ...(flags["model"] !== undefined && { model: resolveBedrockModel(flags["model"]) }), + ...(flags["timeout-seconds"] !== undefined && { timeoutSeconds: flags["timeout-seconds"] }), + }; + + for await (const event of config.projectManager.addResource(project, { + resourceType: "evaluator", + resourceConfig: { name: scaffold.name }, + scaffold, + })) { + config.io.stderr.write(`${event.message}\n`); + } + + config.io.stderr.write(`added evaluator '${flags["name"]}' to '${project.name}'\n`); + if (!hasMetric) + config.io.stderr.write( + `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py\n`, + ); + }, + }); + +function parseMetric(raw: string): { library: EvaluatorLibrary; metricClass: string } { + const dot = raw.indexOf("."); + const library = dot > 0 ? raw.slice(0, dot) : ""; + const metricClass = dot > 0 ? raw.slice(dot + 1) : ""; + if (!(library in EVALUATOR_LIBRARIES)) + throw new InputValidationError( + `invalid --metric "${raw}": expected where library is one of ${Object.keys(EVALUATOR_LIBRARIES).join(", ")} (e.g. deepeval.FaithfulnessMetric)`, + ); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(metricClass)) + throw new InputValidationError( + `invalid metric class "${metricClass}" in --metric "${raw}": expected a single class name like FaithfulnessMetric`, + ); + return { library: library as EvaluatorLibrary, metricClass }; +} + +function resolveBedrockModel(model: string | undefined): string | undefined { + if (!model) return undefined; + const id = model.startsWith("bedrock/") ? model.slice("bedrock/".length) : model; + if (!isValidBedrockModelId(id)) + throw new InputValidationError( + `invalid --model "${model}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN, optionally prefixed with "bedrock/"`, + ); + return id; +} diff --git a/src/handlers/project/add/evaluator/index.ts b/src/handlers/project/add/evaluator/index.ts index 5dbc413ab..1c05ea3be 100644 --- a/src/handlers/project/add/evaluator/index.ts +++ b/src/handlers/project/add/evaluator/index.ts @@ -1,9 +1,11 @@ import { Router } from "../../../../router"; import type { AddProjectResourceConfig } from "../types"; import { createAddLlmAsAJudgeEvaluatorHandler } from "./llm-as-a-judge"; +import { createAddCodeBasedEvaluatorHandler } from "./code-based"; export function createAddEvaluatorHandler(config: AddProjectResourceConfig): Router { const evaluator = new Router("evaluator", "add a custom evaluator to the current project"); evaluator.handler(createAddLlmAsAJudgeEvaluatorHandler(config)); + evaluator.handler(createAddCodeBasedEvaluatorHandler(config)); return evaluator; } diff --git a/src/handlers/project/remove/index.ts b/src/handlers/project/remove/index.ts index 40bfc132a..5c0a49f79 100644 --- a/src/handlers/project/remove/index.ts +++ b/src/handlers/project/remove/index.ts @@ -44,6 +44,7 @@ export const createRemoveProjectHandler = (config: RemoveProjectResourceConfig) "online-eval", "online-insight", "memory", + "evaluator", "gateway", "gateway-target", "gateway-connector", diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index 4e142deb5..fb255068b 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -6,6 +6,7 @@ import type { PaymentConnectorSchema, PaymentManagerSchema } from "../../project import type { ConfigBundleSchema } from "../../projectSchemas/config-bundle"; import { MemorySchema } from "../../projectSchemas/memory"; import type { EvaluatorSchema } from "../../projectSchemas/evaluator"; +import type { ManagedEvaluatorScaffoldInput } from "../../core/project/templates/evaluator"; import type { ProjectSpecSchema } from "../../projectSchemas/project"; import z from "zod"; import type { ImportBedrockAgentInput, RuntimeResourceConfig } from "./add/runtime/types"; @@ -210,6 +211,12 @@ export type AddResourceInput = | { resourceType: "evaluator"; resourceConfig: z.input; + scaffold?: undefined; + } + | { + resourceType: "evaluator"; + resourceConfig: { name: string }; + scaffold: ManagedEvaluatorScaffoldInput; } | { resourceType: "gateway"; @@ -281,6 +288,7 @@ export type RemoveResourceInput = | "online-eval" | "online-insight" | "memory" + | "evaluator" | "gateway" | "policy-engine" | "payment-manager";