feat(project): add project add evaluator code-based

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore[autoevals]",
+    "autoevals>=0.0.80,<1.0.0",
+{{#if ModelProviderBedrock}}
+    # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+    "litellm>=1.60,<1.85",
+{{else}}
+    "openai>=1.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore[deepeval]",
+    "deepeval>=2.0.0,<3.0.0",
+{{#if ModelProviderBedrock}}
+    "aiobotocore>=2.13.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    }
+  ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    custom_code_based_evaluator,
+    EvaluatorInput,
+    EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+    """Evaluate agent behavior with custom logic.
+
+    Args:
+        input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+    Returns:
+        EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+    """
+    # TODO: Replace with your evaluation logic
+    return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore>=1.6.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/project/fsUtils.ts

@@ -20,3 +20,10 @@ export function enclosingProjectRoot(directory: string): string | undefined {
     }
   }
 }
+
+export function toPythonPackageName(name: string): string {
+  return name
+    .replace(/[^a-zA-Z0-9._-]/g, "-")
+    .replace(/^[^a-zA-Z0-9]+/, "")
+    .replace(/[^a-zA-Z0-9]+$/, "");
+}

src/core/project/manager.tsx

@@ -42,6 +42,7 @@ import {
 } from "./templates/export";
 import { HarnessSpecSchema } from "../../projectSchemas/harness";
 import { FsTreeNode } from "./templates/fsTree";
+import { getEvaluatorTemplateResolver } from "./templates/evaluator";
 import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project";
 import { ConfigBundleSchema } from "../../projectSchemas/config-bundle";
 import {
@@ -336,7 +337,23 @@ export class FsProjectManager implements ProjectManager {
         break;
       }
       case "evaluator": {
-        projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+        if (input.scaffold) {
+          yield { message: "Scaffolding evaluator in project" };
+          const outputPath = join(project.rootPath, "app", input.scaffold.name);
+          if (existsSync(outputPath))
+            throw new InputValidationError(
+              `cannot scaffold evaluator '${input.scaffold.name}': 'app/${input.scaffold.name}' already exists (another resource may use this name, or a previous scaffold was left behind)`,
+            );
+          scaffoldedPaths.push(outputPath);
+          const result = await getEvaluatorTemplateResolver({
+            assetSource: this.assetSource,
+            templateRenderer: this.templateRenderer,
+          }).resolve(input.scaffold);
+          await result.tree.write(dirname(outputPath));
+          projectSpec.evaluators.push(...(result.spec.evaluators ?? []));
+        } else {
+          projectSpec.evaluators.push(parseResource(EvaluatorSchema, input.resourceConfig));
+        }
         break;
       }
       case "gateway":

src/core/project/templates/evaluator.ts

@@ -0,0 +1,89 @@
+import { FsTreeNode } from "./fsTree";
+import type { AssetSource } from "../source";
+import type { Evaluator, EvaluationLevel } from "../../../projectSchemas/evaluator";
+import type { TemplateRenderer, TemplateResolver } from "./types";
+import { toPythonPackageName } from "../fsUtils";
+
+const DEFAULT_TIMEOUT = 60;
+
+export const EVALUATOR_LIBRARIES = {
+  deepeval: { assetDir: "evaluators/deepeval-lambda", defaultTimeoutSeconds: 300 },
+  autoevals: { assetDir: "evaluators/autoevals-lambda", defaultTimeoutSeconds: DEFAULT_TIMEOUT },
+} as const;
+
+export type EvaluatorLibrary = keyof typeof EVALUATOR_LIBRARIES;
+
+const EMPTY_ASSET_DIR = "evaluators/python-lambda";
+
+export type ManagedEvaluatorScaffoldInput = {
+  name: string;
+  level: EvaluationLevel;
+  description?: string;
+  kmsKeyArn?: string;
+  tags?: Record<string, string>;
+  metric?: { library: EvaluatorLibrary; metricClass: string };
+  model?: string;
+  timeoutSeconds?: number;
+};
+
+function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator {
+  const timeoutSeconds =
+    input.timeoutSeconds ??
+    (input.metric
+      ? EVALUATOR_LIBRARIES[input.metric.library].defaultTimeoutSeconds
+      : DEFAULT_TIMEOUT);
+  return {
+    name: input.name,
+    level: input.level,
+    ...(input.description && { description: input.description }),
+    config: {
+      codeBased: {
+        managed: {
+          codeLocation: `app/${input.name}`,
+          entrypoint: "lambda_function.handler",
+          timeoutSeconds,
+          additionalPolicies: ["execution-role-policy.json"],
+        },
+      },
+    },
+    ...(input.kmsKeyArn && { kmsKeyArn: input.kmsKeyArn }),
+    ...(input.tags && { tags: input.tags }),
+  };
+}
+
+function buildRenderContext(input: ManagedEvaluatorScaffoldInput): Record<string, unknown> {
+  const context: Record<string, unknown> = { Name: toPythonPackageName(input.name) };
+  if (input.metric) {
+    context["EvaluatorClass"] = input.metric.metricClass;
+    context["Model"] = input.model ?? "";
+    context["ModelProviderBedrock"] = input.model !== undefined;
+    context["EvaluatorParams"] = "";
+  }
+  return context;
+}
+
+type GetEvaluatorTemplateResolverConfig = {
+  assetSource: AssetSource;
+  templateRenderer: TemplateRenderer;
+};
+
+export function getEvaluatorTemplateResolver(
+  config: GetEvaluatorTemplateResolverConfig,
+): TemplateResolver<ManagedEvaluatorScaffoldInput> {
+  return {
+    async resolve(input) {
+      const assetDir = input.metric
+        ? EVALUATOR_LIBRARIES[input.metric.library].assetDir
+        : EMPTY_ASSET_DIR;
+      const tree = await FsTreeNode.fromAssetSource(
+        { assetSource: config.assetSource },
+        { assetDir },
+        {
+          rootDirName: input.name,
+          transformContent: (raw) => config.templateRenderer.render(raw, buildRenderContext(input)),
+        },
+      );
+      return { tree, spec: { evaluators: [buildManagedEvaluatorSpec(input)] } };
+    },
+  };
+}

src/core/project/templates/export.ts

@@ -16,7 +16,7 @@ import { credentialEnvVarName, type Credential } from "../../../projectSchemas/c
 import type { Memory } from "../../../projectSchemas/memory";
 import type { EnvLocalEntry } from "../../../handlers/project/types";
 import { InputValidationError } from "../../../errors/errors";
-import { toPythonPackageName } from "./runtime";
+import { toPythonPackageName } from "../fsUtils";
 
 type ProjectSpec = z.infer<typeof ProjectSpecSchema>;
 

src/core/project/templates/runtime.ts

@@ -5,6 +5,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime";
 import type { TemplateRenderer, TemplateResolver } from "./types";
 import type { ScaffoldRuntimeInput } from "../../../handlers/project/types";
 import { InputValidationError } from "../../../errors";
+import { toPythonPackageName } from "../fsUtils";
 
 function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime {
   const { scaffoldRuntimeInput, name, ...infra } = input;
@@ -38,18 +39,6 @@ function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime {
   };
 }
 
-/**
- * Normalize a name for use as a Python package name per PEP 508.
- * Valid names consist only of ASCII letters, numbers, period, underscore, and
- * hyphen, and must start and end with a letter or number.
- */
-export function toPythonPackageName(name: string): string {
-  return name
-    .replace(/[^a-zA-Z0-9._-]/g, "-")
-    .replace(/^[^a-zA-Z0-9]+/, "")
-    .replace(/[^a-zA-Z0-9]+$/, "");
-}
-
 /**
  * Normalize a name for use as an npm package name.
  *

src/core/project/templates/types.ts

@@ -3,6 +3,7 @@ import type { ProjectRuntime } from "../../../projectSchemas/runtime";
 import type { MemorySchema } from "../../../projectSchemas/memory";
 import type { CredentialSchema } from "../../../projectSchemas/credential";
 import type { HarnessRegistryEntry } from "../../../projectSchemas/harness";
+import type { Evaluator } from "../../../projectSchemas/evaluator";
 import type z from "zod";
 
 /** AgentCore Project Spec Entries that rendered as part of a {@link Template} **/
@@ -11,6 +12,7 @@ export type SpecEntries = {
   credentials?: z.infer<typeof CredentialSchema>[];
   memories?: z.infer<typeof MemorySchema>[];
   harnesses?: HarnessRegistryEntry[];
+  evaluators?: Evaluator[];
 };
 
 /** A group of files and resources that can be rendered into a project **/