From 986ce40f9e4d8ca5ea7a4cb04589fd288bcaa97e Mon Sep 17 00:00:00 2001 From: CAholder Date: Mon, 20 Jul 2026 12:18:48 -0700 Subject: [PATCH] Add versioned pipeline scenario contracts --- .../skills/databricks-demo-generator/SKILL.md | 11 +- CLAUDE.md | 12 + evaluation/README.md | 26 ++ evaluation/__init__.py | 9 + evaluation/cases/__init__.py | 53 +++ evaluation/cases/financial-services.yaml | 94 +++++ evaluation/cases/healthcare.yaml | 91 +++++ evaluation/cases/manufacturing-machinery.yaml | 97 ++++++ evaluation/cases/retail.yaml | 91 +++++ evaluation/cli.py | 65 ++++ evaluation/models.py | 112 ++++++ evaluation/schema/scenario.schema.json | 324 ++++++++++++++++++ evaluation/schema_generator.py | 36 ++ pyproject.toml | 29 ++ tests/evaluation/test_cases.py | 72 ++++ tests/pipeline/README.md | 17 +- tests/pipeline/scenarios.py | 152 ++------ uv.lock | 196 +++++++++++ 18 files changed, 1365 insertions(+), 122 deletions(-) create mode 100644 evaluation/README.md create mode 100644 evaluation/__init__.py create mode 100644 evaluation/cases/__init__.py create mode 100644 evaluation/cases/financial-services.yaml create mode 100644 evaluation/cases/healthcare.yaml create mode 100644 evaluation/cases/manufacturing-machinery.yaml create mode 100644 evaluation/cases/retail.yaml create mode 100644 evaluation/cli.py create mode 100644 evaluation/models.py create mode 100644 evaluation/schema/scenario.schema.json create mode 100644 evaluation/schema_generator.py create mode 100644 pyproject.toml create mode 100644 tests/evaluation/test_cases.py create mode 100644 uv.lock diff --git a/.claude/skills/databricks-demo-generator/SKILL.md b/.claude/skills/databricks-demo-generator/SKILL.md index 3b4a8be..4093f65 100644 --- a/.claude/skills/databricks-demo-generator/SKILL.md +++ b/.claude/skills/databricks-demo-generator/SKILL.md @@ -29,7 +29,16 @@ The main loop lives in this file (SKILL.md) — it describes **the flow**: stage ## Paths -Your system prompt defines `PROJECT`, `SKILLS`, `DEMO_SKILL_DIR`, and `DEMO_SKILL` as absolute paths. This skill refers to sibling files like `DEMO_SKILL_DIR/stages/*.md`, `DEMO_SKILL_DIR/app/app.md`, `DEMO_SKILL_DIR/references/*`. +Your system prompt normally defines `PROJECT`, `SKILLS`, `DEMO_SKILL_DIR`, and `DEMO_SKILL` as absolute paths. This skill refers to sibling files like `DEMO_SKILL_DIR/stages/*.md`, `DEMO_SKILL_DIR/app/app.md`, `DEMO_SKILL_DIR/references/*`. + +When those injected aliases are absent (for example when the skill is used directly from a project checkout), self-locate before reading any references: + +- `PROJECT` is the execution working directory. +- `SKILLS` is `PROJECT/.claude/skills`. +- `DEMO_SKILL_DIR` is `SKILLS/databricks-demo-generator`. +- `DEMO_SKILL` is `DEMO_SKILL_DIR/SKILL.md`. + +Resolve and use absolute paths from those fallbacks. Injected aliases, when present, remain authoritative. **When spawning subagents**, substitute every placeholder (`DEMO_SKILL_DIR/…`, `PROJECT/…`, `SKILLS/…`) with its real absolute path before sending — the subagent has no system prompt defining them. The full spawn prompt is in `DEMO_SKILL_DIR/stages/03-build.md` → Step 2. diff --git a/CLAUDE.md b/CLAUDE.md index bb451a5..063a270 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,6 +16,10 @@ A **system that generates Databricks demos**. Not one app — **three**, plus a Plus: **ai_dev_kit** (`app/ai_dev_kit/`) — a cloned external repo (`github.com/databricks-solutions/ai-dev-kit`) holding ~26 sub-skills for creating individual Databricks resources (pipelines, dashboards, Genie spaces, KAs, MAS, etc.). The generator's agent uses these during the Build stage. +Versioned **pipeline scenario contracts** live under `evaluation/`. They are +runner-neutral test data and are not imported by or packaged with the deployed +generator app, copied into projects, or exposed through application routes. + ## Mental model ``` @@ -81,6 +85,7 @@ industry-demo-prompts/ │ ├── app.md # How to design+spec a demo app (read during Stage 2) │ └── app_template/ # ★ The template app emitted by the generator ├── initial_templates/ # Pre-built seed templates (retail/loyalty-segmentation) +├── evaluation/ # Versioned pipeline scenario contracts + schema ├── tests/ # Playwright E2E for the generator (targets :9000) ├── install.sh # End-user installer (downloads skill + ai-dev-kit) └── docs/ # Screenshots for README @@ -199,6 +204,13 @@ cd app/test/app_template_test/app ./start.sh # Boots the LuxeBeauty test app on :8765 ``` +To validate the shared pipeline scenarios (from the repository root): + +```bash +uv run sb-eval cases validate +uv run python -m evaluation.schema_generator --check +``` + ## Conventions - **Python**: `uv` only, never `pip`. Use `claude-agent-sdk` (NOT the deprecated `Skill` tool name — pass `skills=` to the SDK). diff --git a/evaluation/README.md b/evaluation/README.md new file mode 100644 index 0000000..04b461c --- /dev/null +++ b/evaluation/README.md @@ -0,0 +1,26 @@ +# Scenario contracts + +`evaluation/` contains the versioned, runner-neutral scenarios used by the +Solution Builder pipeline tests. It is outside `app/`, the shipped +`databricks-demo-generator` skill, installers, and deployed application routes. + +## Contents + +- `cases/*.yaml` defines the canonical prompts, stages, capabilities, expected + artifacts, assertions, citations, and cleanup ownership. +- `models.py` validates those files with strict Pydantic models. +- `schema/scenario.schema.json` is the committed JSON Schema representation. +- `tests/pipeline/scenarios.py` exposes a small compatibility view for the + existing pipeline harness. + +## Validation + +```bash +uv run sb-eval cases validate +uv run python -m evaluation.schema_generator --check +uv run pytest tests/evaluation +``` + +These contracts intentionally do not select, install, or integrate an +evaluation runner. Maintainers can consume the YAML through tooling outside +this public repository. diff --git a/evaluation/__init__.py b/evaluation/__init__.py new file mode 100644 index 0000000..3ea0dd2 --- /dev/null +++ b/evaluation/__init__.py @@ -0,0 +1,9 @@ +"""Versioned scenario contracts for Solution Builder tests. + +This package is deliberately rooted outside ``app/`` and is not included in +the generator wheel, installer, or copied demo-generator skill. +""" + +from .models import Scenario + +__all__ = ["Scenario"] diff --git a/evaluation/cases/__init__.py b/evaluation/cases/__init__.py new file mode 100644 index 0000000..5a46653 --- /dev/null +++ b/evaluation/cases/__init__.py @@ -0,0 +1,53 @@ +"""Load and validate canonical Solution Builder evaluation cases.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from evaluation.models import Scenario + + +CASES_DIR = Path(__file__).parent + + +class CaseValidationError(ValueError): + pass + + +def case_paths(cases_dir: Path = CASES_DIR) -> list[Path]: + return sorted(path for path in cases_dir.glob("*.yaml") if path.is_file()) + + +def load_case(path: Path) -> Scenario: + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + return Scenario.model_validate(raw) + except (OSError, yaml.YAMLError, ValidationError, TypeError) as exc: + raise CaseValidationError(f"{path}: {exc}") from exc + + +def load_cases(cases_dir: Path = CASES_DIR) -> list[Scenario]: + paths = case_paths(cases_dir) + if not paths: + raise CaseValidationError(f"no scenario YAML files found in {cases_dir}") + cases = [load_case(path) for path in paths] + ids = [case.id for case in cases] + if len(ids) != len(set(ids)): + raise CaseValidationError("scenario ids must be unique across case files") + return cases + + +def select_cases(selector: str | None, cases_dir: Path = CASES_DIR) -> list[Scenario]: + cases = load_cases(cases_dir) + if selector in (None, "all"): + return cases + selected = [case for case in cases if case.id == selector] + if not selected: + available = ", ".join(case.id for case in cases) + raise CaseValidationError( + f"unknown scenario {selector!r}; available: {available}" + ) + return selected diff --git a/evaluation/cases/financial-services.yaml b/evaluation/cases/financial-services.yaml new file mode 100644 index 0000000..c5d6c43 --- /dev/null +++ b/evaluation/cases/financial-services.yaml @@ -0,0 +1,94 @@ +schema_version: 1 +id: financial-services +skill: databricks-demo-generator +description: >- + Real-time fraud detection demo for a retail bank. Generate synthetic + credit-card transactions, surface anomalies in a Genie space, and visualize + trends in an AI/BI dashboard. +capabilities: [genie, aibi-dashboards, synthetic-data-gen] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, dashboard, genie_space] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a fraud-detection demo for a retail bank. Use synthetic credit-card + transactions, an anomaly-detection pattern, a Genie space for ad-hoc + questions, and an AI/BI dashboard for the trend view. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved business story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The story has a named banking protagonist, a concrete fraud catalyst, and quantified business impact. + - Synthetic transactions, the anomaly narrative, Genie questions, and dashboard measures form one coherent story. + expected_facts: + - The proposed demo uses synthetic credit-card transactions, Genie, and an AI/BI dashboard. + expected_patterns: ['(?i)fraud', '(?i)(revenue|loss|risk|\$)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects generic fraud stories whose selected products or generated data do not support the same investigation. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: 2026-07-17T00:00:00Z + snippet: Story first and coherence above all; every showcased product must earn a clear beat in the walkthrough. + - id: design-architecture + prompt: Looks good — proceed to architect the pipeline and produce architecture.md. + expected_project_stage: ARCHITECTED + required_artifacts: + - {path: architecture.md, description: Renderable architecture diagram JSON} + semantic_assertions: + - The architecture connects transaction generation to governed data, Genie, and the dashboard without orphan components. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(genie|dashboard)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects architecture output that omits a selected capability or breaks the end-to-end data flow. + sources: [*skill_source] + - id: write-specifications + prompt: Now write the specifications for each component (specifications/*.md). + expected_project_stage: SPECIFICATION + required_artifacts: + - {path: specifications/*.md, description: Functional component specifications} + semantic_assertions: + - Specifications define compatible transaction fields, fraud measures, dashboard visuals, and Genie questions. + expected_facts: [Per-component specifications are written under specifications/.] + expected_patterns: ['specifications/', '(?i)(transaction|anomal)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects specs that are individually plausible but disagree on schemas, measures, or the investigation path. + sources: [*skill_source] + - id: build-demo + prompt: "Build it: write the SQL/Python and capture deployed resource IDs in resources.json." + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - Build artifacts implement the approved specifications and resources.json records only resources actually created. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [SQL or Python implementation artifacts and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(sql|python|dashboard|genie)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and implementation code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects claimed builds with placeholders, missing code, dead resource links, or resources outside the evaluation namespace. + sources: [*skill_source] diff --git a/evaluation/cases/healthcare.yaml b/evaluation/cases/healthcare.yaml new file mode 100644 index 0000000..e915e4b --- /dev/null +++ b/evaluation/cases/healthcare.yaml @@ -0,0 +1,91 @@ +schema_version: 1 +id: healthcare +skill: databricks-demo-generator +description: >- + Patient-readmission risk demo for a hospital network. Combine clinical notes + (knowledge assistant and vector search) with a tabular ML model for 30-day + readmission risk. +capabilities: [knowledge-assistant, vector-search, ml-training-serving] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, volume, vector_index, knowledge_assistant, model, serving_endpoint, mlflow_experiment] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a patient-readmission risk demo. Knowledge Assistant over + discharge-summary documents, vector search over the same corpus, and a + tabular ML model that scores 30-day readmission risk on synthetic patient data. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved clinical story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The story explains distinct, complementary roles for document retrieval and tabular risk scoring. + - The clinical narrative uses synthetic data and avoids claims that the demo provides medical advice. + expected_facts: [The demo combines discharge summaries with a 30-day readmission risk model.] + expected_patterns: ['(?i)readmission', '(?i)(discharge|clinical)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects healthcare stories that collapse RAG and predictive ML into an incoherent or unsafe workflow. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: 2026-07-17T00:00:00Z + snippet: Data, pipeline, dashboard, Genie, and agents must align; one broken link ruins the demo. + - id: design-architecture + prompt: "Proceed: architect the components and produce architecture.md." + expected_project_stage: ARCHITECTED + required_artifacts: [{path: architecture.md, description: Renderable architecture diagram JSON}] + semantic_assertions: + - The architecture separates document ingestion and vector retrieval from feature engineering, training, and serving while showing their user-facing convergence. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(vector|model|assistant)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects architectures that omit the shared corpus, model-serving path, or governance boundaries. + sources: [*skill_source] + - id: write-specifications + prompt: Write the per-component specifications under specifications/. + expected_project_stage: SPECIFICATION + required_artifacts: [{path: specifications/*.md, description: Functional component specifications}] + semantic_assertions: + - Specs define a consistent patient identifier, temporal split, discharge corpus, vector index, model target, and serving contract. + expected_facts: [Per-component specifications are written under specifications/.] + expected_patterns: ['specifications/', '(?i)(30-day|readmission)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects temporal leakage, mismatched patient keys, or a Knowledge Assistant corpus disconnected from the story. + sources: [*skill_source] + - id: build-demo + prompt: Build it now — generate the code, training notebook, and resources.json. + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - Training and retrieval artifacts implement the specifications and resources.json contains only validated IDs. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [Training code, retrieval configuration, and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(train|vector|knowledge)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and Python or SQL code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects builds that claim live RAG or serving assets without code, identifiers, or cleanup-safe names. + sources: [*skill_source] diff --git a/evaluation/cases/manufacturing-machinery.yaml b/evaluation/cases/manufacturing-machinery.yaml new file mode 100644 index 0000000..5497b6a --- /dev/null +++ b/evaluation/cases/manufacturing-machinery.yaml @@ -0,0 +1,97 @@ +schema_version: 1 +id: manufacturing-machinery +skill: databricks-demo-generator +description: >- + Warranty optimization demo for a heavy-machinery OEM (Manufacturing > + Machinery sub-vertical). Synthetic warranty claims and telemetry land in + Delta, Genie answers cost questions, a supervisor routes work, and a + Databricks App supports field service. +capabilities: [synthetic-data-gen, genie, supervisor-agent, databricks-apps] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, genie_space, multi_agent_supervisor, serving_endpoint, app] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a warranty-optimization demo for a heavy-machinery OEM + (Manufacturing > Machinery sub-vertical). Use synthetic-data-gen to + create warranty claims and machine-telemetry tables in Delta, stand up a + Genie space over the Gold warranty tables for ad-hoc cost/failure-mode + questions, wire a multi-agent supervisor that routes between Genie + (quantitative) and a claims-triage agent, and ship a Databricks App + (FastAPI + React) where a field-service manager can inspect a claim and + see the supervisor's recommendation. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved warranty story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The field-service workflow gives synthetic telemetry, warranty costs, Genie, supervisor routing, and the app distinct connected roles. + expected_facts: [The demo is for a heavy-machinery OEM and includes Genie, a supervisor, and a field-service app.] + expected_patterns: ['(?i)warranty', '(?i)(field.service|machinery)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects disconnected app, supervisor, or data stories and regressions to a generic manufacturing vertical. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and app guidance + retrieved_at: 2026-07-17T00:00:00Z + snippet: Every showcased product earns a clear beat, and app creation is folded into specification and build stages. + - id: design-architecture + prompt: Looks good — proceed to architect the pipeline and produce architecture.md. + expected_project_stage: ARCHITECTED + required_artifacts: [{path: architecture.md, description: Renderable architecture diagram JSON}] + semantic_assertions: + - The architecture connects claims and telemetry through Gold data to Genie, supervisor routing, and the app. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(genie|supervisor|app)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects missing routing, app, or data-flow nodes in a selected full-stack scenario. + sources: [*skill_source] + - id: write-specifications + prompt: Now write the specifications for each component (specifications/*.md) — synthetic data tables, Genie space, supervisor agent, and the Databricks App. + expected_project_stage: SPECIFICATION + required_artifacts: [{path: specifications/*.md, description: Functional component specifications}] + semantic_assertions: + - Specs preserve claim identifiers, failure modes, cost measures, routing criteria, and app interactions across components. + expected_facts: [Specifications cover synthetic data, Genie, the supervisor, and the Databricks App.] + expected_patterns: ['specifications/', '(?i)(claim|telemetry)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects interface drift between data, Genie, supervisor tools, and the app experience. + sources: [*skill_source] + - id: build-demo + prompt: "Build it: write the SQL/Python for synthetic data, the Genie config, the supervisor agent definition, the app code, and capture deployed resource IDs in resources.json." + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: app/**, description: Databricks App implementation} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - App, supervisor, Genie, and data artifacts implement the approved contracts and resources.json records validated IDs. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [Data code, Genie config, supervisor definition, app code, and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(supervisor|app|genie)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and app plus data code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && test -d app && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects incomplete full-stack builds, stale IDs, or resources that cannot be isolated and cleaned. + sources: [*skill_source] diff --git a/evaluation/cases/retail.yaml b/evaluation/cases/retail.yaml new file mode 100644 index 0000000..8c91074 --- /dev/null +++ b/evaluation/cases/retail.yaml @@ -0,0 +1,91 @@ +schema_version: 1 +id: retail +skill: databricks-demo-generator +description: >- + Demand-forecasting demo for an omnichannel retailer. Stream point-of-sale + events into Delta, aggregate into a Lakebase OLTP store, and serve a forecast + model via Model Serving. +capabilities: [lakebase, ml-training-serving, sdp] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, pipeline, lakebase, model, serving_endpoint, mlflow_experiment] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a demand-forecasting demo for an omnichannel retailer. Use a + Lakeflow Spark Declarative Pipeline (SDP) to ingest synthetic POS events + into Delta, land aggregates in Lakebase for fast lookups, and serve a + Prophet-style forecast via Model Serving. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved retail story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The story explains why governed Delta history, Lakebase lookups, and model serving are all needed by one retail protagonist. + expected_facts: [The demo uses SDP, Lakebase, and Model Serving for omnichannel demand forecasting.] + expected_patterns: ['(?i)(demand|forecast)', '(?i)(retail|point.of.sale|POS)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects retail stories that select SDP, Lakebase, and serving without a coherent latency and user-value rationale. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: 2026-07-17T00:00:00Z + snippet: Match products to moments and make data, pipeline, consumption layers, and story align end to end. + - id: design-architecture + prompt: "Proceed: produce architecture.md." + expected_project_stage: ARCHITECTED + required_artifacts: [{path: architecture.md, description: Renderable architecture diagram JSON}] + semantic_assertions: + - The architecture orders POS ingestion, Delta transformations, forecasting, Lakebase synchronization, and serving dependencies correctly. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(lakebase|serving|pipeline)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects architectures that reverse dependencies or leave Lakebase and model serving disconnected. + sources: [*skill_source] + - id: write-specifications + prompt: Write the per-component specifications. + expected_project_stage: SPECIFICATION + required_artifacts: [{path: specifications/*.md, description: Functional component specifications}] + semantic_assertions: + - Specs align POS event grain, forecast horizon, aggregate keys, Lakebase schema, and serving inputs and outputs. + expected_facts: [Per-component specifications are written under specifications/.] + expected_patterns: ['specifications/', '(?i)(forecast|aggregate)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects grain, key, or temporal-contract disagreements across pipeline, model, and operational store specs. + sources: [*skill_source] + - id: build-demo + prompt: "Build it: generate the SDP pipeline, the Lakebase sync, the forecast notebook, and resources.json." + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - SDP, sync, and forecasting artifacts implement the shared schema and resources.json contains only validated IDs. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [The SDP pipeline, Lakebase sync, forecast code, and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(pipeline|lakebase|forecast)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and Python or SQL code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects placeholder builds, broken sync contracts, stale IDs, or resources outside the evaluation namespace. + sources: [*skill_source] diff --git a/evaluation/cli.py b/evaluation/cli.py new file mode 100644 index 0000000..062deaf --- /dev/null +++ b/evaluation/cli.py @@ -0,0 +1,65 @@ +"""CLI for validating canonical Solution Builder scenarios.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from evaluation.cases import CaseValidationError, load_cases +from evaluation.schema_generator import rendered_schema + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def cases_validate(*, as_json: bool) -> int: + try: + cases = load_cases() + schema_path = REPO_ROOT / "evaluation" / "schema" / "scenario.schema.json" + if ( + not schema_path.is_file() + or schema_path.read_text(encoding="utf-8") != rendered_schema() + ): + raise CaseValidationError( + "committed JSON Schema is stale; run `uv run python -m evaluation.schema_generator`" + ) + except Exception as exc: # noqa: BLE001 - CLI boundary + if as_json: + print(json.dumps({"valid": False, "error": str(exc)})) + else: + print(f"case validation failed: {exc}", file=sys.stderr) + return 1 + payload = { + "valid": True, + "count": len(cases), + "scenarios": [case.id for case in cases], + } + print( + json.dumps(payload, indent=2) + if as_json + else f"validated {len(cases)} scenarios: {', '.join(payload['scenarios'])}" + ) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="sb-eval") + subcommands = parser.add_subparsers(dest="command", required=True) + cases = subcommands.add_parser("cases") + case_commands = cases.add_subparsers(dest="case_command", required=True) + validate = case_commands.add_parser("validate") + validate.add_argument("--json", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "cases" and args.case_command == "validate": + return cases_validate(as_json=args.json) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evaluation/models.py b/evaluation/models.py new file mode 100644 index 0000000..63a80c9 --- /dev/null +++ b/evaluation/models.py @@ -0,0 +1,112 @@ +"""Canonical, runner-neutral evaluation contracts.""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +STAGE_ORDER = ( + "DRAFTING", + "SUMMARIZED", + "ARCHITECTED", + "SPECIFICATION", + "BUILT", + "BUNDLED", +) + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ProjectStage(StrEnum): + DRAFTING = "DRAFTING" + SUMMARIZED = "SUMMARIZED" + ARCHITECTED = "ARCHITECTED" + SPECIFICATION = "SPECIFICATION" + BUILT = "BUILT" + BUNDLED = "BUNDLED" + + +class SourceCitation(StrictModel): + type: Literal["docs", "blog", "glean", "confluence", "slack", "github", "manual"] + uri: str = Field(min_length=1) + title: str = Field(min_length=1) + retrieved_at: datetime + snippet: str = Field(min_length=1, max_length=280) + + +class ArtifactExpectation(StrictModel): + path: str = Field(min_length=1) + description: str = Field(min_length=1) + required: bool = True + + +class VerificationCommand(StrictModel): + fact: str = Field(min_length=1) + command: str = Field(min_length=1) + check: str = Field(min_length=1) + + +class ToolExpectations(StrictModel): + required: list[str] = Field(default_factory=list) + banned: list[str] = Field(default_factory=list) + + +class ScenarioStep(StrictModel): + id: str = Field(pattern=r"^[a-z0-9][a-z0-9-]*$") + prompt: str = Field(min_length=1) + expected_project_stage: ProjectStage + required_artifacts: list[ArtifactExpectation] = Field(min_length=1) + semantic_assertions: list[str] = Field(min_length=1) + expected_facts: list[str] = Field(min_length=1) + expected_patterns: list[str] = Field(min_length=1) + tool_expectations: ToolExpectations + verification_commands: list[VerificationCommand] = Field(default_factory=list) + regression_intent: str = Field(min_length=1) + sources: list[SourceCitation] = Field(min_length=1) + + +class LiveResourceExpectations(StrictModel): + cleanup_owner: Literal["solution-builder"] = "solution-builder" + expected_resource_kinds: list[str] = Field(default_factory=list) + additional_resource_kinds: list[str] = Field(default_factory=list) + evaluation_prefix: str = Field(default="sb_eval_", pattern=r"^[a-z][a-z0-9_]*_$") + + +class Scenario(StrictModel): + schema_version: Literal[1] = 1 + id: str = Field(pattern=r"^[a-z0-9][a-z0-9-]*$") + skill: str = Field(min_length=1) + description: str = Field(min_length=1) + capabilities: list[str] = Field(min_length=1) + timeout_seconds: int = Field(ge=60, le=14_400) + target_stage: ProjectStage + steps: list[ScenarioStep] = Field(min_length=1) + live_resources: LiveResourceExpectations + + @model_validator(mode="after") + def validate_workflow(self) -> "Scenario": + ids = [step.id for step in self.steps] + if len(ids) != len(set(ids)): + raise ValueError("step ids must be unique") + stages = [ + STAGE_ORDER.index(step.expected_project_stage.value) for step in self.steps + ] + if stages != sorted(stages): + raise ValueError("step stages must be ordered monotonically") + if self.steps[-1].expected_project_stage != self.target_stage: + raise ValueError("the final step stage must equal target_stage") + return self + + @property + def initial_prompt(self) -> str: + return self.steps[0].prompt + + @property + def drive_messages(self) -> list[str]: + return [step.prompt for step in self.steps[1:]] diff --git a/evaluation/schema/scenario.schema.json b/evaluation/schema/scenario.schema.json new file mode 100644 index 0000000..74dc385 --- /dev/null +++ b/evaluation/schema/scenario.schema.json @@ -0,0 +1,324 @@ +{ + "$defs": { + "ArtifactExpectation": { + "additionalProperties": false, + "properties": { + "description": { + "minLength": 1, + "title": "Description", + "type": "string" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "path", + "description" + ], + "title": "ArtifactExpectation", + "type": "object" + }, + "LiveResourceExpectations": { + "additionalProperties": false, + "properties": { + "additional_resource_kinds": { + "items": { + "type": "string" + }, + "title": "Additional Resource Kinds", + "type": "array" + }, + "cleanup_owner": { + "const": "solution-builder", + "default": "solution-builder", + "title": "Cleanup Owner", + "type": "string" + }, + "evaluation_prefix": { + "default": "sb_eval_", + "pattern": "^[a-z][a-z0-9_]*_$", + "title": "Evaluation Prefix", + "type": "string" + }, + "expected_resource_kinds": { + "items": { + "type": "string" + }, + "title": "Expected Resource Kinds", + "type": "array" + } + }, + "title": "LiveResourceExpectations", + "type": "object" + }, + "ProjectStage": { + "enum": [ + "DRAFTING", + "SUMMARIZED", + "ARCHITECTED", + "SPECIFICATION", + "BUILT", + "BUNDLED" + ], + "title": "ProjectStage", + "type": "string" + }, + "ScenarioStep": { + "additionalProperties": false, + "properties": { + "expected_facts": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Expected Facts", + "type": "array" + }, + "expected_patterns": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Expected Patterns", + "type": "array" + }, + "expected_project_stage": { + "$ref": "#/$defs/ProjectStage" + }, + "id": { + "pattern": "^[a-z0-9][a-z0-9-]*$", + "title": "Id", + "type": "string" + }, + "prompt": { + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "regression_intent": { + "minLength": 1, + "title": "Regression Intent", + "type": "string" + }, + "required_artifacts": { + "items": { + "$ref": "#/$defs/ArtifactExpectation" + }, + "minItems": 1, + "title": "Required Artifacts", + "type": "array" + }, + "semantic_assertions": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Semantic Assertions", + "type": "array" + }, + "sources": { + "items": { + "$ref": "#/$defs/SourceCitation" + }, + "minItems": 1, + "title": "Sources", + "type": "array" + }, + "tool_expectations": { + "$ref": "#/$defs/ToolExpectations" + }, + "verification_commands": { + "items": { + "$ref": "#/$defs/VerificationCommand" + }, + "title": "Verification Commands", + "type": "array" + } + }, + "required": [ + "id", + "prompt", + "expected_project_stage", + "required_artifacts", + "semantic_assertions", + "expected_facts", + "expected_patterns", + "tool_expectations", + "regression_intent", + "sources" + ], + "title": "ScenarioStep", + "type": "object" + }, + "SourceCitation": { + "additionalProperties": false, + "properties": { + "retrieved_at": { + "format": "date-time", + "title": "Retrieved At", + "type": "string" + }, + "snippet": { + "maxLength": 280, + "minLength": 1, + "title": "Snippet", + "type": "string" + }, + "title": { + "minLength": 1, + "title": "Title", + "type": "string" + }, + "type": { + "enum": [ + "docs", + "blog", + "glean", + "confluence", + "slack", + "github", + "manual" + ], + "title": "Type", + "type": "string" + }, + "uri": { + "minLength": 1, + "title": "Uri", + "type": "string" + } + }, + "required": [ + "type", + "uri", + "title", + "retrieved_at", + "snippet" + ], + "title": "SourceCitation", + "type": "object" + }, + "ToolExpectations": { + "additionalProperties": false, + "properties": { + "banned": { + "items": { + "type": "string" + }, + "title": "Banned", + "type": "array" + }, + "required": { + "items": { + "type": "string" + }, + "title": "Required", + "type": "array" + } + }, + "title": "ToolExpectations", + "type": "object" + }, + "VerificationCommand": { + "additionalProperties": false, + "properties": { + "check": { + "minLength": 1, + "title": "Check", + "type": "string" + }, + "command": { + "minLength": 1, + "title": "Command", + "type": "string" + }, + "fact": { + "minLength": 1, + "title": "Fact", + "type": "string" + } + }, + "required": [ + "fact", + "command", + "check" + ], + "title": "VerificationCommand", + "type": "object" + } + }, + "$id": "https://github.com/databricks-solutions/solution-builder/evaluation/schema/scenario.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "capabilities": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Capabilities", + "type": "array" + }, + "description": { + "minLength": 1, + "title": "Description", + "type": "string" + }, + "id": { + "pattern": "^[a-z0-9][a-z0-9-]*$", + "title": "Id", + "type": "string" + }, + "live_resources": { + "$ref": "#/$defs/LiveResourceExpectations" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "skill": { + "minLength": 1, + "title": "Skill", + "type": "string" + }, + "steps": { + "items": { + "$ref": "#/$defs/ScenarioStep" + }, + "minItems": 1, + "title": "Steps", + "type": "array" + }, + "target_stage": { + "$ref": "#/$defs/ProjectStage" + }, + "timeout_seconds": { + "maximum": 14400, + "minimum": 60, + "title": "Timeout Seconds", + "type": "integer" + } + }, + "required": [ + "id", + "skill", + "description", + "capabilities", + "timeout_seconds", + "target_stage", + "steps", + "live_resources" + ], + "title": "Scenario", + "type": "object" +} diff --git a/evaluation/schema_generator.py b/evaluation/schema_generator.py new file mode 100644 index 0000000..51d3ad3 --- /dev/null +++ b/evaluation/schema_generator.py @@ -0,0 +1,36 @@ +"""Generate/check the committed JSON Schema for canonical scenarios.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from evaluation.models import Scenario + + +SCHEMA_PATH = Path(__file__).parent / "schema" / "scenario.schema.json" + + +def rendered_schema() -> str: + schema = Scenario.model_json_schema() + schema["$schema"] = "https://json-schema.org/draft/2020-12/schema" + schema["$id"] = ( + "https://github.com/databricks-solutions/solution-builder/evaluation/schema/scenario.schema.json" + ) + return json.dumps(schema, indent=2, sort_keys=True) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args(argv) + expected = rendered_schema() + if args.check: + return 0 if SCHEMA_PATH.is_file() and SCHEMA_PATH.read_text() == expected else 1 + SCHEMA_PATH.write_text(expected, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c3534cc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "solution-builder-evaluation" +version = "0.1.0" +description = "Versioned test scenario contracts for Solution Builder" +requires-python = ">=3.12,<3.13" +dependencies = [ + "pydantic>=2.11,<3", + "pyyaml>=6.0,<7", +] + +[project.scripts] +sb-eval = "evaluation.cli:main" + +[dependency-groups] +dev = [ + "pytest>=8.3,<9", + "pytest-asyncio>=0.24,<1", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["evaluation"] + +[tool.pytest.ini_options] +testpaths = ["tests/evaluation"] +asyncio_mode = "auto" diff --git a/tests/evaluation/test_cases.py b/tests/evaluation/test_cases.py new file mode 100644 index 0000000..94b019a --- /dev/null +++ b/tests/evaluation/test_cases.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from evaluation.cases import load_cases +from tests.pipeline.scenarios import SCENARIOS + + +LEGACY = { + "financial-services": { + "capabilities": ["genie", "aibi-dashboards", "synthetic-data-gen"], + "initial_prompt": "Build a fraud-detection demo for a retail bank. Use synthetic credit-card transactions, an anomaly-detection pattern, a Genie space for ad-hoc questions, and an AI/BI dashboard for the trend view.", + "drive_messages": [ + "Looks good — proceed to architect the pipeline and produce architecture.md.", + "Now write the specifications for each component (specifications/*.md).", + "Build it: write the SQL/Python and capture deployed resource IDs in resources.json.", + ], + }, + "healthcare": { + "capabilities": ["knowledge-assistant", "vector-search", "ml-training-serving"], + "initial_prompt": "Build a patient-readmission risk demo. Knowledge Assistant over discharge-summary documents, vector search over the same corpus, and a tabular ML model that scores 30-day readmission risk on synthetic patient data.", + "drive_messages": [ + "Proceed: architect the components and produce architecture.md.", + "Write the per-component specifications under specifications/.", + "Build it now — generate the code, training notebook, and resources.json.", + ], + }, + "manufacturing-machinery": { + "capabilities": [ + "synthetic-data-gen", + "genie", + "supervisor-agent", + "databricks-apps", + ], + "initial_prompt": "Build a warranty-optimization demo for a heavy-machinery OEM (Manufacturing > Machinery sub-vertical). Use synthetic-data-gen to create warranty claims and machine-telemetry tables in Delta, stand up a Genie space over the Gold warranty tables for ad-hoc cost/failure-mode questions, wire a multi-agent supervisor that routes between Genie (quantitative) and a claims-triage agent, and ship a Databricks App (FastAPI + React) where a field-service manager can inspect a claim and see the supervisor's recommendation.", + "drive_messages": [ + "Looks good — proceed to architect the pipeline and produce architecture.md.", + "Now write the specifications for each component (specifications/*.md) — synthetic data tables, Genie space, supervisor agent, and the Databricks App.", + "Build it: write the SQL/Python for synthetic data, the Genie config, the supervisor agent definition, the app code, and capture deployed resource IDs in resources.json.", + ], + }, + "retail": { + "capabilities": ["lakebase", "ml-training-serving", "sdp"], + "initial_prompt": "Build a demand-forecasting demo for an omnichannel retailer. Use a Lakeflow Spark Declarative Pipeline (SDP) to ingest synthetic POS events into Delta, land aggregates in Lakebase for fast lookups, and serve a Prophet-style forecast via Model Serving.", + "drive_messages": [ + "Proceed: produce architecture.md.", + "Write the per-component specifications.", + "Build it: generate the SDP pipeline, the Lakebase sync, the forecast notebook, and resources.json.", + ], + }, +} + + +def test_four_canonical_cases_validate() -> None: + cases = load_cases() + assert [case.id for case in cases] == [ + "financial-services", + "healthcare", + "manufacturing-machinery", + "retail", + ] + assert all(case.timeout_seconds == 3600 for case in cases) + assert all(case.target_stage.value == "BUILT" for case in cases) + + +def test_pipeline_compatibility_preserves_existing_scenarios() -> None: + assert len(SCENARIOS) == 4 + for scenario in SCENARIOS: + expected = LEGACY[scenario.slug] + assert scenario.capabilities == expected["capabilities"] + assert scenario.initial_prompt == expected["initial_prompt"] + assert scenario.drive_messages == expected["drive_messages"] + assert scenario.target_stage == "BUILT" + assert scenario.timeout_seconds == 3600 diff --git a/tests/pipeline/README.md b/tests/pipeline/README.md index 886923a..cd65dd2 100644 --- a/tests/pipeline/README.md +++ b/tests/pipeline/README.md @@ -12,7 +12,7 @@ You'll spend FMAPI tokens. Plan for ~45–60 min per full run. ## Run ```bash -tests/pipeline/run.sh # all 3 scenarios, target=BUILT +tests/pipeline/run.sh # all 4 scenarios, target=BUILT tests/pipeline/run.sh --scenario healthcare # single scenario tests/pipeline/run.sh --target SPECIFICATION # cheaper (~15 min) tests/pipeline/run.sh --scenario-timeout 1800 # 30 min/scenario cap @@ -59,10 +59,15 @@ Otherwise FAIL — the per-scenario `README.md` lists every issue. ## Add a scenario -Append a `Scenario(...)` to `SCENARIOS` in `scenarios.py`. Capability slugs -must match filenames (without `.md`) in -`.claude/skills/databricks-demo-generator/references/blocks/capabilities/`. -A new scenario runs in parallel with the rest automatically — no other wiring. +Add and validate a canonical case under `evaluation/cases/`: + +```bash +uv run sb-eval cases validate +``` + +Capability slugs must match the demo-generator capability blocks. The pipeline +harness loads this versioned YAML, so a new case runs in parallel without a +second hardcoded definition. ## Files @@ -71,7 +76,7 @@ A new scenario runs in parallel with the rest automatically — no other wiring. | `run.sh` | Bootstrap: ensure backend, run pytest, surface summary path. | | `test_pipeline.py` | Pytest entry. Fans out scenarios via `asyncio.gather`. | | `runner.py` | `drive_project()`: create → invoke per turn → snapshot. | -| `scenarios.py` | The 3 hardcoded scenarios + `Scenario` dataclass. | +| `scenarios.py` | Compatibility loader for canonical `evaluation/cases/*.yaml`. | | `assertions.py` | Pure pass/fail helpers (stage, errors, artifacts). | | `api_client.py` | `httpx.AsyncClient` wrapper + SSE consumer with reconnect. | | `conftest.py` | Pytest fixtures: scenario selection, output dir, health check. | diff --git a/tests/pipeline/scenarios.py b/tests/pipeline/scenarios.py index e05812b..dba4ec1 100644 --- a/tests/pipeline/scenarios.py +++ b/tests/pipeline/scenarios.py @@ -1,25 +1,16 @@ -"""Hardcoded end-to-end scenarios. +"""Pipeline compatibility view over canonical scenario contracts. -Each scenario drives a single project from creation through the agent until it -reaches a target stage. To add a new scenario, append a Scenario to SCENARIOS. -Capability slugs must match filenames in -.claude/skills/databricks-demo-generator/references/blocks/capabilities/. +The durable scenario contract lives in ``evaluation/cases/*.yaml``. This +module keeps the pipeline runner's small dataclass interface while ensuring +all pipeline entry points use the same prompts and expectations. """ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace - -# Stage ordering must match backend.models.ProjectStage -STAGE_ORDER = [ - "DRAFTING", - "SUMMARIZED", - "ARCHITECTED", - "SPECIFICATION", - "BUILT", - "BUNDLED", -] +from evaluation.cases import load_cases +from evaluation.models import STAGE_ORDER def stage_index(stage: str) -> int: @@ -37,111 +28,42 @@ class Scenario: initial_prompt: str drive_messages: list[str] = field(default_factory=list) target_stage: str = "BUILT" - timeout_seconds: int = 60 * 60 # 60 min default per scenario + timeout_seconds: int = 60 * 60 def __post_init__(self) -> None: if stage_index(self.target_stage) < 0: raise ValueError(f"unknown target_stage: {self.target_stage}") -SCENARIOS: list[Scenario] = [ - Scenario( - slug="financial-services", - description=( - "Real-time fraud detection demo for a retail bank. " - "Generate synthetic credit-card transactions, surface anomalies in a " - "Genie space, and visualize trends in an AI/BI dashboard." - ), - capabilities=["genie", "aibi-dashboards", "synthetic-data-gen"], - initial_prompt=( - "Build a fraud-detection demo for a retail bank. " - "Use synthetic credit-card transactions, an anomaly-detection pattern, " - "a Genie space for ad-hoc questions, and an AI/BI dashboard for the trend view." - ), - drive_messages=[ - "Looks good — proceed to architect the pipeline and produce architecture.md.", - "Now write the specifications for each component (specifications/*.md).", - "Build it: write the SQL/Python and capture deployed resource IDs in resources.json.", - ], - target_stage="BUILT", - ), - Scenario( - slug="healthcare", - description=( - "Patient-readmission risk demo for a hospital network. " - "Combine clinical notes (knowledge assistant + vector search) with a " - "tabular ML model for 30-day readmission risk." - ), - capabilities=["knowledge-assistant", "vector-search", "ml-training-serving"], - initial_prompt=( - "Build a patient-readmission risk demo. " - "Knowledge Assistant over discharge-summary documents, vector search over the same corpus, " - "and a tabular ML model that scores 30-day readmission risk on synthetic patient data." - ), - drive_messages=[ - "Proceed: architect the components and produce architecture.md.", - "Write the per-component specifications under specifications/.", - "Build it now — generate the code, training notebook, and resources.json.", - ], - target_stage="BUILT", - ), - Scenario( - slug="manufacturing-machinery", - description=( - "Warranty optimization demo for a heavy-machinery OEM (Manufacturing > Machinery sub-vertical). " - "Synthetic warranty claims + telemetry land in Delta, a Genie space answers quantitative " - "warranty-cost questions, a multi-agent supervisor routes between Genie and a claims-triage agent, " - "and a Databricks App gives field service a UI to inspect claims and the supervisor's verdict." - ), - capabilities=["synthetic-data-gen", "genie", "supervisor-agent", "databricks-apps"], - initial_prompt=( - "Build a warranty-optimization demo for a heavy-machinery OEM (Manufacturing > Machinery sub-vertical). " - "Use synthetic-data-gen to create warranty claims and machine-telemetry tables in Delta, " - "stand up a Genie space over the Gold warranty tables for ad-hoc cost/failure-mode questions, " - "wire a multi-agent supervisor that routes between Genie (quantitative) and a claims-triage agent, " - "and ship a Databricks App (FastAPI + React) where a field-service manager can inspect a claim " - "and see the supervisor's recommendation." - ), - drive_messages=[ - "Looks good — proceed to architect the pipeline and produce architecture.md.", - "Now write the specifications for each component (specifications/*.md) — synthetic data tables, Genie space, supervisor agent, and the Databricks App.", - "Build it: write the SQL/Python for synthetic data, the Genie config, the supervisor agent definition, the app code, and capture deployed resource IDs in resources.json.", - ], - target_stage="BUILT", - ), - Scenario( - slug="retail", - description=( - "Demand-forecasting demo for an omnichannel retailer. " - "Stream point-of-sale events into Delta, aggregate into a Lakebase OLTP store, " - "and serve a forecast model via Model Serving." - ), - capabilities=["lakebase", "ml-training-serving", "sdp"], - initial_prompt=( - "Build a demand-forecasting demo for an omnichannel retailer. " - "Use a Lakeflow Spark Declarative Pipeline (SDP) to ingest synthetic POS events into Delta, " - "land aggregates in Lakebase for fast lookups, and serve a Prophet-style forecast via Model Serving." - ), - drive_messages=[ - "Proceed: produce architecture.md.", - "Write the per-component specifications.", - "Build it: generate the SDP pipeline, the Lakebase sync, the forecast notebook, and resources.json.", - ], - target_stage="BUILT", - ), -] +def _pipeline_scenario(case) -> Scenario: + return Scenario( + slug=case.id, + description=case.description, + capabilities=list(case.capabilities), + initial_prompt=case.initial_prompt, + drive_messages=list(case.drive_messages), + target_stage=case.target_stage.value, + timeout_seconds=case.timeout_seconds, + ) + + +SCENARIOS: list[Scenario] = [_pipeline_scenario(case) for case in load_cases()] def get_scenarios(slugs: list[str] | None) -> list[Scenario]: - if not slugs: - return list(SCENARIOS) - by_slug = {s.slug: s for s in SCENARIOS} - out: list[Scenario] = [] - for slug in slugs: - if slug not in by_slug: - raise SystemExit( - f"unknown scenario slug: {slug!r}. " - f"available: {', '.join(by_slug)}" - ) - out.append(by_slug[slug]) - return out + by_slug = {scenario.slug: scenario for scenario in SCENARIOS} + selected = list(by_slug) if not slugs else slugs + unknown = [slug for slug in selected if slug not in by_slug] + if unknown: + raise SystemExit( + f"unknown scenario slug: {unknown[0]!r}. available: {', '.join(by_slug)}" + ) + # Fixtures override target/timeout in-place, so return independent copies. + return [ + replace( + by_slug[slug], + capabilities=list(by_slug[slug].capabilities), + drive_messages=list(by_slug[slug].drive_messages), + ) + for slug in selected + ] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..249ecdc --- /dev/null +++ b/uv.lock @@ -0,0 +1,196 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "0.26.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/8e/c4/453c52c659521066969523e87d85d54139bbd17b78f09532fb8eb8cdb58e/pytest_asyncio-0.26.0.tar.gz", hash = "sha256:c4df2a697648241ff39e7f0e4a73050b03f123f760673956cf0d72a4990e312f", upload-time = "2025-03-25T06:22:28.883Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/20/7f/338843f449ace853647ace35870874f69a764d251872ed1b4de9f234822c/pytest_asyncio-0.26.0-py3-none-any.whl", hash = "sha256:7b51ed894f4fbea1340262bdae5135797ebbe21d8638978e35d31c6d19f72fb0", upload-time = "2025-03-25T06:22:27.807Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "solution-builder-evaluation" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.11,<3" }, + { name = "pyyaml", specifier = ">=6.0,<7" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3,<9" }, + { name = "pytest-asyncio", specifier = ">=0.24,<1" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", upload-time = "2025-10-01T02:14:40.154Z" }, +]