From fa73fa7d4251005933938586f8a6d0596793c260 Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 09:13:32 -0700 Subject: [PATCH 1/7] Add maintainer-only SkillForge evaluation --- .../skills/databricks-demo-generator/SKILL.md | 11 +- .github/workflows/skillforge-evaluation.yml | 148 ++++ .gitignore | 3 + CLAUDE.md | 15 + evaluation/README.md | 47 ++ evaluation/__init__.py | 9 + evaluation/adapter.py | 252 ++++++ evaluation/cases/__init__.py | 53 ++ evaluation/cases/financial-services.yaml | 94 +++ evaluation/cases/healthcare.yaml | 91 +++ evaluation/cases/manufacturing-machinery.yaml | 97 +++ evaluation/cases/retail.yaml | 91 +++ evaluation/cli.py | 171 ++++ evaluation/fixture.py | 139 ++++ evaluation/hashing.py | 59 ++ evaluation/hook_cli.py | 98 +++ evaluation/lifecycle.py | 29 + evaluation/live.py | 772 ++++++++++++++++++ evaluation/models.py | 177 ++++ evaluation/normalize.py | 141 ++++ .../comparison_rubric.md | 11 + .../output_instructions.md | 47 ++ .../thinking_instructions.md | 47 ++ evaluation/runner.py | 414 ++++++++++ evaluation/schema/scenario.schema.json | 324 ++++++++ evaluation/schema_generator.py | 36 + evaluation/skillforge.lock.yaml | 15 + evaluation/toolchain.py | 188 +++++ pyproject.toml | 29 + .../financial-services.ground-truth.yaml | 163 ++++ .../golden/financial-services.manifest.yaml | 28 + tests/evaluation/test_adapter.py | 65 ++ tests/evaluation/test_cases.py | 72 ++ tests/evaluation/test_cleanup.py | 411 ++++++++++ tests/evaluation/test_hashing.py | 24 + tests/evaluation/test_lifecycle.py | 92 +++ tests/evaluation/test_non_live_integration.py | 74 ++ tests/pipeline/README.md | 17 +- tests/pipeline/scenarios.py | 152 +--- uv.lock | 196 +++++ 40 files changed, 4780 insertions(+), 122 deletions(-) create mode 100644 .github/workflows/skillforge-evaluation.yml create mode 100644 evaluation/README.md create mode 100644 evaluation/__init__.py create mode 100644 evaluation/adapter.py create mode 100644 evaluation/cases/__init__.py create mode 100644 evaluation/cases/financial-services.yaml create mode 100644 evaluation/cases/healthcare.yaml create mode 100644 evaluation/cases/manufacturing-machinery.yaml create mode 100644 evaluation/cases/retail.yaml create mode 100644 evaluation/cli.py create mode 100644 evaluation/fixture.py create mode 100644 evaluation/hashing.py create mode 100644 evaluation/hook_cli.py create mode 100644 evaluation/lifecycle.py create mode 100644 evaluation/live.py create mode 100644 evaluation/models.py create mode 100644 evaluation/normalize.py create mode 100644 evaluation/rubrics/databricks-demo-generator/comparison_rubric.md create mode 100644 evaluation/rubrics/databricks-demo-generator/output_instructions.md create mode 100644 evaluation/rubrics/databricks-demo-generator/thinking_instructions.md create mode 100644 evaluation/runner.py create mode 100644 evaluation/schema/scenario.schema.json create mode 100644 evaluation/schema_generator.py create mode 100644 evaluation/skillforge.lock.yaml create mode 100644 evaluation/toolchain.py create mode 100644 pyproject.toml create mode 100644 tests/evaluation/golden/financial-services.ground-truth.yaml create mode 100644 tests/evaluation/golden/financial-services.manifest.yaml create mode 100644 tests/evaluation/test_adapter.py create mode 100644 tests/evaluation/test_cases.py create mode 100644 tests/evaluation/test_cleanup.py create mode 100644 tests/evaluation/test_hashing.py create mode 100644 tests/evaluation/test_lifecycle.py create mode 100644 tests/evaluation/test_non_live_integration.py create mode 100644 uv.lock diff --git a/.claude/skills/databricks-demo-generator/SKILL.md b/.claude/skills/databricks-demo-generator/SKILL.md index 3b4a8be..129697c 100644 --- a/.claude/skills/databricks-demo-generator/SKILL.md +++ b/.claude/skills/databricks-demo-generator/SKILL.md @@ -29,7 +29,16 @@ The main loop lives in this file (SKILL.md) — it describes **the flow**: stage ## Paths -Your system prompt defines `PROJECT`, `SKILLS`, `DEMO_SKILL_DIR`, and `DEMO_SKILL` as absolute paths. This skill refers to sibling files like `DEMO_SKILL_DIR/stages/*.md`, `DEMO_SKILL_DIR/app/app.md`, `DEMO_SKILL_DIR/references/*`. +Your system prompt normally defines `PROJECT`, `SKILLS`, `DEMO_SKILL_DIR`, and `DEMO_SKILL` as absolute paths. This skill refers to sibling files like `DEMO_SKILL_DIR/stages/*.md`, `DEMO_SKILL_DIR/app/app.md`, `DEMO_SKILL_DIR/references/*`. + +When those injected aliases are absent (for example in a maintainer evaluation fixture), self-locate before reading any references: + +- `PROJECT` is the execution working directory. +- `SKILLS` is `PROJECT/.claude/skills`. +- `DEMO_SKILL_DIR` is `SKILLS/databricks-demo-generator`. +- `DEMO_SKILL` is `DEMO_SKILL_DIR/SKILL.md`. + +Resolve and use absolute paths from those fallbacks. Injected aliases, when present, remain authoritative. **When spawning subagents**, substitute every placeholder (`DEMO_SKILL_DIR/…`, `PROJECT/…`, `SKILLS/…`) with its real absolute path before sending — the subagent has no system prompt defining them. The full spawn prompt is in `DEMO_SKILL_DIR/stages/03-build.md` → Step 2. diff --git a/.github/workflows/skillforge-evaluation.yml b/.github/workflows/skillforge-evaluation.yml new file mode 100644 index 0000000..697a1c9 --- /dev/null +++ b/.github/workflows/skillforge-evaluation.yml @@ -0,0 +1,148 @@ +name: SkillForge evaluation + +on: + pull_request: + paths: + - ".claude/skills/databricks-demo-generator/**" + - "evaluation/**" + - "tests/evaluation/**" + - "tests/pipeline/scenarios.py" + - ".github/workflows/skillforge-evaluation.yml" + workflow_dispatch: + inputs: + scenario: + description: Canonical scenario id or all + required: true + default: all + type: string + +concurrency: + group: skillforge-${{ github.event_name == 'workflow_dispatch' && 'live' || github.ref }} + cancel-in-progress: false + +jobs: + validate-and-quick-eval: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6 + with: + enable-cache: true + - name: Install maintainer environment + run: uv sync --group dev --locked + - name: Validate canonical cases + run: uv run sb-eval cases validate + - name: Run maintainer contract tests + run: uv run pytest -q + - name: Read pinned revisions + id: pins + run: | + uv run python - <<'PY' >> "$GITHUB_OUTPUT" + from evaluation.toolchain import load_lock + lock = load_lock() + print(f"skillforge_repo={lock.skillforge_repository.removesuffix('.git').removeprefix('https://github.com/')}") + print(f"skillforge_revision={lock.skillforge_revision}") + print(f"ai_dev_kit_repo={lock.ai_dev_kit_repository.removesuffix('.git').removeprefix('https://github.com/')}") + print(f"ai_dev_kit_revision={lock.ai_dev_kit_revision}") + PY + - name: Checkout pinned SkillForge + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: ${{ steps.pins.outputs.skillforge_repo }} + ref: ${{ steps.pins.outputs.skillforge_revision }} + token: ${{ secrets.SKILLFORGE_READ_TOKEN }} + path: .deps/skillforge + - name: Checkout pinned ai-dev-kit + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: ${{ steps.pins.outputs.ai_dev_kit_repo }} + ref: ${{ steps.pins.outputs.ai_dev_kit_revision }} + path: .deps/ai-dev-kit + - name: Install pinned external SkillForge + run: uv tool install '.deps/skillforge/python[all]' + - name: Informational L1/L3 evaluation + id: quick-eval + continue-on-error: true + env: + SB_EVAL_AI_DEV_KIT_DIR: ${{ github.workspace }}/.deps/ai-dev-kit + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: uv run sb-eval run --levels L1,L3 + - name: Upload evaluation reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: skillforge-quick-${{ github.run_id }} + path: test-runs/skillforge/ + if-no-files-found: warn + + full-live-eval: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + environment: solution-builder-live-evaluation + permissions: + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6 + with: + enable-cache: true + - name: Install maintainer environment + run: uv sync --group dev --locked + - name: Validate canonical cases + run: uv run sb-eval cases validate + - name: Run maintainer contract tests + run: uv run pytest -q + - name: Read pinned revisions + id: pins + run: | + uv run python - <<'PY' >> "$GITHUB_OUTPUT" + from evaluation.toolchain import load_lock + lock = load_lock() + print(f"skillforge_repo={lock.skillforge_repository.removesuffix('.git').removeprefix('https://github.com/')}") + print(f"skillforge_revision={lock.skillforge_revision}") + print(f"ai_dev_kit_repo={lock.ai_dev_kit_repository.removesuffix('.git').removeprefix('https://github.com/')}") + print(f"ai_dev_kit_revision={lock.ai_dev_kit_revision}") + PY + - name: Checkout pinned SkillForge + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: ${{ steps.pins.outputs.skillforge_repo }} + ref: ${{ steps.pins.outputs.skillforge_revision }} + token: ${{ secrets.SKILLFORGE_READ_TOKEN }} + path: .deps/skillforge + - name: Checkout pinned ai-dev-kit + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: ${{ steps.pins.outputs.ai_dev_kit_repo }} + ref: ${{ steps.pins.outputs.ai_dev_kit_revision }} + path: .deps/ai-dev-kit + - name: Install pinned external SkillForge + run: uv tool install '.deps/skillforge/python[all]' + - name: Configure dedicated non-production profile + env: + DATABRICKS_HOST: ${{ secrets.SB_EVAL_DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.SB_EVAL_DATABRICKS_TOKEN }} + run: | + mkdir -p "$HOME/.databricks" + printf '[solution-builder-eval]\nhost = %s\ntoken = %s\n' "$DATABRICKS_HOST" "$DATABRICKS_TOKEN" > "$HOME/.databrickscfg" + chmod 600 "$HOME/.databrickscfg" + - name: Full manual L1-L5 live evaluation + env: + SB_EVAL_AI_DEV_KIT_DIR: ${{ github.workspace }}/.deps/ai-dev-kit + SB_EVAL_DATABRICKS_PROFILE: solution-builder-eval + SB_EVAL_ALLOWED_PROFILES: solution-builder-eval + SB_EVAL_ALLOWED_HOSTS: ${{ secrets.SB_EVAL_DATABRICKS_HOST }} + DATABRICKS_HOST: ${{ secrets.SB_EVAL_DATABRICKS_HOST }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + SB_EVAL_SCENARIO: ${{ inputs.scenario }} + run: uv run sb-eval run --levels all --live --scenario "$SB_EVAL_SCENARIO" + - name: Upload evaluation reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: skillforge-live-${{ github.run_id }} + path: test-runs/skillforge/ + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 7055c06..1e5f4f3 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ screenshot-eval/ # tests/pipeline/ harness output test-runs/ +# Maintainer evaluation dependency checkouts (never packaged or deployed) +.deps/ + # Python caches __pycache__/ *.pyc diff --git a/CLAUDE.md b/CLAUDE.md index bb451a5..180068c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,6 +16,8 @@ A **system that generates Databricks demos**. Not one app — **three**, plus a Plus: **ai_dev_kit** (`app/ai_dev_kit/`) — a cloned external repo (`github.com/databricks-solutions/ai-dev-kit`) holding ~26 sub-skills for creating individual Databricks resources (pipelines, dashboards, Genie spaces, KAs, MAS, etc.). The generator's agent uses these during the Build stage. +Maintainers also have an optional **evaluation harness** (`evaluation/`). It owns canonical scenarios and shells out to an exactly pinned external SkillForge executable. It is not imported by or packaged with the deployed generator app, copied into projects, or exposed through application routes. + ## Mental model ``` @@ -81,6 +83,7 @@ industry-demo-prompts/ │ ├── app.md # How to design+spec a demo app (read during Stage 2) │ └── app_template/ # ★ The template app emitted by the generator ├── initial_templates/ # Pre-built seed templates (retail/loyalty-segmentation) +├── evaluation/ # ★ Maintainer-only scenarios, sb-eval CLI, SkillForge adapter ├── tests/ # Playwright E2E for the generator (targets :9000) ├── install.sh # End-user installer (downloads skill + ai-dev-kit) └── docs/ # Screenshots for README @@ -199,6 +202,18 @@ cd app/test/app_template_test/app ./start.sh # Boots the LuxeBeauty test app on :8765 ``` +For **maintainer evaluation** (from the repository root): + +```bash +uv run sb-eval cases validate +uv run sb-eval doctor +uv run sb-eval run --levels L1,L3 +# Manual, guarded, non-production only: +uv run sb-eval run --levels all --live --scenario +``` + +SkillForge remains separately installed at `evaluation/skillforge.lock.yaml`'s exact revision. Reports and transient fixtures land under gitignored `test-runs/skillforge/`. Scores are advisory; live cleanup failures and leaked resources fail the evaluation command. + ## Conventions - **Python**: `uv` only, never `pip`. Use `claude-agent-sdk` (NOT the deprecated `Skill` tool name — pass `skills=` to the SDK). diff --git a/evaluation/README.md b/evaluation/README.md new file mode 100644 index 0000000..f537b77 --- /dev/null +++ b/evaluation/README.md @@ -0,0 +1,47 @@ +# Maintainer evaluation + +`evaluation/` is the durable, runner-neutral evaluation API for Solution Builder. It is outside `app/`, the shipped `databricks-demo-generator` skill, deployed app wheel artifacts, installers, and all deployed routes. + +## Contracts + +- `cases/*.yaml` contains the versioned canonical scenarios. `tests/pipeline` consumes the same files. +- `models.py` and `schema/scenario.schema.json` define the Pydantic and JSON Schema contracts. +- `adapter.py` converts a scenario losslessly into transient SkillForge v5 assets with `shared_cwd: true`. +- `skillforge.lock.yaml` pins SkillForge and ai-dev-kit by full Git commit. +- `EvalRun` is the normalized result contract. Generated fixtures, raw output, normalized JSON, HTML, and leak reports live under `test-runs/skillforge/`. + +SkillForge is never installed by `sb-eval`. The CLI discovers the external `stf` executable, requires `stf build-info --json`, and verifies its version, revision, and safety features against the lock. + +## Commands + +```bash +uv run sb-eval cases validate +uv run sb-eval doctor +uv run sb-eval run --levels L1,L3 +uv run sb-eval run --levels all --live --scenario financial-services +``` + +`--scenario all` is the default. Scores and gaps are informational; the command does not apply a minimum score. External runner errors, `invalid_eval`, missing live requirements, and cleanup leaks return nonzero. + +## Live guardrails + +Live runs are manual only. Set all of: + +```bash +export SB_EVAL_DATABRICKS_PROFILE=solution-builder-eval +export SB_EVAL_ALLOWED_PROFILES=solution-builder-eval +export SB_EVAL_ALLOWED_HOSTS=https://non-production-workspace.example.com +``` + +The profile name may not be `DEFAULT`, `prod`, or `production`. The resolved host must exactly match the allowlist, and `databricks current-user me` must identify the caller as a service principal. Every run/case/side receives distinct `SB_EVAL_CATALOG`, `SB_EVAL_SCHEMA`, and `SB_EVAL_RESOURCE_PREFIX` values. Setup writes `.skillforge/run-context.json`; cleanup reconciles `resources.json` with SkillForge tracking, deletes in reverse dependency order, retries, and writes `leak-report.json`. Any remainder fails the command. + +## Improvement loop + +1. Run a quick or full evaluation on a feature branch. +2. Inspect normalized gaps, raw SkillForge output, and MLflow traces. +3. Run `/forge-improve --run-id ` locally. +4. Review the skill diff; improvement never writes directly to `main`. +5. Re-run the identical scenarios and confirm no regressions or leaks. +6. Submit the change through the normal reviewed PR process. + +Shared reports belong in MLflow and CI artifacts, not Git. No GitHub credentials, evaluation controls, or reports belong in the deployed app. diff --git a/evaluation/__init__.py b/evaluation/__init__.py new file mode 100644 index 0000000..0b545e1 --- /dev/null +++ b/evaluation/__init__.py @@ -0,0 +1,9 @@ +"""Maintainer-only evaluation tooling for Solution Builder. + +This package is deliberately rooted outside ``app/`` and is not included in +the generator wheel, installer, or copied demo-generator skill. +""" + +from .models import EvalRun, Scenario + +__all__ = ["EvalRun", "Scenario"] diff --git a/evaluation/adapter.py b/evaluation/adapter.py new file mode 100644 index 0000000..d24f5e9 --- /dev/null +++ b/evaluation/adapter.py @@ -0,0 +1,252 @@ +"""Lossless conversion from canonical scenarios to transient SkillForge v5 assets.""" + +from __future__ import annotations + +import json +import re +import shutil +import sys +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from evaluation.live import expected_resource_kinds +from evaluation.models import Scenario + + +_UNRESOLVED = re.compile(r"\$\{[^}]+\}|\{\{[^}]+\}\}") + + +class AdapterError(ValueError): + pass + + +@dataclass(frozen=True) +class RenderedAssets: + eval_dir: Path + ground_truth: Path + manifest: Path + thinking_rubric: Path + output_rubric: Path + comparison_rubric: Path + + +class SkillForgeAdapter: + def __init__(self, repo_root: Path) -> None: + self.repo_root = repo_root.resolve() + self.rubric_dir = ( + self.repo_root / "evaluation" / "rubrics" / "databricks-demo-generator" + ) + + def validate(self, scenario: Scenario) -> None: + self._validate_scenario_sources(scenario) + self._reject_placeholders(self._ground_truth(scenario, live=False)) + self._reject_placeholders(self._manifest(scenario, live=False)) + + def render( + self, + scenario: Scenario, + eval_dir: Path, + *, + live: bool = False, + ) -> RenderedAssets: + self._validate_scenario_sources(scenario) + eval_dir.mkdir(parents=True, exist_ok=True) + + ground_truth_data = self._ground_truth(scenario, live=live) + manifest_data = self._manifest(scenario, live=live) + self._reject_placeholders(ground_truth_data) + self._reject_placeholders(manifest_data) + + ground_truth = eval_dir / "ground_truth.yaml" + manifest = eval_dir / "manifest.yaml" + ground_truth.write_text( + yaml.safe_dump(ground_truth_data, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + manifest.write_text( + yaml.safe_dump(manifest_data, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + + rubric_names = ( + "thinking_instructions.md", + "output_instructions.md", + "comparison_rubric.md", + ) + for name in rubric_names: + source = self.rubric_dir / name + if not source.is_file(): + raise AdapterError(f"missing rubric: {source}") + shutil.copy2(source, eval_dir / name) + + source_dir = eval_dir / "source_of_truth" + source_dir.mkdir(exist_ok=True) + (source_dir / "canonical-scenario.json").write_text( + scenario.model_dump_json(indent=2), encoding="utf-8" + ) + + return RenderedAssets( + eval_dir=eval_dir, + ground_truth=ground_truth, + manifest=manifest, + thinking_rubric=eval_dir / rubric_names[0], + output_rubric=eval_dir / rubric_names[1], + comparison_rubric=eval_dir / rubric_names[2], + ) + + def _ground_truth(self, scenario: Scenario, *, live: bool) -> dict[str, Any]: + cases: list[dict[str, Any]] = [] + for index, step in enumerate(scenario.steps): + expected_facts: list[Any] = list(step.expected_facts) + expected_facts.extend( + { + "fact": verification.fact, + "verify_cmd": verification.command, + "verify_check": verification.check, + } + for verification in step.verification_commands + ) + prompt = step.prompt + if live: + prompt = ( + f"{prompt}\n\n" + "LIVE EVALUATION SAFETY: use the SB_EVAL_CATALOG, " + "SB_EVAL_SCHEMA, and SB_EVAL_RESOURCE_PREFIX environment " + "values for every created resource. Record every created " + "resource in resources.json. Do not mutate resources outside " + "that namespace." + ) + cases.append( + { + "id": f"{scenario.id}--{index + 1:02d}-{step.id}", + "inputs": {"prompt": prompt}, + "expectations": { + "expected_facts": expected_facts, + "assertions": step.semantic_assertions, + "expected_patterns": step.expected_patterns, + "trace_expectations": { + "required_tools": step.tool_expectations.required, + "banned_tools": step.tool_expectations.banned, + "token_budget": {"max_total": 80_000}, + }, + "guidelines": [ + f"Expected project stage after this step: {step.expected_project_stage.value}", + "Required artifacts: " + + ", ".join(item.path for item in step.required_artifacts), + ], + }, + "metadata": { + "category": "happy_path", + "difficulty": "hard" + if index == len(scenario.steps) - 1 + else "intermediate", + "generation_session_id": str( + uuid.uuid5( + uuid.NAMESPACE_URL, + f"solution-builder:{scenario.id}:{step.id}", + ) + ), + "regression_intent": step.regression_intent, + "sources": [ + { + **source.model_dump(mode="json"), + "uri": self._skillforge_source_uri(source.uri), + } + for source in step.sources + ], + }, + } + ) + return {"version": "5", "test_cases": cases} + + def _manifest(self, scenario: Scenario, *, live: bool) -> dict[str, Any]: + declared_resource_kinds = [ + *scenario.live_resources.expected_resource_kinds, + *scenario.live_resources.additional_resource_kinds, + ] + derived_resource_kinds = expected_resource_kinds(scenario.capabilities) + manifest_resource_kinds = [ + *dict.fromkeys(declared_resource_kinds), + *( + kind + for kind in derived_resource_kinds + if kind not in declared_resource_kinds + ), + ] + manifest: dict[str, Any] = { + "skill_name": scenario.skill, + "description": f"Solution Builder canonical scenario: {scenario.id}", + "shared_cwd": True, + "tool_modules": [], + "comparison_judge": { + "rubric_file": "comparison_rubric.md", + "dimensions": [ + "correctness", + "coherence", + "safety", + "artifact_quality", + ], + "anti_bias": [ + "Treat the target skill as WITH and the control skill as WITHOUT.", + "Do not reward verbosity or tool-call count.", + "Attribute shared-cwd cascading failures to their earliest causal case.", + ], + }, + "solution_builder": { + "scenario_id": scenario.id, + "capabilities": scenario.capabilities, + "expected_resource_kinds": manifest_resource_kinds, + "cleanup_owner": scenario.live_resources.cleanup_owner, + }, + } + if live: + hook_command = [ + sys.executable, + "-m", + "evaluation.hook_cli", + ] + manifest["lifecycle"] = { + "setup": {"command": [*hook_command, "setup"]}, + "cleanup": {"command": [*hook_command, "cleanup"], "always": True}, + } + return manifest + + def _validate_scenario_sources(self, scenario: Scenario) -> None: + for step in scenario.steps: + if not step.regression_intent.strip(): + raise AdapterError( + f"{scenario.id}/{step.id}: missing regression intent" + ) + if not ( + step.expected_facts + and step.semantic_assertions + and step.expected_patterns + and step.required_artifacts + ): + raise AdapterError(f"{scenario.id}/{step.id}: incomplete expectations") + for source in step.sources: + if not source.uri.startswith("repo://"): + continue + path = self.repo_root / source.uri.removeprefix("repo://") + if not path.is_file(): + raise AdapterError( + f"{scenario.id}/{step.id}: source does not exist: {source.uri}" + ) + + def _skillforge_source_uri(self, uri: str) -> str: + # Citations are identity/provenance, not files SkillForge opens. Keep + # repository URIs portable while validating their local targets above. + return uri + + @staticmethod + def _reject_placeholders(data: Any) -> None: + rendered = json.dumps(data, default=str) + match = _UNRESOLVED.search(rendered) + if match: + raise AdapterError( + f"unresolved placeholder in generated assets: {match.group(0)}" + ) diff --git a/evaluation/cases/__init__.py b/evaluation/cases/__init__.py new file mode 100644 index 0000000..5a46653 --- /dev/null +++ b/evaluation/cases/__init__.py @@ -0,0 +1,53 @@ +"""Load and validate canonical Solution Builder evaluation cases.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from evaluation.models import Scenario + + +CASES_DIR = Path(__file__).parent + + +class CaseValidationError(ValueError): + pass + + +def case_paths(cases_dir: Path = CASES_DIR) -> list[Path]: + return sorted(path for path in cases_dir.glob("*.yaml") if path.is_file()) + + +def load_case(path: Path) -> Scenario: + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + return Scenario.model_validate(raw) + except (OSError, yaml.YAMLError, ValidationError, TypeError) as exc: + raise CaseValidationError(f"{path}: {exc}") from exc + + +def load_cases(cases_dir: Path = CASES_DIR) -> list[Scenario]: + paths = case_paths(cases_dir) + if not paths: + raise CaseValidationError(f"no scenario YAML files found in {cases_dir}") + cases = [load_case(path) for path in paths] + ids = [case.id for case in cases] + if len(ids) != len(set(ids)): + raise CaseValidationError("scenario ids must be unique across case files") + return cases + + +def select_cases(selector: str | None, cases_dir: Path = CASES_DIR) -> list[Scenario]: + cases = load_cases(cases_dir) + if selector in (None, "all"): + return cases + selected = [case for case in cases if case.id == selector] + if not selected: + available = ", ".join(case.id for case in cases) + raise CaseValidationError( + f"unknown scenario {selector!r}; available: {available}" + ) + return selected diff --git a/evaluation/cases/financial-services.yaml b/evaluation/cases/financial-services.yaml new file mode 100644 index 0000000..c5d6c43 --- /dev/null +++ b/evaluation/cases/financial-services.yaml @@ -0,0 +1,94 @@ +schema_version: 1 +id: financial-services +skill: databricks-demo-generator +description: >- + Real-time fraud detection demo for a retail bank. Generate synthetic + credit-card transactions, surface anomalies in a Genie space, and visualize + trends in an AI/BI dashboard. +capabilities: [genie, aibi-dashboards, synthetic-data-gen] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, dashboard, genie_space] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a fraud-detection demo for a retail bank. Use synthetic credit-card + transactions, an anomaly-detection pattern, a Genie space for ad-hoc + questions, and an AI/BI dashboard for the trend view. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved business story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The story has a named banking protagonist, a concrete fraud catalyst, and quantified business impact. + - Synthetic transactions, the anomaly narrative, Genie questions, and dashboard measures form one coherent story. + expected_facts: + - The proposed demo uses synthetic credit-card transactions, Genie, and an AI/BI dashboard. + expected_patterns: ['(?i)fraud', '(?i)(revenue|loss|risk|\$)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects generic fraud stories whose selected products or generated data do not support the same investigation. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: 2026-07-17T00:00:00Z + snippet: Story first and coherence above all; every showcased product must earn a clear beat in the walkthrough. + - id: design-architecture + prompt: Looks good — proceed to architect the pipeline and produce architecture.md. + expected_project_stage: ARCHITECTED + required_artifacts: + - {path: architecture.md, description: Renderable architecture diagram JSON} + semantic_assertions: + - The architecture connects transaction generation to governed data, Genie, and the dashboard without orphan components. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(genie|dashboard)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects architecture output that omits a selected capability or breaks the end-to-end data flow. + sources: [*skill_source] + - id: write-specifications + prompt: Now write the specifications for each component (specifications/*.md). + expected_project_stage: SPECIFICATION + required_artifacts: + - {path: specifications/*.md, description: Functional component specifications} + semantic_assertions: + - Specifications define compatible transaction fields, fraud measures, dashboard visuals, and Genie questions. + expected_facts: [Per-component specifications are written under specifications/.] + expected_patterns: ['specifications/', '(?i)(transaction|anomal)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects specs that are individually plausible but disagree on schemas, measures, or the investigation path. + sources: [*skill_source] + - id: build-demo + prompt: "Build it: write the SQL/Python and capture deployed resource IDs in resources.json." + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - Build artifacts implement the approved specifications and resources.json records only resources actually created. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [SQL or Python implementation artifacts and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(sql|python|dashboard|genie)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and implementation code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects claimed builds with placeholders, missing code, dead resource links, or resources outside the evaluation namespace. + sources: [*skill_source] diff --git a/evaluation/cases/healthcare.yaml b/evaluation/cases/healthcare.yaml new file mode 100644 index 0000000..e915e4b --- /dev/null +++ b/evaluation/cases/healthcare.yaml @@ -0,0 +1,91 @@ +schema_version: 1 +id: healthcare +skill: databricks-demo-generator +description: >- + Patient-readmission risk demo for a hospital network. Combine clinical notes + (knowledge assistant and vector search) with a tabular ML model for 30-day + readmission risk. +capabilities: [knowledge-assistant, vector-search, ml-training-serving] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, volume, vector_index, knowledge_assistant, model, serving_endpoint, mlflow_experiment] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a patient-readmission risk demo. Knowledge Assistant over + discharge-summary documents, vector search over the same corpus, and a + tabular ML model that scores 30-day readmission risk on synthetic patient data. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved clinical story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The story explains distinct, complementary roles for document retrieval and tabular risk scoring. + - The clinical narrative uses synthetic data and avoids claims that the demo provides medical advice. + expected_facts: [The demo combines discharge summaries with a 30-day readmission risk model.] + expected_patterns: ['(?i)readmission', '(?i)(discharge|clinical)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects healthcare stories that collapse RAG and predictive ML into an incoherent or unsafe workflow. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: 2026-07-17T00:00:00Z + snippet: Data, pipeline, dashboard, Genie, and agents must align; one broken link ruins the demo. + - id: design-architecture + prompt: "Proceed: architect the components and produce architecture.md." + expected_project_stage: ARCHITECTED + required_artifacts: [{path: architecture.md, description: Renderable architecture diagram JSON}] + semantic_assertions: + - The architecture separates document ingestion and vector retrieval from feature engineering, training, and serving while showing their user-facing convergence. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(vector|model|assistant)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects architectures that omit the shared corpus, model-serving path, or governance boundaries. + sources: [*skill_source] + - id: write-specifications + prompt: Write the per-component specifications under specifications/. + expected_project_stage: SPECIFICATION + required_artifacts: [{path: specifications/*.md, description: Functional component specifications}] + semantic_assertions: + - Specs define a consistent patient identifier, temporal split, discharge corpus, vector index, model target, and serving contract. + expected_facts: [Per-component specifications are written under specifications/.] + expected_patterns: ['specifications/', '(?i)(30-day|readmission)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects temporal leakage, mismatched patient keys, or a Knowledge Assistant corpus disconnected from the story. + sources: [*skill_source] + - id: build-demo + prompt: Build it now — generate the code, training notebook, and resources.json. + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - Training and retrieval artifacts implement the specifications and resources.json contains only validated IDs. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [Training code, retrieval configuration, and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(train|vector|knowledge)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and Python or SQL code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects builds that claim live RAG or serving assets without code, identifiers, or cleanup-safe names. + sources: [*skill_source] diff --git a/evaluation/cases/manufacturing-machinery.yaml b/evaluation/cases/manufacturing-machinery.yaml new file mode 100644 index 0000000..5497b6a --- /dev/null +++ b/evaluation/cases/manufacturing-machinery.yaml @@ -0,0 +1,97 @@ +schema_version: 1 +id: manufacturing-machinery +skill: databricks-demo-generator +description: >- + Warranty optimization demo for a heavy-machinery OEM (Manufacturing > + Machinery sub-vertical). Synthetic warranty claims and telemetry land in + Delta, Genie answers cost questions, a supervisor routes work, and a + Databricks App supports field service. +capabilities: [synthetic-data-gen, genie, supervisor-agent, databricks-apps] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, genie_space, multi_agent_supervisor, serving_endpoint, app] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a warranty-optimization demo for a heavy-machinery OEM + (Manufacturing > Machinery sub-vertical). Use synthetic-data-gen to + create warranty claims and machine-telemetry tables in Delta, stand up a + Genie space over the Gold warranty tables for ad-hoc cost/failure-mode + questions, wire a multi-agent supervisor that routes between Genie + (quantitative) and a claims-triage agent, and ship a Databricks App + (FastAPI + React) where a field-service manager can inspect a claim and + see the supervisor's recommendation. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved warranty story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The field-service workflow gives synthetic telemetry, warranty costs, Genie, supervisor routing, and the app distinct connected roles. + expected_facts: [The demo is for a heavy-machinery OEM and includes Genie, a supervisor, and a field-service app.] + expected_patterns: ['(?i)warranty', '(?i)(field.service|machinery)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects disconnected app, supervisor, or data stories and regressions to a generic manufacturing vertical. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and app guidance + retrieved_at: 2026-07-17T00:00:00Z + snippet: Every showcased product earns a clear beat, and app creation is folded into specification and build stages. + - id: design-architecture + prompt: Looks good — proceed to architect the pipeline and produce architecture.md. + expected_project_stage: ARCHITECTED + required_artifacts: [{path: architecture.md, description: Renderable architecture diagram JSON}] + semantic_assertions: + - The architecture connects claims and telemetry through Gold data to Genie, supervisor routing, and the app. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(genie|supervisor|app)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects missing routing, app, or data-flow nodes in a selected full-stack scenario. + sources: [*skill_source] + - id: write-specifications + prompt: Now write the specifications for each component (specifications/*.md) — synthetic data tables, Genie space, supervisor agent, and the Databricks App. + expected_project_stage: SPECIFICATION + required_artifacts: [{path: specifications/*.md, description: Functional component specifications}] + semantic_assertions: + - Specs preserve claim identifiers, failure modes, cost measures, routing criteria, and app interactions across components. + expected_facts: [Specifications cover synthetic data, Genie, the supervisor, and the Databricks App.] + expected_patterns: ['specifications/', '(?i)(claim|telemetry)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects interface drift between data, Genie, supervisor tools, and the app experience. + sources: [*skill_source] + - id: build-demo + prompt: "Build it: write the SQL/Python for synthetic data, the Genie config, the supervisor agent definition, the app code, and capture deployed resource IDs in resources.json." + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: app/**, description: Databricks App implementation} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - App, supervisor, Genie, and data artifacts implement the approved contracts and resources.json records validated IDs. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [Data code, Genie config, supervisor definition, app code, and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(supervisor|app|genie)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and app plus data code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && test -d app && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects incomplete full-stack builds, stale IDs, or resources that cannot be isolated and cleaned. + sources: [*skill_source] diff --git a/evaluation/cases/retail.yaml b/evaluation/cases/retail.yaml new file mode 100644 index 0000000..8c91074 --- /dev/null +++ b/evaluation/cases/retail.yaml @@ -0,0 +1,91 @@ +schema_version: 1 +id: retail +skill: databricks-demo-generator +description: >- + Demand-forecasting demo for an omnichannel retailer. Stream point-of-sale + events into Delta, aggregate into a Lakebase OLTP store, and serve a forecast + model via Model Serving. +capabilities: [lakebase, ml-training-serving, sdp] +timeout_seconds: 3600 +target_stage: BUILT +live_resources: + cleanup_owner: solution-builder + expected_resource_kinds: [catalog, schema, table, pipeline, lakebase, model, serving_endpoint, mlflow_experiment] + additional_resource_kinds: [] + evaluation_prefix: sb_eval_ +steps: + - id: capture-story + prompt: >- + Build a demand-forecasting demo for an omnichannel retailer. Use a + Lakeflow Spark Declarative Pipeline (SDP) to ingest synthetic POS events + into Delta, land aggregates in Lakebase for fast lookups, and serve a + Prophet-style forecast via Model Serving. + expected_project_stage: SUMMARIZED + required_artifacts: + - {path: README.md, description: Approved retail story} + - {path: resources.json, description: Capability and resource manifest} + semantic_assertions: + - The story explains why governed Delta history, Lakebase lookups, and model serving are all needed by one retail protagonist. + expected_facts: [The demo uses SDP, Lakebase, and Model Serving for omnichannel demand forecasting.] + expected_patterns: ['(?i)(demand|forecast)', '(?i)(retail|point.of.sale|POS)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: README.md and resources.json exist after story capture. + command: "bash -lc 'test -f README.md && test -f resources.json && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects retail stories that select SDP, Lakebase, and serving without a coherent latency and user-value rationale. + sources: + - &skill_source + type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: 2026-07-17T00:00:00Z + snippet: Match products to moments and make data, pipeline, consumption layers, and story align end to end. + - id: design-architecture + prompt: "Proceed: produce architecture.md." + expected_project_stage: ARCHITECTED + required_artifacts: [{path: architecture.md, description: Renderable architecture diagram JSON}] + semantic_assertions: + - The architecture orders POS ingestion, Delta transformations, forecasting, Lakebase synchronization, and serving dependencies correctly. + expected_facts: [The project contains architecture.md.] + expected_patterns: ['architecture\.md', '(?i)(lakebase|serving|pipeline)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: architecture.md exists and is not empty. + command: "bash -lc 'test -s architecture.md && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects architectures that reverse dependencies or leave Lakebase and model serving disconnected. + sources: [*skill_source] + - id: write-specifications + prompt: Write the per-component specifications. + expected_project_stage: SPECIFICATION + required_artifacts: [{path: specifications/*.md, description: Functional component specifications}] + semantic_assertions: + - Specs align POS event grain, forecast horizon, aggregate keys, Lakebase schema, and serving inputs and outputs. + expected_facts: [Per-component specifications are written under specifications/.] + expected_patterns: ['specifications/', '(?i)(forecast|aggregate)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: At least one Markdown specification exists. + command: "bash -lc 'find specifications -name \"*.md\" -type f -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects grain, key, or temporal-contract disagreements across pipeline, model, and operational store specs. + sources: [*skill_source] + - id: build-demo + prompt: "Build it: generate the SDP pipeline, the Lakebase sync, the forecast notebook, and resources.json." + expected_project_stage: BUILT + required_artifacts: + - {path: resources.json, description: Populated deployed resource manifest} + - {path: '**/*.{py,sql,json}', description: Buildable implementation artifacts} + semantic_assertions: + - SDP, sync, and forecasting artifacts implement the shared schema and resources.json contains only validated IDs. + - Live resource names use the evaluation namespace supplied in the environment when present. + expected_facts: [The SDP pipeline, Lakebase sync, forecast code, and resource identifiers are produced.] + expected_patterns: ['resources\.json', '(?i)(pipeline|lakebase|forecast)'] + tool_expectations: {required: [Read, Write], banned: []} + verification_commands: + - fact: resources.json is valid JSON and Python or SQL code exists. + command: "bash -lc 'python3 -m json.tool resources.json >/dev/null && find . -type f -name \"*.py\" -print -quit | grep -q . && echo {\"ok\":true}'" + check: .ok == true + regression_intent: Detects placeholder builds, broken sync contracts, stale IDs, or resources outside the evaluation namespace. + sources: [*skill_source] diff --git a/evaluation/cli.py b/evaluation/cli.py new file mode 100644 index 0000000..8a42afa --- /dev/null +++ b/evaluation/cli.py @@ -0,0 +1,171 @@ +"""Maintainer CLI for canonical cases and pinned external SkillForge.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timezone + +from evaluation.adapter import SkillForgeAdapter +from evaluation.cases import CaseValidationError, load_cases, select_cases +from evaluation.runner import new_run_id, normalize_levels, run_scenario +from evaluation.schema_generator import rendered_schema +from evaluation.toolchain import ( + REPO_ROOT, + Check, + find_executable, + load_lock, + run_doctor, +) + + +def _print_checks(checks: list[Check], *, as_json: bool) -> None: + if as_json: + print(json.dumps([check.__dict__ for check in checks], indent=2)) + return + for check in checks: + marker = "PASS" if check.ok else "FAIL" + print(f"[{marker}] {check.name}: {check.detail}") + + +def cases_validate(*, as_json: bool) -> int: + try: + cases = load_cases() + adapter = SkillForgeAdapter(REPO_ROOT) + for case in cases: + adapter.validate(case) + schema_path = REPO_ROOT / "evaluation" / "schema" / "scenario.schema.json" + if ( + not schema_path.is_file() + or schema_path.read_text(encoding="utf-8") != rendered_schema() + ): + raise CaseValidationError( + "committed JSON Schema is stale; run `uv run python -m evaluation.schema_generator`" + ) + except Exception as exc: # noqa: BLE001 - CLI boundary + if as_json: + print(json.dumps({"valid": False, "error": str(exc)})) + else: + print(f"case validation failed: {exc}", file=sys.stderr) + return 1 + payload = { + "valid": True, + "count": len(cases), + "scenarios": [case.id for case in cases], + } + print( + json.dumps(payload, indent=2) + if as_json + else f"validated {len(cases)} scenarios: {', '.join(payload['scenarios'])}" + ) + return 0 + + +def doctor(*, as_json: bool) -> int: + checks = run_doctor(REPO_ROOT) + lock = load_lock() + executable = find_executable(lock) + if executable: + completed = subprocess.run( + [executable, "doctor", "--json", "--skip-judge"], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + checks.append( + Check( + "skillforge-doctor", + completed.returncode == 0, + (completed.stdout or completed.stderr).strip()[-4000:], + ) + ) + _print_checks(checks, as_json=as_json) + return 0 if all(check.ok or not check.required for check in checks) else 1 + + +def run_command(args: argparse.Namespace) -> int: + try: + levels = normalize_levels(args.levels) + cases = select_cases(args.scenario) + except (ValueError, CaseValidationError) as exc: + print(str(exc), file=sys.stderr) + return 2 + run_id = new_run_id() + stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") + output_dir = REPO_ROOT / "test-runs" / "skillforge" / f"{stamp}-{run_id[:8]}" + output_dir.mkdir(parents=True, exist_ok=False) + results = [] + failed = False + for case in cases: + print( + f"[sb-eval] scenario={case.id} levels={','.join(levels)} live={args.live}" + ) + try: + result = run_scenario( + case, + levels=levels, + output_dir=output_dir, + live=args.live, + agent_model=args.agent_model, + judge_model=args.judge_model, + run_id=run_id, + ) + results.append(result) + print(f"[sb-eval] {case.id}: {result.status}") + failed = failed or result.status != "passed" + except Exception as exc: # noqa: BLE001 - continue to retain other reports + failed = True + print(f"[sb-eval] {case.id}: fatal: {exc}", file=sys.stderr) + summary = { + "run_id": run_id, + "output_dir": str(output_dir), + "results": [ + { + "scenario": result.scenario_id, + "status": result.status, + "report": result.reports.get("html"), + } + for result in results + ], + } + (output_dir / "summary.json").write_text( + json.dumps(summary, indent=2), encoding="utf-8" + ) + print(f"[sb-eval] reports: {output_dir}") + return 1 if failed else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="sb-eval") + subcommands = parser.add_subparsers(dest="command", required=True) + cases = subcommands.add_parser("cases") + case_commands = cases.add_subparsers(dest="case_command", required=True) + validate = case_commands.add_parser("validate") + validate.add_argument("--json", action="store_true") + doctor_parser = subcommands.add_parser("doctor") + doctor_parser.add_argument("--json", action="store_true") + run = subcommands.add_parser("run") + run.add_argument("--levels", required=True, help="L1,L3 or all") + run.add_argument("--live", action="store_true") + run.add_argument("--scenario", default="all") + run.add_argument("--agent-model") + run.add_argument("--judge-model") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "cases" and args.case_command == "validate": + return cases_validate(as_json=args.json) + if args.command == "doctor": + return doctor(as_json=args.json) + if args.command == "run": + return run_command(args) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evaluation/fixture.py b/evaluation/fixture.py new file mode 100644 index 0000000..9101d16 --- /dev/null +++ b/evaluation/fixture.py @@ -0,0 +1,139 @@ +"""Build isolated project fixtures from the target and pinned sibling skills.""" + +from __future__ import annotations + +import io +import os +import shutil +import subprocess +import tarfile +from dataclasses import dataclass +from pathlib import Path + +from evaluation.toolchain import ( + REPO_ROOT, + ToolchainLock, + load_lock, + resolve_ai_dev_kit, +) + + +EXCLUDED_AI_DEV_KIT_SKILLS = frozenset({"TEMPLATE"}) +COPY_IGNORE = shutil.ignore_patterns( + ".git", + ".venv", + "__pycache__", + ".pytest_cache", + "dist", + "eval", + "test-runs", +) + + +@dataclass(frozen=True) +class ProjectFixture: + root: Path + project_dir: Path + target_skill: Path + control_skill: Path + skills_dir: Path + + +class FixtureError(RuntimeError): + pass + + +def _extract_pinned_skills( + checkout: Path, + lock: ToolchainLock, + destination: Path, +) -> Path: + if os.environ.get("SB_EVAL_ALLOW_UNPINNED_FIXTURE") == "test-only": + source = checkout / lock.ai_dev_kit_skills_subdirectory + if not source.is_dir(): + raise FixtureError(f"test fixture has no skills directory: {source}") + shutil.copytree(source, destination / lock.ai_dev_kit_skills_subdirectory) + return destination / lock.ai_dev_kit_skills_subdirectory + + completed = subprocess.run( + [ + "git", + "-C", + str(checkout), + "archive", + "--format=tar", + lock.ai_dev_kit_revision, + lock.ai_dev_kit_skills_subdirectory, + ], + check=False, + capture_output=True, + ) + if completed.returncode != 0: + raise FixtureError( + f"cannot materialize ai-dev-kit revision {lock.ai_dev_kit_revision}: " + + completed.stderr.decode("utf-8", errors="replace") + ) + with tarfile.open(fileobj=io.BytesIO(completed.stdout), mode="r:") as archive: + archive.extractall(destination, filter="data") + return destination / lock.ai_dev_kit_skills_subdirectory + + +def build_fixture( + destination: Path, + *, + repo_root: Path = REPO_ROOT, + lock: ToolchainLock | None = None, +) -> ProjectFixture: + lock = lock or load_lock() + target_source = repo_root / ".claude" / "skills" / "databricks-demo-generator" + if not (target_source / "SKILL.md").is_file(): + raise FixtureError(f"target skill not found: {target_source}") + ai_dev_kit = resolve_ai_dev_kit(repo_root) + if ai_dev_kit is None: + raise FixtureError("ai-dev-kit checkout not found; set SB_EVAL_AI_DEV_KIT_DIR") + + project_dir = destination / "project" + skills_dir = project_dir / ".claude" / "skills" + skills_dir.mkdir(parents=True, exist_ok=False) + target_skill = skills_dir / "databricks-demo-generator" + shutil.copytree(target_source, target_skill, ignore=COPY_IGNORE) + + extracted = destination / ".ai-dev-kit-pinned" + extracted.mkdir() + sibling_skills = _extract_pinned_skills(ai_dev_kit, lock, extracted) + for source in sorted(sibling_skills.iterdir()): + if ( + not source.is_dir() + or source.name in EXCLUDED_AI_DEV_KIT_SKILLS + or not (source / "SKILL.md").is_file() + ): + continue + shutil.copytree(source, skills_dir / source.name, ignore=COPY_IGNORE) + shutil.rmtree(extracted) + + control_skill = skills_dir / "solution-builder-without-skill" + control_skill.mkdir() + (control_skill / "SKILL.md").write_text( + "---\n" + "name: solution-builder-without-skill\n" + "description: Neutral control for WITH/WITHOUT evaluation.\n" + "---\n\n" + "Complete the user's task using general reasoning. Do not claim access " + "to the databricks-demo-generator workflow.\n", + encoding="utf-8", + ) + (project_dir / "CLAUDE.md").write_text( + "# Evaluation fixture\n\n" + "The execution cwd is the project root. Skills are under " + "`.claude/skills/`. In live runs, every resource must use the " + "`SB_EVAL_*` namespace supplied in the environment and must be " + "recorded in resources.json.\n", + encoding="utf-8", + ) + return ProjectFixture( + root=destination, + project_dir=project_dir, + target_skill=target_skill, + control_skill=control_skill, + skills_dir=skills_dir, + ) diff --git a/evaluation/hashing.py b/evaluation/hashing.py new file mode 100644 index 0000000..bd53154 --- /dev/null +++ b/evaluation/hashing.py @@ -0,0 +1,59 @@ +"""Stable content identity for complete distributed skills.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + + +EXCLUDED_PARTS = frozenset( + { + ".git", + ".hg", + ".svn", + ".venv", + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "node_modules", + "dist", + "build", + "eval", + "test-runs", + "playwright-report", + "test-results", + } +) + + +def distributed_skill_files(skill_dir: Path) -> list[Path]: + """Return every distributed regular file in stable relative-path order.""" + root = skill_dir.resolve() + if not (root / "SKILL.md").is_file(): + raise ValueError(f"not a skill directory: {root}") + files: list[Path] = [] + for path in root.rglob("*"): + if not path.is_file(): + continue + relative = path.relative_to(root) + if any(part in EXCLUDED_PARTS for part in relative.parts): + continue + if path.is_symlink(): + continue + files.append(path) + return sorted(files, key=lambda path: path.relative_to(root).as_posix()) + + +def hash_skill(skill_dir: Path) -> str: + """Hash relative paths and bytes for every distributed skill file.""" + root = skill_dir.resolve() + digest = hashlib.sha256() + for path in distributed_skill_files(root): + relative = path.relative_to(root).as_posix().encode("utf-8") + payload = path.read_bytes() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(len(payload).to_bytes(8, "big")) + digest.update(payload) + return digest.hexdigest() diff --git a/evaluation/hook_cli.py b/evaluation/hook_cli.py new file mode 100644 index 0000000..68fcf83 --- /dev/null +++ b/evaluation/hook_cli.py @@ -0,0 +1,98 @@ +"""Generic SkillForge lifecycle hook entrypoint (JSON stdin/stdout).""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +from evaluation.live import ( + DatabricksCliCleaner, + LiveNamespace, + LivePolicy, + ResourceCleaner, + reconcile_resources, + resources_from_manifest, + resources_from_tracked, + write_leak_report, +) + + +def _read_context() -> dict[str, Any]: + payload = json.load(sys.stdin) + if not isinstance(payload, dict): + raise ValueError("lifecycle context must be a JSON object") + required = {"run_id", "case_id", "side", "cwd", "project_dir", "tracked_resources"} + missing = sorted(required - payload.keys()) + if missing: + raise ValueError(f"lifecycle context missing: {', '.join(missing)}") + return payload + + +def setup(context: dict[str, Any]) -> dict[str, Any]: + prefix = os.environ.get("SB_EVAL_PREFIX", "sb_eval_") + policy = LivePolicy.from_env(prefix) + namespace = LiveNamespace.allocate( + run_id=str(context["run_id"]), + case_id=str(context["case_id"]), + side=str(context["side"]), + evaluation_prefix=prefix, + ) + policy.validate_namespace(namespace) + run_context = { + **context, + "namespace": namespace.__dict__, + "profile": policy.profile, + "host": policy.host, + } + context_path = Path(context["cwd"]) / ".skillforge" / "run-context.json" + context_path.parent.mkdir(parents=True, exist_ok=True) + context_path.write_text(json.dumps(run_context, indent=2), encoding="utf-8") + return { + "run_context_file": str(context_path), + "environment": namespace.environment(policy.profile, policy.host), + } + + +def cleanup(context: dict[str, Any]) -> dict[str, Any]: + prefix = os.environ.get("SB_EVAL_PREFIX", "sb_eval_") + policy = LivePolicy.from_env(prefix) + manifest_resources = resources_from_manifest( + Path(context["cwd"]) / "resources.json", side=str(context["side"]) + ) + tracked_resources = resources_from_tracked(context.get("tracked_resources") or []) + resources = reconcile_resources(manifest_resources, tracked_resources) + backend = DatabricksCliCleaner(policy) + report = ResourceCleaner( + backend.delete, backend.exists, retries=3, retry_delay=1 + ).cleanup(resources) + leak_path = Path(context["cwd"]) / ".skillforge" / "leak-report.json" + write_leak_report(leak_path, report, resources) + result = { + "complete": report.complete, + "deleted": report.deleted, + "remaining": report.remaining, + "errors": report.errors, + "leak_report": str(leak_path), + } + if not report.complete: + print(json.dumps(result)) + raise SystemExit(1) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=("setup", "cleanup")) + args = parser.parse_args(argv) + context = _read_context() + result = setup(context) if args.action == "setup" else cleanup(context) + print(json.dumps(result)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evaluation/lifecycle.py b/evaluation/lifecycle.py new file mode 100644 index 0000000..28e60b2 --- /dev/null +++ b/evaluation/lifecycle.py @@ -0,0 +1,29 @@ +"""Runner-neutral lifecycle primitive used by adapter and contract tests.""" + +from __future__ import annotations + +import inspect +from collections.abc import Awaitable, Callable +from typing import Any, TypeVar + + +T = TypeVar("T") + + +async def _await(value: Any) -> Any: + return await value if inspect.isawaitable(value) else value + + +async def run_with_lifecycle( + *, + context: dict[str, Any], + setup: Callable[[dict[str, Any]], Any], + operation: Callable[[dict[str, Any], Any], T | Awaitable[T]], + cleanup: Callable[[dict[str, Any], Any], Any], +) -> T: + """Run setup/operation/cleanup with cleanup guaranteed by ``finally``.""" + setup_output = await _await(setup(context)) + try: + return await _await(operation(context, setup_output)) + finally: + await _await(cleanup(context, setup_output)) diff --git a/evaluation/live.py b/evaluation/live.py new file mode 100644 index 0000000..14815d0 --- /dev/null +++ b/evaluation/live.py @@ -0,0 +1,772 @@ +"""Live-evaluation isolation, resource reconciliation, and cleanup.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Iterable +from urllib.parse import quote + + +CAPABILITY_RESOURCE_KINDS: dict[str, set[str]] = { + "aibi-dashboards": {"dashboard"}, + "databricks-apps": {"app"}, + "genie": {"genie_space"}, + "knowledge-assistant": {"knowledge_assistant", "serving_endpoint"}, + "lakebase": {"lakebase"}, + "metric-views": {"metric_view"}, + "ml-training-serving": {"model", "serving_endpoint", "mlflow_experiment"}, + "sdp": {"pipeline"}, + "supervisor-agent": {"multi_agent_supervisor", "serving_endpoint"}, + "synthetic-data-gen": {"catalog", "schema", "table"}, + "vector-search": {"vector_endpoint", "vector_index"}, +} + +ACTIVITY_RESOURCE_KINDS = frozenset( + { + "bundle", + "bundle_run", + "dashboard_published", + "job_run", + "pipeline_run", + } +) + + +def expected_resource_kinds( + capabilities: Iterable[str], + *, + overrides: Iterable[str] = (), +) -> list[str]: + kinds: set[str] = set(overrides) + for capability in capabilities: + kinds.update(CAPABILITY_RESOURCE_KINDS.get(capability, set())) + return sorted(kinds) + + +def _slug(value: str, limit: int = 24) -> str: + cleaned = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") + return (cleaned or "case")[:limit] + + +@dataclass(frozen=True) +class LiveNamespace: + run_id: str + case_id: str + side: str + prefix: str + catalog: str + schema: str + resource_prefix: str + + @classmethod + def allocate( + cls, + *, + run_id: str, + case_id: str, + side: str, + evaluation_prefix: str, + ) -> "LiveNamespace": + digest = hashlib.sha256( + f"{run_id}:{case_id}:{side}".encode("utf-8") + ).hexdigest()[:8] + stem = f"{evaluation_prefix}{_slug(run_id, 12)}_{_slug(case_id, 16)}_{_slug(side, 8)}_{digest}" + return cls( + run_id=run_id, + case_id=case_id, + side=side, + prefix=evaluation_prefix, + catalog=stem, + schema=f"{evaluation_prefix}{_slug(case_id, 12)}_{_slug(side, 8)}_{digest}", + resource_prefix=f"{stem}_", + ) + + def environment(self, profile: str, host: str) -> dict[str, str]: + return { + "SB_EVAL_RUN_ID": self.run_id, + "SB_EVAL_CASE_ID": self.case_id, + "SB_EVAL_SIDE": self.side, + "SB_EVAL_CATALOG": self.catalog, + "SB_EVAL_SCHEMA": self.schema, + "SB_EVAL_RESOURCE_PREFIX": self.resource_prefix, + "DATABRICKS_CONFIG_PROFILE": profile, + "DATABRICKS_HOST": host, + } + + +@dataclass(frozen=True) +class LivePolicy: + profile: str + host: str + allowed_hosts: tuple[str, ...] + evaluation_prefix: str + + @classmethod + def from_env(cls, evaluation_prefix: str = "sb_eval_") -> "LivePolicy": + profile = os.environ.get("SB_EVAL_DATABRICKS_PROFILE", "").strip() + if not profile or profile.lower() in {"default", "prod", "production"}: + raise ValueError( + "--live requires a dedicated non-production SB_EVAL_DATABRICKS_PROFILE" + ) + allowed_profiles = { + item.strip() + for item in os.environ.get("SB_EVAL_ALLOWED_PROFILES", "").split(",") + if item.strip() + } + if profile not in allowed_profiles: + raise ValueError("live profile is not listed in SB_EVAL_ALLOWED_PROFILES") + allowed_hosts = tuple( + item.strip().rstrip("/") + for item in os.environ.get("SB_EVAL_ALLOWED_HOSTS", "").split(",") + if item.strip() + ) + if not allowed_hosts: + raise ValueError("--live requires SB_EVAL_ALLOWED_HOSTS") + host = os.environ.get("DATABRICKS_HOST", "").rstrip("/") + if not host: + completed = subprocess.run( + ["databricks", "auth", "env", "--profile", profile], + check=False, + capture_output=True, + text=True, + timeout=20, + ) + if completed.returncode != 0: + raise ValueError( + "could not resolve host for the live evaluation profile" + ) + try: + payload = json.loads(completed.stdout) + host = str(payload.get("env", {}).get("DATABRICKS_HOST", "")).rstrip( + "/" + ) + except json.JSONDecodeError as exc: + raise ValueError("databricks auth env returned invalid JSON") from exc + if host not in allowed_hosts: + raise ValueError(f"workspace host {host!r} is not allowlisted") + identity = subprocess.run( + [ + "databricks", + "current-user", + "me", + "--profile", + profile, + "-o", + "json", + ], + check=False, + capture_output=True, + text=True, + timeout=20, + ) + if identity.returncode != 0: + raise ValueError("could not verify the live evaluation service principal") + try: + identity_payload = json.loads(identity.stdout) + except json.JSONDecodeError as exc: + raise ValueError("current-user me returned invalid JSON") from exc + application_id = ( + identity_payload.get("application_id") + or identity_payload.get("applicationId") + if isinstance(identity_payload, dict) + else None + ) + if not application_id: + raise ValueError( + "--live requires credentials for a dedicated Databricks service principal" + ) + if not re.fullmatch(r"[a-z][a-z0-9_]*_", evaluation_prefix): + raise ValueError( + "evaluation prefix must be lowercase and end in underscore" + ) + return cls( + profile=profile, + host=host, + allowed_hosts=allowed_hosts, + evaluation_prefix=evaluation_prefix, + ) + + def validate_namespace(self, namespace: LiveNamespace) -> None: + for label, value in ( + ("catalog", namespace.catalog), + ("schema", namespace.schema), + ("resource prefix", namespace.resource_prefix), + ): + if not value.startswith(self.evaluation_prefix): + raise ValueError(f"{label} {value!r} is outside the evaluation prefix") + + +@dataclass(frozen=True) +class ResourceRecord: + resource_type: str + resource_id: str + name: str = "" + parent_id: str | None = None + side: str | None = None + source: str = "resources.json" + + @property + def key(self) -> str: + return f"{self.resource_type}:{self.resource_id}" + + +RESOURCE_KEYS: dict[str, str] = { + "catalog": "catalog", + "schema": "schema", + "pipeline_id": "pipeline", + "metric_view_name": "metric_view", + "dashboard_id": "dashboard", + "genie_space_id": "genie_space", + "knowledge_assistant_id": "knowledge_assistant", + "knowledge_assistant_endpoint": "serving_endpoint", + "multi_agent_supervisor_id": "multi_agent_supervisor", + "multi_agent_supervisor_endpoint": "serving_endpoint", + "ml_model_name": "model", + "mlflow_experiment_path": "mlflow_experiment", +} + + +def resources_from_manifest( + path: Path, *, side: str | None = None +) -> list[ResourceRecord]: + if not path.is_file(): + return [] + raw = json.loads(path.read_text(encoding="utf-8")) + created = raw.get("created_resources", raw) if isinstance(raw, dict) else {} + if not isinstance(created, dict): + return [] + records: list[ResourceRecord] = [] + for key, resource_type in RESOURCE_KEYS.items(): + value = created.get(key) + if isinstance(value, (str, int)) and str(value): + resource_id = str(value) + if key == "schema" and created.get("catalog") and "." not in resource_id: + resource_id = f"{created['catalog']}.{resource_id}" + records.append( + ResourceRecord( + resource_type=resource_type, + resource_id=resource_id, + name=resource_id, + side=side, + ) + ) + lakebase_name = created.get("lakebase_project_slug") or created.get( + "lakebase_project_id" + ) + if isinstance(lakebase_name, (str, int)) and str(lakebase_name): + resource_id = str(lakebase_name) + if not resource_id.startswith("projects/"): + resource_id = f"projects/{resource_id}" + records.append( + ResourceRecord( + resource_type="lakebase", + resource_id=resource_id, + name=str(created.get("lakebase_project_slug") or resource_id), + side=side, + ) + ) + app = created.get("app") + if isinstance(app, dict): + app_id = app.get("name") or app.get("id") + if app_id: + records.append( + ResourceRecord( + resource_type="app", + resource_id=str(app_id), + name=str(app.get("name") or app_id), + side=side, + ) + ) + return records + + +def resources_from_tracked(rows: Iterable[dict[str, Any]]) -> list[ResourceRecord]: + records: list[ResourceRecord] = [] + for row in rows: + if row.get("removed"): + continue + resource_type = row.get("resource_type") or row.get("asset_type") + resource_id = row.get("resource_id") or row.get("asset_id") + if ( + not resource_type + or not resource_id + or resource_type == "pending" + or resource_type in ACTIVITY_RESOURCE_KINDS + or row.get("ephemeral") + ): + continue + resource_type = str(resource_type) + resource_id = str(resource_id) + resource_name = str(row.get("name") or resource_id) + if resource_type == "lakebase": + # Lakebase APIs delete by projects/{slug}; tool results may expose + # an opaque uid while the request name retains the deletable slug. + resource_id = resource_name if resource_name != "unknown" else resource_id + if not resource_id.startswith("projects/"): + resource_id = f"projects/{resource_id}" + side = row.get("side") + if isinstance(side, str) and (side == "A" or side.startswith("A-")): + side = "with" + elif isinstance(side, str) and (side == "B" or side.startswith("B-")): + side = "without" + records.append( + ResourceRecord( + resource_type=resource_type, + resource_id=resource_id, + name=resource_name, + parent_id=str(row["parent_id"]) if row.get("parent_id") else None, + side=side, + source="skillforge", + ) + ) + return records + + +def reconcile_resources(*groups: Iterable[ResourceRecord]) -> list[ResourceRecord]: + by_key: dict[str, ResourceRecord] = {} + for record in (record for group in groups for record in group): + existing = by_key.get(record.key) + if existing is None or ( + record.source == "skillforge" and existing.source != "skillforge" + ): + by_key[record.key] = record + return list(by_key.values()) + + +DELETE_PRIORITY: dict[str, int] = { + "app": 100, + "multi_agent_supervisor": 95, + "knowledge_assistant": 94, + "serving_endpoint": 90, + "genie_space": 85, + "dashboard": 84, + "vector_index": 80, + "vector_endpoint": 79, + "model": 75, + "metric_view": 70, + "materialized_view": 69, + "streaming_table": 68, + "function": 67, + "view": 65, + "table": 60, + "pipeline": 72, + "job": 50, + "cluster": 49, + "mlflow_experiment": 45, + "notebook": 44, + "volume": 40, + "schema": 20, + "catalog": 10, + "warehouse": 5, + "lakebase": 4, +} + + +@dataclass +class CleanupReport: + deleted: list[str] = field(default_factory=list) + remaining: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + @property + def complete(self) -> bool: + return not self.remaining and not self.errors + + +class ResourceCleaner: + def __init__( + self, + delete: Callable[[ResourceRecord], None], + exists: Callable[[ResourceRecord], bool], + *, + retries: int = 3, + retry_delay: float = 0.0, + ) -> None: + self.delete = delete + self.exists = exists + self.retries = retries + self.retry_delay = retry_delay + + def cleanup(self, resources: Iterable[ResourceRecord]) -> CleanupReport: + report = CleanupReport() + ordered = sorted( + reconcile_resources(resources), + key=lambda resource: DELETE_PRIORITY.get(resource.resource_type, 50), + reverse=True, + ) + for resource in ordered: + try: + if not self.exists(resource): + report.deleted.append(resource.key) + continue + except Exception: # noqa: BLE001 - attempt deletion on uncertain state + pass + last_error: Exception | None = None + for attempt in range(self.retries): + try: + self.delete(resource) + last_error = None + if not self.exists(resource): + report.deleted.append(resource.key) + break + except Exception as exc: # noqa: BLE001 - retries are intentional + last_error = exc + if attempt + 1 < self.retries and self.retry_delay: + time.sleep(self.retry_delay) + else: + report.remaining.append(resource.key) + if last_error is not None: + report.errors.append(f"{resource.key}: {last_error}") + return report + + +class DatabricksCliCleaner: + """Delete/verify resources through the external Databricks CLI.""" + + def __init__(self, policy: LivePolicy) -> None: + self.policy = policy + + @staticmethod + def _is_not_found(completed: subprocess.CompletedProcess[str]) -> bool: + combined = (completed.stdout + completed.stderr).lower() + markers = ( + "not found", + "not_found", + "does not exist", + "resource_does_not_exist", + '"status_code":404', + '"status_code": 404', + "status code 404", + ) + return any(marker in combined for marker in markers) + + def _run( + self, args: list[str], *, allow_not_found: bool = False + ) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + [*args, "--profile", self.policy.profile], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + if completed.returncode and not ( + allow_not_found and self._is_not_found(completed) + ): + raise RuntimeError((completed.stderr or completed.stdout).strip()) + return completed + + def _require_evaluation_scope(self, resource: ResourceRecord) -> None: + """Refuse destructive named-resource operations outside the eval prefix.""" + prefix = self.policy.evaluation_prefix + rid = resource.resource_id + scoped_parts: tuple[str, ...] | None = None + minimum_parts = 0 + if resource.resource_type == "catalog": + scoped_parts = (rid,) + minimum_parts = 1 + elif resource.resource_type == "schema": + parts = tuple(rid.split(".")) + scoped_parts = parts[:2] + minimum_parts = 2 + elif resource.resource_type in { + "metric_view", + "view", + "table", + "streaming_table", + "materialized_view", + "volume", + "model", + "function", + "vector_index", + }: + parts = tuple(rid.split(".")) + scoped_parts = parts[:2] + minimum_parts = 3 + elif resource.resource_type == "lakebase": + scoped_parts = (rid.removeprefix("projects/"),) + minimum_parts = 1 + elif resource.resource_type in {"cluster", "warehouse", "vector_endpoint"}: + scoped_parts = (resource.name or rid,) + minimum_parts = 1 + elif resource.resource_type in {"notebook", "mlflow_experiment"}: + path_parts = tuple( + part for part in (resource.name or rid).split("/") if part + ) + if not any(part.startswith(prefix) for part in path_parts): + raise RuntimeError( + f"refusing to delete {resource.key}: outside evaluation prefix {prefix!r}" + ) + return + if scoped_parts is None: + return + normalized = tuple(part.strip("`") for part in scoped_parts) + actual_parts = len(tuple(rid.split("."))) + if ( + actual_parts < minimum_parts + or not normalized + or any(not part.startswith(prefix) for part in normalized) + ): + raise RuntimeError( + f"refusing to delete {resource.key}: outside evaluation prefix {prefix!r}" + ) + + @staticmethod + def _active_payload(completed: subprocess.CompletedProcess[str]) -> bool: + if completed.returncode != 0: + return False + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError: + return True + if not isinstance(payload, dict): + return True + item = payload.get("experiment", payload) + if not isinstance(item, dict): + return True + lifecycle = str( + item.get("lifecycle_stage") or item.get("lifecycle_state") or "" + ).upper() + return lifecycle not in {"DELETED", "TRASHED"} + + def _experiment(self, resource_id: str) -> tuple[str | None, bool]: + command = ( + [ + "databricks", + "experiments", + "get-experiment", + resource_id, + "-o", + "json", + ] + if resource_id.isdigit() + else [ + "databricks", + "experiments", + "get-by-name", + resource_id, + "-o", + "json", + ] + ) + completed = self._run(command, allow_not_found=True) + if self._is_not_found(completed) or completed.returncode != 0: + return None, False + try: + payload = json.loads(completed.stdout) + item = payload.get("experiment", payload) + experiment_id = str(item["experiment_id"]) + except (json.JSONDecodeError, KeyError, TypeError) as exc: + raise RuntimeError( + f"could not resolve MLflow experiment {resource_id!r}" + ) from exc + return experiment_id, self._active_payload(completed) + + def delete(self, resource: ResourceRecord) -> None: + self._require_evaluation_scope(resource) + rid = resource.resource_id + commands: dict[str, list[str]] = { + "app": ["databricks", "apps", "delete", rid], + "serving_endpoint": ["databricks", "serving-endpoints", "delete", rid], + "pipeline": ["databricks", "pipelines", "delete", rid], + "job": ["databricks", "jobs", "delete", rid], + "warehouse": ["databricks", "warehouses", "delete", rid], + "cluster": ["databricks", "clusters", "permanent-delete", rid], + "lakebase": ["databricks", "postgres", "delete-project", rid], + "vector_index": [ + "databricks", + "vector-search-indexes", + "delete-index", + rid, + ], + "vector_endpoint": [ + "databricks", + "vector-search-endpoints", + "delete-endpoint", + rid, + ], + "dashboard": ["databricks", "lakeview", "trash", rid], + "genie_space": ["databricks", "genie", "trash-space", rid], + "knowledge_assistant": [ + "databricks", + "knowledge-assistants", + "delete-knowledge-assistant", + rid + if rid.startswith("knowledge-assistants/") + else f"knowledge-assistants/{rid}", + ], + "multi_agent_supervisor": [ + "databricks", + "api", + "delete", + f"/api/2.0/tiles/{quote(rid, safe='')}", + ], + "metric_view": ["databricks", "tables", "delete", rid], + "view": ["databricks", "tables", "delete", rid], + "table": ["databricks", "tables", "delete", rid], + "streaming_table": ["databricks", "tables", "delete", rid], + "materialized_view": ["databricks", "tables", "delete", rid], + "volume": ["databricks", "volumes", "delete", rid], + "function": ["databricks", "functions", "delete", rid, "--force"], + "notebook": ["databricks", "workspace", "delete", rid], + "schema": ["databricks", "schemas", "delete", rid, "--force"], + "catalog": ["databricks", "catalogs", "delete", rid, "--force"], + "model": ["databricks", "registered-models", "delete", rid], + } + if resource.resource_type == "mlflow_experiment": + experiment_id, active = self._experiment(rid) + if not active or experiment_id is None: + return + self._run( + [ + "databricks", + "experiments", + "delete-experiment", + experiment_id, + ], + allow_not_found=True, + ) + return + if resource.resource_type in commands: + self._run(commands[resource.resource_type], allow_not_found=True) + return + raise RuntimeError(f"no cleanup command for {resource.resource_type}") + + def exists(self, resource: ResourceRecord) -> bool: + rid = resource.resource_id + commands: dict[str, list[str]] = { + "app": ["databricks", "apps", "get", rid, "-o", "json"], + "serving_endpoint": [ + "databricks", + "serving-endpoints", + "get", + rid, + "-o", + "json", + ], + "pipeline": ["databricks", "pipelines", "get", rid, "-o", "json"], + "job": ["databricks", "jobs", "get", rid, "-o", "json"], + "warehouse": ["databricks", "warehouses", "get", rid, "-o", "json"], + "cluster": ["databricks", "clusters", "get", rid, "-o", "json"], + "lakebase": [ + "databricks", + "postgres", + "get-project", + rid, + "-o", + "json", + ], + "vector_index": [ + "databricks", + "vector-search-indexes", + "get-index", + rid, + "-o", + "json", + ], + "vector_endpoint": [ + "databricks", + "vector-search-endpoints", + "get-endpoint", + rid, + "-o", + "json", + ], + "dashboard": ["databricks", "lakeview", "get", rid, "-o", "json"], + "genie_space": [ + "databricks", + "genie", + "get-space", + rid, + "-o", + "json", + ], + "knowledge_assistant": [ + "databricks", + "knowledge-assistants", + "get-knowledge-assistant", + rid + if rid.startswith("knowledge-assistants/") + else f"knowledge-assistants/{rid}", + "-o", + "json", + ], + "multi_agent_supervisor": [ + "databricks", + "api", + "get", + f"/api/2.0/multi-agent-supervisors/{quote(rid, safe='')}", + "-o", + "json", + ], + "metric_view": ["databricks", "tables", "get", rid, "-o", "json"], + "view": ["databricks", "tables", "get", rid, "-o", "json"], + "table": ["databricks", "tables", "get", rid, "-o", "json"], + "streaming_table": [ + "databricks", + "tables", + "get", + rid, + "-o", + "json", + ], + "materialized_view": [ + "databricks", + "tables", + "get", + rid, + "-o", + "json", + ], + "volume": ["databricks", "volumes", "read", rid, "-o", "json"], + "function": ["databricks", "functions", "get", rid, "-o", "json"], + "notebook": [ + "databricks", + "workspace", + "get-status", + rid, + "-o", + "json", + ], + "schema": ["databricks", "schemas", "get", rid, "-o", "json"], + "catalog": ["databricks", "catalogs", "get", rid, "-o", "json"], + "model": [ + "databricks", + "registered-models", + "get", + rid, + "-o", + "json", + ], + } + if resource.resource_type == "mlflow_experiment": + _, active = self._experiment(rid) + return active + command = commands.get(resource.resource_type) + if command: + completed = self._run(command, allow_not_found=True) + if self._is_not_found(completed): + return False + return self._active_payload(completed) + return True + + +def write_leak_report( + path: Path, report: CleanupReport, resources: Iterable[ResourceRecord] +) -> None: + payload = { + "complete": report.complete, + "deleted": report.deleted, + "remaining": report.remaining, + "errors": report.errors, + "resources": [resource.__dict__ for resource in resources], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") diff --git a/evaluation/models.py b/evaluation/models.py new file mode 100644 index 0000000..ba32ac9 --- /dev/null +++ b/evaluation/models.py @@ -0,0 +1,177 @@ +"""Canonical, runner-neutral evaluation contracts.""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +STAGE_ORDER = ( + "DRAFTING", + "SUMMARIZED", + "ARCHITECTED", + "SPECIFICATION", + "BUILT", + "BUNDLED", +) + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ProjectStage(StrEnum): + DRAFTING = "DRAFTING" + SUMMARIZED = "SUMMARIZED" + ARCHITECTED = "ARCHITECTED" + SPECIFICATION = "SPECIFICATION" + BUILT = "BUILT" + BUNDLED = "BUNDLED" + + +class SourceCitation(StrictModel): + type: Literal["docs", "blog", "glean", "confluence", "slack", "github", "manual"] + uri: str = Field(min_length=1) + title: str = Field(min_length=1) + retrieved_at: datetime + snippet: str = Field(min_length=1, max_length=280) + + +class ArtifactExpectation(StrictModel): + path: str = Field(min_length=1) + description: str = Field(min_length=1) + required: bool = True + + +class VerificationCommand(StrictModel): + fact: str = Field(min_length=1) + command: str = Field(min_length=1) + check: str = Field(min_length=1) + + +class ToolExpectations(StrictModel): + required: list[str] = Field(default_factory=list) + banned: list[str] = Field(default_factory=list) + + +class ScenarioStep(StrictModel): + id: str = Field(pattern=r"^[a-z0-9][a-z0-9-]*$") + prompt: str = Field(min_length=1) + expected_project_stage: ProjectStage + required_artifacts: list[ArtifactExpectation] = Field(min_length=1) + semantic_assertions: list[str] = Field(min_length=1) + expected_facts: list[str] = Field(min_length=1) + expected_patterns: list[str] = Field(min_length=1) + tool_expectations: ToolExpectations + verification_commands: list[VerificationCommand] = Field(default_factory=list) + regression_intent: str = Field(min_length=1) + sources: list[SourceCitation] = Field(min_length=1) + + +class LiveResourceExpectations(StrictModel): + cleanup_owner: Literal["solution-builder"] = "solution-builder" + expected_resource_kinds: list[str] = Field(default_factory=list) + additional_resource_kinds: list[str] = Field(default_factory=list) + evaluation_prefix: str = Field(default="sb_eval_", pattern=r"^[a-z][a-z0-9_]*_$") + + +class Scenario(StrictModel): + schema_version: Literal[1] = 1 + id: str = Field(pattern=r"^[a-z0-9][a-z0-9-]*$") + skill: str = Field(min_length=1) + description: str = Field(min_length=1) + capabilities: list[str] = Field(min_length=1) + timeout_seconds: int = Field(ge=60, le=14_400) + target_stage: ProjectStage + steps: list[ScenarioStep] = Field(min_length=1) + live_resources: LiveResourceExpectations + + @model_validator(mode="after") + def validate_workflow(self) -> "Scenario": + ids = [step.id for step in self.steps] + if len(ids) != len(set(ids)): + raise ValueError("step ids must be unique") + stages = [ + STAGE_ORDER.index(step.expected_project_stage.value) for step in self.steps + ] + if stages != sorted(stages): + raise ValueError("step stages must be ordered monotonically") + if self.steps[-1].expected_project_stage != self.target_stage: + raise ValueError("the final step stage must equal target_stage") + return self + + @property + def initial_prompt(self) -> str: + return self.steps[0].prompt + + @property + def drive_messages(self) -> list[str]: + return [step.prompt for step in self.steps[1:]] + + +class EvalModels(StrictModel): + agent: str | None = None + judge: str | None = None + + +class EvalScore(StrictModel): + name: str + value: float | None = None + side: Literal["with", "without", "comparison"] = "comparison" + + +class EvalGap(StrictModel): + code: str + message: str + level: str | None = None + side: Literal["with", "without", "comparison"] = "comparison" + + +class EvalResource(StrictModel): + resource_type: str + resource_id: str + name: str | None = None + side: Literal["with", "without"] | None = None + source: str + removed: bool = False + + +class CleanupStatus(StrictModel): + attempted: bool = False + complete: bool = False + deleted: list[str] = Field(default_factory=list) + remaining: list[str] = Field(default_factory=list) + errors: list[str] = Field(default_factory=list) + leak_report: str | None = None + + +class MlflowIdentity(StrictModel): + experiment_id: str | None = None + run_ids: list[str] = Field(default_factory=list) + trace_ids: list[str] = Field(default_factory=list) + + +class EvalRun(StrictModel): + schema_version: Literal[1] = 1 + run_id: str + runner: Literal["skillforge"] = "skillforge" + scenario_id: str + git_sha: str + skill_hash: str + skillforge_version: str + skillforge_revision: str + levels: list[str] + models: EvalModels + scores: list[EvalScore] = Field(default_factory=list) + gaps: list[EvalGap] = Field(default_factory=list) + resources: list[EvalResource] = Field(default_factory=list) + cleanup: CleanupStatus = Field(default_factory=CleanupStatus) + mlflow: MlflowIdentity = Field(default_factory=MlflowIdentity) + reports: dict[str, str] = Field(default_factory=dict) + status: Literal["passed", "failed", "invalid_eval", "leaked"] + started_at: datetime + completed_at: datetime + raw_result: dict[str, Any] = Field(default_factory=dict) diff --git a/evaluation/normalize.py b/evaluation/normalize.py new file mode 100644 index 0000000..386b8fa --- /dev/null +++ b/evaluation/normalize.py @@ -0,0 +1,141 @@ +"""Normalize external SkillForge JSON into the stable EvalRun contract.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from evaluation.models import EvalGap, EvalScore, MlflowIdentity + + +def walk(value: Any) -> Iterator[tuple[str | None, Any]]: + if isinstance(value, dict): + for key, child in value.items(): + yield key, child + yield from walk(child) + elif isinstance(value, list): + for child in value: + yield None, child + yield from walk(child) + + +def contains_invalid_eval(raw: dict[str, Any]) -> bool: + for key, value in walk(raw): + if isinstance(value, str) and value.lower() == "invalid_eval": + return True + if key == "invalid_eval" and bool(value): + return True + return False + + +def skill_was_invoked(raw: dict[str, Any], skill_name: str) -> bool: + found_signal = False + for key, value in walk(raw): + if key not in {"skills_invoked", "invoked_skills", "skill_invocations"}: + continue + found_signal = True + if isinstance(value, str) and skill_name in value: + return True + if isinstance(value, list) and any(skill_name in str(item) for item in value): + return True + if isinstance(value, dict) and skill_name in str(value): + return True + return False if found_signal else False + + +def extract_scores(raw: dict[str, Any]) -> list[EvalScore]: + scores: list[EvalScore] = [] + for result_key, side in (("result_a", "with"), ("result_b", "without")): + result = raw.get(result_key) + if not isinstance(result, dict): + continue + composite = result.get("composite_score") + if isinstance(composite, (int, float)): + scores.append( + EvalScore(name="composite", value=float(composite), side=side) + ) + levels = result.get("levels") + if isinstance(levels, dict): + for level, level_result in levels.items(): + if not isinstance(level_result, dict): + continue + value = level_result.get("score") + if isinstance(value, (int, float)): + scores.append( + EvalScore(name=str(level), value=float(value), side=side) + ) + verdict = raw.get("verdict") + if isinstance(verdict, dict): + for dimension in verdict.get("dimensions") or []: + if not isinstance(dimension, dict): + continue + value = dimension.get("score") or dimension.get("score_a") + if isinstance(value, (int, float)): + scores.append( + EvalScore( + name=f"comparison/{dimension.get('name', 'dimension')}", + value=float(value), + side="comparison", + ) + ) + return scores + + +def extract_gaps(raw: dict[str, Any]) -> list[EvalGap]: + gaps: list[EvalGap] = [] + for result_key, side in (("result_a", "with"), ("result_b", "without")): + result = raw.get(result_key) + if not isinstance(result, dict): + continue + suggestions = result.get("suggestions") or [] + for index, suggestion in enumerate(suggestions): + gaps.append( + EvalGap( + code=f"suggestion_{index + 1}", + message=str(suggestion), + side=side, + ) + ) + structured = result.get("gaps") + if not structured: + continue + for key, value in walk(structured): + if key not in { + "message", + "detail", + "rationale", + "summary", + } or not isinstance(value, str): + continue + gaps.append( + EvalGap( + code="skillforge_gap", + message=value, + side=side, + ) + ) + # Stable de-duplication prevents nested gap summaries appearing repeatedly. + unique: dict[tuple[str, str, str], EvalGap] = {} + for gap in gaps: + unique[(gap.code, gap.message, gap.side)] = gap + return list(unique.values()) + + +def extract_mlflow(raw: dict[str, Any]) -> MlflowIdentity: + run_ids: set[str] = set() + trace_ids: set[str] = set() + experiment_id: str | None = None + for key, value in walk(raw): + if not isinstance(value, str) or not value: + continue + if key in {"mlflow_run_id", "run_id"} and key == "mlflow_run_id": + run_ids.add(value) + elif key in {"mlflow_trace_id", "trace_id"}: + trace_ids.add(value) + elif key in {"mlflow_experiment_id", "experiment_id"} and experiment_id is None: + experiment_id = value + return MlflowIdentity( + experiment_id=experiment_id, + run_ids=sorted(run_ids), + trace_ids=sorted(trace_ids), + ) diff --git a/evaluation/rubrics/databricks-demo-generator/comparison_rubric.md b/evaluation/rubrics/databricks-demo-generator/comparison_rubric.md new file mode 100644 index 0000000..2d67d2b --- /dev/null +++ b/evaluation/rubrics/databricks-demo-generator/comparison_rubric.md @@ -0,0 +1,11 @@ +# WITH/WITHOUT comparison rubric + +Compare the target-skill side (WITH) with the control side (WITHOUT) on: + +- coherence across story, data, architecture, specifications, and implementation; +- fidelity to the requested capabilities and industry details; +- completeness and validity of required artifacts; +- safe handling of live-resource namespaces and resource manifests; +- actionable reasoning that avoids fabricated deployment claims. + +Do not reward verbosity, tool-call count, or skill-name awareness. When a later shared-cwd case fails because an earlier artifact was wrong, attribute the regression to the earliest causal case and avoid counting the same defect repeatedly. diff --git a/evaluation/rubrics/databricks-demo-generator/output_instructions.md b/evaluation/rubrics/databricks-demo-generator/output_instructions.md new file mode 100644 index 0000000..d5a12e9 --- /dev/null +++ b/evaluation/rubrics/databricks-demo-generator/output_instructions.md @@ -0,0 +1,47 @@ +# Databricks demo-generator output rubric + +Judge output and filesystem artifacts together. Require: + +- The requested business protagonist, catalyst, measurable value, and Databricks capabilities appear in one coherent walkthrough. +- `resources.json`, `README.md`, `architecture.md`, and specifications agree on names, schemas, capabilities, and dependencies. +- Every required artifact for the case exists and is substantive; placeholders and dead resource IDs fail. +- Functional specifications describe outcomes and contracts, while build artifacts contain the implementation. +- Selected capabilities are neither omitted nor added as disconnected ornamentation. +- Live resources use the supplied evaluation prefix and record enough identity for deterministic cleanup. + +Custom or qualitative judgments are diagnostic. Deterministic facts, patterns, assertions, trace expectations, and verification commands carry the hard requirements. + +## Dimension checklist + +1. `README.md` names a protagonist, challenge, catalyst, and resolution. +2. The story includes a measurable business value statement. +3. The requested industry and sub-vertical remain recognizable. +4. Every requested capability appears in the walkthrough. +5. Unrequested capabilities do not distract from the main question. +6. `resources.json.capabilities` agrees with the story and specifications. +7. `created_resources` starts empty before a live build. +8. Created IDs are syntactically plausible and not placeholder text. +9. `architecture.md` uses the renderer's required JSON schema. +10. Architecture edges represent real data or control dependencies. +11. Architecture nodes cover all buildable capabilities. +12. Lakeflow specifications define sources, targets, grain, and quality rules. +13. Governance specifications name securable objects and access boundaries. +14. ML specifications define labels, features, temporal splits, and metrics. +15. Dashboard specifications define business measures and visible insights. +16. Genie specifications define tables, joins, measures, and example questions. +17. Knowledge Assistant specifications define a real source corpus. +18. Supervisor specifications define routable agents and decision rules. +19. App specifications define pages, data contracts, and agent interactions. +20. Shared identifiers use the same names and types across component specs. +21. Build artifacts are present for each implementation requirement. +22. SQL and Python are executable rather than illustrative pseudocode. +23. Generated configs reference the approved catalog and schema. +24. Resource manifests record endpoints and experiment paths when required. +25. App artifacts follow the shipped template's stack and structural contracts. +26. No output claims a deployment that the trace or manifest cannot substantiate. +27. Errors preserve partial resource identity for cleanup. +28. Live catalogs and schemas start with the required evaluation prefix. +29. Live opaque IDs have corresponding tracked creation evidence. +30. WITH and WITHOUT outputs are both available for comparison. +31. The report exposes gaps without turning advisory scores into merge gates. +32. The cumulative shared-cwd output remains coherent after the final case. diff --git a/evaluation/rubrics/databricks-demo-generator/thinking_instructions.md b/evaluation/rubrics/databricks-demo-generator/thinking_instructions.md new file mode 100644 index 0000000..aeca0f0 --- /dev/null +++ b/evaluation/rubrics/databricks-demo-generator/thinking_instructions.md @@ -0,0 +1,47 @@ +# Databricks demo-generator reasoning rubric + +Evaluate the reasoning process, not verbosity. A strong run: + +1. Reads the platform architecture and only the domain/capability references needed for the request. +2. Preserves the staged workflow and does not skip user gates unless the prompt explicitly authorizes continuation. +3. Treats story, data, specifications, and resources as one coherence contract; downstream decisions cite upstream facts. +4. Distinguishes proposed resources from resources actually created. It never invents IDs or pre-seeds `created_resources`. +5. For live evaluation, discovers and obeys `SB_EVAL_*` namespace variables and does not mutate resources outside them. +6. Uses the project filesystem as durable state between ordered cases without assuming conversation history persists. + +Diagnostic findings should identify the earliest step that caused a downstream failure so shared-cwd cascades are not double-counted. + +## Dimension checklist + +1. Intent capture identifies whether the request is vague, moderate, or detailed. +2. Explicitly requested capabilities are preserved rather than replaced by defaults. +3. Platform dependencies are checked before a story commits to components. +4. Reference reads are targeted and relevant to the active stage. +5. The business protagonist has a role that can act on the demo insight. +6. The catalyst is observable in the proposed synthetic data. +7. Business value is quantified in dollars, time, risk, or another decision metric. +8. The wow moment follows from the data rather than appearing as unsupported narration. +9. Each selected product has a distinct walkthrough moment. +10. No component is included only as architectural ornament. +11. Architecture dependencies flow from sources through processing to consumption. +12. Governance and identity boundaries are considered where the scenario needs them. +13. Specifications inherit exact entity names and keys from the approved story. +14. Temporal fields and model cutoffs avoid future-data leakage. +15. Synthetic data supports every dashboard, query, model, and agent claim. +16. Dashboard metrics can visibly prove the promised business moment. +17. Genie questions are answerable from the specified tables and measures. +18. Agent routing criteria are explicit and non-overlapping. +19. App interactions call resources that the specifications actually define. +20. Build order respects sequential data prerequisites. +21. Independent post-pipeline work is identified for safe parallel execution. +22. Created resource IDs are recorded only after creation and validation. +23. Failed or skipped resources are explained without fabricated identifiers. +24. Filesystem state is inspected before each ordered prompt is answered. +25. The run does not assume prior conversation state across SkillForge cases. +26. Existing artifacts are updated consistently instead of recreated blindly. +27. Live evaluation variables are discovered before any mutation. +28. Every live name remains within the supplied catalog, schema, and prefix. +29. Cleanup identity is retained in `resources.json` even when deployment fails later. +30. Reasoning distinguishes diagnostic recommendations from hard acceptance requirements. +31. A downstream failure is traced to its earliest causal artifact. +32. The final reasoning checks coherence across all generated deliverables. diff --git a/evaluation/runner.py b/evaluation/runner.py new file mode 100644 index 0000000..c698872 --- /dev/null +++ b/evaluation/runner.py @@ -0,0 +1,414 @@ +"""Orchestrate pinned external SkillForge runs without importing SkillForge.""" + +from __future__ import annotations + +import json +import os +import subprocess +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from evaluation.adapter import SkillForgeAdapter +from evaluation.fixture import build_fixture +from evaluation.hashing import hash_skill +from evaluation.live import ( + DatabricksCliCleaner, + LivePolicy, + ResourceCleaner, + ResourceRecord, + expected_resource_kinds, + reconcile_resources, + resources_from_manifest, + resources_from_tracked, + write_leak_report, +) +from evaluation.models import ( + CleanupStatus, + EvalGap, + EvalModels, + EvalResource, + EvalRun, + Scenario, +) +from evaluation.normalize import ( + contains_invalid_eval, + extract_gaps, + extract_mlflow, + extract_scores, + skill_was_invoked, +) +from evaluation.toolchain import ( + REPO_ROOT, + find_executable, + load_lock, + skillforge_build_info, +) + + +def normalize_levels(value: str) -> list[str]: + aliases = { + "L1": "unit", + "L2": "integration", + "L3": "static", + "L4": "thinking", + "L5": "output", + "UNIT": "unit", + "INTEGRATION": "integration", + "STATIC": "static", + "THINKING": "thinking", + "OUTPUT": "output", + } + if value.strip().lower() == "all": + return ["unit", "integration", "static", "thinking", "output"] + levels: list[str] = [] + for item in value.split(","): + key = item.strip().upper() + if key not in aliases: + raise ValueError(f"unknown evaluation level: {item!r}") + mapped = aliases[key] + if mapped not in levels: + levels.append(mapped) + if not levels: + raise ValueError("at least one evaluation level is required") + return levels + + +def _git_sha(repo_root: Path) -> str: + completed = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + +def _load_tracked_resources(home: Path) -> list[ResourceRecord]: + resources: list[ResourceRecord] = [] + for path in home.glob("runs/**/resources.json"): + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + rows = ( + raw + if isinstance(raw, list) + else raw.get("resources", []) + if isinstance(raw, dict) + else [] + ) + if isinstance(rows, list): + resources.extend( + resources_from_tracked(row for row in rows if isinstance(row, dict)) + ) + return resources + + +def _load_project_resources(search_root: Path) -> list[ResourceRecord]: + resources: list[ResourceRecord] = [] + for path in search_root.glob("**/resources.json"): + if ".skillforge" in path.parts or "skillforge-home" in path.parts: + continue + try: + resources.extend(resources_from_manifest(path)) + except (OSError, json.JSONDecodeError): + continue + return resources + + +def _html_summary(result: EvalRun) -> str: + score_rows = "".join( + f"{score.side}{score.name}{score.value}" + for score in result.scores + ) + gap_rows = "".join( + f"
  • {gap.side}/{gap.code}: {gap.message}
    • " + for gap in result.gaps + ) + return ( + "Solution Builder evaluation" + f"

    {result.scenario_id}: {result.status}

    " + f"

    Run {result.run_id}; skill hash {result.skill_hash}

    " + "

    Scores

    " + f"{score_rows}
    SideNameValue

    Gaps

      {gap_rows}
    " + f"

    Cleanup

    {json.dumps(result.cleanup.model_dump(), indent=2)}
    " + ) + + +def _tag_mlflow_runs( + raw: dict[str, Any], + *, + scenario: Scenario, + git_sha: str, + skill_hash: str, + skillforge_revision: str, + levels: list[str], + agent_model: str | None, + judge_model: str | None, +) -> list[str]: + profile = os.environ.get("SB_EVAL_DATABRICKS_PROFILE", "").strip() + if not profile: + return [] + errors: list[str] = [] + for result_key, side in (("result_a", "with"), ("result_b", "without")): + result = raw.get(result_key) + run_id = result.get("mlflow_run_id") if isinstance(result, dict) else None + if not run_id: + continue + tags = { + "sb_eval.scenario_id": scenario.id, + "sb_eval.git_sha": git_sha, + "sb_eval.skill_hash": skill_hash, + "sb_eval.skillforge_revision": skillforge_revision, + "sb_eval.agent_model": agent_model or "default", + "sb_eval.judge_model": judge_model or "default", + "sb_eval.levels": ",".join(levels), + "sb_eval.side": side, + "sb_eval.capabilities": ",".join(scenario.capabilities), + } + for key, value in tags.items(): + completed = subprocess.run( + [ + "databricks", + "api", + "post", + "/api/2.0/mlflow/runs/set-tag", + "--json", + json.dumps({"run_id": run_id, "key": key, "value": value}), + "--profile", + profile, + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if completed.returncode: + errors.append( + f"{run_id}/{key}: {(completed.stderr or completed.stdout).strip()}" + ) + return errors + + +def run_scenario( + scenario: Scenario, + *, + levels: list[str], + output_dir: Path, + live: bool, + agent_model: str | None, + judge_model: str | None, + run_id: str, + repo_root: Path = REPO_ROOT, +) -> EvalRun: + started_at = datetime.now(timezone.utc) + lock = load_lock() + executable = find_executable(lock) + if executable is None: + raise RuntimeError( + f"external executable {lock.skillforge_executable!r} was not found" + ) + build_info = skillforge_build_info(executable) + scenario_dir = output_dir / scenario.id + scenario_dir.mkdir(parents=True, exist_ok=False) + fixture = build_fixture(scenario_dir / "fixture", repo_root=repo_root, lock=lock) + assets = SkillForgeAdapter(repo_root).render( + scenario, scenario_dir / "eval-assets", live=live + ) + raw_path = scenario_dir / "skillforge-result.json" + skillforge_home = scenario_dir / "skillforge-home" + command = [ + executable, + "compare", + str(fixture.target_skill), + str(fixture.control_skill), + "--levels", + ",".join(levels), + "--timeout", + str(scenario.timeout_seconds), + "--comparison-id", + f"{run_id}-{scenario.id}", + "--project-dir", + str(fixture.project_dir), + "--eval-dir", + str(assets.eval_dir), + "--output", + str(raw_path), + ] + if agent_model: + command.extend(["--agent-model", agent_model]) + if judge_model: + command.extend(["--judge-model", judge_model]) + env = os.environ.copy() + env["SKILLFORGE_HOME"] = str(skillforge_home) + env["SB_EVAL_PREFIX"] = scenario.live_resources.evaluation_prefix + if live: + env["SB_EVAL_LIVE"] = "1" + # Validate before the external runner can make a tool call. + LivePolicy.from_env(scenario.live_resources.evaluation_prefix) + completed = subprocess.run( + command, + cwd=repo_root, + env=env, + check=False, + capture_output=True, + text=True, + ) + (scenario_dir / "skillforge.stdout.log").write_text( + completed.stdout, encoding="utf-8" + ) + (scenario_dir / "skillforge.stderr.log").write_text( + completed.stderr, encoding="utf-8" + ) + raw: dict[str, Any] = {} + if raw_path.is_file(): + try: + loaded = json.loads(raw_path.read_text(encoding="utf-8")) + if isinstance(loaded, dict): + raw = loaded + except json.JSONDecodeError: + raw = {"parse_error": "SkillForge result was not valid JSON"} + + gaps = extract_gaps(raw) + git_sha = _git_sha(repo_root) + skill_hash = hash_skill( + repo_root / ".claude" / "skills" / "databricks-demo-generator" + ) + tag_errors = _tag_mlflow_runs( + raw, + scenario=scenario, + git_sha=git_sha, + skill_hash=skill_hash, + skillforge_revision=str(build_info["revision"]), + levels=levels, + agent_model=agent_model, + judge_model=judge_model, + ) + if tag_errors: + gaps.append( + EvalGap( + code="mlflow_tagging_failed", + message="; ".join(tag_errors), + ) + ) + invalid = contains_invalid_eval(raw) + if completed.returncode != 0: + gaps.append( + EvalGap( + code="skillforge_failed", + message=(completed.stderr or completed.stdout).strip()[-4000:] + or f"exit {completed.returncode}", + ) + ) + if invalid: + gaps.append( + EvalGap(code="invalid_eval", message="SkillForge reported invalid_eval") + ) + + tracked = _load_tracked_resources(skillforge_home) + manifests = _load_project_resources(scenario_dir) + resources = reconcile_resources(tracked, manifests) + cleanup = CleanupStatus(attempted=live, complete=not live) + status = "passed" + if completed.returncode != 0: + status = "failed" + if invalid: + status = "invalid_eval" + if live and tag_errors: + status = "failed" + + if live: + if not skill_was_invoked(raw, scenario.skill): + gaps.append( + EvalGap( + code="skill_not_invoked", + message=f"WITH traces did not prove invocation of {scenario.skill}", + side="with", + ) + ) + status = "failed" + actual_kinds = {resource.resource_type for resource in resources} + expected_kinds = set( + expected_resource_kinds( + scenario.capabilities, + overrides=( + *scenario.live_resources.expected_resource_kinds, + *scenario.live_resources.additional_resource_kinds, + ), + ) + ) + missing_kinds = sorted(expected_kinds - actual_kinds) + if missing_kinds: + gaps.append( + EvalGap( + code="missing_live_resource_kinds", + message="missing expected resource kinds: " + + ", ".join(missing_kinds), + ) + ) + status = "failed" + policy = LivePolicy.from_env(scenario.live_resources.evaluation_prefix) + backend = DatabricksCliCleaner(policy) + cleanup_report = ResourceCleaner( + backend.delete, backend.exists, retries=3, retry_delay=1 + ).cleanup(resources) + leak_path = scenario_dir / "leak-report.json" + write_leak_report(leak_path, cleanup_report, resources) + cleanup = CleanupStatus( + attempted=True, + complete=cleanup_report.complete, + deleted=cleanup_report.deleted, + remaining=cleanup_report.remaining, + errors=cleanup_report.errors, + leak_report=str(leak_path), + ) + if not cleanup.complete: + status = "leaked" + + result = EvalRun( + run_id=run_id, + scenario_id=scenario.id, + git_sha=git_sha, + skill_hash=skill_hash, + skillforge_version=str(build_info["version"]), + skillforge_revision=str(build_info["revision"]), + levels=levels, + models=EvalModels(agent=agent_model, judge=judge_model), + scores=extract_scores(raw), + gaps=gaps, + resources=[ + EvalResource( + resource_type=resource.resource_type, + resource_id=resource.resource_id, + name=resource.name or None, + side=resource.side if resource.side in {"with", "without"} else None, + source=resource.source, + ) + for resource in resources + ], + cleanup=cleanup, + mlflow=extract_mlflow(raw), + reports={ + "skillforge_json": str(raw_path), + "stdout": str(scenario_dir / "skillforge.stdout.log"), + "stderr": str(scenario_dir / "skillforge.stderr.log"), + }, + status=status, + started_at=started_at, + completed_at=datetime.now(timezone.utc), + raw_result=raw, + ) + normalized_path = scenario_dir / "eval-run.json" + html_path = scenario_dir / "report.html" + result.reports["normalized_json"] = str(normalized_path) + result.reports["html"] = str(html_path) + normalized_path.write_text(result.model_dump_json(indent=2), encoding="utf-8") + html_path.write_text(_html_summary(result), encoding="utf-8") + return result + + +def new_run_id() -> str: + return uuid.uuid4().hex diff --git a/evaluation/schema/scenario.schema.json b/evaluation/schema/scenario.schema.json new file mode 100644 index 0000000..74dc385 --- /dev/null +++ b/evaluation/schema/scenario.schema.json @@ -0,0 +1,324 @@ +{ + "$defs": { + "ArtifactExpectation": { + "additionalProperties": false, + "properties": { + "description": { + "minLength": 1, + "title": "Description", + "type": "string" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "path", + "description" + ], + "title": "ArtifactExpectation", + "type": "object" + }, + "LiveResourceExpectations": { + "additionalProperties": false, + "properties": { + "additional_resource_kinds": { + "items": { + "type": "string" + }, + "title": "Additional Resource Kinds", + "type": "array" + }, + "cleanup_owner": { + "const": "solution-builder", + "default": "solution-builder", + "title": "Cleanup Owner", + "type": "string" + }, + "evaluation_prefix": { + "default": "sb_eval_", + "pattern": "^[a-z][a-z0-9_]*_$", + "title": "Evaluation Prefix", + "type": "string" + }, + "expected_resource_kinds": { + "items": { + "type": "string" + }, + "title": "Expected Resource Kinds", + "type": "array" + } + }, + "title": "LiveResourceExpectations", + "type": "object" + }, + "ProjectStage": { + "enum": [ + "DRAFTING", + "SUMMARIZED", + "ARCHITECTED", + "SPECIFICATION", + "BUILT", + "BUNDLED" + ], + "title": "ProjectStage", + "type": "string" + }, + "ScenarioStep": { + "additionalProperties": false, + "properties": { + "expected_facts": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Expected Facts", + "type": "array" + }, + "expected_patterns": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Expected Patterns", + "type": "array" + }, + "expected_project_stage": { + "$ref": "#/$defs/ProjectStage" + }, + "id": { + "pattern": "^[a-z0-9][a-z0-9-]*$", + "title": "Id", + "type": "string" + }, + "prompt": { + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "regression_intent": { + "minLength": 1, + "title": "Regression Intent", + "type": "string" + }, + "required_artifacts": { + "items": { + "$ref": "#/$defs/ArtifactExpectation" + }, + "minItems": 1, + "title": "Required Artifacts", + "type": "array" + }, + "semantic_assertions": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Semantic Assertions", + "type": "array" + }, + "sources": { + "items": { + "$ref": "#/$defs/SourceCitation" + }, + "minItems": 1, + "title": "Sources", + "type": "array" + }, + "tool_expectations": { + "$ref": "#/$defs/ToolExpectations" + }, + "verification_commands": { + "items": { + "$ref": "#/$defs/VerificationCommand" + }, + "title": "Verification Commands", + "type": "array" + } + }, + "required": [ + "id", + "prompt", + "expected_project_stage", + "required_artifacts", + "semantic_assertions", + "expected_facts", + "expected_patterns", + "tool_expectations", + "regression_intent", + "sources" + ], + "title": "ScenarioStep", + "type": "object" + }, + "SourceCitation": { + "additionalProperties": false, + "properties": { + "retrieved_at": { + "format": "date-time", + "title": "Retrieved At", + "type": "string" + }, + "snippet": { + "maxLength": 280, + "minLength": 1, + "title": "Snippet", + "type": "string" + }, + "title": { + "minLength": 1, + "title": "Title", + "type": "string" + }, + "type": { + "enum": [ + "docs", + "blog", + "glean", + "confluence", + "slack", + "github", + "manual" + ], + "title": "Type", + "type": "string" + }, + "uri": { + "minLength": 1, + "title": "Uri", + "type": "string" + } + }, + "required": [ + "type", + "uri", + "title", + "retrieved_at", + "snippet" + ], + "title": "SourceCitation", + "type": "object" + }, + "ToolExpectations": { + "additionalProperties": false, + "properties": { + "banned": { + "items": { + "type": "string" + }, + "title": "Banned", + "type": "array" + }, + "required": { + "items": { + "type": "string" + }, + "title": "Required", + "type": "array" + } + }, + "title": "ToolExpectations", + "type": "object" + }, + "VerificationCommand": { + "additionalProperties": false, + "properties": { + "check": { + "minLength": 1, + "title": "Check", + "type": "string" + }, + "command": { + "minLength": 1, + "title": "Command", + "type": "string" + }, + "fact": { + "minLength": 1, + "title": "Fact", + "type": "string" + } + }, + "required": [ + "fact", + "command", + "check" + ], + "title": "VerificationCommand", + "type": "object" + } + }, + "$id": "https://github.com/databricks-solutions/solution-builder/evaluation/schema/scenario.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "capabilities": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Capabilities", + "type": "array" + }, + "description": { + "minLength": 1, + "title": "Description", + "type": "string" + }, + "id": { + "pattern": "^[a-z0-9][a-z0-9-]*$", + "title": "Id", + "type": "string" + }, + "live_resources": { + "$ref": "#/$defs/LiveResourceExpectations" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "skill": { + "minLength": 1, + "title": "Skill", + "type": "string" + }, + "steps": { + "items": { + "$ref": "#/$defs/ScenarioStep" + }, + "minItems": 1, + "title": "Steps", + "type": "array" + }, + "target_stage": { + "$ref": "#/$defs/ProjectStage" + }, + "timeout_seconds": { + "maximum": 14400, + "minimum": 60, + "title": "Timeout Seconds", + "type": "integer" + } + }, + "required": [ + "id", + "skill", + "description", + "capabilities", + "timeout_seconds", + "target_stage", + "steps", + "live_resources" + ], + "title": "Scenario", + "type": "object" +} diff --git a/evaluation/schema_generator.py b/evaluation/schema_generator.py new file mode 100644 index 0000000..51d3ad3 --- /dev/null +++ b/evaluation/schema_generator.py @@ -0,0 +1,36 @@ +"""Generate/check the committed JSON Schema for canonical scenarios.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from evaluation.models import Scenario + + +SCHEMA_PATH = Path(__file__).parent / "schema" / "scenario.schema.json" + + +def rendered_schema() -> str: + schema = Scenario.model_json_schema() + schema["$schema"] = "https://json-schema.org/draft/2020-12/schema" + schema["$id"] = ( + "https://github.com/databricks-solutions/solution-builder/evaluation/schema/scenario.schema.json" + ) + return json.dumps(schema, indent=2, sort_keys=True) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args(argv) + expected = rendered_schema() + if args.check: + return 0 if SCHEMA_PATH.is_file() and SCHEMA_PATH.read_text() == expected else 1 + SCHEMA_PATH.write_text(expected, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evaluation/skillforge.lock.yaml b/evaluation/skillforge.lock.yaml new file mode 100644 index 0000000..d579df0 --- /dev/null +++ b/evaluation/skillforge.lock.yaml @@ -0,0 +1,15 @@ +schema_version: 1 +skillforge: + repository: https://github.com/databricks-field-eng/skillforge.git + revision: ef9b6fb851e7f80c3765bb1eba4e6a5b16581ff9 + version: 0.1.0 + executable: stf + required_features: + - recursive-skill-discovery + - complete-skill-content-hash + - per-side-lifecycle-hooks + - extended-databricks-resource-tracking +ai_dev_kit: + repository: https://github.com/databricks-solutions/ai-dev-kit.git + revision: 8cb4456543d6413e8a12923d4d4b5394fea89b25 + skills_subdirectory: databricks-skills diff --git a/evaluation/toolchain.py b/evaluation/toolchain.py new file mode 100644 index 0000000..c201c86 --- /dev/null +++ b/evaluation/toolchain.py @@ -0,0 +1,188 @@ +"""Pinned external-tool discovery and feature preflight.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parent.parent +LOCK_PATH = Path(__file__).parent / "skillforge.lock.yaml" + + +@dataclass(frozen=True) +class ToolchainLock: + skillforge_repository: str + skillforge_revision: str + skillforge_version: str + skillforge_executable: str + required_features: tuple[str, ...] + ai_dev_kit_repository: str + ai_dev_kit_revision: str + ai_dev_kit_skills_subdirectory: str + + +@dataclass(frozen=True) +class Check: + name: str + ok: bool + detail: str + required: bool = True + + +def load_lock(path: Path = LOCK_PATH) -> ToolchainLock: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + skillforge = raw["skillforge"] + ai_dev_kit = raw["ai_dev_kit"] + return ToolchainLock( + skillforge_repository=skillforge["repository"], + skillforge_revision=skillforge["revision"], + skillforge_version=skillforge["version"], + skillforge_executable=skillforge["executable"], + required_features=tuple(skillforge["required_features"]), + ai_dev_kit_repository=ai_dev_kit["repository"], + ai_dev_kit_revision=ai_dev_kit["revision"], + ai_dev_kit_skills_subdirectory=ai_dev_kit["skills_subdirectory"], + ) + + +def find_executable(lock: ToolchainLock) -> str | None: + override = os.environ.get("SB_EVAL_STF") + if override: + path = Path(override).expanduser() + return ( + str(path.resolve()) if path.is_file() and os.access(path, os.X_OK) else None + ) + return shutil.which(lock.skillforge_executable) + + +def skillforge_build_info(executable: str) -> dict[str, Any]: + completed = subprocess.run( + [executable, "build-info", "--json"], + check=False, + capture_output=True, + text=True, + timeout=20, + ) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout).strip() + raise RuntimeError( + "pinned SkillForge must support `stf build-info --json`: " + detail + ) + try: + info = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("SkillForge build-info did not return JSON") from exc + if not isinstance(info, dict): + raise RuntimeError("SkillForge build-info JSON must be an object") + return info + + +def resolve_ai_dev_kit(repo_root: Path = REPO_ROOT) -> Path | None: + candidates = [ + os.environ.get("SB_EVAL_AI_DEV_KIT_DIR"), + os.environ.get("AI_DEV_KIT_PATH"), + str(repo_root / "app" / "ai_dev_kit"), + str(repo_root.parent / "ai-dev-kit"), + ] + for candidate in candidates: + if not candidate: + continue + path = Path(candidate).expanduser().resolve() + if path.is_dir(): + return path + return None + + +def git_has_revision(repo: Path, revision: str) -> bool: + completed = subprocess.run( + ["git", "-C", str(repo), "cat-file", "-e", f"{revision}^{{commit}}"], + check=False, + capture_output=True, + text=True, + ) + return completed.returncode == 0 + + +def run_doctor(repo_root: Path = REPO_ROOT) -> list[Check]: + from evaluation.cases import load_cases + + lock = load_lock() + checks: list[Check] = [] + try: + cases = load_cases() + checks.append(Check("canonical-cases", True, f"{len(cases)} valid scenarios")) + except Exception as exc: # noqa: BLE001 - doctor aggregates failures + checks.append(Check("canonical-cases", False, str(exc))) + + executable = find_executable(lock) + if executable is None: + checks.append( + Check( + "skillforge-executable", + False, + f"{lock.skillforge_executable!r} not found; install revision {lock.skillforge_revision}", + ) + ) + else: + checks.append(Check("skillforge-executable", True, executable)) + try: + info = skillforge_build_info(executable) + version_ok = info.get("version") == lock.skillforge_version + revision_ok = info.get("revision") == lock.skillforge_revision + features = set(info.get("features") or []) + missing = sorted(set(lock.required_features) - features) + checks.extend( + [ + Check( + "skillforge-version", + version_ok, + f"expected {lock.skillforge_version}, got {info.get('version')}", + ), + Check( + "skillforge-revision", + revision_ok, + f"expected {lock.skillforge_revision}, got {info.get('revision')}", + ), + Check( + "skillforge-features", + not missing, + "all required features present" + if not missing + else f"missing: {', '.join(missing)}", + ), + ] + ) + except Exception as exc: # noqa: BLE001 + checks.append(Check("skillforge-build-info", False, str(exc))) + + ai_dev_kit = resolve_ai_dev_kit(repo_root) + if ai_dev_kit is None: + checks.append( + Check( + "ai-dev-kit", + False, + "checkout not found; set SB_EVAL_AI_DEV_KIT_DIR", + ) + ) + elif os.environ.get("SB_EVAL_ALLOW_UNPINNED_FIXTURE") == "test-only": + checks.append(Check("ai-dev-kit", True, f"test-only fixture: {ai_dev_kit}")) + else: + has_revision = git_has_revision(ai_dev_kit, lock.ai_dev_kit_revision) + checks.append( + Check( + "ai-dev-kit", + has_revision, + f"{ai_dev_kit} contains pinned revision {lock.ai_dev_kit_revision}" + if has_revision + else f"{ai_dev_kit} does not contain {lock.ai_dev_kit_revision}", + ) + ) + return checks diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d6993d5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "solution-builder-evaluation" +version = "0.1.0" +description = "Maintainer-only evaluation contracts and SkillForge adapter for Solution Builder" +requires-python = ">=3.12,<3.13" +dependencies = [ + "pydantic>=2.11,<3", + "pyyaml>=6.0,<7", +] + +[project.scripts] +sb-eval = "evaluation.cli:main" + +[dependency-groups] +dev = [ + "pytest>=8.3,<9", + "pytest-asyncio>=0.24,<1", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["evaluation"] + +[tool.pytest.ini_options] +testpaths = ["tests/evaluation"] +asyncio_mode = "auto" diff --git a/tests/evaluation/golden/financial-services.ground-truth.yaml b/tests/evaluation/golden/financial-services.ground-truth.yaml new file mode 100644 index 0000000..b2e4453 --- /dev/null +++ b/tests/evaluation/golden/financial-services.ground-truth.yaml @@ -0,0 +1,163 @@ +version: '5' +test_cases: +- id: financial-services--01-capture-story + inputs: + prompt: Build a fraud-detection demo for a retail bank. Use synthetic credit-card + transactions, an anomaly-detection pattern, a Genie space for ad-hoc questions, + and an AI/BI dashboard for the trend view. + expectations: + expected_facts: + - The proposed demo uses synthetic credit-card transactions, Genie, and an AI/BI + dashboard. + - fact: README.md and resources.json exist after story capture. + verify_cmd: bash -lc 'test -f README.md && test -f resources.json && echo {"ok":true}' + verify_check: .ok == true + assertions: + - The story has a named banking protagonist, a concrete fraud catalyst, and quantified + business impact. + - Synthetic transactions, the anomaly narrative, Genie questions, and dashboard + measures form one coherent story. + expected_patterns: + - (?i)fraud + - (?i)(revenue|loss|risk|\$) + trace_expectations: + required_tools: + - Read + - Write + banned_tools: [] + token_budget: + max_total: 80000 + guidelines: + - 'Expected project stage after this step: SUMMARIZED' + - 'Required artifacts: README.md, resources.json' + metadata: + category: happy_path + difficulty: intermediate + generation_session_id: 5edf0889-9b05-5e3d-ab3d-326591ec1a17 + regression_intent: Detects generic fraud stories whose selected products or generated + data do not support the same investigation. + sources: + - type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: '2026-07-17T00:00:00Z' + snippet: Story first and coherence above all; every showcased product must earn + a clear beat in the walkthrough. +- id: financial-services--02-design-architecture + inputs: + prompt: Looks good — proceed to architect the pipeline and produce architecture.md. + expectations: + expected_facts: + - The project contains architecture.md. + - fact: architecture.md exists and is not empty. + verify_cmd: bash -lc 'test -s architecture.md && echo {"ok":true}' + verify_check: .ok == true + assertions: + - The architecture connects transaction generation to governed data, Genie, and + the dashboard without orphan components. + expected_patterns: + - architecture\.md + - (?i)(genie|dashboard) + trace_expectations: + required_tools: + - Read + - Write + banned_tools: [] + token_budget: + max_total: 80000 + guidelines: + - 'Expected project stage after this step: ARCHITECTED' + - 'Required artifacts: architecture.md' + metadata: + category: happy_path + difficulty: intermediate + generation_session_id: cdae271d-b326-5c95-9b8d-a8853f2c8f99 + regression_intent: Detects architecture output that omits a selected capability + or breaks the end-to-end data flow. + sources: + - type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: '2026-07-17T00:00:00Z' + snippet: Story first and coherence above all; every showcased product must earn + a clear beat in the walkthrough. +- id: financial-services--03-write-specifications + inputs: + prompt: Now write the specifications for each component (specifications/*.md). + expectations: + expected_facts: + - Per-component specifications are written under specifications/. + - fact: At least one Markdown specification exists. + verify_cmd: bash -lc 'find specifications -name "*.md" -type f -print -quit + | grep -q . && echo {"ok":true}' + verify_check: .ok == true + assertions: + - Specifications define compatible transaction fields, fraud measures, dashboard + visuals, and Genie questions. + expected_patterns: + - specifications/ + - (?i)(transaction|anomal) + trace_expectations: + required_tools: + - Read + - Write + banned_tools: [] + token_budget: + max_total: 80000 + guidelines: + - 'Expected project stage after this step: SPECIFICATION' + - 'Required artifacts: specifications/*.md' + metadata: + category: happy_path + difficulty: intermediate + generation_session_id: e49ead5c-4903-5c55-a315-9b62c6591b68 + regression_intent: Detects specs that are individually plausible but disagree + on schemas, measures, or the investigation path. + sources: + - type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: '2026-07-17T00:00:00Z' + snippet: Story first and coherence above all; every showcased product must earn + a clear beat in the walkthrough. +- id: financial-services--04-build-demo + inputs: + prompt: 'Build it: write the SQL/Python and capture deployed resource IDs in resources.json.' + expectations: + expected_facts: + - SQL or Python implementation artifacts and resource identifiers are produced. + - fact: resources.json is valid JSON and implementation code exists. + verify_cmd: bash -lc 'python3 -m json.tool resources.json >/dev/null && find + . -type f -name "*.py" -print -quit | grep -q . && echo {"ok":true}' + verify_check: .ok == true + assertions: + - Build artifacts implement the approved specifications and resources.json records + only resources actually created. + - Live resource names use the evaluation namespace supplied in the environment + when present. + expected_patterns: + - resources\.json + - (?i)(sql|python|dashboard|genie) + trace_expectations: + required_tools: + - Read + - Write + banned_tools: [] + token_budget: + max_total: 80000 + guidelines: + - 'Expected project stage after this step: BUILT' + - 'Required artifacts: resources.json, **/*.{py,sql,json}' + metadata: + category: happy_path + difficulty: hard + generation_session_id: c883ee3d-ce79-5194-8273-98423bfd17ce + regression_intent: Detects claimed builds with placeholders, missing code, dead + resource links, or resources outside the evaluation namespace. + sources: + - type: github + uri: repo://.claude/skills/databricks-demo-generator/SKILL.md + title: Databricks Demo Generator workflow and coherence contract + retrieved_at: '2026-07-17T00:00:00Z' + snippet: Story first and coherence above all; every showcased product must earn + a clear beat in the walkthrough. diff --git a/tests/evaluation/golden/financial-services.manifest.yaml b/tests/evaluation/golden/financial-services.manifest.yaml new file mode 100644 index 0000000..3712986 --- /dev/null +++ b/tests/evaluation/golden/financial-services.manifest.yaml @@ -0,0 +1,28 @@ +skill_name: databricks-demo-generator +description: 'Solution Builder canonical scenario: financial-services' +shared_cwd: true +tool_modules: [] +comparison_judge: + rubric_file: comparison_rubric.md + dimensions: + - correctness + - coherence + - safety + - artifact_quality + anti_bias: + - Treat the target skill as WITH and the control skill as WITHOUT. + - Do not reward verbosity or tool-call count. + - Attribute shared-cwd cascading failures to their earliest causal case. +solution_builder: + scenario_id: financial-services + capabilities: + - genie + - aibi-dashboards + - synthetic-data-gen + expected_resource_kinds: + - catalog + - schema + - table + - dashboard + - genie_space + cleanup_owner: solution-builder diff --git a/tests/evaluation/test_adapter.py b/tests/evaluation/test_adapter.py new file mode 100644 index 0000000..5d20174 --- /dev/null +++ b/tests/evaluation/test_adapter.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from evaluation.adapter import AdapterError, SkillForgeAdapter +from evaluation.cases import load_cases +from evaluation.models import SourceCitation + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def test_financial_services_v5_golden(tmp_path: Path) -> None: + case = next(case for case in load_cases() if case.id == "financial-services") + rendered = SkillForgeAdapter(REPO_ROOT).render(case, tmp_path / "eval") + golden_dir = Path(__file__).parent / "golden" + assert ( + rendered.ground_truth.read_text().rstrip() + == (golden_dir / "financial-services.ground-truth.yaml").read_text().rstrip() + ) + assert ( + rendered.manifest.read_text().rstrip() + == (golden_dir / "financial-services.manifest.yaml").read_text().rstrip() + ) + + +def test_adapter_rejects_missing_source_file(tmp_path: Path) -> None: + case = load_cases()[0] + bad_source = SourceCitation( + type="github", + uri="repo://does/not/exist.md", + title="Missing", + retrieved_at="2026-07-17T00:00:00Z", + snippet="This source is intentionally missing for the negative test.", + ) + bad_step = case.steps[0].model_copy(update={"sources": [bad_source]}) + bad_case = case.model_copy(update={"steps": [bad_step, *case.steps[1:]]}) + with pytest.raises(AdapterError, match="source does not exist"): + SkillForgeAdapter(REPO_ROOT).render(bad_case, tmp_path / "eval") + + +def test_adapter_rejects_unresolved_placeholders(tmp_path: Path) -> None: + case = load_cases()[0] + bad_step = case.steps[0].model_copy(update={"prompt": "Use ${UNRESOLVED_VALUE}"}) + bad_case = case.model_copy(update={"steps": [bad_step, *case.steps[1:]]}) + with pytest.raises(AdapterError, match="unresolved placeholder"): + SkillForgeAdapter(REPO_ROOT).render(bad_case, tmp_path / "eval") + + +def test_v5_contains_sources_regression_and_expectations(tmp_path: Path) -> None: + case = load_cases()[1] + rendered = SkillForgeAdapter(REPO_ROOT).render(case, tmp_path / "eval") + raw = yaml.safe_load(rendered.ground_truth.read_text()) + assert raw["version"] == "5" + for item in raw["test_cases"]: + assert item["metadata"]["sources"] + assert item["metadata"]["regression_intent"] + expectations = item["expectations"] + assert expectations["expected_facts"] + assert expectations["assertions"] + assert expectations["expected_patterns"] + assert expectations["trace_expectations"] diff --git a/tests/evaluation/test_cases.py b/tests/evaluation/test_cases.py new file mode 100644 index 0000000..3d75d0a --- /dev/null +++ b/tests/evaluation/test_cases.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from evaluation.cases import load_cases +from tests.pipeline.scenarios import SCENARIOS + + +LEGACY = { + "financial-services": { + "capabilities": ["genie", "aibi-dashboards", "synthetic-data-gen"], + "initial_prompt": "Build a fraud-detection demo for a retail bank. Use synthetic credit-card transactions, an anomaly-detection pattern, a Genie space for ad-hoc questions, and an AI/BI dashboard for the trend view.", + "drive_messages": [ + "Looks good — proceed to architect the pipeline and produce architecture.md.", + "Now write the specifications for each component (specifications/*.md).", + "Build it: write the SQL/Python and capture deployed resource IDs in resources.json.", + ], + }, + "healthcare": { + "capabilities": ["knowledge-assistant", "vector-search", "ml-training-serving"], + "initial_prompt": "Build a patient-readmission risk demo. Knowledge Assistant over discharge-summary documents, vector search over the same corpus, and a tabular ML model that scores 30-day readmission risk on synthetic patient data.", + "drive_messages": [ + "Proceed: architect the components and produce architecture.md.", + "Write the per-component specifications under specifications/.", + "Build it now — generate the code, training notebook, and resources.json.", + ], + }, + "manufacturing-machinery": { + "capabilities": [ + "synthetic-data-gen", + "genie", + "supervisor-agent", + "databricks-apps", + ], + "initial_prompt": "Build a warranty-optimization demo for a heavy-machinery OEM (Manufacturing > Machinery sub-vertical). Use synthetic-data-gen to create warranty claims and machine-telemetry tables in Delta, stand up a Genie space over the Gold warranty tables for ad-hoc cost/failure-mode questions, wire a multi-agent supervisor that routes between Genie (quantitative) and a claims-triage agent, and ship a Databricks App (FastAPI + React) where a field-service manager can inspect a claim and see the supervisor's recommendation.", + "drive_messages": [ + "Looks good — proceed to architect the pipeline and produce architecture.md.", + "Now write the specifications for each component (specifications/*.md) — synthetic data tables, Genie space, supervisor agent, and the Databricks App.", + "Build it: write the SQL/Python for synthetic data, the Genie config, the supervisor agent definition, the app code, and capture deployed resource IDs in resources.json.", + ], + }, + "retail": { + "capabilities": ["lakebase", "ml-training-serving", "sdp"], + "initial_prompt": "Build a demand-forecasting demo for an omnichannel retailer. Use a Lakeflow Spark Declarative Pipeline (SDP) to ingest synthetic POS events into Delta, land aggregates in Lakebase for fast lookups, and serve a Prophet-style forecast via Model Serving.", + "drive_messages": [ + "Proceed: produce architecture.md.", + "Write the per-component specifications.", + "Build it: generate the SDP pipeline, the Lakebase sync, the forecast notebook, and resources.json.", + ], + }, +} + + +def test_four_canonical_cases_validate() -> None: + cases = load_cases() + assert [case.id for case in cases] == [ + "financial-services", + "healthcare", + "manufacturing-machinery", + "retail", + ] + assert all(case.timeout_seconds == 3600 for case in cases) + assert all(case.target_stage.value == "BUILT" for case in cases) + + +def test_pipeline_compatibility_is_lossless() -> None: + assert len(SCENARIOS) == 4 + for scenario in SCENARIOS: + expected = LEGACY[scenario.slug] + assert scenario.capabilities == expected["capabilities"] + assert scenario.initial_prompt == expected["initial_prompt"] + assert scenario.drive_messages == expected["drive_messages"] + assert scenario.target_stage == "BUILT" + assert scenario.timeout_seconds == 3600 diff --git a/tests/evaluation/test_cleanup.py b/tests/evaluation/test_cleanup.py new file mode 100644 index 0000000..b03e553 --- /dev/null +++ b/tests/evaluation/test_cleanup.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from evaluation.live import ( + DatabricksCliCleaner, + LivePolicy, + ResourceCleaner, + ResourceRecord, + expected_resource_kinds, + reconcile_resources, + resources_from_manifest, + resources_from_tracked, +) + + +def _policy() -> LivePolicy: + return LivePolicy( + profile="solution-builder-eval", + host="https://eval.example.com", + allowed_hosts=("https://eval.example.com",), + evaluation_prefix="sb_eval_", + ) + + +def _completed( + args: list[str], + *, + returncode: int = 0, + stdout: str = "{}", + stderr: str = "", +) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(args, returncode, stdout, stderr) + + +def _set_live_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SB_EVAL_DATABRICKS_PROFILE", "solution-builder-eval") + monkeypatch.setenv("SB_EVAL_ALLOWED_PROFILES", "solution-builder-eval") + monkeypatch.setenv("SB_EVAL_ALLOWED_HOSTS", " https://eval.example.com/ ") + monkeypatch.setenv("DATABRICKS_HOST", "https://eval.example.com") + + +def test_reconcile_deduplicates_and_prefers_skillforge() -> None: + manifest = ResourceRecord("pipeline", "p1", source="resources.json") + tracked = ResourceRecord("pipeline", "p1", name="tracked", source="skillforge") + assert reconcile_resources([manifest], [tracked]) == [tracked] + + +def test_cleanup_uses_reverse_dependency_order_and_deduplicates() -> None: + seen: list[str] = [] + existing = {"app:a", "table:t", "catalog:c"} + + def delete(resource: ResourceRecord) -> None: + seen.append(resource.key) + existing.remove(resource.key) + + cleaner = ResourceCleaner(delete, lambda resource: resource.key in existing) + report = cleaner.cleanup( + [ + ResourceRecord("catalog", "c"), + ResourceRecord("table", "t"), + ResourceRecord("app", "a"), + ResourceRecord("app", "a"), + ] + ) + assert report.complete + assert seen == ["app:a", "table:t", "catalog:c"] + + +def test_cleanup_retries_then_succeeds() -> None: + attempts = 0 + exists = True + + def delete(resource: ResourceRecord) -> None: + nonlocal attempts, exists + attempts += 1 + if attempts < 3: + raise RuntimeError("transient") + exists = False + + report = ResourceCleaner(delete, lambda _: exists, retries=3).cleanup( + [ResourceRecord("pipeline", "p1")] + ) + assert report.complete + assert attempts == 3 + + +def test_partial_deletion_emits_remaining_and_error() -> None: + def delete(resource: ResourceRecord) -> None: + raise RuntimeError("permission denied") + + report = ResourceCleaner(delete, lambda _: True, retries=2).cleanup( + [ResourceRecord("dashboard", "d1")] + ) + assert not report.complete + assert report.remaining == ["dashboard:d1"] + assert "permission denied" in report.errors[0] + + +def test_cleanup_is_idempotent_when_resource_is_already_absent() -> None: + calls = 0 + + def delete(resource: ResourceRecord) -> None: + nonlocal calls + calls += 1 + + report = ResourceCleaner(delete, lambda _: False).cleanup( + [ResourceRecord("pipeline", "gone")] + ) + assert report.complete + assert calls == 0 + + +def test_live_policy_requires_allowlisted_service_principal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_live_env(monkeypatch) + monkeypatch.setattr( + "evaluation.live.subprocess.run", + lambda args, **kwargs: _completed( + args, stdout=json.dumps({"application_id": "service-principal-id"}) + ), + ) + policy = LivePolicy.from_env() + assert policy.profile == "solution-builder-eval" + assert policy.host == "https://eval.example.com" + + +def test_live_policy_rejects_human_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_live_env(monkeypatch) + monkeypatch.setattr( + "evaluation.live.subprocess.run", + lambda args, **kwargs: _completed( + args, stdout=json.dumps({"user_name": "maintainer@example.com"}) + ), + ) + with pytest.raises(ValueError, match="service principal"): + LivePolicy.from_env() + + +def test_manifest_normalizes_schema_and_lakebase_slug(tmp_path: Path) -> None: + path = tmp_path / "resources.json" + path.write_text( + json.dumps( + { + "created_resources": { + "catalog": "sb_eval_catalog", + "schema": "sb_eval_schema", + "warehouse_id": "shared-warehouse", + "lakebase_project_id": "opaque-uid", + "lakebase_project_slug": "sb_eval_lakebase", + } + } + ) + ) + resources = resources_from_manifest(path) + assert {resource.key for resource in resources} == { + "catalog:sb_eval_catalog", + "schema:sb_eval_catalog.sb_eval_schema", + "lakebase:projects/sb_eval_lakebase", + } + + +def test_tracked_comparison_sides_are_normalized() -> None: + resources = resources_from_tracked( + [ + {"asset_type": "pipeline", "asset_id": "p1", "side": "A-with"}, + {"asset_type": "pipeline", "asset_id": "p2", "side": "B-without"}, + { + "asset_type": "lakebase", + "asset_id": "opaque-uid", + "name": "sb_eval_lakebase", + "side": "A-with", + }, + {"asset_type": "pipeline_run", "asset_id": "update-id"}, + ] + ) + assert [resource.side for resource in resources] == [ + "with", + "without", + "with", + ] + assert resources[-1].key == "lakebase:projects/sb_eval_lakebase" + + +def test_capabilities_derive_resource_kinds_with_overrides() -> None: + assert expected_resource_kinds( + ["sdp", "ml-training-serving", "vector-search"], overrides=["volume"] + ) == [ + "mlflow_experiment", + "model", + "pipeline", + "serving_endpoint", + "vector_endpoint", + "vector_index", + "volume", + ] + + +@pytest.mark.parametrize( + ("resource", "expected_command"), + [ + ( + ResourceRecord("catalog", "sb_eval_catalog"), + ["databricks", "catalogs", "delete", "sb_eval_catalog", "--force"], + ), + ( + ResourceRecord("schema", "sb_eval_catalog.sb_eval_schema"), + [ + "databricks", + "schemas", + "delete", + "sb_eval_catalog.sb_eval_schema", + "--force", + ], + ), + ( + ResourceRecord("table", "sb_eval_catalog.sb_eval_schema.table"), + [ + "databricks", + "tables", + "delete", + "sb_eval_catalog.sb_eval_schema.table", + ], + ), + ( + ResourceRecord("model", "sb_eval_catalog.sb_eval_schema.model"), + [ + "databricks", + "registered-models", + "delete", + "sb_eval_catalog.sb_eval_schema.model", + ], + ), + ( + ResourceRecord("vector_index", "sb_eval_catalog.sb_eval_schema.index"), + [ + "databricks", + "vector-search-indexes", + "delete-index", + "sb_eval_catalog.sb_eval_schema.index", + ], + ), + ( + ResourceRecord( + "vector_endpoint", "sb_eval_endpoint", name="sb_eval_endpoint" + ), + [ + "databricks", + "vector-search-endpoints", + "delete-endpoint", + "sb_eval_endpoint", + ], + ), + ( + ResourceRecord("cluster", "cluster-id", name="sb_eval_cluster"), + ["databricks", "clusters", "permanent-delete", "cluster-id"], + ), + ( + ResourceRecord("function", "sb_eval_catalog.sb_eval_schema.function"), + [ + "databricks", + "functions", + "delete", + "sb_eval_catalog.sb_eval_schema.function", + "--force", + ], + ), + ( + ResourceRecord("notebook", "/Shared/sb_eval_notebook"), + [ + "databricks", + "workspace", + "delete", + "/Shared/sb_eval_notebook", + ], + ), + ( + ResourceRecord("lakebase", "projects/sb_eval_project"), + [ + "databricks", + "postgres", + "delete-project", + "projects/sb_eval_project", + ], + ), + ( + ResourceRecord("dashboard", "dashboard-id"), + ["databricks", "lakeview", "trash", "dashboard-id"], + ), + ( + ResourceRecord("genie_space", "space-id"), + ["databricks", "genie", "trash-space", "space-id"], + ), + ( + ResourceRecord("knowledge_assistant", "ka-id"), + [ + "databricks", + "knowledge-assistants", + "delete-knowledge-assistant", + "knowledge-assistants/ka-id", + ], + ), + ( + ResourceRecord("multi_agent_supervisor", "mas-id"), + [ + "databricks", + "api", + "delete", + "/api/2.0/tiles/mas-id", + ], + ), + ], +) +def test_databricks_cleaner_uses_supported_delete_commands( + resource: ResourceRecord, + expected_command: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + cleaner = DatabricksCliCleaner(_policy()) + calls: list[list[str]] = [] + + def fake_run(args: list[str], *, allow_not_found: bool = False): + calls.append(args) + return _completed(args) + + monkeypatch.setattr(cleaner, "_run", fake_run) + cleaner.delete(resource) + assert calls == [expected_command] + + +def test_databricks_cleaner_verifies_absence_after_delete( + monkeypatch: pytest.MonkeyPatch, +) -> None: + existing = True + calls: list[list[str]] = [] + + def fake_subprocess_run(args, **kwargs): + nonlocal existing + calls.append(args) + if args[1:3] == ["pipelines", "delete"]: + existing = False + return _completed(args) + if existing: + return _completed(args) + return _completed( + args, + returncode=1, + stdout="", + stderr="RESOURCE_DOES_NOT_EXIST: pipeline not found", + ) + + monkeypatch.setattr("evaluation.live.subprocess.run", fake_subprocess_run) + backend = DatabricksCliCleaner(_policy()) + report = ResourceCleaner(backend.delete, backend.exists, retries=1).cleanup( + [ResourceRecord("pipeline", "pipeline-id")] + ) + assert report.complete + assert [call[1:3] for call in calls] == [ + ["pipelines", "get"], + ["pipelines", "delete"], + ["pipelines", "get"], + ] + + +def test_mlflow_experiment_cleanup_resolves_id_and_checks_lifecycle( + monkeypatch: pytest.MonkeyPatch, +) -> None: + deleted = False + calls: list[list[str]] = [] + + def fake_subprocess_run(args, **kwargs): + nonlocal deleted + calls.append(args) + if args[1:3] == ["experiments", "delete-experiment"]: + deleted = True + return _completed(args) + lifecycle = "DELETED" if deleted else "ACTIVE" + return _completed( + args, + stdout=json.dumps( + { + "experiment": { + "experiment_id": "42", + "lifecycle_stage": lifecycle, + } + } + ), + ) + + monkeypatch.setattr("evaluation.live.subprocess.run", fake_subprocess_run) + backend = DatabricksCliCleaner(_policy()) + report = ResourceCleaner(backend.delete, backend.exists, retries=1).cleanup( + [ResourceRecord("mlflow_experiment", "/Shared/sb_eval_experiment")] + ) + assert report.complete + assert any(call[1:3] == ["experiments", "delete-experiment"] for call in calls) + + +def test_destructive_uc_cleanup_refuses_out_of_scope_name() -> None: + backend = DatabricksCliCleaner(_policy()) + with pytest.raises(RuntimeError, match="outside evaluation prefix"): + backend.delete(ResourceRecord("table", "production.default.customers")) + with pytest.raises(RuntimeError, match="outside evaluation prefix"): + backend.delete(ResourceRecord("mlflow_experiment", "/Shared/production")) diff --git a/tests/evaluation/test_hashing.py b/tests/evaluation/test_hashing.py new file mode 100644 index 0000000..3eaf817 --- /dev/null +++ b/tests/evaluation/test_hashing.py @@ -0,0 +1,24 @@ +from pathlib import Path + +from evaluation.hashing import hash_skill + + +def test_template_typescript_changes_complete_hash(tmp_path: Path) -> None: + skill = tmp_path / "skill" + template = skill / "app" / "template" + template.mkdir(parents=True) + (skill / "SKILL.md").write_text("# Skill\n") + source = template / "main.ts" + source.write_text("export const value = 1;\n") + before = hash_skill(skill) + source.write_text("export const value = 2;\n") + assert hash_skill(skill) != before + + +def test_eval_changes_do_not_change_hash(tmp_path: Path) -> None: + skill = tmp_path / "skill" + (skill / "eval").mkdir(parents=True) + (skill / "SKILL.md").write_text("# Skill\n") + before = hash_skill(skill) + (skill / "eval" / "ground_truth.yaml").write_text("version: '5'\n") + assert hash_skill(skill) == before diff --git a/tests/evaluation/test_lifecycle.py b/tests/evaluation/test_lifecycle.py new file mode 100644 index 0000000..2e00a2c --- /dev/null +++ b/tests/evaluation/test_lifecycle.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from evaluation.lifecycle import run_with_lifecycle +from evaluation.live import LiveNamespace + + +@pytest.mark.parametrize("failure", [None, RuntimeError("boom")]) +async def test_cleanup_runs_on_success_and_exception(failure: Exception | None) -> None: + events: list[str] = [] + + async def setup(context): + events.append("setup") + return {"ready": True} + + async def operation(context, setup_output): + events.append("operation") + if failure: + raise failure + return "ok" + + async def cleanup(context, setup_output): + events.append("cleanup") + + if failure: + with pytest.raises(RuntimeError, match="boom"): + await run_with_lifecycle( + context={}, setup=setup, operation=operation, cleanup=cleanup + ) + else: + assert ( + await run_with_lifecycle( + context={}, setup=setup, operation=operation, cleanup=cleanup + ) + == "ok" + ) + assert events == ["setup", "operation", "cleanup"] + + +async def test_cleanup_runs_on_timeout() -> None: + cleaned = asyncio.Event() + + async def operation(context, setup_output): + await asyncio.sleep(10) + + async def cleanup(context, setup_output): + cleaned.set() + + with pytest.raises(TimeoutError): + await asyncio.wait_for( + run_with_lifecycle( + context={}, setup=lambda _: None, operation=operation, cleanup=cleanup + ), + timeout=0.01, + ) + assert cleaned.is_set() + + +async def test_cleanup_runs_on_cancellation() -> None: + cleaned = asyncio.Event() + + async def operation(context, setup_output): + await asyncio.sleep(10) + + async def cleanup(context, setup_output): + cleaned.set() + + task = asyncio.create_task( + run_with_lifecycle( + context={}, setup=lambda _: None, operation=operation, cleanup=cleanup + ) + ) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert cleaned.is_set() + + +def test_with_without_namespaces_are_distinct() -> None: + with_side = LiveNamespace.allocate( + run_id="run-1", case_id="case-1", side="with", evaluation_prefix="sb_eval_" + ) + without_side = LiveNamespace.allocate( + run_id="run-1", case_id="case-1", side="without", evaluation_prefix="sb_eval_" + ) + assert with_side.catalog != without_side.catalog + assert with_side.schema != without_side.schema + assert with_side.resource_prefix != without_side.resource_prefix diff --git a/tests/evaluation/test_non_live_integration.py b/tests/evaluation/test_non_live_integration.py new file mode 100644 index 0000000..ad0e24e --- /dev/null +++ b/tests/evaluation/test_non_live_integration.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from pathlib import Path + +from evaluation.cases import load_cases +from evaluation.runner import run_scenario +from evaluation.toolchain import load_lock + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _fake_stf(path: Path) -> None: + lock = load_lock() + path.write_text( + "#!/usr/bin/env python3\n" + "import json, pathlib, sys\n" + f"REVISION = {lock.skillforge_revision!r}\n" + f"VERSION = {lock.skillforge_version!r}\n" + "args = sys.argv[1:]\n" + "if args[:2] == ['build-info', '--json']:\n" + " print(json.dumps({'version': VERSION, 'revision': REVISION, 'features': " + + repr(list(lock.required_features)) + + "})); raise SystemExit(0)\n" + "if args and args[0] == 'doctor':\n" + " print(json.dumps({'ok': True})); raise SystemExit(0)\n" + "if args and args[0] == 'compare':\n" + " output = pathlib.Path(args[args.index('--output') + 1])\n" + " levels = args[args.index('--levels') + 1].split(',')\n" + " level_data = {level: {'score': 1.0, 'task_results': []} for level in levels}\n" + " payload = {'mode': 'comparison', 'comparison_id': 'fake', " + "'result_a': {'skill_name': 'databricks-demo-generator', 'composite_score': 1.0, 'levels': level_data, 'suggestions': [], 'gaps': {}, 'mlflow_run_id': 'mlflow-with', 'skills_invoked': ['databricks-demo-generator']}, " + "'result_b': {'skill_name': 'solution-builder-without-skill', 'composite_score': 0.4, 'levels': level_data, 'suggestions': ['less coherent'], 'gaps': {}, 'mlflow_run_id': 'mlflow-without'}, " + "'verdict': {'winner': 'A', 'dimensions': []}}\n" + " output.parent.mkdir(parents=True, exist_ok=True); output.write_text(json.dumps(payload)); raise SystemExit(0)\n" + "raise SystemExit(2)\n", + encoding="utf-8", + ) + path.chmod(0o755) + + +def test_non_live_l1_to_l5_adapter_run(tmp_path: Path, monkeypatch) -> None: + fake_stf = tmp_path / "stf" + _fake_stf(fake_stf) + ai_dev_kit = tmp_path / "ai-dev-kit" + for skill_name in ("databricks-jobs", "databricks-lakebase-provisioned"): + sibling = ai_dev_kit / "databricks-skills" / skill_name + sibling.mkdir(parents=True) + (sibling / "SKILL.md").write_text( + f"---\nname: {skill_name}\ndescription: fixture\n---\n" + ) + monkeypatch.setenv("SB_EVAL_STF", str(fake_stf)) + monkeypatch.setenv("SB_EVAL_AI_DEV_KIT_DIR", str(ai_dev_kit)) + monkeypatch.setenv("SB_EVAL_ALLOW_UNPINNED_FIXTURE", "test-only") + case = load_cases()[0] + result = run_scenario( + case, + levels=["unit", "integration", "static", "thinking", "output"], + output_dir=tmp_path / "run", + live=False, + agent_model="fixture-agent", + judge_model="fixture-judge", + run_id="fixture-run", + repo_root=REPO_ROOT, + ) + assert result.status == "passed" + assert {score.side for score in result.scores} >= {"with", "without"} + assert result.cleanup.complete + assert result.mlflow.run_ids == ["mlflow-with", "mlflow-without"] + assert Path(result.reports["html"]).is_file() + fixture_skills = ( + tmp_path / "run" / case.id / "fixture" / "project" / ".claude" / "skills" + ) + assert (fixture_skills / "databricks-lakebase-provisioned" / "SKILL.md").is_file() diff --git a/tests/pipeline/README.md b/tests/pipeline/README.md index 886923a..7d82f94 100644 --- a/tests/pipeline/README.md +++ b/tests/pipeline/README.md @@ -12,7 +12,7 @@ You'll spend FMAPI tokens. Plan for ~45–60 min per full run. ## Run ```bash -tests/pipeline/run.sh # all 3 scenarios, target=BUILT +tests/pipeline/run.sh # all 4 scenarios, target=BUILT tests/pipeline/run.sh --scenario healthcare # single scenario tests/pipeline/run.sh --target SPECIFICATION # cheaper (~15 min) tests/pipeline/run.sh --scenario-timeout 1800 # 30 min/scenario cap @@ -59,10 +59,15 @@ Otherwise FAIL — the per-scenario `README.md` lists every issue. ## Add a scenario -Append a `Scenario(...)` to `SCENARIOS` in `scenarios.py`. Capability slugs -must match filenames (without `.md`) in -`.claude/skills/databricks-demo-generator/references/blocks/capabilities/`. -A new scenario runs in parallel with the rest automatically — no other wiring. +Add and validate a canonical case under `evaluation/cases/`: + +```bash +uv run sb-eval cases validate +``` + +Capability slugs must match the demo-generator capability blocks. The pipeline +harness and SkillForge adapter both load the same versioned YAML, so a new case +runs in parallel here without a second hardcoded definition. ## Files @@ -71,7 +76,7 @@ A new scenario runs in parallel with the rest automatically — no other wiring. | `run.sh` | Bootstrap: ensure backend, run pytest, surface summary path. | | `test_pipeline.py` | Pytest entry. Fans out scenarios via `asyncio.gather`. | | `runner.py` | `drive_project()`: create → invoke per turn → snapshot. | -| `scenarios.py` | The 3 hardcoded scenarios + `Scenario` dataclass. | +| `scenarios.py` | Compatibility loader for canonical `evaluation/cases/*.yaml`. | | `assertions.py` | Pure pass/fail helpers (stage, errors, artifacts). | | `api_client.py` | `httpx.AsyncClient` wrapper + SSE consumer with reconnect. | | `conftest.py` | Pytest fixtures: scenario selection, output dir, health check. | diff --git a/tests/pipeline/scenarios.py b/tests/pipeline/scenarios.py index e05812b..3e9ec57 100644 --- a/tests/pipeline/scenarios.py +++ b/tests/pipeline/scenarios.py @@ -1,25 +1,16 @@ -"""Hardcoded end-to-end scenarios. +"""Pipeline compatibility view over canonical evaluation scenarios. -Each scenario drives a single project from creation through the agent until it -reaches a target stage. To add a new scenario, append a Scenario to SCENARIOS. -Capability slugs must match filenames in -.claude/skills/databricks-demo-generator/references/blocks/capabilities/. +The durable scenario contract lives in ``evaluation/cases/*.yaml``. This +module keeps the pipeline runner's small dataclass interface while preventing +the E2E harness and SkillForge adapter from drifting onto separate prompts. """ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace - -# Stage ordering must match backend.models.ProjectStage -STAGE_ORDER = [ - "DRAFTING", - "SUMMARIZED", - "ARCHITECTED", - "SPECIFICATION", - "BUILT", - "BUNDLED", -] +from evaluation.cases import load_cases +from evaluation.models import STAGE_ORDER def stage_index(stage: str) -> int: @@ -37,111 +28,42 @@ class Scenario: initial_prompt: str drive_messages: list[str] = field(default_factory=list) target_stage: str = "BUILT" - timeout_seconds: int = 60 * 60 # 60 min default per scenario + timeout_seconds: int = 60 * 60 def __post_init__(self) -> None: if stage_index(self.target_stage) < 0: raise ValueError(f"unknown target_stage: {self.target_stage}") -SCENARIOS: list[Scenario] = [ - Scenario( - slug="financial-services", - description=( - "Real-time fraud detection demo for a retail bank. " - "Generate synthetic credit-card transactions, surface anomalies in a " - "Genie space, and visualize trends in an AI/BI dashboard." - ), - capabilities=["genie", "aibi-dashboards", "synthetic-data-gen"], - initial_prompt=( - "Build a fraud-detection demo for a retail bank. " - "Use synthetic credit-card transactions, an anomaly-detection pattern, " - "a Genie space for ad-hoc questions, and an AI/BI dashboard for the trend view." - ), - drive_messages=[ - "Looks good — proceed to architect the pipeline and produce architecture.md.", - "Now write the specifications for each component (specifications/*.md).", - "Build it: write the SQL/Python and capture deployed resource IDs in resources.json.", - ], - target_stage="BUILT", - ), - Scenario( - slug="healthcare", - description=( - "Patient-readmission risk demo for a hospital network. " - "Combine clinical notes (knowledge assistant + vector search) with a " - "tabular ML model for 30-day readmission risk." - ), - capabilities=["knowledge-assistant", "vector-search", "ml-training-serving"], - initial_prompt=( - "Build a patient-readmission risk demo. " - "Knowledge Assistant over discharge-summary documents, vector search over the same corpus, " - "and a tabular ML model that scores 30-day readmission risk on synthetic patient data." - ), - drive_messages=[ - "Proceed: architect the components and produce architecture.md.", - "Write the per-component specifications under specifications/.", - "Build it now — generate the code, training notebook, and resources.json.", - ], - target_stage="BUILT", - ), - Scenario( - slug="manufacturing-machinery", - description=( - "Warranty optimization demo for a heavy-machinery OEM (Manufacturing > Machinery sub-vertical). " - "Synthetic warranty claims + telemetry land in Delta, a Genie space answers quantitative " - "warranty-cost questions, a multi-agent supervisor routes between Genie and a claims-triage agent, " - "and a Databricks App gives field service a UI to inspect claims and the supervisor's verdict." - ), - capabilities=["synthetic-data-gen", "genie", "supervisor-agent", "databricks-apps"], - initial_prompt=( - "Build a warranty-optimization demo for a heavy-machinery OEM (Manufacturing > Machinery sub-vertical). " - "Use synthetic-data-gen to create warranty claims and machine-telemetry tables in Delta, " - "stand up a Genie space over the Gold warranty tables for ad-hoc cost/failure-mode questions, " - "wire a multi-agent supervisor that routes between Genie (quantitative) and a claims-triage agent, " - "and ship a Databricks App (FastAPI + React) where a field-service manager can inspect a claim " - "and see the supervisor's recommendation." - ), - drive_messages=[ - "Looks good — proceed to architect the pipeline and produce architecture.md.", - "Now write the specifications for each component (specifications/*.md) — synthetic data tables, Genie space, supervisor agent, and the Databricks App.", - "Build it: write the SQL/Python for synthetic data, the Genie config, the supervisor agent definition, the app code, and capture deployed resource IDs in resources.json.", - ], - target_stage="BUILT", - ), - Scenario( - slug="retail", - description=( - "Demand-forecasting demo for an omnichannel retailer. " - "Stream point-of-sale events into Delta, aggregate into a Lakebase OLTP store, " - "and serve a forecast model via Model Serving." - ), - capabilities=["lakebase", "ml-training-serving", "sdp"], - initial_prompt=( - "Build a demand-forecasting demo for an omnichannel retailer. " - "Use a Lakeflow Spark Declarative Pipeline (SDP) to ingest synthetic POS events into Delta, " - "land aggregates in Lakebase for fast lookups, and serve a Prophet-style forecast via Model Serving." - ), - drive_messages=[ - "Proceed: produce architecture.md.", - "Write the per-component specifications.", - "Build it: generate the SDP pipeline, the Lakebase sync, the forecast notebook, and resources.json.", - ], - target_stage="BUILT", - ), -] +def _pipeline_scenario(case) -> Scenario: + return Scenario( + slug=case.id, + description=case.description, + capabilities=list(case.capabilities), + initial_prompt=case.initial_prompt, + drive_messages=list(case.drive_messages), + target_stage=case.target_stage.value, + timeout_seconds=case.timeout_seconds, + ) + + +SCENARIOS: list[Scenario] = [_pipeline_scenario(case) for case in load_cases()] def get_scenarios(slugs: list[str] | None) -> list[Scenario]: - if not slugs: - return list(SCENARIOS) - by_slug = {s.slug: s for s in SCENARIOS} - out: list[Scenario] = [] - for slug in slugs: - if slug not in by_slug: - raise SystemExit( - f"unknown scenario slug: {slug!r}. " - f"available: {', '.join(by_slug)}" - ) - out.append(by_slug[slug]) - return out + by_slug = {scenario.slug: scenario for scenario in SCENARIOS} + selected = list(by_slug) if not slugs else slugs + unknown = [slug for slug in selected if slug not in by_slug] + if unknown: + raise SystemExit( + f"unknown scenario slug: {unknown[0]!r}. available: {', '.join(by_slug)}" + ) + # Fixtures override target/timeout in-place, so return independent copies. + return [ + replace( + by_slug[slug], + capabilities=list(by_slug[slug].capabilities), + drive_messages=list(by_slug[slug].drive_messages), + ) + for slug in selected + ] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..249ecdc --- /dev/null +++ b/uv.lock @@ -0,0 +1,196 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "0.26.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/8e/c4/453c52c659521066969523e87d85d54139bbd17b78f09532fb8eb8cdb58e/pytest_asyncio-0.26.0.tar.gz", hash = "sha256:c4df2a697648241ff39e7f0e4a73050b03f123f760673956cf0d72a4990e312f", upload-time = "2025-03-25T06:22:28.883Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/20/7f/338843f449ace853647ace35870874f69a764d251872ed1b4de9f234822c/pytest_asyncio-0.26.0-py3-none-any.whl", hash = "sha256:7b51ed894f4fbea1340262bdae5135797ebbe21d8638978e35d31c6d19f72fb0", upload-time = "2025-03-25T06:22:27.807Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://pypi-proxy.cloud.databricks.com/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "solution-builder-evaluation" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "pydantic", specifier = ">=2.11,<3" }, + { name = "pyyaml", specifier = ">=6.0,<7" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3,<9" }, + { name = "pytest-asyncio", specifier = ">=0.24,<1" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi-proxy.cloud.databricks.com/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://pypi-proxy.cloud.databricks.com/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", upload-time = "2025-10-01T02:14:40.154Z" }, +] From 3606dfe3d0b464c375661657068d004535bbb07e Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 09:15:56 -0700 Subject: [PATCH 2/7] Keep fork evaluation checks secret-safe --- .github/workflows/skillforge-evaluation.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/skillforge-evaluation.yml b/.github/workflows/skillforge-evaluation.yml index 697a1c9..8f58a05 100644 --- a/.github/workflows/skillforge-evaluation.yml +++ b/.github/workflows/skillforge-evaluation.yml @@ -26,6 +26,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + env: + HAS_QUICK_EVAL_SECRETS: ${{ secrets.SKILLFORGE_READ_TOKEN != '' && secrets.ANTHROPIC_API_KEY != '' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6 @@ -49,6 +51,7 @@ jobs: print(f"ai_dev_kit_revision={lock.ai_dev_kit_revision}") PY - name: Checkout pinned SkillForge + if: env.HAS_QUICK_EVAL_SECRETS == 'true' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: repository: ${{ steps.pins.outputs.skillforge_repo }} @@ -56,14 +59,20 @@ jobs: token: ${{ secrets.SKILLFORGE_READ_TOKEN }} path: .deps/skillforge - name: Checkout pinned ai-dev-kit + if: env.HAS_QUICK_EVAL_SECRETS == 'true' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: repository: ${{ steps.pins.outputs.ai_dev_kit_repo }} ref: ${{ steps.pins.outputs.ai_dev_kit_revision }} path: .deps/ai-dev-kit - name: Install pinned external SkillForge + if: env.HAS_QUICK_EVAL_SECRETS == 'true' run: uv tool install '.deps/skillforge/python[all]' + - name: Note unavailable fork secrets + if: env.HAS_QUICK_EVAL_SECRETS != 'true' + run: echo "External L1/L3 evaluation skipped because fork-safe maintainer secrets are unavailable." - name: Informational L1/L3 evaluation + if: env.HAS_QUICK_EVAL_SECRETS == 'true' id: quick-eval continue-on-error: true env: From acece95cf854d44350eae2b2717fa6fdf1c4e83d Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 10:31:52 -0700 Subject: [PATCH 3/7] Recognize OAuth M2M evaluation profiles --- evaluation/live.py | 89 +++++++++++++++++++++++++------- tests/evaluation/test_cleanup.py | 26 ++++++++++ 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/evaluation/live.py b/evaluation/live.py index 14815d0..c669752 100644 --- a/evaluation/live.py +++ b/evaluation/live.py @@ -129,26 +129,28 @@ def from_env(cls, evaluation_prefix: str = "sb_eval_") -> "LivePolicy": ) if not allowed_hosts: raise ValueError("--live requires SB_EVAL_ALLOWED_HOSTS") - host = os.environ.get("DATABRICKS_HOST", "").rstrip("/") + auth_context = subprocess.run( + ["databricks", "auth", "env", "--profile", profile], + check=False, + capture_output=True, + text=True, + timeout=20, + ) + if auth_context.returncode != 0: + raise ValueError("could not resolve the live evaluation profile") + try: + auth_payload = json.loads(auth_context.stdout) + profile_env = auth_payload.get("env", {}) + except json.JSONDecodeError as exc: + raise ValueError("databricks auth env returned invalid JSON") from exc + if not isinstance(profile_env, dict): + raise ValueError("databricks auth env returned an invalid environment") + host = ( + os.environ.get("DATABRICKS_HOST", "") + or str(profile_env.get("DATABRICKS_HOST", "")) + ).rstrip("/") if not host: - completed = subprocess.run( - ["databricks", "auth", "env", "--profile", profile], - check=False, - capture_output=True, - text=True, - timeout=20, - ) - if completed.returncode != 0: - raise ValueError( - "could not resolve host for the live evaluation profile" - ) - try: - payload = json.loads(completed.stdout) - host = str(payload.get("env", {}).get("DATABRICKS_HOST", "")).rstrip( - "/" - ) - except json.JSONDecodeError as exc: - raise ValueError("databricks auth env returned invalid JSON") from exc + raise ValueError("could not resolve host for the live evaluation profile") if host not in allowed_hosts: raise ValueError(f"workspace host {host!r} is not allowlisted") identity = subprocess.run( @@ -178,7 +180,54 @@ def from_env(cls, evaluation_prefix: str = "sb_eval_") -> "LivePolicy": if isinstance(identity_payload, dict) else None ) - if not application_id: + user_name = ( + identity_payload.get("user_name") or identity_payload.get("userName") + if isinstance(identity_payload, dict) + else None + ) + profile_client_id = profile_env.get("DATABRICKS_CLIENT_ID") + is_service_principal = bool(application_id) or bool( + profile_client_id and user_name == profile_client_id + ) + if ( + not is_service_principal + and isinstance(user_name, str) + and re.fullmatch( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", + user_name, + ) + ): + lookup = subprocess.run( + [ + "databricks", + "service-principals", + "list", + "--profile", + profile, + "--filter", + f'applicationId eq "{user_name}"', + "-o", + "json", + ], + check=False, + capture_output=True, + text=True, + timeout=20, + ) + if lookup.returncode == 0: + try: + principals = json.loads(lookup.stdout) + except json.JSONDecodeError: + principals = [] + if isinstance(principals, list): + is_service_principal = any( + isinstance(item, dict) + and (item.get("application_id") or item.get("applicationId")) + == user_name + and item.get("active", True) + for item in principals + ) + if not is_service_principal: raise ValueError( "--live requires credentials for a dedicated Databricks service principal" ) diff --git a/tests/evaluation/test_cleanup.py b/tests/evaluation/test_cleanup.py index b03e553..8a7e593 100644 --- a/tests/evaluation/test_cleanup.py +++ b/tests/evaluation/test_cleanup.py @@ -144,6 +144,32 @@ def test_live_policy_rejects_human_identity( LivePolicy.from_env() +def test_live_policy_accepts_oauth_m2m_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _set_live_env(monkeypatch) + client_id = "f8854bec-358f-40d8-bd30-ea315cc43411" + + def fake_subprocess_run(args, **kwargs): + if args[1:3] == ["auth", "env"]: + return _completed( + args, + stdout=json.dumps( + { + "env": { + "DATABRICKS_HOST": "https://eval.example.com", + "DATABRICKS_AUTH_TYPE": "oauth-m2m", + "DATABRICKS_CLIENT_ID": client_id, + } + } + ), + ) + return _completed(args, stdout=json.dumps({"userName": client_id})) + + monkeypatch.setattr("evaluation.live.subprocess.run", fake_subprocess_run) + assert LivePolicy.from_env().profile == "solution-builder-eval" + + def test_manifest_normalizes_schema_and_lakebase_slug(tmp_path: Path) -> None: path = tmp_path / "resources.json" path.write_text( From bd85186108c1f0cf16751014e7417ba5c2411e90 Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 10:37:54 -0700 Subject: [PATCH 4/7] Pin live SkillForge workspace routing --- evaluation/runner.py | 36 ++++++++++++++++++++++- tests/evaluation/test_runner.py | 52 +++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tests/evaluation/test_runner.py diff --git a/evaluation/runner.py b/evaluation/runner.py index c698872..f49d3b9 100644 --- a/evaluation/runner.py +++ b/evaluation/runner.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Any +import yaml + from evaluation.adapter import SkillForgeAdapter from evaluation.fixture import build_fixture from evaluation.hashing import hash_skill @@ -86,6 +88,35 @@ def _git_sha(repo_root: Path) -> str: return completed.stdout.strip() +def _write_live_skillforge_config(home: Path, policy: LivePolicy) -> Path: + """Pin SkillForge's workspace and MLflow routing for a guarded live run.""" + experiment = os.environ.get( + "SB_EVAL_MLFLOW_EXPERIMENT", "/Shared/sb_eval_skillforge_evals" + ).strip() + experiment_name = experiment.rsplit("/", 1)[-1] + if not experiment.startswith("/") or not experiment_name.startswith( + policy.evaluation_prefix + ): + raise ValueError( + "live MLflow experiment must be an absolute path whose name starts " + f"with {policy.evaluation_prefix!r}" + ) + home.mkdir(parents=True, exist_ok=True) + path = home / "config.yaml" + path.write_text( + yaml.safe_dump( + { + "databricks": {"profile": policy.profile}, + "mlflow": {"enabled": True, "experiment": experiment}, + "llm": {"backend": "fmapi"}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + return path + + def _load_tracked_resources(home: Path) -> list[ResourceRecord]: resources: list[ResourceRecord] = [] for path in home.glob("runs/**/resources.json"): @@ -248,7 +279,10 @@ def run_scenario( if live: env["SB_EVAL_LIVE"] = "1" # Validate before the external runner can make a tool call. - LivePolicy.from_env(scenario.live_resources.evaluation_prefix) + policy = LivePolicy.from_env(scenario.live_resources.evaluation_prefix) + _write_live_skillforge_config(skillforge_home, policy) + env["DATABRICKS_CONFIG_PROFILE"] = policy.profile + env["DATABRICKS_HOST"] = policy.host completed = subprocess.run( command, cwd=repo_root, diff --git a/tests/evaluation/test_runner.py b/tests/evaluation/test_runner.py new file mode 100644 index 0000000..1c003ba --- /dev/null +++ b/tests/evaluation/test_runner.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from evaluation.live import LivePolicy +from evaluation.runner import _write_live_skillforge_config + + +def _policy() -> LivePolicy: + return LivePolicy( + profile="solution-builder-test", + host="https://dais-demo.cloud.databricks.com", + allowed_hosts=("https://dais-demo.cloud.databricks.com",), + evaluation_prefix="sb_eval_", + ) + + +def test_live_skillforge_config_pins_profile_and_mlflow( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv( + "SB_EVAL_MLFLOW_EXPERIMENT", "/Shared/sb_eval_skillforge_acceptance" + ) + + path = _write_live_skillforge_config(tmp_path / "skillforge-home", _policy()) + + assert yaml.safe_load(path.read_text(encoding="utf-8")) == { + "databricks": {"profile": "solution-builder-test"}, + "mlflow": { + "enabled": True, + "experiment": "/Shared/sb_eval_skillforge_acceptance", + }, + "llm": {"backend": "fmapi"}, + } + + +@pytest.mark.parametrize( + "experiment", + ["/Shared/skillforge-evals", "sb_eval_relative", "/Shared/production"], +) +def test_live_skillforge_config_rejects_unsafe_experiment( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + experiment: str, +) -> None: + monkeypatch.setenv("SB_EVAL_MLFLOW_EXPERIMENT", experiment) + + with pytest.raises(ValueError, match="absolute path whose name starts"): + _write_live_skillforge_config(tmp_path, _policy()) From 147c12fc7552df1f4d94f6d59716fe2887a34374 Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 10:49:16 -0700 Subject: [PATCH 5/7] Isolate pinned SkillForge run state --- evaluation/runner.py | 7 +++- .../skillforge_runtime/sitecustomize.py | 22 +++++++++++ tests/evaluation/test_runner.py | 37 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 evaluation/skillforge_runtime/sitecustomize.py diff --git a/evaluation/runner.py b/evaluation/runner.py index f49d3b9..09fcf70 100644 --- a/evaluation/runner.py +++ b/evaluation/runner.py @@ -274,7 +274,6 @@ def run_scenario( if judge_model: command.extend(["--judge-model", judge_model]) env = os.environ.copy() - env["SKILLFORGE_HOME"] = str(skillforge_home) env["SB_EVAL_PREFIX"] = scenario.live_resources.evaluation_prefix if live: env["SB_EVAL_LIVE"] = "1" @@ -283,6 +282,12 @@ def run_scenario( _write_live_skillforge_config(skillforge_home, policy) env["DATABRICKS_CONFIG_PROFILE"] = policy.profile env["DATABRICKS_HOST"] = policy.host + env["SKILLFORGE_HOME"] = str(skillforge_home) + shim_dir = Path(__file__).resolve().parent / "skillforge_runtime" + python_path = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = os.pathsep.join( + item for item in (str(shim_dir), python_path) if item + ) completed = subprocess.run( command, cwd=repo_root, diff --git a/evaluation/skillforge_runtime/sitecustomize.py b/evaluation/skillforge_runtime/sitecustomize.py new file mode 100644 index 0000000..7312379 --- /dev/null +++ b/evaluation/skillforge_runtime/sitecustomize.py @@ -0,0 +1,22 @@ +"""Keep pinned SkillForge run state inside the adapter-owned transient home.""" + +from __future__ import annotations + +import os +from pathlib import Path + + +def _patch_skillforge_run_roots() -> None: + home = os.environ.get("SKILLFORGE_HOME", "").strip() + if not home: + return + try: + from skillforge.eval import run_isolation, unified_runner + except ImportError: + return + runs = Path(home) / "runs" + run_isolation._SKILLFORGE_RUNS = runs + unified_runner._SKILLFORGE_RUNS = runs + + +_patch_skillforge_run_roots() diff --git a/tests/evaluation/test_runner.py b/tests/evaluation/test_runner.py index 1c003ba..159b712 100644 --- a/tests/evaluation/test_runner.py +++ b/tests/evaluation/test_runner.py @@ -1,5 +1,8 @@ from __future__ import annotations +import runpy +import sys +import types from pathlib import Path import pytest @@ -50,3 +53,37 @@ def test_live_skillforge_config_rejects_unsafe_experiment( with pytest.raises(ValueError, match="absolute path whose name starts"): _write_live_skillforge_config(tmp_path, _policy()) + + +def test_skillforge_runtime_shim_redirects_hard_coded_run_roots( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + skillforge = types.ModuleType("skillforge") + skillforge_eval = types.ModuleType("skillforge.eval") + run_isolation = types.ModuleType("skillforge.eval.run_isolation") + unified_runner = types.ModuleType("skillforge.eval.unified_runner") + run_isolation._SKILLFORGE_RUNS = Path.home() / ".skillforge" / "runs" + unified_runner._SKILLFORGE_RUNS = Path.home() / ".skillforge" / "runs" + skillforge_eval.run_isolation = run_isolation + skillforge_eval.unified_runner = unified_runner + monkeypatch.setitem(sys.modules, "skillforge", skillforge) + monkeypatch.setitem(sys.modules, "skillforge.eval", skillforge_eval) + monkeypatch.setitem( + sys.modules, "skillforge.eval.run_isolation", run_isolation + ) + monkeypatch.setitem( + sys.modules, "skillforge.eval.unified_runner", unified_runner + ) + monkeypatch.setenv("SKILLFORGE_HOME", str(tmp_path / "skillforge-home")) + + shim = ( + Path(__file__).resolve().parents[2] + / "evaluation" + / "skillforge_runtime" + / "sitecustomize.py" + ) + runpy.run_path(shim) + + expected = tmp_path / "skillforge-home" / "runs" + assert run_isolation._SKILLFORGE_RUNS == expected + assert unified_runner._SKILLFORGE_RUNS == expected From ed31f07b247238306df4a6aae412b760e183ef97 Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 10:57:52 -0700 Subject: [PATCH 6/7] Allow skill invocation in SkillForge runs --- evaluation/adapter.py | 10 +++++++++- evaluation/live.py | 6 ++++++ .../golden/financial-services.ground-truth.yaml | 4 ++++ tests/evaluation/test_adapter.py | 1 + tests/evaluation/test_cleanup.py | 13 +++++++++++++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/evaluation/adapter.py b/evaluation/adapter.py index d24f5e9..53a872d 100644 --- a/evaluation/adapter.py +++ b/evaluation/adapter.py @@ -129,7 +129,15 @@ def _ground_truth(self, scenario: Scenario, *, live: bool) -> dict[str, Any]: "assertions": step.semantic_assertions, "expected_patterns": step.expected_patterns, "trace_expectations": { - "required_tools": step.tool_expectations.required, + # SkillForge derives the agent's allowed tool list + # from this field. Keep the canonical expectations + # and add the built-in Skill tool so WITH runs can + # actually invoke the skill under evaluation. + "required_tools": list( + dict.fromkeys( + ["Skill", *step.tool_expectations.required] + ) + ), "banned_tools": step.tool_expectations.banned, "token_budget": {"max_total": 80_000}, }, diff --git a/evaluation/live.py b/evaluation/live.py index c669752..c7b8b65 100644 --- a/evaluation/live.py +++ b/evaluation/live.py @@ -353,6 +353,12 @@ def resources_from_tracked(rows: Iterable[dict[str, Any]]) -> list[ResourceRecor continue resource_type = str(resource_type) resource_id = str(resource_id) + if resource_id.strip().lower() in {"unknown", "none", "null"}: + # SkillForge may emit medium-confidence activity detections before + # a CLI response contains an actual identifier. They are useful + # diagnostics but are not actionable resources and must not count + # toward expected-kind checks or cleanup. + continue resource_name = str(row.get("name") or resource_id) if resource_type == "lakebase": # Lakebase APIs delete by projects/{slug}; tool results may expose diff --git a/tests/evaluation/golden/financial-services.ground-truth.yaml b/tests/evaluation/golden/financial-services.ground-truth.yaml index b2e4453..456c2aa 100644 --- a/tests/evaluation/golden/financial-services.ground-truth.yaml +++ b/tests/evaluation/golden/financial-services.ground-truth.yaml @@ -22,6 +22,7 @@ test_cases: - (?i)(revenue|loss|risk|\$) trace_expectations: required_tools: + - Skill - Read - Write banned_tools: [] @@ -60,6 +61,7 @@ test_cases: - (?i)(genie|dashboard) trace_expectations: required_tools: + - Skill - Read - Write banned_tools: [] @@ -99,6 +101,7 @@ test_cases: - (?i)(transaction|anomal) trace_expectations: required_tools: + - Skill - Read - Write banned_tools: [] @@ -140,6 +143,7 @@ test_cases: - (?i)(sql|python|dashboard|genie) trace_expectations: required_tools: + - Skill - Read - Write banned_tools: [] diff --git a/tests/evaluation/test_adapter.py b/tests/evaluation/test_adapter.py index 5d20174..f8ade52 100644 --- a/tests/evaluation/test_adapter.py +++ b/tests/evaluation/test_adapter.py @@ -63,3 +63,4 @@ def test_v5_contains_sources_regression_and_expectations(tmp_path: Path) -> None assert expectations["assertions"] assert expectations["expected_patterns"] assert expectations["trace_expectations"] + assert "Skill" in expectations["trace_expectations"]["required_tools"] diff --git a/tests/evaluation/test_cleanup.py b/tests/evaluation/test_cleanup.py index 8a7e593..df1f6bb 100644 --- a/tests/evaluation/test_cleanup.py +++ b/tests/evaluation/test_cleanup.py @@ -215,6 +215,19 @@ def test_tracked_comparison_sides_are_normalized() -> None: assert resources[-1].key == "lakebase:projects/sb_eval_lakebase" +def test_tracked_resources_ignore_unresolved_activity_detections() -> None: + assert resources_from_tracked( + [ + { + "asset_type": "catalog", + "asset_id": "unknown", + "confidence": "medium", + "detection_source": "cli", + } + ] + ) == [] + + def test_capabilities_derive_resource_kinds_with_overrides() -> None: assert expected_resource_kinds( ["sdp", "ml-training-serving", "vector-search"], overrides=["volume"] From 2fd76fe0138d726f33687c63441d94844fa2e612 Mon Sep 17 00:00:00 2001 From: CAholder Date: Fri, 17 Jul 2026 12:19:22 -0700 Subject: [PATCH 7/7] Harden live evaluation reconciliation --- evaluation/live.py | 31 +++++++++++++++- evaluation/runner.py | 8 +++-- .../skillforge_runtime/sitecustomize.py | 10 ++++++ tests/evaluation/test_cleanup.py | 22 +++++++++--- tests/evaluation/test_runner.py | 36 ++++++++++++++++++- 5 files changed, 98 insertions(+), 9 deletions(-) diff --git a/evaluation/live.py b/evaluation/live.py index c7b8b65..58abbcd 100644 --- a/evaluation/live.py +++ b/evaluation/live.py @@ -281,6 +281,16 @@ def key(self) -> str: "mlflow_experiment_path": "mlflow_experiment", } +RESOURCE_NAME_KEYS: dict[str, str] = { + "pipeline_id": "pipeline_name", + "dashboard_id": "dashboard_name", + "genie_space_id": "genie_space_name", + "knowledge_assistant_id": "knowledge_assistant_name", + "knowledge_assistant_endpoint": "knowledge_assistant_endpoint", + "multi_agent_supervisor_id": "multi_agent_supervisor_name", + "multi_agent_supervisor_endpoint": "multi_agent_supervisor_endpoint", +} + def resources_from_manifest( path: Path, *, side: str | None = None @@ -298,11 +308,14 @@ def resources_from_manifest( resource_id = str(value) if key == "schema" and created.get("catalog") and "." not in resource_id: resource_id = f"{created['catalog']}.{resource_id}" + resource_name = str( + created.get(RESOURCE_NAME_KEYS.get(key, "")) or resource_id + ) records.append( ResourceRecord( resource_type=resource_type, resource_id=resource_id, - name=resource_id, + name=resource_name, side=side, ) ) @@ -551,6 +564,22 @@ def _require_evaluation_scope(self, resource: ResourceRecord) -> None: elif resource.resource_type in {"cluster", "warehouse", "vector_endpoint"}: scoped_parts = (resource.name or rid,) minimum_parts = 1 + elif resource.resource_type in { + "app", + "serving_endpoint", + "pipeline", + "job", + "dashboard", + "genie_space", + "knowledge_assistant", + "multi_agent_supervisor", + }: + name = (resource.name or "").strip() + if not name.startswith(prefix): + raise RuntimeError( + f"refusing to delete {resource.key}: outside evaluation prefix {prefix!r}" + ) + return elif resource.resource_type in {"notebook", "mlflow_experiment"}: path_parts = tuple( part for part in (resource.name or rid).split("/") if part diff --git a/evaluation/runner.py b/evaluation/runner.py index 09fcf70..a00e50d 100644 --- a/evaluation/runner.py +++ b/evaluation/runner.py @@ -141,7 +141,8 @@ def _load_tracked_resources(home: Path) -> list[ResourceRecord]: def _load_project_resources(search_root: Path) -> list[ResourceRecord]: resources: list[ResourceRecord] = [] for path in search_root.glob("**/resources.json"): - if ".skillforge" in path.parts or "skillforge-home" in path.parts: + relative_parts = path.relative_to(search_root).parts + if ".skillforge" in relative_parts or ".claude" in relative_parts: continue try: resources.extend(resources_from_manifest(path)) @@ -348,7 +349,10 @@ def run_scenario( ) tracked = _load_tracked_resources(skillforge_home) - manifests = _load_project_resources(scenario_dir) + # Agent-authored manifests live in the per-side run cwd. Never scan the + # copied fixture skill: it contains reference-demo resources.json files + # that are examples, not resources created by this evaluation. + manifests = _load_project_resources(skillforge_home / "runs") resources = reconcile_resources(tracked, manifests) cleanup = CleanupStatus(attempted=live, complete=not live) status = "passed" diff --git a/evaluation/skillforge_runtime/sitecustomize.py b/evaluation/skillforge_runtime/sitecustomize.py index 7312379..00dc74e 100644 --- a/evaluation/skillforge_runtime/sitecustomize.py +++ b/evaluation/skillforge_runtime/sitecustomize.py @@ -17,6 +17,16 @@ def _patch_skillforge_run_roots() -> None: runs = Path(home) / "runs" run_isolation._SKILLFORGE_RUNS = runs unified_runner._SKILLFORGE_RUNS = runs + # The pinned revision's legacy workspace loader does not honor + # SKILLFORGE_HOME and otherwise reads ~/.skillforge/config.yaml. Point it + # at the same transient config used by skillforge.config.get_config(). + try: + from skillforge.databricks import auth + except ImportError: + return + config_root = Path(home) + auth._SF_CONFIG_DIR = config_root + auth._SF_CONFIG_PATH = config_root / "config.yaml" _patch_skillforge_run_roots() diff --git a/tests/evaluation/test_cleanup.py b/tests/evaluation/test_cleanup.py index df1f6bb..49a5023 100644 --- a/tests/evaluation/test_cleanup.py +++ b/tests/evaluation/test_cleanup.py @@ -330,15 +330,17 @@ def test_capabilities_derive_resource_kinds_with_overrides() -> None: ], ), ( - ResourceRecord("dashboard", "dashboard-id"), + ResourceRecord("dashboard", "dashboard-id", name="sb_eval_dashboard"), ["databricks", "lakeview", "trash", "dashboard-id"], ), ( - ResourceRecord("genie_space", "space-id"), + ResourceRecord("genie_space", "space-id", name="sb_eval_genie"), ["databricks", "genie", "trash-space", "space-id"], ), ( - ResourceRecord("knowledge_assistant", "ka-id"), + ResourceRecord( + "knowledge_assistant", "ka-id", name="sb_eval_knowledge_assistant" + ), [ "databricks", "knowledge-assistants", @@ -347,7 +349,9 @@ def test_capabilities_derive_resource_kinds_with_overrides() -> None: ], ), ( - ResourceRecord("multi_agent_supervisor", "mas-id"), + ResourceRecord( + "multi_agent_supervisor", "mas-id", name="sb_eval_supervisor" + ), [ "databricks", "api", @@ -398,7 +402,7 @@ def fake_subprocess_run(args, **kwargs): monkeypatch.setattr("evaluation.live.subprocess.run", fake_subprocess_run) backend = DatabricksCliCleaner(_policy()) report = ResourceCleaner(backend.delete, backend.exists, retries=1).cleanup( - [ResourceRecord("pipeline", "pipeline-id")] + [ResourceRecord("pipeline", "pipeline-id", name="sb_eval_pipeline")] ) assert report.complete assert [call[1:3] for call in calls] == [ @@ -448,3 +452,11 @@ def test_destructive_uc_cleanup_refuses_out_of_scope_name() -> None: backend.delete(ResourceRecord("table", "production.default.customers")) with pytest.raises(RuntimeError, match="outside evaluation prefix"): backend.delete(ResourceRecord("mlflow_experiment", "/Shared/production")) + with pytest.raises(RuntimeError, match="outside evaluation prefix"): + backend.delete( + ResourceRecord("pipeline", "opaque-id", name="production-pipeline") + ) + with pytest.raises(RuntimeError, match="outside evaluation prefix"): + backend.delete( + ResourceRecord("dashboard", "opaque-id", name="production-dashboard") + ) diff --git a/tests/evaluation/test_runner.py b/tests/evaluation/test_runner.py index 159b712..2319a49 100644 --- a/tests/evaluation/test_runner.py +++ b/tests/evaluation/test_runner.py @@ -9,7 +9,7 @@ import yaml from evaluation.live import LivePolicy -from evaluation.runner import _write_live_skillforge_config +from evaluation.runner import _load_project_resources, _write_live_skillforge_config def _policy() -> LivePolicy: @@ -62,10 +62,15 @@ def test_skillforge_runtime_shim_redirects_hard_coded_run_roots( skillforge_eval = types.ModuleType("skillforge.eval") run_isolation = types.ModuleType("skillforge.eval.run_isolation") unified_runner = types.ModuleType("skillforge.eval.unified_runner") + skillforge_databricks = types.ModuleType("skillforge.databricks") + auth = types.ModuleType("skillforge.databricks.auth") run_isolation._SKILLFORGE_RUNS = Path.home() / ".skillforge" / "runs" unified_runner._SKILLFORGE_RUNS = Path.home() / ".skillforge" / "runs" + auth._SF_CONFIG_DIR = Path.home() / ".skillforge" + auth._SF_CONFIG_PATH = auth._SF_CONFIG_DIR / "config.yaml" skillforge_eval.run_isolation = run_isolation skillforge_eval.unified_runner = unified_runner + skillforge_databricks.auth = auth monkeypatch.setitem(sys.modules, "skillforge", skillforge) monkeypatch.setitem(sys.modules, "skillforge.eval", skillforge_eval) monkeypatch.setitem( @@ -74,6 +79,8 @@ def test_skillforge_runtime_shim_redirects_hard_coded_run_roots( monkeypatch.setitem( sys.modules, "skillforge.eval.unified_runner", unified_runner ) + monkeypatch.setitem(sys.modules, "skillforge.databricks", skillforge_databricks) + monkeypatch.setitem(sys.modules, "skillforge.databricks.auth", auth) monkeypatch.setenv("SKILLFORGE_HOME", str(tmp_path / "skillforge-home")) shim = ( @@ -87,3 +94,30 @@ def test_skillforge_runtime_shim_redirects_hard_coded_run_roots( expected = tmp_path / "skillforge-home" / "runs" assert run_isolation._SKILLFORGE_RUNS == expected assert unified_runner._SKILLFORGE_RUNS == expected + assert auth._SF_CONFIG_DIR == tmp_path / "skillforge-home" + assert auth._SF_CONFIG_PATH == tmp_path / "skillforge-home" / "config.yaml" + + +def test_project_resource_scan_excludes_copied_skills(tmp_path: Path) -> None: + authored = tmp_path / "comparison" / "a" / "with" / "resources.json" + authored.parent.mkdir(parents=True) + authored.write_text( + '{"catalog":"sb_eval_created"}\n', encoding="utf-8" + ) + reference = ( + tmp_path + / "comparison" + / "a" + / "with" + / ".claude" + / "skills" + / "demo" + / "references" + / "resources.json" + ) + reference.parent.mkdir(parents=True) + reference.write_text('{"catalog":"production"}\n', encoding="utf-8") + + assert [resource.key for resource in _load_project_resources(tmp_path)] == [ + "catalog:sb_eval_created" + ]